Change some practice

2026-06-27 13:50:24 +00:00 · 2021-09-27 16:59:35 +03:00
parent f44beb51a9
commit f41e7a6611
187 changed files with 33 additions and 72 deletions
@@ -0,0 +1,70 @@
+variables:
+  K8S_API_URL: https://172.20.100.2:6443
+
+stages:
+  - build
+  - test
+  - cleanup
+  - push
+  - deploy
+
+build:
+  stage: build
+  script:
+    - docker build -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_REF_SLUG.$CI_PIPELINE_ID .
+
+test:
+  stage: test
+  image:
+    name: docker/compose:1.27.4
+    entrypoint: [""]
+  script:
+    - docker-compose 
+        -p "$CI_PROJECT_NAME"_"$CI_PIPELINE_ID"
+      up
+        --abort-on-container-exit
+        --exit-code-from app
+        --quiet-pull
+
+cleanup:
+  stage: cleanup
+  image:
+    name: docker/compose:1.27.4
+    entrypoint: [""]
+  script:
+    - docker-compose -p "$CI_PROJECT_NAME"_"$CI_PIPELINE_ID" down
+  when: always
+
+push:
+  stage: push
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+  script:
+    - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_REF_SLUG.$CI_PIPELINE_ID
+  only:
+    - master
+
+deploy:
+  stage: deploy
+  image: centosadmin/kubernetes-helm:3.3.4
+  environment:
+    name: production
+  resource_group: deploy_production
+  script:
+    - kubectl config set-cluster k8s --insecure-skip-tls-verify=true --server=$K8S_API_URL
+    - kubectl config set-credentials ci --token=$K8S_CI_TOKEN
+    - kubectl config set-context ci --cluster=k8s --user=ci
+    - kubectl config use-context ci
+    - helm upgrade --install $CI_PROJECT_PATH_SLUG .helm
+        --set image=$CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME
+        --set imageTag=$CI_COMMIT_REF_SLUG.$CI_PIPELINE_ID
+        --timeout 180s
+        --atomic
+        --debug
+        --namespace $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME
+  after_script:
+    - kubectl -n $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME logs -lcomponent=atomiclog --tail=-1
+    - kubectl -n $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME delete job -lcomponent=atomiclog
+  only:
+    - master
+
@@ -0,0 +1,38 @@
+# Добавляем helm hook
+
+## 1. Добавляем манифест job
+
+```bash
+cp job.yaml ~/xpaste/.helm/templates/job.yaml
+
+cd ~/xpaste
+```
+
+## 2. Добавляем просмотр результатов работы job в CI/CD
+
+Добавляем в `.gitlab-ci.yml` в шаг `deploy:` раздел `after_script:`
+
+```yaml
+deploy:
+  ...
+  after_script:
+    - kubectl -n $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME logs -lcomponent=atomiclog --tail=-1
+    - kubectl -n $CI_PROJECT_PATH_SLUG-$CI_ENVIRONMENT_NAME delete job -lcomponent=atomiclog
+```
+
+## 3. Пушим, смотрим в вывод CI/CD
+
+```
+git add -A
+git commit -am "Add job template for getting k8s logs if deploy has failed"
+git push
+
+```
+
+## 4. Исправление настроек приложения
+
+Ищем ошибку в выводе логов пода.
+
+* Для исправления ошибки в работе приложения необходимо внести изменения в `values.yml` чарта, описанные в [snippet](https://gitlab.slurm.io/-/snippets/83)
+
+Для проверки открываем в браузере URL: `http://xpaste.s<Ваш номер логина>.k8s.slurm.io`. `<Ваш номер логина>` необходимо заменить на номер своего студента. Открывать нужно в режиме `инкогнито`. Теперь приложение должно быть доступно.
@@ -0,0 +1,43 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ .Release.Name }}-atomiclog"
+  annotations:
+    "helm.sh/hook": pre-delete,pre-rollback
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": before-hook-creation
+  labels:
+    app: {{ .Chart.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+    component: atomiclog
+spec:
+  activeDeadlineSeconds: 100
+  template:
+    metadata:
+      labels:
+        app: {{ .Chart.Name }}
+        release: {{ .Release.Name }}
+        component: atomiclog
+    spec:
+      containers:
+      - name: atomiclog
+        image: centosadmin/kubernetes-helm:3.3.4
+        imagePullPolicy: IfNotPresent
+        command:
+        - /bin/bash
+        args:
+        - /usr/local/bin/hooklog.sh
+        - {{ .Release.Namespace }}
+        - {{ .Release.Name }}
+        resources:
+          limits:
+            cpu: 50m
+            memory: 256Mi
+          requests:
+            cpu: 50m
+            memory: 128Mi
+      dnsPolicy: ClusterFirst
+      restartPolicy: Never
+      serviceAccountName: {{ .Release.Namespace }}
@@ -0,0 +1,111 @@
+#!/bin/bash
+set -u
+#
+# file: https://galaxy.southbridge.io/templates/antools/-/blob/master/scripts/hooklog.sh
+# version: 0.1.0
+#
+# Use this script with Helm pre-rollback hook for logs and events printing.
+#
+# Nikolay Mesropyan and Southbridge LLC team, 2020 A.D.
+#
+
+_main() {
+    local fn=${FUNCNAME[0]}
+
+    trap '_except $LINENO' ERR
+
+    if [[ "${1:-NOP}" != NOP ]]; then
+	local ns="$1"
+    else
+	_help; exit 0
+    fi
+
+    if [[ "${2:-NOP}" != NOP ]]; then
+	local release="$2"
+    else
+	_help; exit 0
+    fi
+
+    printf '\033[1;31m%s\033[1;35m' "Get pods status: "
+    printf -- '-%.0s' {1..130}
+    printf '\033[0m\n'
+    kubectl -n "$ns" get po -lrelease="$release" -o wide | grep -Fv atomiclog
+
+    printf '\033[1;31m%s\033[1;35m' "Tail of overall 'Warning' events: "
+    printf -- '-%.0s' {1..108}
+    printf '\033[0m\n'
+    kubectl -n "$ns" get events --field-selector type=Warning --sort-by='.metadata.creationTimestamp' | tail
+
+    local -a Daemonsets=() Deployments=() Jobs=() Statefulsets=()
+
+    mapfile -t Daemonsets < <( kubectl -n "$ns" get daemonset -lrelease="$release" --no-headers -o custom-columns=":metadata.name" )
+    mapfile -t Deployments < <( kubectl -n "$ns" get deployment -lrelease="$release" --no-headers -o custom-columns=":metadata.name" )
+    mapfile -t Jobs < <( kubectl -n "$ns" get job -lrelease="$release" --no-headers -o custom-columns=":metadata.name" )
+    mapfile -t Statefulsets < <( kubectl -n "$ns" get statefulset -lrelease="$release" --no-headers -o custom-columns=":metadata.name" )
+
+    for (( i = 0; i < ${#Daemonsets[@]}; i++ )); do
+	__not_ready DaemonSet "${Daemonsets[i]}"
+    done
+
+    for (( i = 0; i < ${#Deployments[@]}; i++ )); do
+	__not_ready Deployment "${Deployments[i]}"
+    done
+
+    for (( i = 0; i < ${#Jobs[@]}; i++ )); do
+	__not_ready Job "${Jobs[i]}"
+    done
+
+    for (( i = 0; i < ${#Statefulsets[@]}; i++ )); do
+	__not_ready StatefulSet "${Statefulsets[i]}"
+    done
+
+    exit 0
+}
+
+__not_ready() {
+    local not_ready_pod=""
+    local text="of the first not-ready pod"
+
+    not_ready_pod=$(kubectl -n "$ns" get po -lrelease="$release" --no-headers | grep "^$2" \
+	| gawk -F' *|/' '$4 !~ "Completed|Evicted" { print $0 }' \
+	| gawk -F' *|/' '$2 != $3 || $4 != "Running" { print $1; exit }')
+
+    if [[ -n "$not_ready_pod" ]]; then
+	__events "$@"
+	__logs "$@"
+    fi
+}
+
+__events() {
+    printf '\033[1;31m%s\033[1;35m' "$1 ${2}: events ${text}: "
+    printf -- '-%.0s' {1..76}
+    printf '\033[0m\n'
+    kubectl -n "$ns" get events --field-selector involvedObject.name="$not_ready_pod" || :
+}
+
+__logs() {
+    local -a Containers=()
+
+    mapfile -t Containers < <( kubectl -n "$ns" get po "$not_ready_pod" --no-headers -o jsonpath="{.spec.containers[*].name}" | sed 's/\s\+/\n/g' )
+
+    for (( i = 0; i < ${#Containers[@]}; i++ )); do
+	printf '\033[1;31m%s\033[1;35m' "$1 ${2}: logs ${text}, container '${Containers[i]}': "
+	printf -- '-%.0s' {1..56}
+	printf '\033[0m\n'
+	kubectl -n "$ns" logs "$not_ready_pod" "${Containers[i]}" --ignore-errors=true --tail=-1 || :
+    done
+}
+
+_except() {
+    local ret=$?
+    local no=${1:-no_line}
+
+    echo "error occured in function '$fn' near line ${no}, exit code ${ret}. Continuing..."
+}
+
+_help() {
+    echo "Usage: $0 <metadata.namespace> <metadata.labels.release>" >&2
+}
+
+_main "$@"
+