Merge branch 'refactor-ci' into 'develop'

Refactor ci

See merge request !202

.gitlab-ci.yml

@@ -2,14 +2,6 @@ variables:
   SECURE_ANALYZERS_PREFIX: registry.gitlab.com/gitlab-org/security-products/analyzers
   DOCKER_IMAGE_SHA: "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}"
 
-workflow:
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-    - if: '$CI_PIPELINE_SOURCE == "schedule"'
-    - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH != "develop" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
-      when: never
-    - if: '$CI_COMMIT_BRANCH'
-
 stages:
   - .pre
   - build
@@ -19,15 +11,22 @@ stages:
   - deploy
   - .post
 
+workflow:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: never
+    - when: always
+
 default:
   before_script:
     - git config --global url."https://$GO_MODULES_USER:$GO_MODULES_ACCESS_TOKEN@code.fbi.h-da.de".insteadOf "https://code.fbi.h-da.de"
 
 include:
-  - local: '/build/ci/.code-quality-ci.yml'
-  - local: '/build/ci/.security-and-compliance-ci.yml'
-  - local: '/build/ci/.build-container.yml'
-  - local: '/build/ci/.test.yml'
-  - local: '/build/ci/.containerlab-ci.yml'
-  - local: '/build/ci/.uml-autogen-ci.yml'
-  - local: '/build/ci/.deploy-k8s.yml'
+  - local: '/.gitlab/ci/.build-container.yml'
+  - local: '/.gitlab/ci/.code-quality-ci.yml'
+  - local: '/.gitlab/ci/.security-and-compliance-ci.yml'
+  - local: '/.gitlab/ci/.test.yml'
+  - local: '/.gitlab/ci/.containerlab-ci.yml'
+  - local: '/.gitlab/ci/.integration-test.yml'
+  - local: '/.gitlab/ci/.uml-autogen-ci.yml'
+  - local: '/.gitlab/ci/.deploy-k8s.yml'

build/ci/.build-container.yml → .gitlab/ci/.build-container.yml

@@ -7,32 +7,23 @@ build-docker:
   stage: build
   allow_failure: false
   needs: []
+  variables:
+    TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH
   tags:
     - shell-builder
 
   rules:
-    - if: $CI_COMMIT_BRANCH == "develop" && $CI_NIGHTLY == null
-      variables:
-        TAG: $CI_REGISTRY_IMAGE:develop
-        BUILDARGS: -race
-    - if: $CI_NIGHTLY == "develop"
-      variables:
-        TAG: $CI_REGISTRY_IMAGE:nightly-develop
-        BUILDARGS: -race
-    - if: $CI_NIGHTLY == "mainline"
-      variables:
-        TAG: $CI_REGISTRY_IMAGE:nightly
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      variables:
-        TAG: $CI_REGISTRY_IMAGE:merge-request
-        BUILDARGS: -race
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_NIGHTLY == null
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
       variables:
         TAG: $CI_REGISTRY_IMAGE:latest
-    - if: '$CI_COMMIT_BRANCH'
+    - if: $CI_COMMIT_BRANCH == "develop"
       variables:
-        TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH
-      
+        TAG: $CI_REGISTRY_IMAGE:develop
+        BUILDARGS: -race
+    - when: always
+    - variables:
+        TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+ 
   script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker build --build-arg GITLAB_USER=$GO_MODULES_USER --build-arg GITLAB_TOKEN=$GO_MODULES_ACCESS_TOKEN --build-arg BUILDARGS=$BUILDARGS -t $DOCKER_IMAGE_SHA .

build/ci/.code-quality-ci.yml → .gitlab/ci/.code-quality-ci.yml

 code-quality:
-  image: golangci/golangci-lint:v1.42-alpine
+  image: golangci/golangci-lint:latest-alpine
   stage: test
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_COMMIT_BRANCH == "develop"
   script:
     # writes golangci-lint output to gl-code-quality-report.json
-    - golangci-lint run --config build/ci/.golangci-config/.golangci.yml --out-format code-climate | tee gl-code-quality-report.json
+    - golangci-lint run --config .gitlab/ci/.golangci-config/.golangci.yml --out-format code-climate | tee gl-code-quality-report.json
   artifacts:
     reports:
       codequality: gl-code-quality-report.json

build/ci/.containerlab-ci.yml → .gitlab/ci/.containerlab-ci.yml

@@ -11,7 +11,7 @@ variables:
     - shell
   before_script:
     - cd ${CLAB_DIR}
-    - export PATH="${PATH}:${CI_PROJECT_DIR}/build/ci/scripts"
+    - export PATH="${PATH}:${CI_PROJECT_DIR}/.gitlab/ci/scripts"
     - echo "$CI_REGISTRY_PASSWORD" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
     - echo ${GOSDN_CONTAINER_IMAGE}
     - docker pull ${GOSDN_CONTAINER_IMAGE}
@@ -21,7 +21,7 @@ variables:
   image: alpine:latest
   stage: build
   before_script:
-    - export PATH="${PATH}:${CI_PROJECT_DIR}/build/ci/scripts"
+    - export PATH="${PATH}:${CI_PROJECT_DIR}/.gitlab/ci/scripts"
     - firstOctet=$(generate_octet.sh $CI_COMMIT_SHA)
     - secondOctet=$(generate_octet.sh $CI_PIPELINE_ID)
     - export CLAB_MGMT_SUBNET="172.$firstOctet.$secondOctet.0/24"
@@ -43,10 +43,6 @@ containerlab:template:integration:
   extends: .containerlab_template
   variables:
     CLAB_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/int01.clab.tmpl.yml"
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_NIGHTLY
 
 
 containerlab:deploy:integration:
@@ -64,16 +60,13 @@ containerlab:deploy:integration:
   artifacts:
     reports:
       dotenv: ${CI_PROJECT_DIR}/build.env
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_NIGHTLY
 
 
 containerlab:destroy:
   stage: .post
   tags:
     - shell
+  needs: ["containerlab:deploy:integration", "integration-test:nucleus", "integration-test:api", "containerlab:template:integration"]
   before_script:
     - cd ${CLAB_DIR}
   script:
@@ -81,36 +74,30 @@ containerlab:destroy:
     - docker volume rm -f ${CLAB_NAME}-volume
     - docker image rm -f ${GOSDN_CONTAINER_IMAGE}
   allow_failure: true
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
-      when: always
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-      when: always
-    - if: $CI_NIGHTLY
-      when: always
+  when: always
 
 
-containerlab:template:develop:
-  extends: .containerlab_template
-  variables:
-    CLAB_NAME: "dev"
-    CLAB_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/dev.clab.tmpl.yml"
-    GOSDN_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE:develop"
-  rules:
-    - if: $CI_COMMIT_BRANCH == 'develop' && $CI_NIGHTLY == null
-
-
-containerlab:deploy:develop:
-  extends:
-    - .containerlab_deploy
-  needs: ["containerlab:template:develop", "build-docker"]
-  variables:
-    CLAB_NAME: "dev"
-    GOSDN_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE:develop"
-  script:
-    - sudo containerlab deploy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml --reconfigure
-  rules:
-    - if: $CI_COMMIT_BRANCH == 'develop' && $CI_NIGHTLY == null
+#containerlab:template:develop:
+#  extends: .containerlab_template
+#  variables:
+#    CLAB_NAME: "dev"
+#    CLAB_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/dev.clab.tmpl.yml"
+#    GOSDN_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE:develop"
+#  rules:
+#    - if: $CI_COMMIT_BRANCH == 'develop' && $CI_NIGHTLY == null
+#
+#
+#containerlab:deploy:develop:
+#  extends:
+#    - .containerlab_deploy
+#  needs: ["containerlab:template:develop", "build-docker"]
+#  variables:
+#    CLAB_NAME: "dev"
+#    GOSDN_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE:develop"
+#  script:
+#    - sudo containerlab deploy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml --reconfigure
+#  rules:
+#    - if: $CI_COMMIT_BRANCH == 'develop' && $CI_NIGHTLY == null
 
 
 containerlab:clean:

build/ci/.golangci-config/.golangci.yml → .gitlab/ci/.golangci-config/.golangci.yml

@@ -27,13 +27,9 @@ linters:
   enable:
     - gofmt
     - goimports
+    - revive 
     - gocyclo
     - govet
-    - unused
-    - staticcheck
-    - typecheck
-    - revive
-    - whitespace
 issues:
   exclude-use-default: false
   max-issues-per-linter: 0

build/ci/.test.yml → .gitlab/ci/.integration-test.yml

@@ -9,16 +9,13 @@
     GOSDN_TEST_ENDPOINT: "141.100.70.178:${CEOS1_PORT}"
     GOSDN_TEST_USER: "admin"
     GOSDN_TEST_PASSWORD: "admin"
-  rules:
-    - if: $CI_NIGHTLY
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME != $CI_DEFAULT_BRANCH
-      allow_failure: true
+
 
 integration-test:nucleus:
   <<: *integration-test
   script:
-    - ${CI_PROJECT_DIR}/build/ci/scripts/wait-for-it.sh ${GOSDN_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
+    - ${CI_PROJECT_DIR}/.gitlab/ci/scripts/wait-for-it.sh ${GOSDN_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
+    - sleep 30
     - cd ./test/integration
     - go test -race -v -run TestGnmi_SetIntegration
     - go test -race -v -run TestGnmi_GetIntegration
@@ -32,30 +29,3 @@ integration-test:api:
   script:
     - cd ./api
     - go test -race -v -run TestApiIntegration
-
-.test: &test
-  image: ${CI_REGISTRY_IMAGE}:testing_${CI_PIPELINE_ID}
-  stage: test
-  allow_failure: true
-  variables:
-    GOSDN_LOG: "nolog"
-    GOSDN_CHANGE_TIMEOUT: "100ms"
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH
-    - if: $CI_NIGHTLY
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
-      allow_failure: true
-
-unit-test:
-  script:
-    - go test -short -race $(go list ./... | grep -v /forks/ | grep -v /mocks ) -v -coverprofile=coverage.out
-  after_script:
-    - go tool cover -func=coverage.out
-  <<: *test
-
-controller-test:
-  image: golang:1.16
-  script:
-    - go test -race -v -run TestRun
-  <<: *test

.gitlab/ci/.test.yml

+.test: &test
+  image: ${CI_REGISTRY_IMAGE}:testing_${CI_PIPELINE_ID}
+  stage: test
+  variables:
+    GOSDN_LOG: "nolog"
+    GOSDN_CHANGE_TIMEOUT: "100ms"
+
+unit-test:
+  script:
+    - go test -short -race $(go list ./... | grep -v /forks/ | grep -v /mocks ) -v -coverprofile=coverage.out
+  after_script:
+    - go tool cover -func=coverage.out
+  <<: *test
+
+controller-test:
+  image: golang:1.16
+  script:
+    - go test -race -v -run TestRun
+  <<: *test

build/ci/scripts/generate_octet.sh → .gitlab/ci/scripts/generate_octet.sh

@@ -10,4 +10,4 @@ decimal_value=${decimal_value/-/}
 
 octet=$(((decimal_value % 255)))
 
-echo -n $octet
\ No newline at end of file
+echo -n $octet

build/ci/scripts/generate_octet.bash

-#!/bin/bash
-
-input=$1
-#trailing_backspace=$2
-
-hashed_value=$(echo $input | sha256sum | awk '{print $1}' )
-
-decimal_value=$((16#$hashed_value))
-decimal_value=${decimal_value/-/}
-
-octet=$(((decimal_value % 255)))
-
-echo -n $octet
\ No newline at end of file