Merge branch 'develop' into istaester/update-deps

.gitlab-ci.yml

@@ -5,9 +5,9 @@ variables:
   GOLANG_VERSION: "1.16"
 
 stages:
-  - .pre
   - build
   - test
+  - analyze
   - apply
   - integration-test
   - deploy
@@ -15,11 +15,17 @@ stages:
 
 workflow:
   rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_OPEN_MERGE_REQUESTS'
       when: never
-    - when: always
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: always
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+      when: always
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: always
 
 include:
+  - local: '/.gitlab/ci/.ruleset.yml'
   - local: '/.gitlab/ci/.build-container.yml'
   - local: '/.gitlab/ci/.code-quality-ci.yml'
   - local: '/.gitlab/ci/.security-and-compliance-ci.yml'

.gitlab/ci/.build-container.yml

@@ -5,6 +5,14 @@
     entrypoint: [ "" ]
   variables:
     TAG: $CI_COMMIT_BRANCH
+  before_script:
+      # replace all slashes in the tag with hyphen, because slashes are not allowed in tags
+    - TAG=${TAG//\//-}
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$CI_DEPENDENCY_PROXY_SERVER\":{\"username\":\"$CI_DEPENDENCY_PROXY_USER\",\"password\":\"$CI_DEPENDENCY_PROXY_TOKEN\"}}}" > /kaniko/.docker/config.json
+  needs: []
+
+build-testing-image:
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
       variables:
@@ -13,9 +21,10 @@
       variables:
         TAG: develop
         BUILDARGS: -race
-    - when: always
+    - !reference [.push_event, rules]
+    - !reference [.merge_request, rules]
   before_script:
-      # replace all slashes in the tag with hyphen, because slashes are not allowed in tags
+    # replace all slashes in the tag with hyphen, because slashes are not allowed in tags
     - TAG=${TAG//\//-}
     - mkdir -p /kaniko/.docker
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" >/kaniko/.docker/config.json
@@ -29,11 +38,23 @@ build-testing-image:
       --dockerfile "Dockerfile"
       --build-arg "GOLANG_VERSION=$GOLANG_VERSION"
       --build-arg "BUILDARGS=$BUILDARGS"
+      --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/"
       --destination "$GOSDN_TESTING_IMAGE"
       --target "installer"
   <<: *build
 
 build-image:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
+      when: never
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      variables:
+        TAG: latest
+    - if: $CI_COMMIT_BRANCH == "develop"
+      variables:
+        TAG: develop
+        BUILDARGS: -race
+    - when: always
   script:
     - /kaniko/executor
       --cache=true
@@ -41,6 +62,9 @@ build-image:
       --dockerfile "Dockerfile"
       --build-arg "GOLANG_VERSION=$GOLANG_VERSION"
       --build-arg "BUILDARGS=$BUILDARGS"
+      --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/"
       --destination "$GOSDN_IMAGE"
       --destination "$CI_REGISTRY_IMAGE:$TAG"
   <<: *build
+
+      #--build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/"

.gitlab/ci/.code-quality-ci.yml

 code-quality:
-  image: golangci/golangci-lint:latest-alpine
-  stage: test
+  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/golangci/golangci-lint:latest-alpine
+  stage: analyze
   script:
     # writes golangci-lint output to gl-code-quality-report.json
     - golangci-lint run --config .gitlab/ci/.golangci-config/.golangci.yml --out-format code-climate | tee gl-code-quality-report.json
@@ -9,5 +9,6 @@ code-quality:
       codequality: gl-code-quality-report.json
     paths:
       - gl-code-quality-report.json
+  rules:
+    - !reference [.merge_request, rules]
   needs: []
-  
\ No newline at end of file

.gitlab/ci/.containerlab-ci.yml

@@ -5,6 +5,10 @@ variables:
 # Templates for Job Types
 .containerlab_deploy: &containerlab_deploy
   stage: apply
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
+      when: never
+    - when: on_success
   tags:
     - shell
   before_script:
@@ -16,8 +20,12 @@ variables:
     - docker pull ${CEOS_IMAGE}
 
 .containerlab_template: &containerlab_template
-  image: alpine:latest
+  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/alpine:latest
   stage: build
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
+      when: never
+    - when: on_success
   before_script:
     - export PATH="${PATH}:${CI_PROJECT_DIR}/.gitlab/ci/scripts"
     - firstOctet=$(generate_octet.sh $CI_COMMIT_SHA)
@@ -35,6 +43,8 @@ variables:
     name: ${CLAB_NAME}
     paths:
       - ${CLAB_NAME}.clab.yml
+  rules:
+    - !reference [.merge_request, rules]
 
 # JOBS
 containerlab:template:integration:
@@ -58,9 +68,16 @@ containerlab:deploy:integration:
   artifacts:
     reports:
       dotenv: ${CI_PROJECT_DIR}/build.env
+  rules:
+    - !reference [.merge_request, rules]
+
 
 
 containerlab:destroy:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
+      when: never
+    - when: always
   stage: .post
   tags:
     - shell
@@ -72,7 +89,8 @@ containerlab:destroy:
     - docker volume rm -f ${CLAB_NAME}-volume
     - docker image rm -f ${GOSDN_IMAGE}
   allow_failure: true
-  when: always
+  rules:
+    - !reference [.containerlab_cleanup, rules]
 
 
 #containerlab:template:develop:

.gitlab/ci/.deploy-k8s.yml

 build:k8s-bot:
   stage: build
-  image: golang:$GOLANG_VERSION
+  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/golang:$GOLANG_VERSION
   rules:
     - if: $CI_COMMIT_BRANCH == "develop"
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@@ -14,7 +14,7 @@ build:k8s-bot:
 
 .deploy: &deploy
   image: 
-    name: bitnami/kubectl:latest
+    name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/bitnami/kubectl:latest
     entrypoint: [""]
   before_script:
     - echo "override global before script"

.gitlab/ci/.integration-test.yml

 .integration-test: &integration-test
   image: $GOSDN_TESTING_IMAGE
   stage: integration-test
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
+      when: never
+    - when: on_success
   needs:
     - job: "containerlab:deploy:integration"
   tags:
@@ -22,6 +26,8 @@ integration-test:nucleus:
     - go test -race -v -run TestGnmi_GetIntegration
     - go test -race -v -run TestGnmi_SubscribeIntegration
     - go test -race -v -run TestGnmi_CapabilitiesIntegration
+  rules:
+    - !reference [.merge_request, rules]
 
 integration-test:api:
   <<: *integration-test
@@ -30,3 +36,5 @@ integration-test:api:
   script:
     - cd ./api
     - go test -race -v -run TestApiIntegration
+  rules:
+    - !reference [.merge_request, rules]

.gitlab/ci/.ruleset.yml

+.push_event:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+      when: on_success
+
+.merge_request:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: on_success
+
+.containerlab_cleanup:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: always
+
+.merge_request_and_changed_dependency:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      changes:
+        - go.mod
+        - go.sum
+      when: always
+
+.nightly_pipeline:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: always
+
+.nightly_develop_pipeline:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"  && $CI_NIGHTLY == "develop"'
+      when: always
+
+.nightly_main_pipeline:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"  && $CI_NIGHTLY == "mainline"'
+      when: always

.gitlab/ci/.security-and-compliance-ci.yml

+.rules: &rules
+  stage: analyze
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
+      when: never
+    - when: always
+  needs: []
+
 sast:
   variables:
     SAST_ANALYZER_IMAGE_TAG: '2'
@@ -8,3 +16,34 @@ include:
   - template: Security/SAST.gitlab-ci.yml
   - template: Dependency-Scanning.gitlab-ci.yml
   - template: Security/License-Scanning.gitlab-ci.yml
+  #  - template: Security/Secret-Detection.gitlab-ci.yml
+  - template: Security/Container-Scanning.gitlab-ci.yml
+
+license_scanning:
+  rules:
+    - !reference [.merge_request_and_changed_dependency, rules]
+
+gemnasium-dependency_scanning:
+  rules:
+    - !reference [.merge_request_and_changed_dependency, rules]
+
+gosec-sast:
+  rules:
+    - !reference [.nightly_pipeline, rules]
+
+semgrep-sast:
+  rules:
+    - !reference [.nightly_pipeline, rules]
+
+container_scanning:
+  stage: analyze
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
+      when: never
+    - when: always
+  variables:
+    DOCKER_IMAGE: "${GOSDN_IMAGE}"
+    DOCKER_USER: "${CI_REGISTRY_USER}"
+    DOCKER_PASSWORD: "${CI_REGISTRY_PASSWORD}"
+  needs:
+    - build-image

.gitlab/ci/.test.yml

 .test: &test
   image: $GOSDN_TESTING_IMAGE
   stage: test
+  rules:
+    - when: on_success
   variables:
     GOSDN_LOG: "nolog"
     GOSDN_CHANGE_TIMEOUT: "100ms"
@@ -19,8 +21,20 @@ unit-test:
   after_script:
     - go tool cover -func=coverage.out
   <<: *test
+  rules:
+    - !reference [.push_event, rules]
+    - !reference [.merge_request, rules]
 
 controller-test:
   script:
     - gotestsum --junitfile report.xml --format testname -- -race -v -run TestRun
   <<: *test
+  rules:
+    - !reference [.merge_request, rules]
+
+test-build:
+  artifacts:
+    when: never
+  script:
+    - GOOS=linux go build $BUILDARGS ./cmd/gosdn
+  <<: *test

.gitlab/ci/.uml-autogen-ci.yml

 goplantuml:
-    image: golang:$GOLANG_VERSION
+    image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/golang:$GOLANG_VERSION
     stage: .post
     only:
         - develop

Dockerfile

 ARG GOLANG_VERSION=1.16
 ARG BUILDARGS
+ARG $GITLAB_PROXY
 
-FROM golang:$GOLANG_VERSION-buster AS installer
+FROM ${GITLAB_PROXY}golang:$GOLANG_VERSION-buster AS installer
 
 WORKDIR /src/gosdn
 COPY go.* ./
@@ -13,7 +14,7 @@ COPY . ./
 
 RUN GOOS=linux go build $BUILDARGS ./cmd/gosdn
 
-FROM debian:bullseye
+FROM ${GITLAB_PROXY}debian:bullseye
 EXPOSE 8080
 EXPOSE 55055
 COPY --from=builder /src/gosdn/gosdn .