Merge branch '133-Deploy-develop-to-Containerlab' into 'develop'

Resolve "Deploy develop to Containerlab" See merge request !189

Merge branch '133-Deploy-develop-to-Containerlab' into 'develop'
773e97fb · Manuel Kieweg · 94b5c6de · 20013e4f · 773e97fb · 773e97fb
Commit 773e97fb authored 3 years ago by Manuel Kieweg
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
 variables:
  SECURE_ANALYZERS_PREFIX: registry.gitlab.com/gitlab-org/security-products/analyzers
-  DOCKER_IMAGE_SHA: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}
+  DOCKER_IMAGE_SHA: "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}"
 workflow:
  rules:
@@ -31,4 +31,3 @@ include:
  - local: '/build/ci/.containerlab-ci.yml'
  - local: '/build/ci/.uml-autogen-ci.yml'
  - local: '/build/ci/.deploy-k8s.yml'
--- a/build/ci/.containerlab-ci.yml
+++ b/build/ci/.containerlab-ci.yml
 variables:
  CEOS_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE/ceos:latest"
-  CLAB_INT1_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/int01.clab.yml"
+  GOSDN_CONTAINER_IMAGE: "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}"
-  CLAB_NAME: "clab${CI_PIPELINE_IID}"
  CLAB_DIR: "/mnt"
+  CLAB_NAME: "clab${CI_PIPELINE_IID}"
+# Templates for Job Types
+.containerlab_deploy: &containerlab_deploy
+  stage: apply
+  tags:
+    - shell
+  before_script:
+    - cd ${CLAB_DIR}
+    - export PATH="${PATH}:${CI_PROJECT_DIR}/build/ci/scripts"
+    - echo "$CI_REGISTRY_PASSWORD" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+    - echo ${GOSDN_CONTAINER_IMAGE}
+    - docker pull ${GOSDN_CONTAINER_IMAGE}
+    - docker pull ${CEOS_CONTAINER_IMAGE}
-.containerlab_rules: &containerlab_rules
+.containerlab_template: &containerlab_template
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_NIGHTLY
-containerlab:template:
-  extends: .containerlab_rules
  image: alpine:latest
  stage: build
  before_script:
-    - echo "Override global before_script"
+    - export PATH="${PATH}:${CI_PROJECT_DIR}/build/ci/scripts"
-  script:
+    - firstOctet=$(generate_octet.sh $CI_COMMIT_SHA)
-    - ./build/ci/generate_octet.sh $CI_COMMIT_SHA >> $(pwd)/firstOctet
+    - secondOctet=$(generate_octet.sh $CI_PIPELINE_ID)
-    - ./build/ci/generate_octet.sh $CI_PIPELINE_ID >> $(pwd)/secondOctet
-    - export firstOctet=$(cat $(pwd)/firstOctet)
-    - export secondOctet=$(cat $(pwd)/secondOctet)
    - export CLAB_MGMT_SUBNET="172.$firstOctet.$secondOctet.0/24"
+  script:
    - |
      sed -e "s|@@CEOS_CONTAINER_IMAGE@@|${CEOS_CONTAINER_IMAGE}|g" \
-          -e "s|@@GOSDN_CONTAINER_IMAGE@@|${DOCKER_IMAGE_SHA}|g" \
+          -e "s|@@GOSDN_CONTAINER_IMAGE@@|${GOSDN_CONTAINER_IMAGE}|g" \
          -e "s|@@CLAB_NAME@@|${CLAB_NAME}|g" \
          -e "s|@@CLAB_MGMT_SUBNET@@|${CLAB_MGMT_SUBNET}|g" \
-          ${CLAB_INT1_TEMPLATE} > ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml
+          ${CLAB_TEMPLATE} > ${CLAB_NAME}.clab.yml
    - cat ${CLAB_NAME}.clab.yml
  artifacts:
    name: ${CLAB_NAME}
    paths:
-      - ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml
+      - ${CLAB_NAME}.clab.yml
-containerlab:deploy:
+# JOBS
-  extends:
+containerlab:template:integration:
-    - .containerlab_rules
+  extends: .containerlab_template
-  stage: apply
+  variables:
-  needs: ["containerlab:template", "build-docker"]
+    CLAB_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/int01.clab.tmpl.yml"
-  tags:
+  rules:
-    - shell
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
-  before_script:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - cd ${CLAB_DIR}
+    - if: $CI_NIGHTLY
-    - echo "$CI_REGISTRY_PASSWORD" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
-    - echo $DOCKER_IMAGE_SHA
-    - docker pull $DOCKER_IMAGE_SHA
+containerlab:deploy:integration:
-    - docker pull ${CEOS_CONTAINER_IMAGE}
+  extends: .containerlab_deploy
+  needs: ["containerlab:template:integration", "build-docker"]
  script:
    - sudo containerlab deploy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml --reconfigure
-    - echo "GOSDN_HTTP_PORT=$(docker inspect -f '{{ (index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort }}' clab-${CLAB_NAME}-gosdn)" >> ${CI_PROJECT_DIR}/build.env
+    - |
-    - echo "GOSDN_GRPC_PORT=$(docker inspect -f '{{ (index (index .NetworkSettings.Ports "55055/tcp") 0).HostPort }}' clab-${CLAB_NAME}-gosdn)" >> ${CI_PROJECT_DIR}/build.env
+      echo -e "\
-    - echo "CEOS1_PORT=$(docker inspect -f '{{ (index (index .NetworkSettings.Ports "6030/tcp") 0).HostPort }}' clab-${CLAB_NAME}-ceos1)" >> ${CI_PROJECT_DIR}/build.env
+      GOSDN_HTTP_PORT=$(docker_host_port 8080 clab-${CLAB_NAME}-gosdn)\n\
-  dependencies:
+      GOSDN_GRPC_PORT=$(docker_host_port 55055 clab-${CLAB_NAME}-gosdn)\n\
-    - containerlab:template
+      CEOS1_PORT=$(docker_host_port 6030 clab-${CLAB_NAME}-ceos1)" \
+      > ${CI_PROJECT_DIR}/build.env
+    - cat ${CI_PROJECT_DIR}/build.env
  artifacts:
    reports:
      dotenv: ${CI_PROJECT_DIR}/build.env
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_NIGHTLY
 containerlab:destroy:
+  stage: .post
+  tags:
+    - shell
+  before_script:
+    - cd ${CLAB_DIR}
+  script:
+    - sudo containerlab destroy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml
+    - docker volume rm -f ${CLAB_NAME}-volume
+    - docker image rm -f ${GOSDN_CONTAINER_IMAGE}
+  allow_failure: true
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
      when: always
@@ -68,15 +88,44 @@ containerlab:destroy:
      when: always
    - if: $CI_NIGHTLY
      when: always
+containerlab:template:develop:
+  extends: .containerlab_template
+  variables:
+    CLAB_NAME: "dev"
+    CLAB_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/dev.clab.tmpl.yml"
+    GOSDN_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE:develop"
+  rules:
+    - if: $CI_COMMIT_BRANCH == 'develop' && $CI_NIGHTLY == null
+containerlab:deploy:develop:
+  extends:
+    - .containerlab_deploy
+  needs: ["containerlab:template:develop", "build-docker"]
+  variables:
+    CLAB_NAME: "dev"
+    GOSDN_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE:develop"
+  script:
+    - sudo containerlab deploy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml --reconfigure
+  rules:
+    - if: $CI_COMMIT_BRANCH == 'develop' && $CI_NIGHTLY == null
+containerlab:clean:
  stage: .post
  tags:
    - shell
  before_script:
    - cd ${CLAB_DIR}
  script:
-    - sudo containerlab destroy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml
+    - docker kill $(docker ps -q) || true
-    - docker volume rm -f ${CLAB_NAME}-volume
+    - docker rm $(docker ps -a -q) || true
-    - docker image rm -f ${DOCKER_IMAGE_SHA}
+    - docker rmi $(docker images | grep 'registry.code.fbi.h-da.de/danet/gosdn ' | awk '{print $3}') || true
+    - sudo rm -rf ${CLAB_DIR}/* || true
+    - sudo sed -i 's|.*clab.*||g' /etc/hosts
  allow_failure: true
-  dependencies:
+  rules:
-    - containerlab:template
+    - if: $CI_SCHEDULE_CLEAN
+      when: always
--- a/build/ci/.test.yml
+++ b/build/ci/.test.yml
@@ -2,7 +2,7 @@
  image: ${CI_REGISTRY_IMAGE}:testing_${CI_PIPELINE_ID}
  stage: integration-test
  needs:
-    - job: "containerlab:deploy"
+    - job: "containerlab:deploy:integration"
  variables:
    GOSDN_LOG: "nolog"
    GOSDN_TEST_API_ENDPOINT: "141.100.70.178:${GOSDN_GRPC_PORT}"
@@ -18,7 +18,7 @@
 integration-test:nucleus:
  <<: *integration-test
  script:
-    - ${CI_PROJECT_DIR}/build/ci/wait-for-it.sh ${GOSDN_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
+    - ${CI_PROJECT_DIR}/build/ci/scripts/wait-for-it.sh ${GOSDN_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
    - cd ./test/integration
    - go test -race -v -run TestGnmi_SetIntegration
    - go test -race -v -run TestGnmi_GetIntegration

--- a/build/ci/scripts/docker_host_port.sh
+++ b/build/ci/scripts/docker_host_port.sh
+#!/bin/sh
+TEMPLATE="{{ (index (index .NetworkSettings.Ports \"${1}/tcp\") 0).HostPort }}"
+docker inspect -f "${TEMPLATE}" ${2}
--- a/build/ci/generate_octet.bash
+++ b/build/ci/generate_octet.bash
--- a/build/ci/generate_octet.sh
+++ b/build/ci/generate_octet.sh
--- a/build/ci/wait-for-it.sh
+++ b/build/ci/wait-for-it.sh
--- a/test/containerlab/dev.clab.tmpl.yml
+++ b/test/containerlab/dev.clab.tmpl.yml
+# topology documentation: http://containerlab.srlinux.dev/lab-examples/srl-ceos/
+name: @@CLAB_NAME@@
+mgmt:
+  network: clab-@@CLAB_NAME@@
+  ipv4_subnet: @@CLAB_MGMT_SUBNET@@
+topology:
+  kinds:
+    ceos:
+      image: @@CEOS_CONTAINER_IMAGE@@
+  nodes:
+    ceos1:
+      kind: ceos
+    ceos2:
+      kind: ceos
+    ceos3:
+      kind: ceos
+    gosdn:
+      kind: linux
+      image: @@GOSDN_CONTAINER_IMAGE@@
+      ports:
+        - 8080:8080
+        - 55055:55055
+  links:
+    - endpoints: ["ceos1:eth1", "ceos2:eth1"]
+    - endpoints: ["ceos1:eth2", "ceos3:eth1"]
+    - endpoints: ["ceos2:eth2", "ceos3:eth2"]
--- a/test/containerlab/int01.clab.yml
+++ b/test/containerlab/int01.clab.yml