Resolve "Deploy develop to Containerlab"

.gitlab-ci.yml

 variables:
   SECURE_ANALYZERS_PREFIX: registry.gitlab.com/gitlab-org/security-products/analyzers
-  DOCKER_IMAGE_SHA: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}
+  DOCKER_IMAGE_SHA: "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}"
 
 workflow:
   rules:
@@ -31,4 +31,3 @@ include:
   - local: '/build/ci/.containerlab-ci.yml'
   - local: '/build/ci/.uml-autogen-ci.yml'
   - local: '/build/ci/.deploy-k8s.yml'
-

build/ci/.containerlab-ci.yml

 variables:
   CEOS_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE/ceos:latest"
-  CLAB_INT1_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/int01.clab.yml"
-  CLAB_NAME: "clab${CI_PIPELINE_IID}"
+  GOSDN_CONTAINER_IMAGE: "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}"
   CLAB_DIR: "/mnt"
+  CLAB_NAME: "clab${CI_PIPELINE_IID}"
 
+# Templates for Job Types
+.containerlab_deploy: &containerlab_deploy
+  stage: apply
+  tags:
+    - shell
+  before_script:
+    - cd ${CLAB_DIR}
+    - export PATH="${PATH}:${CI_PROJECT_DIR}/build/ci/scripts"
+    - echo "$CI_REGISTRY_PASSWORD" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+    - echo ${GOSDN_CONTAINER_IMAGE}
+    - docker pull ${GOSDN_CONTAINER_IMAGE}
+    - docker pull ${CEOS_CONTAINER_IMAGE}
 
-.containerlab_rules: &containerlab_rules
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_NIGHTLY
-
-
-containerlab:template:
-  extends: .containerlab_rules
+.containerlab_template: &containerlab_template
   image: alpine:latest
   stage: build
   before_script:
-    - echo "Override global before_script"
-  script:
-    - ./build/ci/generate_octet.sh $CI_COMMIT_SHA >> $(pwd)/firstOctet
-    - ./build/ci/generate_octet.sh $CI_PIPELINE_ID >> $(pwd)/secondOctet
-    - export firstOctet=$(cat $(pwd)/firstOctet)
-    - export secondOctet=$(cat $(pwd)/secondOctet)
+    - export PATH="${PATH}:${CI_PROJECT_DIR}/build/ci/scripts"
+    - firstOctet=$(generate_octet.sh $CI_COMMIT_SHA)
+    - secondOctet=$(generate_octet.sh $CI_PIPELINE_ID)
     - export CLAB_MGMT_SUBNET="172.$firstOctet.$secondOctet.0/24"
+  script:
     - |
       sed -e "s|@@CEOS_CONTAINER_IMAGE@@|${CEOS_CONTAINER_IMAGE}|g" \
-          -e "s|@@GOSDN_CONTAINER_IMAGE@@|${DOCKER_IMAGE_SHA}|g" \
+          -e "s|@@GOSDN_CONTAINER_IMAGE@@|${GOSDN_CONTAINER_IMAGE}|g" \
           -e "s|@@CLAB_NAME@@|${CLAB_NAME}|g" \
           -e "s|@@CLAB_MGMT_SUBNET@@|${CLAB_MGMT_SUBNET}|g" \
-          ${CLAB_INT1_TEMPLATE} > ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml
+          ${CLAB_TEMPLATE} > ${CLAB_NAME}.clab.yml
     - cat ${CLAB_NAME}.clab.yml
   artifacts:
     name: ${CLAB_NAME}
     paths:
-      - ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml
+      - ${CLAB_NAME}.clab.yml
 
-containerlab:deploy:
-  extends:
-    - .containerlab_rules
-  stage: apply
-  needs: ["containerlab:template", "build-docker"]
-  tags:
-    - shell
-  before_script:
-    - cd ${CLAB_DIR}
-    - echo "$CI_REGISTRY_PASSWORD" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
-    - echo $DOCKER_IMAGE_SHA
-    - docker pull $DOCKER_IMAGE_SHA
-    - docker pull ${CEOS_CONTAINER_IMAGE}
+# JOBS
+containerlab:template:integration:
+  extends: .containerlab_template
+  variables:
+    CLAB_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/int01.clab.tmpl.yml"
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_NIGHTLY
+
+
+containerlab:deploy:integration:
+  extends: .containerlab_deploy
+  needs: ["containerlab:template:integration", "build-docker"]
   script:
     - sudo containerlab deploy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml --reconfigure
-    - echo "GOSDN_HTTP_PORT=$(docker inspect -f '{{ (index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort }}' clab-${CLAB_NAME}-gosdn)" >> ${CI_PROJECT_DIR}/build.env
-    - echo "GOSDN_GRPC_PORT=$(docker inspect -f '{{ (index (index .NetworkSettings.Ports "55055/tcp") 0).HostPort }}' clab-${CLAB_NAME}-gosdn)" >> ${CI_PROJECT_DIR}/build.env
-    - echo "CEOS1_PORT=$(docker inspect -f '{{ (index (index .NetworkSettings.Ports "6030/tcp") 0).HostPort }}' clab-${CLAB_NAME}-ceos1)" >> ${CI_PROJECT_DIR}/build.env
-  dependencies:
-    - containerlab:template
+    - |
+      echo -e "\
+      GOSDN_HTTP_PORT=$(docker_host_port 8080 clab-${CLAB_NAME}-gosdn)\n\
+      GOSDN_GRPC_PORT=$(docker_host_port 55055 clab-${CLAB_NAME}-gosdn)\n\
+      CEOS1_PORT=$(docker_host_port 6030 clab-${CLAB_NAME}-ceos1)" \
+      > ${CI_PROJECT_DIR}/build.env
+    - cat ${CI_PROJECT_DIR}/build.env
   artifacts:
     reports:
       dotenv: ${CI_PROJECT_DIR}/build.env
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_NIGHTLY
+
 
 containerlab:destroy:
+  stage: .post
+  tags:
+    - shell
+  before_script:
+    - cd ${CLAB_DIR}
+  script:
+    - sudo containerlab destroy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml
+    - docker volume rm -f ${CLAB_NAME}-volume
+    - docker image rm -f ${GOSDN_CONTAINER_IMAGE}
+  allow_failure: true
   rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
       when: always
@@ -68,15 +88,44 @@ containerlab:destroy:
       when: always
     - if: $CI_NIGHTLY
       when: always
+
+
+containerlab:template:develop:
+  extends: .containerlab_template
+  variables:
+    CLAB_NAME: "dev"
+    CLAB_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/dev.clab.tmpl.yml"
+    GOSDN_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE:develop"
+  rules:
+    - if: $CI_COMMIT_BRANCH == 'develop' && $CI_NIGHTLY == null
+
+
+containerlab:deploy:develop:
+  extends:
+    - .containerlab_deploy
+  needs: ["containerlab:template:develop", "build-docker"]
+  variables:
+    CLAB_NAME: "dev"
+    GOSDN_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE:develop"
+  script:
+    - sudo containerlab deploy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml --reconfigure
+  rules:
+    - if: $CI_COMMIT_BRANCH == 'develop' && $CI_NIGHTLY == null
+
+
+containerlab:clean:
   stage: .post
   tags:
     - shell
   before_script:
     - cd ${CLAB_DIR}
   script:
-    - sudo containerlab destroy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml
-    - docker volume rm -f ${CLAB_NAME}-volume
-    - docker image rm -f ${DOCKER_IMAGE_SHA}
+    - docker kill $(docker ps -q) || true
+    - docker rm $(docker ps -a -q) || true
+    - docker rmi $(docker images | grep 'registry.code.fbi.h-da.de/danet/gosdn ' | awk '{print $3}') || true
+    - sudo rm -rf ${CLAB_DIR}/* || true
+    - sudo sed -i 's|.*clab.*||g' /etc/hosts
   allow_failure: true
-  dependencies:
-    - containerlab:template
+  rules:
+    - if: $CI_SCHEDULE_CLEAN
+      when: always

build/ci/.test.yml

@@ -2,7 +2,7 @@
   image: ${CI_REGISTRY_IMAGE}:testing_${CI_PIPELINE_ID}
   stage: integration-test
   needs:
-    - job: "containerlab:deploy"
+    - job: "containerlab:deploy:integration"
   variables:
     GOSDN_LOG: "nolog"
     GOSDN_TEST_API_ENDPOINT: "141.100.70.178:${GOSDN_GRPC_PORT}"
@@ -18,7 +18,7 @@
 integration-test:nucleus:
   <<: *integration-test
   script:
-    - ${CI_PROJECT_DIR}/build/ci/wait-for-it.sh ${GOSDN_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
+    - ${CI_PROJECT_DIR}/build/ci/scripts/wait-for-it.sh ${GOSDN_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
     - cd ./test/integration
     - go test -race -v -run TestGnmi_SetIntegration
     - go test -race -v -run TestGnmi_GetIntegration

build/ci/scripts/docker_host_port.sh

+#!/bin/sh
+TEMPLATE="{{ (index (index .NetworkSettings.Ports \"${1}/tcp\") 0).HostPort }}"
+docker inspect -f "${TEMPLATE}" ${2}

test/containerlab/dev.clab.tmpl.yml

+# topology documentation: http://containerlab.srlinux.dev/lab-examples/srl-ceos/
+name: @@CLAB_NAME@@
+
+mgmt:
+  network: clab-@@CLAB_NAME@@
+  ipv4_subnet: @@CLAB_MGMT_SUBNET@@
+
+topology:
+  kinds:
+    ceos:
+      image: @@CEOS_CONTAINER_IMAGE@@
+  nodes:
+    ceos1:
+      kind: ceos
+    ceos2:
+      kind: ceos
+    ceos3:
+      kind: ceos
+
+    gosdn:
+      kind: linux
+      image: @@GOSDN_CONTAINER_IMAGE@@
+      ports:
+        - 8080:8080
+        - 55055:55055
+
+  links:
+    - endpoints: ["ceos1:eth1", "ceos2:eth1"]
+    - endpoints: ["ceos1:eth2", "ceos3:eth1"]
+    - endpoints: ["ceos2:eth2", "ceos3:eth2"]