Resolve "Migrate integration test ci environment to containerlab"

.gitlab-ci.yml

@@ -28,6 +28,6 @@ include:
   - local: '/build/ci/.security-and-compliance-ci.yml'
   - local: '/build/ci/.build-container.yml'
   - local: '/build/ci/.test.yml'
-  - local: '/build/ci/.terraform-ci.yml'
+  - local: '/build/ci/.containerlab-ci.yml'
   - local: '/build/ci/.deploy-k8s.yml'
   - local: '/build/ci/.uml-autogen-ci.yml'

build/ci/.build-container.yml

@@ -12,7 +12,6 @@ variables:
   tags:
     - dind
   script:
-    - docker info
     - >
       docker build \
         --build-arg GITLAB_USER=$GO_MODULES_USER \

build/ci/.containerlab-ci.yml

+
+variables:
+  CEOS_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE/ceos:latest"
+  CLAB_INT1_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/int01.clab.yml"
+  CLAB_NAME: "clab${CI_PIPELINE_IID}"
+  CLAB_DIR: "/mnt"
+
+
+.containerlab_rules: &containerlab_rules
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_NIGHTLY
+
+
+.containerlab_template: &containerlab_template
+  tags:
+    - shell
+  before_script:
+    - cd ${CLAB_DIR}
+    - echo "$CI_REGISTRY_PASSWORD" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+    - echo $DOCKER_IMAGE_SHA
+    - docker pull $DOCKER_IMAGE_SHA
+    - docker pull ${CEOS_CONTAINER_IMAGE}
+
+
+containerlab:template:
+  extends: .containerlab_rules
+  image: alpine:latest
+  stage: build
+  before_script:
+    - echo "Override global before_script"
+  script:
+    - ./build/ci/generate_octet.sh $CI_COMMIT_SHA >> $(pwd)/firstOctet
+    - ./build/ci/generate_octet.sh $CI_PIPELINE_ID >> $(pwd)/secondOctet
+    - export firstOctet=$(cat $(pwd)/firstOctet)
+    - export secondOctet=$(cat $(pwd)/secondOctet)
+    - export CLAB_MGMT_SUBNET="172.$firstOctet.$secondOctet.0/24"
+    - |
+      sed -e "s|@@CEOS_CONTAINER_IMAGE@@|${CEOS_CONTAINER_IMAGE}|g" \
+          -e "s|@@GOSDN_CONTAINER_IMAGE@@|${DOCKER_IMAGE_SHA}|g" \
+          -e "s|@@CLAB_NAME@@|${CLAB_NAME}|g" \
+          -e "s|@@CLAB_MGMT_SUBNET@@|${CLAB_MGMT_SUBNET}|g" \
+          ${CLAB_INT1_TEMPLATE} > ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml
+    - cat ${CLAB_NAME}.clab.yml
+  artifacts:
+    name: ${CLAB_NAME}
+    paths:
+      - ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml
+
+
+containerlab:deploy:
+  extends:
+    - .containerlab_template
+    - .containerlab_rules
+  stage: apply
+  script:
+    - sudo containerlab deploy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml --reconfigure
+    - echo "GOSDN_HTTP_PORT=$(docker inspect -f '{{ (index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort }}' clab-${CLAB_NAME}-gosdn)" >> ${CI_PROJECT_DIR}/build.env
+    - echo "GOSDN_GRPC_PORT=$(docker inspect -f '{{ (index (index .NetworkSettings.Ports "55055/tcp") 0).HostPort }}' clab-${CLAB_NAME}-gosdn)" >> ${CI_PROJECT_DIR}/build.env
+    - echo "CEOS1_PORT=$(docker inspect -f '{{ (index (index .NetworkSettings.Ports "6030/tcp") 0).HostPort }}' clab-${CLAB_NAME}-ceos1)" >> ${CI_PROJECT_DIR}/build.env
+  dependencies:
+    - containerlab:template
+  artifacts:
+    reports:
+      dotenv: ${CI_PROJECT_DIR}/build.env
+
+containerlab:destroy:
+  stage: .post
+  tags:
+    - shell
+  before_script:
+    - cd ${CLAB_DIR}
+  script:
+    - sudo containerlab destroy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml
+    - docker volume rm -f ${CLAB_NAME}-volume
+    - docker image rm -f ${DOCKER_IMAGE_SHA}
+  allow_failure: true
+  dependencies:
+    - containerlab:template
+  when: always

build/ci/.deploy-k8s.yml

@@ -4,7 +4,6 @@ build:k8s-bot:
   rules:
     - if: $CI_COMMIT_BRANCH == "develop"
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
   script:
     - cd build/cd
     - go build -o k8s-bot
@@ -24,16 +23,6 @@ build:k8s-bot:
   script:
     - ./build/cd/k8s-bot
 
-deploy:integration-test:
-  <<: *deploy
-  stage: apply
-  needs:
-    - job: "build:merge-request"
-    - job: "build:k8s-bot"
-      artifacts: true
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-
 deploy:develop:
   <<: *deploy
   stage: deploy
@@ -73,20 +62,3 @@ deploy:nightly:develop:
       artifacts: true
   rules:
     - if: $CI_COMMIT_BRANCH == "develop" && $CI_NIGHTLY == "mainline"
-
-destroy:k8s:
-  image: 
-    name: bitnami/kubectl:latest
-    entrypoint: [""]
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: always
-  before_script:
-    - echo "override global before script"
-  stage: .post
-  variables:
-    K8S_OP: "delete"
-  script:
-    - ./build/cd/k8s-bot
-  dependencies:
-    - build:k8s-bot

build/ci/.terraform-ci.yml

-
-variables:
-  TF_ROOT: ${CI_PROJECT_DIR}/test/terraform
-  TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_PIPELINE_ID}
-  DOCKER_IMAGE_SHA: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}
-
-cache:
-  key: ${CI_PIPELINE_ID}
-  paths:
-    - ${TF_ROOT}/.terraform
-
-.terraform_prefab: &tf
-  image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest
-  variables:
-    CI_DEBUG_TRACE: "false"
-  before_script:
-    - ./build/ci/generate_octet.sh $CI_COMMIT_SHA >> ${TF_ROOT}/firstOctet
-    - ./build/ci/generate_octet.sh $CI_PIPELINE_ID >> ${TF_ROOT}/secondOctet
-    - cd ${TF_ROOT}
-    - export TF_VAR_integration_username=terraform
-    - export TF_VAR_integration_access_token=${TERRAFORM_API_TOKEN}
-    - export TF_VAR_integration_registry=${CI_REGISTRY}
-    - export TF_VAR_ceos_tag=registry.code.fbi.h-da.de/cocsn/gosdn/ceos:${CI_PIPELINE_ID}
-    - export TF_VAR_container_tag=registry.code.fbi.h-da.de/cocsn/gosdn:${CI_PIPELINE_ID}
-    - export TF_VAR_tls_key=${DOCKER_TLS_KEY}
-    - export TF_VAR_tls_cert=${DOCKER_TLS_CERT}
-    - export TF_VAR_tls_ca_cert=${DOCKER_TLS_CA}
-    - export TF_VAR_ceos_address=172.24.$(cat firstOctet).$(cat secondOctet)
-    - export TF_VAR_gosdn_address=172.24.$(cat secondOctet).$(cat firstOctet)
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
-    - if: $CI_COMMIT_BRANCH == "integration-test"
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_NIGHTLY
-
-init:
-  stage: .pre
-  script:
-    - gitlab-terraform init
-  <<: *tf
-
-tag-images:
-  stage: .pre
-  before_script:
-    - echo "override global before script"
-  image: docker:19.03.12
-  tags:
-    - dind
-  services:
-  - name: docker:19.03.12-dind
-    command: ["--registry-mirror", "http://141.100.70.170:6000", "--dns", "1.1.1.1"]
-  variables:
-    DOCKER_TLS_CERTDIR: "/certs"
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop')
-      variables:
-        TF_VAR_container_tag: $DOCKER_IMAGE_SHA
-    - if: $CI_COMMIT_BRANCH == "integration-test"
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_NIGHTLY
-  script:
-    - docker info
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - docker pull registry.code.fbi.h-da.de/cocsn/gosdn/ceos:latest
-    - docker tag registry.code.fbi.h-da.de/cocsn/gosdn/ceos:latest registry.code.fbi.h-da.de/cocsn/gosdn/ceos:${CI_PIPELINE_ID}
-
-validate:
-  stage: test
-  script:
-    - gitlab-terraform validate
-  needs: ["init"]
-  <<: *tf
-
-plan:
-  before_script:
-    - cd ${TF_ROOT}
-  stage: build
-  script:
-    - gitlab-terraform plan
-    - gitlab-terraform plan-json
-  artifacts:
-    name: plan
-    paths:
-      - ${TF_ROOT}/plan.cache
-    reports:
-      terraform: ${TF_ROOT}/plan.json
-  needs: ["validate"]
-  <<: *tf
-
-apply:
-  stage: apply
-  script:
-    - gitlab-terraform apply
-  dependencies:
-    - plan
-  <<: *tf
-
-destroy:tf:
-  stage: .post
-  script:
-    - gitlab-terraform destroy
-  <<: *tf

build/ci/.test.yml

@@ -2,30 +2,23 @@
   image: golang:1.16
   stage: integration-test
   needs:
-    - job: "apply"
-    - job: "deploy:integration-test"
-
+    - job: "containerlab:deploy"
   variables:
     GOSDN_LOG: "nolog"
-    GOSDN_CHANGE_TIMEOUT: "100ms"
+    GOSDN_TEST_API_ENDPOINT: "141.100.70.178:${GOSDN_GRPC_PORT}"
+    GOSDN_TEST_ENDPOINT: "141.100.70.178:${CEOS1_PORT}"
+    GOSDN_TEST_USER: "admin"
+    GOSDN_TEST_PASSWORD: "admin"
   rules:
     - if: $CI_NIGHTLY
-      when: delayed
-      start_in: 2 minutes
     - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH
-      when: delayed
-      start_in: 2 minutes
     - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME != $CI_DEFAULT_BRANCH
       allow_failure: true
-      when: delayed
-      start_in: 2 minutes
 
 integration-test:nucleus:
   <<: *integration-test
   script:
-    - ./build/ci/generate_octet.bash $CI_COMMIT_SHA >> firstOctet
-    - ./build/ci/generate_octet.bash $CI_PIPELINE_ID >> secondOctet
-    - export GOSDN_TEST_ENDPOINT=172.24.$(cat firstOctet).$(cat secondOctet):6030
+    - ${CI_PROJECT_DIR}/build/ci/wait-for-it.sh ${GOSDN_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
     - cd ./test/integration
     - go test -race -v -run TestGnmi_SetIntegration
     - go test -race -v -run TestGnmi_GetIntegration
@@ -36,16 +29,7 @@ integration-test:api:
   <<: *integration-test
   variables:
     K8S_OP: "getenv"
-  needs:
-    - job: "build:merge-request"
-    - job: "apply"
-    - job: "deploy:integration-test"
-    - job: "build:k8s-bot"
-      artifacts: true
   script:
-    - ./build/ci/generate_octet.bash $CI_COMMIT_SHA >> firstOctet
-    - ./build/ci/generate_octet.bash $CI_PIPELINE_ID >> secondOctet
-    - export GOSDN_TEST_API_ENDPOINT=172.24.$(cat secondOctet).$(cat firstOctet):55055
     - cd ./api
     - go test -race -v -run TestApiIntegration
 
@@ -74,3 +58,8 @@ controller-test:
   script:
     - go test -race -v -run TestRun
   <<: *test
+
+controller-test:
+  script:
+    - go test -race -v -run TestRun
+  <<: *test

build/ci/wait-for-it.sh

+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

test/containerlab/int01.clab.yml

+# topology documentation: http://containerlab.srlinux.dev/lab-examples/srl-ceos/
+name: @@CLAB_NAME@@
+
+mgmt:
+  network: @@CLAB_NAME@@
+  ipv4_subnet: @@CLAB_MGMT_SUBNET@@
+
+topology:
+  kinds:
+    ceos:
+      image: @@CEOS_CONTAINER_IMAGE@@
+  nodes:
+    ceos1:
+      kind: ceos
+      ports:
+        - 0:6030
+
+    gosdn:
+      kind: linux
+      image: @@GOSDN_CONTAINER_IMAGE@@
+      ports:
+        - 0:8080
+        - 0:55055