Create new integration-test setup

See merge request !574

Co-authored-by: Malte Bauch <malte.bauch@stud.h-da.de>
Co-authored-by: Fabian Seidl <fabian.seidl@h-da.de>

.devcontainer/Dockerfile

-# The devcontainer will be based on Python
+# The devcontainer will be based on debian
 # The base container already has entrypoint, vscode user account, etc. out of the box
 FROM mcr.microsoft.com/vscode/devcontainers/base:bullseye
 
@@ -6,17 +6,17 @@ FROM mcr.microsoft.com/vscode/devcontainers/base:bullseye
 ARG _CLAB_VERSION
 
 # Set permissions for mounts in devcontainer.json
-RUN mkdir /home/vscode/.vscode-server/
-RUN mkdir /home/vscode/.vscode-server/bin
+RUN mkdir -p /home/vscode/.vscode-server/bin
 RUN chown -R vscode:vscode /home/vscode/.vscode-server
 
 # Update and install some basic tools inside the container
 # Adjust this list based on your demands
 RUN apt-get update \
     && apt-get upgrade -y \
-    && apt-get install -y --no-install-recommends \
+    && apt-get install -y \
     sshpass \
     curl \
+    wget \
     iputils-ping \
     htop \
     zsh \
@@ -29,7 +29,7 @@ RUN apt-get update \
     && rm -Rf /usr/share/doc && rm -Rf /usr/share/man \
     && apt-get clean
 
-# Install oh-my-zsh for more terminal features and set is as primary shell
+# Install oh-my-zsh for more terminal features and set it as primary shell
 ENV SHELL /bin/zsh
 RUN wget https://github.com/robbyrussell/oh-my-zsh/raw/master/tools/install.sh -O - | zsh || true
 
@@ -39,4 +39,3 @@ RUN echo "export EDITOR='nano'" >> /home/vscode/.zshrc
 
 # Install preferred version of the containerlab
 RUN bash -c "$(curl -sL https://get.containerlab.dev)" -- -v ${_CLAB_VERSION}
-    #&& pip3 install --user yamllint

.devcontainer/devcontainer.json

 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
-    // README at: https://github.com/devcontainers/templates/tree/main/src/python
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
     {
         "name": "goSDN-develop-clab-go",
         "build": {
             "dockerfile": "Dockerfile",
             "args": {
-                "_CLAB_VERSION": "0.47.2"
+                "_CLAB_VERSION": "0.48.6"
             }
         },
         "features": {
@@ -16,7 +16,7 @@
             },
             "ghcr.io/devcontainers/features/go:1": {
                     "version": "1.21"
-                }
+            }
         },
         // Add any required extensions that must be pre-installed in the devcontainer
         "customizations": {

.dockerignore

 .git
-.gitlab
 build
 documentation
 mocks

.gitignore

@@ -4,8 +4,8 @@ artifacts/
 
 # containerlab
 clab-gosdn*/
-*clab.yml.bak
-dev_env_data/clab/*clab.yaml.bak
+*.yml.bak
+dev_env_data/clab/*.yaml.bak
 
 # non vimmers
 .vscode/
@@ -74,4 +74,4 @@ plugin-registry/plugins/
 
 # lab-vm
 lab-vm/vm-with-packer/VM/
-lab-vm/vm-with-vagrant/.vagrant/
+lab-vm/vm-with-vagrant/.vagrant/
\ No newline at end of file

.gitlab-ci.yml

 variables:
-    GOSDN_IMAGE: "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}"
-    GOSDN_TESTING_IMAGE: "${CI_REGISTRY_IMAGE}:testing_${CI_COMMIT_SHA}"
+    DOCKER_TAG: $CI_COMMIT_SHA
+    GOSDN_IMAGE: "${CI_REGISTRY_IMAGE}:${DOCKER_TAG}"
+    GOSDN_TESTING_IMAGE: "${CI_REGISTRY_IMAGE}:testing_${DOCKER_TAG}"
+    GOSDN_INTEGRATION_TEST_IMAGE: "${CI_REGISTRY_IMAGE}/integration-tests:${DOCKER_TAG}"
     CEOS_IMAGE: "${CI_PCONTAINERS_REGISTRY_IMAGE}/ceos:4.28.2F"
+    CLAB_VERSION: "0.48.6"
     GOLANG_VERSION: "1.21"
+    GNMI_TARGET_IMAGE: registry.code.fbi.h-da.de/danet/gnmi-target/debian:master
+    GNMI_TARGET_INTEGRATION_IMAGE: "registry.code.fbi.h-da.de/danet/gosdn/gnmi-target-integration-test:${CI_COMMIT_SHA}"
 
 workflow:
   rules:
@@ -12,10 +17,11 @@ workflow:
 stages:
     - tools
     - build
+    - build-testing
+    - build-release
     - test
     - analyze
-    - integration-test
-    - build-release
+    - test-legacy
     - deploy
     - .post
 
@@ -25,8 +31,9 @@ include:
     - local: "/.gitlab/ci/.code-quality-ci.yml"
     - local: "/.gitlab/ci/.security-and-compliance-ci.yml"
     - local: "/.gitlab/ci/.test.yml"
-    - local: "/.gitlab/ci/.integration-test-containerlab.yml"
-    - local: "/.gitlab/ci/.integration-test.yml"
     - local: "/.gitlab/ci/.uml-autogen-ci.yml"
     - local: "/.gitlab/ci/.renovate.yml"
     - local: "/.gitlab/ci/.mk-docs-deploy.yml"
+    - local: "/.gitlab/ci/legacy/.test.yml"
+    #- local: "/.gitlab/ci/legacy/.integration-test-containerlab.yml"
+    #- local: "/.gitlab/ci/legacy/.integration-test.yml"

.gitlab/ci/.build-container-images.yml

 .build: &build
     stage: build
-    tags:
-      - shell-builder
+    image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:latest
+    services:
+        - name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:latest
+          alias: docker
     variables:
-        TAG: $CI_COMMIT_SHA
+      DOCKER_TLS_CERTDIR: "/certs"
+    before_script:
+      - apk add git
+      - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+      - docker login -u $CI_DEPENDENCY_PROXY_USER -p $CI_DEPENDENCY_PROXY_PASSWORD $CI_DEPENDENCY_PROXY_SERVER
     needs: []
 
-build-controller-testing-image:
-    script:
-        - docker buildx build -t "$GOSDN_TESTING_IMAGE" -f "${CI_PROJECT_DIR}/controller/controller.Dockerfile" --target "builder" --build-arg "GOLANG_VERSION=$GOLANG_VERSION" --build-arg "BUILDARGS=$BUILDARGS" --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/" .
-        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-        - docker push "$GOSDN_TESTING_IMAGE"
-    <<: *build
+
+.build-testing: &build-testing
+    stage: build-testing
+    image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:latest
+    services:
+        - name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:latest
+          alias: docker
+    variables:
+      DOCKER_TLS_CERTDIR: "/certs"
+    before_script:
+      - apk add git
+      - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+      - docker login -u $CI_DEPENDENCY_PROXY_USER -p $CI_DEPENDENCY_PROXY_PASSWORD $CI_DEPENDENCY_PROXY_SERVER
 
 build-controller-image:
     script:
-        - docker buildx build -t "$CI_REGISTRY_IMAGE:$TAG" -f "${CI_PROJECT_DIR}/controller/controller.Dockerfile" --build-arg "GOLANG_VERSION=$GOLANG_VERSION" --build-arg "BUILDARGS=$BUILDARGS" --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/" .
-        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-        - docker push "$CI_REGISTRY_IMAGE:$TAG"
+        - docker buildx build -t "$CI_REGISTRY_IMAGE:$DOCKER_TAG" -f "${CI_PROJECT_DIR}/controller/controller.Dockerfile" --build-arg "GOLANG_VERSION=$GOLANG_VERSION" --build-arg "BUILDARGS=$BUILDARGS" --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/" .
+        - docker push "$CI_REGISTRY_IMAGE:$DOCKER_TAG"
     <<: *build
 
 build-cli-image:
     script:
         - CLI_IMAGE_NAME="${CI_REGISTRY_IMAGE}/gosdnc"
-        - docker buildx build -t "$CLI_IMAGE_NAME:$TAG" -f "${CI_PROJECT_DIR}/cli/cli.Dockerfile" --build-arg "GOLANG_VERSION=$GOLANG_VERSION" --build-arg "BUILDARGS=$BUILDARGS" --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/" .
-        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-        - docker push "$CLI_IMAGE_NAME:$TAG"
+        - docker buildx build -t "$CLI_IMAGE_NAME:$DOCKER_TAG" -f "${CI_PROJECT_DIR}/cli/cli.Dockerfile" --build-arg "GOLANG_VERSION=$GOLANG_VERSION" --build-arg "BUILDARGS=$BUILDARGS" --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/" .
+        - docker push "$CLI_IMAGE_NAME:$DOCKER_TAG"
     <<: *build
 
 build-venv-manager-image:
     script:
         - VENV_MANAGER_IMAGE_NAME="${CI_REGISTRY_IMAGE}/venv-manager"
-        - docker buildx build -t "$VENV_MANAGER_IMAGE_NAME:$TAG" -f "${CI_PROJECT_DIR}/applications/venv-manager/venv-manager.Dockerfile" --build-arg "GOLANG_VERSION=$GOLANG_VERSION" --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/" .
-        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-        - docker push "$VENV_MANAGER_IMAGE_NAME:$TAG"
+        - docker buildx build -t "$VENV_MANAGER_IMAGE_NAME:$DOCKER_TAG" -f "${CI_PROJECT_DIR}/applications/venv-manager/venv-manager.Dockerfile" --build-arg "GOLANG_VERSION=$GOLANG_VERSION" --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/" .
+        - docker push "$VENV_MANAGER_IMAGE_NAME:$DOCKER_TAG"
     <<: *build
 
 build-arista-routing-engine-image:
     script:
         - ARISTA_ROUTING_ENGINE_IMAGE_NAME="${CI_REGISTRY_IMAGE}/arista-routing-engine"
-        - docker buildx build -t "$ARISTA_ROUTING_ENGINE_IMAGE_NAME:$TAG" -f "${CI_PROJECT_DIR}/applications/arista-routing-engine/arista-routing-engine.Dockerfile" --build-arg "GOLANG_VERSION=$GOLANG_VERSION" --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/" .
-        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-        - docker push "$ARISTA_ROUTING_ENGINE_IMAGE_NAME:$TAG"
+        - docker buildx build -t "$ARISTA_ROUTING_ENGINE_IMAGE_NAME:$DOCKER_TAG" -f "${CI_PROJECT_DIR}/applications/arista-routing-engine/arista-routing-engine.Dockerfile" --build-arg "GOLANG_VERSION=$GOLANG_VERSION" --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/" .
+        - docker push "$ARISTA_ROUTING_ENGINE_IMAGE_NAME:$DOCKER_TAG"
     <<: *build
 
 build-plugin-registry-image:
     script:
         - PLUGIN_REGISTRY_IMAGE_NAME="${CI_REGISTRY_IMAGE}/plugin-registry"
-        - docker buildx build -t "$PLUGIN_REGISTRY_IMAGE_NAME:$TAG" -f "${CI_PROJECT_DIR}/plugin-registry/plugin-registry.Dockerfile" --build-arg "GOLANG_VERSION=$GOLANG_VERSION" --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/" .
-        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-        - docker push "$PLUGIN_REGISTRY_IMAGE_NAME:$TAG"
+        - docker buildx build -t "$PLUGIN_REGISTRY_IMAGE_NAME:$DOCKER_TAG" -f "${CI_PROJECT_DIR}/plugin-registry/plugin-registry.Dockerfile" --build-arg "GOLANG_VERSION=$GOLANG_VERSION" --build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/" .
+        - docker push "$PLUGIN_REGISTRY_IMAGE_NAME:$DOCKER_TAG"
     <<: *build
+
+build-integration-test-images:
+    needs: ["build-controller-image"]
+    script:
+        # Build certs
+        - apk add openssl make
+        - make generate-certs
+        # Build gosdn with ssl certificates
+        - docker buildx build -t "$CI_REGISTRY_IMAGE:${DOCKER_TAG}_integration-test" -f "${CI_PROJECT_DIR}/.gitlab/ci/gosdn-integration.Dockerfile" --build-arg "GOSDN_IMAGE=$CI_REGISTRY_IMAGE:${DOCKER_TAG}" .
+        # Update gnmi-target with ssl certificates
+        - docker buildx build -t $GNMI_TARGET_INTEGRATION_IMAGE -f "${CI_PROJECT_DIR}/.gitlab/ci/gnmi-cert-integration.Dockerfile" --build-arg "GNMI_TARGET_IMAGE=${GNMI_TARGET_IMAGE}" .
+        # Push images
+        - docker push "$CI_REGISTRY_IMAGE:${DOCKER_TAG}_integration-test"
+        - docker push $GNMI_TARGET_INTEGRATION_IMAGE
+    <<: *build-testing

.gitlab/ci/.code-quality-ci.yml

 variables:
     GOLANG_VERSION: "1.21"
 code-quality:
-    image: golangci/golangci-lint:v1.55.2-alpine
+    image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/golangci/golangci-lint:v1.55.2-alpine
     stage: analyze
     script:
         # writes golangci-lint output to gl-code-quality-report.json

.gitlab/ci/.test.yml

 .test: &test
-    image: $GOSDN_TESTING_IMAGE
     stage: test
-    rules:
-        - when: on_success
-    variables:
-        GOSDN_LOG: "nolog"
-        GOSDN_CHANGE_TIMEOUT: "5000ms"
-    coverage: '/total:\s+\(statements\)\s+(\d+.\d+\%)/'
-    artifacts:
-        when: always
-        reports:
-            junit: report.xml
-            coverage_report:
-                coverage_format: cobertura
-                path: ./controller/coverage.xml
-    needs:
-        - build-controller-testing-image
 
-unit-test:
-    script:
-        - cd controller
-        - make ci-unit-test
-    after_script:
-        - cd controller
-        - go tool cover -func=coverage.out
-        - gocover-cobertura < coverage.out > coverage.xml
-    <<: *test
-
-controller-test:
+integration-test-controller:
+    image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/golang:$GOLANG_VERSION-bookworm
+    needs:
+        - ["build-integration-test-images"]
+    #image: debian:latest
+    variables:
+        MONGO_INITDB_ROOT_USERNAME: root
+        MONGO_INITDB_ROOT_PASSWORD: example
+        FF_NETWORK_PER_BUILD: 1
+    services:
+      - name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/mongo:5
+        alias: mongo
+      - name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/rabbitmq:3-management
+        alias: rabbitmq
+        variables:
+          HEALTHCHECK_TCP_PORT: "5672"
+      - name: $PLUGIN_REGISTRY_IMAGE_NAME:$DOCKER_TAG
+      - name: $GNMI_TARGET_INTEGRATION_IMAGE
+        alias: gnmi-target_1
+        command: ["start", "--cert", "/etc/gnmi-target/ssl/certs/gnmi-target-selfsigned.crt", "--key", "/etc/gnmi-target/ssl/private/gnmi-target-selfsigned.key", "--ca_file", "/etc/gnmi-target/ssl/ca.crt"]
+      - name: $GNMI_TARGET_INTEGRATION_IMAGE
+        alias: gnmi-target_2
+        command: ["start", "--cert", "/etc/gnmi-target/ssl/certs/gnmi-target-selfsigned.crt", "--key", "/etc/gnmi-target/ssl/private/gnmi-target-selfsigned.key", "--ca_file", "/etc/gnmi-target/ssl/ca.crt"]
+      # Use gosdn docker image from this branch
+      # Uncomment later when fixed. See: https://code.fbi.h-da.de/danet/gosdn/-/issues/335
+      #- name: "$CI_REGISTRY_IMAGE:${DOCKER_TAG}_integration-test"
+      #  alias: gosdn
+      #  variables:
+      #    GOSDN_ADMIN_PASSWORD: TestPassword
+      #  command: ["--config", "/app/configs/integration-test-gosdn.toml", "--security", "insecure"]
     script:
-        - cd controller
-        - make ci-controller-test
+        # Remove later when fixed. See: https://code.fbi.h-da.de/danet/gosdn/-/issues/335
+        - make build-gosdn
+        - cd artifacts && GOSDN_ADMIN_PASSWORD=TestPassword ./gosdn --config ../controller/configs/integration-test-gosdn.toml --security insecure &
+        - INTEGRATION_TEST_CONTROLLER_URL="localhost:55055" go test ./integration-tests/*
     <<: *test

.gitlab/ci/gnmi-cert-integration.Dockerfile

+ARG GNMI_TARGET_IMAGE=registry.code.fbi.h-da.de/danet/gnmi-target/debian:latest
+
+FROM ${GNMI_TARGET_IMAGE}
+
+RUN mkdir -p /etc/gnmi-target/ssl/certs/
+RUN mkdir -p /etc/gnmi-target/ssl/private/
+
+COPY ./artifacts/ssl/gnmi-target /etc/gnmi-target/ssl

.gitlab/ci/gosdn-integration.Dockerfile

+ARG GOSDN_IMAGE
+
+FROM ${GOSDN_IMAGE}
+
+RUN mkdir /app/ssl
+COPY ./artifacts/ssl/gosdn /app/ssl
+
+#COPY .gitlab/ci/test-start.sh test-start.sh
+#RUN chmod +x test-start.sh
+#ENTRYPOINT ["./test-start.sh"]

.gitlab/ci/.integration-test-containerlab.yml → .gitlab/ci/legacy/.integration-test-containerlab.yml

@@ -3,8 +3,8 @@ variables:
     CLAB_DIR: "/home/gitlab-runner/clab"
     CLAB_NAME: "clab${CI_PIPELINE_IID}"
 
-containerlab-deploy:
-    stage: integration-test
+legacy-containerlab-deploy:
+    stage: test-legacy
     tags:
         - shell-containerlab
     needs: ["build-controller-image", "build-controller-testing-image"]
@@ -38,16 +38,15 @@ containerlab-deploy:
             dotenv: ${CI_PROJECT_DIR}/build.env
 
 
-containerlab-destroy:
-    stage: integration-test
+legacy-containerlab-destroy:
+    stage: test-legacy
     tags:
         - shell-containerlab
     needs:
         [
-            "containerlab-deploy",
-            #"integration-test:nucleus",
-            "integration-test:api"
-
+            "legacy-containerlab-deploy",
+            "legacy-integration-test-legacy:nucleus",
+            "legacy-integration-test-legacy:api"
         ]
     before_script:
         - cd ${CLAB_DIR}

.gitlab/ci/.integration-test.yml → .gitlab/ci/legacy/.integration-test.yml

-.integration-test: &integration-test
+.legacy-integration-test: &legacy-integration-test
     image: $GOSDN_TESTING_IMAGE
-    stage: integration-test
+    stage: test-legacy
     needs:
-        ["containerlab-deploy"]
+        ["legacy-containerlab-deploy"]
     variables:
         GOSDN_LOG: "nolog"
         GOSDN_TEST_API_ENDPOINT: "10.254.254.105:${GOSDN_GRPC_PORT}"
@@ -10,24 +10,23 @@
         GOSDN_TEST_USER: "admin"
         GOSDN_TEST_PASSWORD: "admin"
 
-            # NOTE: Current test setup and runners are changed, therefore this test is not possible to run at the moment
-            #integration-test:nucleus:
-            #    <<: *integration-test
-            #    script:
-            #        - ${CI_PROJECT_DIR}/.gitlab/ci/scripts/wait-for-it.sh ${CEOS_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
-            #        - cd controller/
-            #        - make integration-test-nucleus
+legacy-integration-test-legacy:nucleus:
+    <<: *legacy-integration-test
+    script:
+        - ${CI_PROJECT_DIR}/.gitlab/ci/scripts/wait-for-it.sh ${CEOS_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
+        - cd controller/
+        - make integration-test-nucleus
 
-integration-test:api:
-    <<: *integration-test
+legacy-integration-test-legacy:api:
+    <<: *legacy-integration-test
     variables:
         K8S_OP: "getenv"
     script:
         - cd controller/
         - make integration-test-api
 
-integration-test:cli:
-    <<: *integration-test
+legacy-integration-test:cli:
+    <<: *legacy-integration-test
     script:
         - cd cli/
         - echo "Your tests here"

.gitlab/ci/legacy/.test.yml

+.legacy-test: &legacy-test
+    image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/golang:$GOLANG_VERSION-bookworm
+    stage: test-legacy
+    rules:
+        - when: on_success
+    variables:
+        GOSDN_LOG: "nolog"
+        GOSDN_CHANGE_TIMEOUT: "5000ms"
+    coverage: '/total:\s+\(statements\)\s+(\d+.\d+\%)/'
+    artifacts:
+        when: always
+        reports:
+            junit: report.xml
+            coverage_report:
+                coverage_format: cobertura
+                path: ./controller/coverage.xml
+    needs: []
+
+legacy-unit-test:
+    script:
+        - cd controller
+        - make ci-unit-test
+    after_script:
+        - cd controller
+        - go tool cover -func=coverage.out
+        - gocover-cobertura < coverage.out > coverage.xml
+    <<: *legacy-test
+
+#legacy-controller-test:
+#    script:
+#        - cd controller
+#        - make ci-controller-test
+#    <<: *legacy-test

Makefile

@@ -11,7 +11,7 @@ MAKEFILE_SUBDIR := ./makefiles
 
 GOCMD=go
 GOBUILD=$(GOCMD) build
-GOCLEAN=$(GOCMD) clean
+GOCLEAN=$(GOCMD) clean -cache -fuzzcache -testcache -modcache
 BUILD_ARTIFACTS_PATH=artifacts
 
 PLUGIN_NAME= bundled_plugin.zip
@@ -73,20 +73,34 @@ shell-orchestrator:
 start: build-gosdn
 	./$(BUILD_ARTIFACTS_PATH)/gosdn -l debug --config ./controller/configs/gosdn.toml
 
-start-virt-manager-example: build containerize-plugin-registry
+virt-manager-example-start: build containerize-plugin-registry
 	./scripts/manage_virt_env.sh --mode start --topology dev_env_data/clab/basic_two_aristas.yaml --sdnconfig dev_env_data/sdn/basic_two_aristas.json
 
-stop-virt-manager-example:
+virt-manager-example-stop:
 	./scripts/manage_virt_env.sh --mode stop --topology dev_env_data/clab/basic_two_aristas.yaml
 
-start-dev-env: containerize-gosdn containerize-plugin-registry
-	./scripts/simple-dev-setup.sh --mode start --topology dev_env_data/clab/basic_two_aristas.yaml
+dev-env-start: containerize-gosdn containerize-plugin-registry
+	./scripts/simple-dev-setup.sh --mode start --topology dev_env_data/clab/basic_two_gnmi_targets.yaml
 
-stop-dev-env:
-	./scripts/simple-dev-setup.sh --mode stop --topology dev_env_data/clab/basic_two_aristas.yaml
+dev-env-stop:
+	./scripts/simple-dev-setup.sh --mode stop --topology dev_env_data/clab/basic_two_gnmi_targets.yaml
 
 generate-certs: generate-gnmi-target-certs generate-gosdn-certs
 
+controller-integration-tests: generate-certs containerize-gosdn containerize-plugin-registry
+	docker-compose -f dev_env_data/docker-compose/integration-test_docker-compose.yml down
+	docker-compose -f dev_env_data/docker-compose/integration-test_docker-compose.yml up -d
+	go test ./integration-tests/*
+	docker-compose -f dev_env_data/docker-compose/integration-test_docker-compose.yml down
+
+controller-integration-tests-debug-up: generate-certs containerize-gosdn containerize-plugin-registry
+	docker-compose -f dev_env_data/docker-compose/integration-test_docker-compose.yml up -d
+
+controller-integration-tests-debug-down:
+	docker-compose -f dev_env_data/docker-compose/integration-test_docker-compose.yml down
+
+# Warning: Depending on you go configuration might also clean caches, modules and docker containers from your other projects.
 clean:
 	$(GOCLEAN)
-	rm -rf $(BUILD_ARTIFACTS_PATH)
+	rm -rf $(BUILD_ARTIFACTS_PATH) $(TOOLS_DIR)
+	docker system prune -af

README.md

@@ -34,10 +34,10 @@ There you can also find some tutorials to get to know the SDN controller:
   - [Additional services](#additional-services)
   - [Configuration file](#configuration-file)
 - [Development-Tutorial](#development-tutorial)
+- [Integration tests](#integration-tests)
 - [Contributing](CONTRIBUTING.md)
 - [License](LICENSE)
 - [CI Status](#ci-status)
-- [Development Tutorial](#development-tutorial)
 
 ---
 
@@ -56,7 +56,7 @@ network using one unified controller.
 
 ## Example
 
-A simple showcase how the controller can be adressed after
+A simple showcase how the controller can be addressed after
 `make containerlab-start` is shown below:
 
 ![](gosdn-cli-showcase.webm)
@@ -68,7 +68,7 @@ A simple showcase how the controller can be adressed after
 - The `cli` is the CLI to manage the `controller`.
 - `csbi` is the implementation of Containerised-Southbound-Interfaces (based on
   the idea and the proof of concept of Manuel Kieweg). Allowing to request
-  capabilities of MNEs and generate a containerised Southbound-Interface based
+  capabilities of MNEs and generate a containerized Southbound-Interface based
   on them. They are currently unsupported.
 - `controller` represents the `goSDN-controller`.
 
@@ -96,7 +96,7 @@ Any network element directly configured by `goSDN`
 ## Launch goSDN Controller local
 In this chapter, you learn how to launch the goSDN controller.
 
-Firstly, make sure that you're located in the root directory of gosdn.
+Firstly, make sure that you're located in the root directory of goSDN.
 `goSDN` provides a `Makefile` for all common use cases.
 
 ```sh
@@ -125,7 +125,7 @@ running `./gosdn` from the shell:
 
 ## Getting Started
 
-If you want to use the the [playground](#playground) you have to make sure you
+If you want to use the [playground](#playground) you have to make sure you
 have [containerlab](https://containerlab.dev/install/) installed on your
 system.
 
@@ -201,9 +201,9 @@ For the storage system:
 For the event system (RabbitMQ broker):
 
 - `amqpprefix`: the URI prefix that is needed to connect to the RabbitMQ broker.
-- `amqphost`: ip adress of the broker
+- `amqphost`: IP address of the broker
 - `amqpport`: port of the broker
-- `amqpuser`: user name of credentials to access the broker, default value is `guest`
+- `amqpuser`: username of credentials to access the broker, default value is `guest`
 - `amqppassword`: user password of credentials to access the broker, default is `guest`
 
 For the plugin registry:
@@ -300,3 +300,13 @@ To develop applications, we provide a framework that can be found [here](https:/
 The framework provides some basic code to easily set up the subscription to the event system of the controller, so that your application can get notified via the events published by the RabbitMQ server about information you would like to receive. This includes functions to register and subscribe to the used server. The information you receive includes changes on network elements like if a value of a YANG path changes or other changes like additions or deletions of entities.
 
 Examples where the application framework is used can be found [here](https://code.fbi.h-da.de/danet/gosdn/-/tree/master/applications).
+
+## Integration tests
+
+The integration tests are currently in its own folder named `integration-tests`, as they use a complete black box design.
+The idea is to test as many of our code with as little tests as possible to get a "catch all" approach where a test may fail and hint for a problem, without really telling us what the problem is.
+As of the nature of this project many people contribute only a small part and often don't have time to write good unit-tests.
+Therefore, we simply want to make sure that a change doesn't break anything important.
+
+There exists a `example_tests` folder where examples of using the `integrationTestUtils` are shown. Each test should use the appropriate set-up and tear-down/clean-up functions, so that each test operates independent of each other.
+

applications/arista-routing-engine/arista-routing-engine.Dockerfile

 ARG GOLANG_VERSION=1.21
 ARG BUILDARGS
-ARG $GITLAB_PROXY
+ARG GITLAB_PROXY
 
 FROM ${GITLAB_PROXY}golang:$GOLANG_VERSION-alpine as builder
 WORKDIR /gosdn/

applications/venv-manager/venv-manager.Dockerfile

 ARG GOLANG_VERSION=1.21
 ARG BUILDARGS
-ARG $GITLAB_PROXY=code.fbi.h-da.de:443/danet/dependency_proxy/containers
+ARG GITLAB_PROXY=code.fbi.h-da.de:443/danet/dependency_proxy/containers
 
 FROM ${GITLAB_PROXY}golang:$GOLANG_VERSION-bookworm AS builder
 

applications/venv-manager/venv-manager/venv-manager.go

@@ -137,7 +137,7 @@ func (v *VenvManager) CreateSDNConfigFile() error {
 	return nil
 }
 
-// getSDNConfigData gets the sDN configuration data.
+// getSDNConfigData gets the SDN configuration data.
 func (v *VenvManager) getSDNConfigData() (*string, error) {
 	conn, err := v.createConnection()
 	if err != nil {

applications/ws-events/ws-events.Dockerfile

 ARG GOLANG_VERSION=1.21
 ARG BUILDARGS
-ARG $GITLAB_PROXY
+ARG GITLAB_PROXY
 
 FROM ${GITLAB_PROXY}golang:$GOLANG_VERSION-alpine as builder
 WORKDIR /app/

controller/Makefile

@@ -9,7 +9,7 @@ GOBUILD=$(GOCMD) build
 GOCLEAN=$(GOCMD) clean
 BINARY_NAME=gosdn
 
-# Tool Vesions
+# Tool Versions
 GOTESTSUM_VERSION=v1.8.1
 MOCKERY_VERSION=v2.20.0
 GOCOVER_COBERTURA=v1.2.0
@@ -43,27 +43,11 @@ start-insecure: clean build
 unit-test: install-tools
 	ENVIRONMENT=testing ./$(TOOLS_DIR)/gotestsum --junitfile report.xml --format testname -- -short -race $$( go list ./... | grep -v /forks/ | grep -v /mocks ) -v -trimpath -coverprofile=coverage.out
 
-controller-test: install-tools
-	ENVIRONMENT=testing ./$(TOOLS_DIR)/gotestsum --junitfile report.xml --format testname -- -race -v -run TestRun
+show-unit-test-coverage: unit-test
+	go tool cover -html=coverage.out
 
 ci-unit-test: ci-install-tools
 	ENVIRONMENT=testing gotestsum --junitfile report.xml --format testname -- -short -race $$( go list ./... | grep -v /forks/ | grep -v /mocks ) -v -trimpath -coverprofile=coverage.out -covermode atomic -timeout 30m
 
-ci-controller-test: ci-install-tools
-	ENVIRONMENT=testing gotestsum --junitfile report.xml --format testname -- -race -v -run TestRun -trimpath -coverprofile=coverage.out
-
-integration-test-nucleus:
-	ENVIRONMENT=testing  &&\
-	cd ./test/integration &&\
-	go test -race -v -run TestGnmi_SetIntegration &&\
-	go test -race -v -run TestGnmi_GetIntegration &&\
-	go test -race -v -run TestGnmi_SubscribeIntegration &&\
-	go test -race -v -run TestGnmi_CapabilitiesIntegration
-
-integration-test-api:
-	ENVIRONMENT=testing  &&\
-	cd ./api &&\
-	go test -race -v -run TestApiIntegration
-
 generate-mocks: install-tools
 	./$(TOOLS_DIR)/mockery --all --dir "./interfaces/"