Skip to content
Snippets Groups Projects
Commit afa2a7fa authored by Malte Bauch's avatar Malte Bauch
Browse files

Merge branch 'master-into-develop2' into 'master'

Master into develop2 into master

See merge request !244
parents db2d6379 c5f06abd
No related branches found
No related tags found
No related merge requests found
Pipeline #96694 passed with warnings
Showing
with 476 additions and 48 deletions
.git
.gitlab
build
documentation
mocks
test
.cobra.yaml
.dockeringore
.gitlab-ci.yaml
ARCHITECTURE.md
CONTRIBUTING.md
README.md
root = true
[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf
[{*.yaml, *.yml}]
indent_size = 2
.vscode/
.vscode/launch.json
.DS_Store
documentation/design-documentation/
documentation/design/*.pdf
......@@ -14,4 +15,16 @@ documentation/design/*.pdf
.idea/workspace.xml
restconf/bin/bin
test/.terraform.local/
configs/gosdn.toml
\ No newline at end of file
configs/gosdn.toml
api/api_test.toml
debug.test
# test files
report.xml
nucleus/util/proto/*_test
# Binary
gosdn
# persistent data
**/stores/**
variables:
SECURE_ANALYZERS_PREFIX: registry.gitlab.com/gitlab-org/security-products/analyzers
DOCKER_IMAGE_SHA: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_PIPELINE_SOURCE == "schedule"'
- if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH != "develop" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
when: never
- if: '$CI_COMMIT_BRANCH'
GOSDN_IMAGE: "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}"
GOSDN_TESTING_IMAGE: "${CI_REGISTRY_IMAGE}:testing_${CI_COMMIT_SHA}"
CEOS_IMAGE: "$CI_REGISTRY_IMAGE/ceos:latest"
GOLANG_VERSION: "1.17"
stages:
- .pre
- build
- test
- analyze
- apply
- integration-test
- deploy
- build-release
- .post
default:
before_script:
- git config --global url."https://$GO_MODULES_USER:$GO_MODULES_ACCESS_TOKEN@code.fbi.h-da.de".insteadOf "https://code.fbi.h-da.de"
include:
- local: '/build/ci/.code-quality-ci.yml'
- local: '/build/ci/.security-and-compliance-ci.yml'
- local: '/build/ci/.build-container.yml'
- local: '/build/ci/.test.yml'
- local: '/build/ci/.containerlab-ci.yml'
- local: '/build/ci/.uml-autogen-ci.yml'
- local: '/build/ci/.deploy-k8s.yml'
- local: '/.gitlab/ci/.build-container.yml'
- local: '/.gitlab/ci/.code-quality-ci.yml'
- local: '/.gitlab/ci/.security-and-compliance-ci.yml'
- local: '/.gitlab/ci/.test.yml'
- local: '/.gitlab/ci/.containerlab-ci.yml'
- local: '/.gitlab/ci/.integration-test.yml'
- local: '/.gitlab/ci/.uml-autogen-ci.yml'
- local: '/.gitlab/ci/.build-release.yml'
.build: &build
stage: build
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [ "" ]
variables:
TAG: $CI_COMMIT_BRANCH
before_script:
# replace all slashes in the tag with hyphen, because slashes are not allowed in tags
- TAG=${TAG//\//-}
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$CI_DEPENDENCY_PROXY_SERVER\":{\"username\":\"$CI_DEPENDENCY_PROXY_USER\",\"password\":\"$CI_DEPENDENCY_PROXY_TOKEN\"}}}" > /kaniko/.docker/config.json
needs: []
build-testing-image:
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
variables:
TAG: latest
- if: $CI_COMMIT_BRANCH == "develop"
variables:
TAG: $CI_COMMIT_BRANCH
BUILDARGS: -race
- when: always
script:
- /kaniko/executor
--cache=true
--context "$CI_PROJECT_DIR"
--dockerfile "Dockerfile"
--build-arg "GOLANG_VERSION=$GOLANG_VERSION"
--build-arg "BUILDARGS=$BUILDARGS"
--build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/"
--destination "$GOSDN_TESTING_IMAGE"
--target "installer"
<<: *build
build-image:
rules:
- if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
when: never
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
variables:
TAG: $CI_DEFAULT_BRANCH
- if: $CI_COMMIT_BRANCH == "develop"
variables:
TAG: develop
BUILDARGS: -race
- when: always
script:
- /kaniko/executor
--cache=true
--context "$CI_PROJECT_DIR"
--dockerfile "Dockerfile"
--build-arg "GOLANG_VERSION=$GOLANG_VERSION"
--build-arg "BUILDARGS=$BUILDARGS"
--build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/"
--destination "$GOSDN_IMAGE"
--destination "$CI_REGISTRY_IMAGE:$TAG"
<<: *build
.build-release-docker: &build-release-docker
stage: build-release
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [ "" ]
rules:
# TODO: Implement later when we need it
#- if: $CI_COMMIT_TAG =~ .+ && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
# variables:
# TAG: $CI_COMMIT_TAG
# when: on_success
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
variables:
TAG: $CI_DEFAULT_BRANCH
when: on_success
- when: never
before_script:
# replace all slashes in the tag with hyphen, because slashes are not allowed in tags
- TAG=${TAG//\//-}
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$CI_DEPENDENCY_PROXY_SERVER\":{\"username\":\"$CI_DEPENDENCY_PROXY_USER\",\"password\":\"$CI_DEPENDENCY_PROXY_TOKEN\"}}}" > /kaniko/.docker/config.json
build-release-image:
script:
- /kaniko/executor
--cache=true
--context "$CI_PROJECT_DIR"
--dockerfile "Dockerfile"
--build-arg "GOLANG_VERSION=$GOLANG_VERSION"
--build-arg "BUILDARGS=$BUILDARGS"
--build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/"
--destination "$GOSDN_IMAGE"
--destination "$CI_REGISTRY_IMAGE:$TAG"
- /kaniko/executor
--cache=true
--context "$CI_PROJECT_DIR"
--dockerfile "Dockerfile"
--build-arg "GOLANG_VERSION=$GOLANG_VERSION"
--build-arg "BUILDARGS=$BUILDARGS"
--build-arg "GITLAB_PROXY=${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/"
--destination "$GOSDN_IMAGE"
--destination "$CI_REGISTRY_IMAGE:latest"
<<: *build-release-docker
.build-release-binary: &build-release-binary
stage: build-release
image:
name: ${GITLAB_PROXY}golang:$GOLANG_VERSION
rules:
# TODO: Implement later when we need it
#- if: $CI_COMMIT_TAG =~ .+ && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
# variables:
# TAG: $CI_COMMIT_TAG
# when: on_success
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
variables:
TAG: $CI_DEFAULT_BRANCH
when: on_success
- if: $CI_COMMIT_BRANCH == "develop"
variables:
TAG: $CI_COMMIT_BRANCH
when: on_success
- when: never
build-release-linux:
script:
- GOOS=linux go build ./cmd/gosdn
- mv gosdn gosdn-$TAG
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file gosdn-$TAG "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/gosdn/$TAG/gosdn-$TAG-linux"'
artifacts:
paths:
- gosdn-$TAG
expire_in: 4 weeks
<<: *build-release-binary
build-release-freebsd:
script:
- GOOS=freebsd go build ./cmd/gosdn
- mv gosdn gosdn-$TAG
- - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file gosdn-$TAG "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/gosdn/$TAG/gosdn-$TAG-freebsd"'
artifacts:
paths:
- gosdn-$TAG
expire_in: 4 weeks
<<: *build-release-binary
\ No newline at end of file
code-quality:
image: golangci/golangci-lint:latest-alpine
stage: test
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_COMMIT_BRANCH == "develop"
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/golangci/golangci-lint:v1.42-alpine
stage: analyze
script:
# writes golangci-lint output to gl-code-quality-report.json
- golangci-lint run --config build/ci/.golangci-config/.golangci.yml --out-format code-climate | tee gl-code-quality-report.json
- golangci-lint run --config .gitlab/ci/.golangci-config/.golangci.yml --out-format code-climate | tee gl-code-quality-report.json
artifacts:
reports:
codequality: gl-code-quality-report.json
paths:
- gl-code-quality-report.json
needs: []
variables:
CLAB_DIR: "/mnt"
CLAB_NAME: "clab${CI_PIPELINE_IID}"
# Templates for Job Types
.containerlab_deploy: &containerlab_deploy
stage: apply
rules:
- if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
when: never
- when: on_success
tags:
- shell
before_script:
- cd ${CLAB_DIR}
- export PATH="${PATH}:${CI_PROJECT_DIR}/.gitlab/ci/scripts"
- echo "$CI_REGISTRY_PASSWORD" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- echo ${GOSDN_IMAGE}
- docker pull ${GOSDN_IMAGE}
- docker pull ${CEOS_IMAGE}
.containerlab_template: &containerlab_template
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/alpine:latest
stage: build
rules:
- if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
when: never
- when: on_success
before_script:
- export PATH="${PATH}:${CI_PROJECT_DIR}/.gitlab/ci/scripts"
- firstOctet=$(generate_octet.sh $CI_COMMIT_SHA)
- secondOctet=$(generate_octet.sh $CI_PIPELINE_ID)
- export CLAB_MGMT_SUBNET="172.$firstOctet.$secondOctet.0/24"
script:
- |
sed -e "s|@@CEOS_CONTAINER_IMAGE@@|${CEOS_IMAGE}|g" \
-e "s|@@GOSDN_CONTAINER_IMAGE@@|${GOSDN_IMAGE}|g" \
-e "s|@@CLAB_NAME@@|${CLAB_NAME}|g" \
-e "s|@@CLAB_MGMT_SUBNET@@|${CLAB_MGMT_SUBNET}|g" \
${CLAB_TEMPLATE} > ${CLAB_NAME}.clab.yml
- cat ${CLAB_NAME}.clab.yml
artifacts:
name: ${CLAB_NAME}
paths:
- ${CLAB_NAME}.clab.yml
# JOBS
containerlab:template:integration:
extends: .containerlab_template
variables:
CLAB_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/int01.clab.tmpl.yml"
containerlab:deploy:integration:
extends: .containerlab_deploy
needs: ["containerlab:template:integration", "build-image"]
script:
- sudo containerlab deploy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml --reconfigure
- |
echo -e "\
GOSDN_HTTP_PORT=$(docker_host_port 8080 clab-${CLAB_NAME}-gosdn)\n\
GOSDN_GRPC_PORT=$(docker_host_port 55055 clab-${CLAB_NAME}-gosdn)\n\
CEOS1_PORT=$(docker_host_port 6030 clab-${CLAB_NAME}-ceos1)" \
> ${CI_PROJECT_DIR}/build.env
- cat ${CI_PROJECT_DIR}/build.env
artifacts:
reports:
dotenv: ${CI_PROJECT_DIR}/build.env
containerlab:destroy:
rules:
- if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
when: never
- when: always
stage: .post
tags:
- shell
needs: ["containerlab:deploy:integration", "integration-test:nucleus", "integration-test:api", "containerlab:template:integration"]
before_script:
- cd ${CLAB_DIR}
script:
- sudo containerlab destroy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml
- docker volume rm -f ${CLAB_NAME}-volume
- docker image rm -f ${GOSDN_IMAGE}
allow_failure: true
#containerlab:template:develop:
# extends: .containerlab_template
# variables:
# CLAB_NAME: "dev"
# CLAB_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/dev.clab.tmpl.yml"
# GOSDN_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE:develop"
# rules:
# - if: $CI_COMMIT_BRANCH == 'develop' && $CI_NIGHTLY == null
#
#
#containerlab:deploy:develop:
# extends:
# - .containerlab_deploy
# needs: ["containerlab:template:develop", "build-image"]
# variables:
# CLAB_NAME: "dev"
# GOSDN_CONTAINER_IMAGE: "$CI_REGISTRY_IMAGE:develop"
# script:
# - sudo containerlab deploy --topo ${CI_PROJECT_DIR}/${CLAB_NAME}.clab.yml --reconfigure
# rules:
# - if: $CI_COMMIT_BRANCH == 'develop' && $CI_NIGHTLY == null
containerlab:clean:
stage: .post
tags:
- shell
before_script:
- cd ${CLAB_DIR}
script:
- docker kill $(docker ps -q) || true
- docker rm $(docker ps -a -q) || true
- docker rmi $(docker images | grep 'registry.code.fbi.h-da.de/danet/gosdn ' | awk '{print $3}') || true
- sudo rm -rf ${CLAB_DIR}/clab-* || true
- sudo sed -i 's|.*clab.*||g' /etc/hosts
allow_failure: true
rules:
- if: $CI_SCHEDULE_CLEAN
when: always
build:k8s-bot:
stage: build
image: golang:1.16
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/golang:$GOLANG_VERSION
rules:
- if: $CI_COMMIT_BRANCH == "develop"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
......@@ -14,7 +14,7 @@ build:k8s-bot:
.deploy: &deploy
image:
name: bitnami/kubectl:latest
name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/bitnami/kubectl:latest
entrypoint: [""]
before_script:
- echo "override global before script"
......
......@@ -27,9 +27,13 @@ linters:
enable:
- gofmt
- goimports
- revive
- gocyclo
- govet
- unused
- staticcheck
- typecheck
- revive
- whitespace
issues:
exclude-use-default: false
max-issues-per-linter: 0
......
.integration-test: &integration-test
image: ${CI_REGISTRY_IMAGE}:testing_${CI_PIPELINE_ID}
image: $GOSDN_TESTING_IMAGE
stage: integration-test
rules:
- if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
when: never
- when: on_success
needs:
- job: "containerlab:deploy"
- job: "containerlab:deploy:integration"
tags:
- integration-test-docker
variables:
GOSDN_LOG: "nolog"
GOSDN_TEST_API_ENDPOINT: "141.100.70.178:${GOSDN_GRPC_PORT}"
GOSDN_TEST_ENDPOINT: "141.100.70.178:${CEOS1_PORT}"
CEOS_TEST_ENDPOINT: "141.100.70.178:${CEOS1_PORT}"
GOSDN_TEST_USER: "admin"
GOSDN_TEST_PASSWORD: "admin"
rules:
- if: $CI_NIGHTLY
- if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH
- if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME != $CI_DEFAULT_BRANCH
allow_failure: true
integration-test:nucleus:
<<: *integration-test
script:
- ${CI_PROJECT_DIR}/build/ci/wait-for-it.sh ${GOSDN_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
- ${CI_PROJECT_DIR}/.gitlab/ci/scripts/wait-for-it.sh ${CEOS_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
- cd ./test/integration
- go test -race -v -run TestGnmi_SetIntegration
- go test -race -v -run TestGnmi_GetIntegration
- go test -race -v -run TestGnmi_SubscribeIntegration
- go test -race -v -run TestGnmi_CapabilitiesIntegration
.test: &test
image: ${CI_REGISTRY_IMAGE}:testing_${CI_PIPELINE_ID}
stage: test
allow_failure: true
integration-test:api:
<<: *integration-test
variables:
GOSDN_LOG: "nolog"
GOSDN_CHANGE_TIMEOUT: "100ms"
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH
- if: $CI_NIGHTLY
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
allow_failure: true
unit-test:
K8S_OP: "getenv"
script:
- go test -short -race $(go list ./... | grep -v /forks/ | grep -v /mocks ) -v -coverprofile=coverage.out
after_script:
- go tool cover -func=coverage.out
<<: *test
- cd ./api
- go test -race -v -run TestApiIntegration
.rules: &rules
stage: analyze
rules:
- if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
when: never
- when: always
needs: []
sast:
variables:
SAST_ANALYZER_IMAGE_TAG: '2'
SAST_EXCLUDED_PATHS: spec, test, tests, tmp
SEARCH_MAX_DEPTH: '4'
include:
- template: Security/SAST.gitlab-ci.yml
- template: Dependency-Scanning.gitlab-ci.yml
- template: Security/License-Scanning.gitlab-ci.yml
# - template: Security/Secret-Detection.gitlab-ci.yml
- template: Security/Container-Scanning.gitlab-ci.yml
gemnasium-dependency_scanning:
<<: *rules
gosec-sast:
<<: *rules
license_scanning:
<<: *rules
semgrep-sast:
<<: *rules
#secret_detection:
# <<: *rules
container_scanning:
stage: analyze
rules:
- if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
when: never
- when: always
variables:
DOCKER_IMAGE: "${GOSDN_IMAGE}"
DOCKER_USER: "${CI_REGISTRY_USER}"
DOCKER_PASSWORD: "${CI_REGISTRY_PASSWORD}"
needs:
- build-image
.test: &test
image: $GOSDN_TESTING_IMAGE
stage: test
rules:
- when: on_success
variables:
GOSDN_LOG: "nolog"
GOSDN_CHANGE_TIMEOUT: "100ms"
before_script:
- go get gotest.tools/gotestsum
artifacts:
when: always
reports:
junit: report.xml
needs:
- build-testing-image
unit-test:
script:
- gotestsum --junitfile report.xml --format testname -- -short -race $(go list ./... | grep -v /forks/ | grep -v /mocks ) -v -coverprofile=coverage.out
after_script:
- go tool cover -func=coverage.out
<<: *test
controller-test:
script:
- gotestsum --junitfile report.xml --format testname -- -race -v -run TestRun
<<: *test
test-build:
artifacts:
when: never
script:
- GOOS=linux go build $BUILDARGS ./cmd/gosdn
<<: *test
goplantuml:
image: golang:1.14
image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/golang:$GOLANG_VERSION
stage: .post
only:
- develop
......
#!/bin/sh
TEMPLATE="{{ (index (index .NetworkSettings.Ports \"${1}/tcp\") 0).HostPort }}"
docker inspect -f "${TEMPLATE}" ${2}
......@@ -10,4 +10,4 @@ decimal_value=${decimal_value/-/}
octet=$(((decimal_value % 255)))
echo -n $octet
\ No newline at end of file
echo -n $octet
File moved
[submodule "documentation/manual"]
path = documentation/manual
url = https://git.overleaf.com/620142e0fbf7476b92f7240a
FROM golang:1.16-buster AS installer
ARG GITLAB_USER
ARG GITLAB_TOKEN
ARG GOLANG_VERSION=1.17
ARG BUILDARGS
ARG $GITLAB_PROXY
RUN apt-get update && apt-get install -y git
RUN git config --global url."https://$GITLAB_USER:$GITLAB_TOKEN@code.fbi.h-da.de".insteadOf "https://code.fbi.h-da.de"
RUN go env -w GOPRIVATE=code.fbi.h-da.de/*
FROM ${GITLAB_PROXY}golang:$GOLANG_VERSION-buster AS installer
WORKDIR /src/gosdn
COPY go.* ./
......@@ -13,20 +10,11 @@ RUN go mod download
FROM installer as builder
COPY ./api ./api
COPY ./build ./build
COPY ./cmd ./cmd
COPY ./cli ./cli
COPY ./configs ./configs
COPY ./database ./database
COPY ./mocks ./mocks
COPY ./nucleus ./nucleus
COPY ./test ./test
COPY . ./
RUN GOOS=linux go build $BUILDARGS ./cmd/gosdn
FROM debian:latest
FROM ${GITLAB_PROXY}debian:bullseye
EXPOSE 8080
EXPOSE 55055
COPY --from=builder /src/gosdn/gosdn .
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment