Merge branch 'prepare-k8s-deployment' into 'master'

Prepare k8s deployment

See merge request cocsn/gosdn!128

.gitlab-ci.yml

@@ -6,9 +6,9 @@ stages:
   - .pre
   - test
   - build
-  - deploy
   - apply
   - integration-test
+  - deploy
   - .post
 
 default:
@@ -21,4 +21,5 @@ include:
   - local: '/build/ci/.security-and-compliance-ci.yml'
   - local: '/build/ci/.build-container.yml'
   - local: '/build/ci/.test.yml'
-  - local: '/build/ci/.terraform-ci.yml'
+  - local: '/build/ci/.terraform-ci.yml'
\ No newline at end of file
+  - local: '/build/ci/.deploy-k8s.yml'
\ No newline at end of file

build/cd/deploy.go

+package main
+
+import (
+	"code.fbi.h-da.de/cocsn/gosdn/nucleus"
+	"context"
+	log "github.com/sirupsen/logrus"
+	appv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	netv1 "k8s.io/api/networking/v1beta1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"os"
+)
+
+func main() {
+	log.SetFormatter(&log.JSONFormatter{})
+	kubeconfig, err := clientcmd.BuildConfigFromFlags("https://api.ocp.fbi.h-da.de:6443", "")
+	if err != nil {
+		log.Fatal(err)
+	}
+	kubeconfig.BearerToken = os.Getenv("K8S_DEPLOY_TOKEN")
+	clientset, err := kubernetes.NewForConfig(kubeconfig)
+	if err != nil {
+		log.Fatal(err)
+	}
+	var tag string
+	switch os.Getenv("CI_COMMIT_BRANCH") {
+	case "master":
+		tag = "latest"
+	case "develop":
+		tag = "develop"
+	default:
+		tag = os.Getenv("CI_COMMIT_SHA")
+	}
+
+	switch os.Getenv("K8S_OP") {
+	case "create":
+		if err := create(clientset, tag); err != nil {
+			log.Fatal(err)
+		}
+	case "delete":
+		if err := remove(clientset, tag); err != nil {
+			log.Fatal(err)
+		}
+	default:
+		log.Fatal("invalid option")
+	}
+}
+
+// nolint
+func create(clientset *kubernetes.Clientset, tag string) error {
+	env := "gosdn-" + tag
+	service := createService(env)
+	ingress := createIngress(env)
+	config := createConfigMap(env)
+	deployment := createDeployment(env, tag)
+	opts := metav1.CreateOptions{}
+	ctx := context.Background()
+	_, err := clientset.CoreV1().Services("cocsn").Create(ctx, service, opts)
+	if err != nil {
+		switch err.(type) {
+		case *errors.StatusError:
+			if err.(*errors.StatusError).ErrStatus.Code == 409 {
+				if err := update(clientset, service, env); err != nil {
+					return err
+				}
+			} else {
+				log.Error(err)
+			}
+		default:
+			log.Error(err)
+		}
+	} else {
+		log.Printf("service %v created", service.Name)
+	}
+	_, err = clientset.NetworkingV1beta1().Ingresses("cocsn").Create(ctx, ingress, opts)
+	if err != nil {
+		switch err.(type) {
+		case *errors.StatusError:
+			if err.(*errors.StatusError).ErrStatus.Code == 409 {
+				if err := update(clientset, ingress, env); err != nil {
+					log.Error(err)
+				}
+			} else {
+				log.Error(err)
+			}
+		default:
+			log.Error(err)
+		}
+	} else {
+		log.Printf("ingress %v created", ingress.Name)
+	}
+	_, err = clientset.CoreV1().ConfigMaps("cocsn").Create(ctx, config, opts)
+	if err != nil {
+		switch err.(type) {
+		case *errors.StatusError:
+			if err.(*errors.StatusError).ErrStatus.Code == 409 {
+				if err := update(clientset, config, env); err != nil {
+					log.Error(err)
+				}
+			} else {
+				log.Error(err)
+			}
+		default:
+			log.Error(err)
+		}
+	} else {
+		log.Printf("configMap %v created", config.Name)
+	}
+	_, err = clientset.AppsV1().Deployments("cocsn").Create(ctx, deployment, opts)
+	if err != nil {
+		switch err.(type) {
+		case *errors.StatusError:
+			if err.(*errors.StatusError).ErrStatus.Code == 409 {
+				if err := update(clientset, deployment, env); err != nil {
+					log.Error(err)
+				}
+			} else {
+				log.Error(err)
+			}
+		default:
+			log.Error(err)
+		}
+	} else {
+		log.Printf("deployment %v created", deployment.Name)
+	}
+	return nil
+}
+
+func update(clientset *kubernetes.Clientset, resource metav1.Common, env string) error {
+	opts := metav1.UpdateOptions{}
+	getOpts := metav1.GetOptions{}
+	ctx := context.Background()
+	switch resource.(type) {
+	case *corev1.Service:
+		service := resource.(*corev1.Service)
+		s, err := clientset.CoreV1().Services("cocsn").Get(ctx, env, getOpts)
+		if err != nil {
+			return err
+		}
+		s.DeepCopyInto(service)
+		_, err = clientset.CoreV1().Services("cocsn").Update(ctx, service, opts)
+		if err != nil {
+			return err
+		}
+		log.Printf("service %v updated", service.Name)
+	case *netv1.Ingress:
+		ingress := resource.(*netv1.Ingress)
+		i, err := clientset.NetworkingV1beta1().Ingresses("cocsn").Get(ctx, env, getOpts)
+		if err != nil {
+			return err
+		}
+		i.DeepCopyInto(ingress)
+		_, err = clientset.NetworkingV1beta1().Ingresses("cocsn").Update(ctx, ingress, opts)
+		if err != nil {
+			return err
+		}
+		log.Printf("ingress %v updated", ingress.Name)
+	case *corev1.ConfigMap:
+		config := resource.(*corev1.ConfigMap)
+		c, err := clientset.CoreV1().ConfigMaps("cocsn").Get(ctx, env+"-config", getOpts)
+		if err != nil {
+			return err
+		}
+		c.DeepCopyInto(config)
+		_, err = clientset.CoreV1().ConfigMaps("cocsn").Update(ctx, config, opts)
+		if err != nil {
+			return err
+		}
+		log.Printf("configMap %v updated", config.Name)
+	case *appv1.Deployment:
+		deployment := resource.(*appv1.Deployment)
+		d, err := clientset.AppsV1().Deployments("cocsn").Get(ctx, env, getOpts)
+		if err != nil {
+			return err
+		}
+		d.DeepCopyInto(deployment)
+		_, err = clientset.AppsV1().Deployments("cocsn").Update(ctx, deployment, opts)
+		if err != nil {
+			return err
+		}
+		log.Printf("deployment %v updated", deployment.Name)
+	default:
+		return &nucleus.ErrInvalidParameters{}
+	}
+	return nil
+}
+
+func remove(clientset *kubernetes.Clientset, tag string) error {
+	env := "gosdn-" + tag
+	opts := metav1.DeleteOptions{}
+	ctx := context.Background()
+	err := clientset.CoreV1().Services("cocsn").Delete(ctx, env, opts)
+	if err != nil {
+		log.Error(err)
+	} else {
+		log.Printf("service %v deleted", env)
+	}
+	err = clientset.CoreV1().ConfigMaps("cocsn").Delete(ctx, env+"-config", opts)
+	if err != nil {
+		log.Error(err)
+	} else {
+		log.Printf("config %v deleted", env+"-config")
+	}
+	err = clientset.AppsV1().Deployments("cocsn").Delete(ctx, env, opts)
+	if err != nil {
+		log.Error(err)
+	} else {
+		log.Printf("deployment %v deleted", env)
+	}
+	err = clientset.NetworkingV1beta1().Ingresses("cocsn").Delete(ctx, env, opts)
+	if err != nil {
+		log.Error(err)
+	} else {
+		log.Printf("ingress %v deleted", env)
+	}
+	return err
+}
+
+func createService(environment string) *corev1.Service {
+	return &corev1.Service{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Service",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      environment,
+			Namespace: "cocsn",
+			Labels:    map[string]string{"run": environment},
+		},
+		Spec: corev1.ServiceSpec{
+			Ports: []corev1.ServicePort{
+				{
+					Name:       "http",
+					Port:       8080,
+					TargetPort: intstr.IntOrString{IntVal: 8080},
+				},
+				{
+					Name:       "grpc",
+					Port:       55055,
+					TargetPort: intstr.IntOrString{IntVal: 55055},
+				},
+			},
+			Selector:     map[string]string{"run": environment},
+			Type:         "NodePort",
+			ExternalName: environment + ".apps.ocp.fbi.h-da.de",
+		},
+	}
+}
+
+func createDeployment(environment, hash string) *appv1.Deployment {
+	return &appv1.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Deployment",
+			APIVersion: "apps/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: environment,
+		},
+		Spec: appv1.DeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"run": environment},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"run": environment},
+				},
+				Spec: corev1.PodSpec{
+					Volumes: []corev1.Volume{
+						{
+							Name: "gosdn-config-volume",
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: "gosdn-develop-config",
+									},
+								},
+							},
+						},
+					},
+					Containers: []corev1.Container{
+						{
+							Name:       "gosdn",
+							Image:      "registry.code.fbi.h-da.de/cocsn/gosdn:" + hash,
+							Command:    nil,
+							Args:       nil,
+							WorkingDir: "",
+							Ports: []corev1.ContainerPort{
+								{
+									Name:          "grpc",
+									ContainerPort: 55055,
+								},
+								{
+									Name:          "http",
+									ContainerPort: 8080,
+								},
+							},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "gosdn-config-volume",
+									MountPath: "/usr/local/etc/gosdn/gosdn.toml",
+								},
+							},
+							LivenessProbe: &corev1.Probe{
+								Handler: corev1.Handler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/livez",
+										Port: intstr.IntOrString{IntVal: 8080},
+									},
+								},
+								InitialDelaySeconds: 5,
+								PeriodSeconds:       2,
+							},
+							ReadinessProbe: &corev1.Probe{
+								Handler: corev1.Handler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/readyz",
+										Port: intstr.IntOrString{IntVal: 8080},
+									},
+								},
+								InitialDelaySeconds: 10,
+								PeriodSeconds:       2,
+							},
+							ImagePullPolicy: "Always",
+						},
+					},
+					ImagePullSecrets: []corev1.LocalObjectReference{
+						{Name: "k8s-gosdn-test"},
+					},
+				},
+			},
+			Strategy: appv1.DeploymentStrategy{
+				Type:          "RollingUpdate",
+				RollingUpdate: &appv1.RollingUpdateDeployment{},
+			},
+		},
+	}
+}
+
+func createConfigMap(env string) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ConfigMap",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: env + "-config",
+		},
+		Data: map[string]string{"gosdn.toml": "#empty"},
+	}
+}
+
+func createIngress(env string) *netv1.Ingress {
+	return &netv1.Ingress{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Ingress",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      env,
+			Namespace: "cocsn",
+		},
+		Spec: netv1.IngressSpec{
+			Rules: []netv1.IngressRule{
+				{
+					Host: env + ".apps.ocp.fbi.h-da.de",
+					IngressRuleValue: netv1.IngressRuleValue{
+						HTTP: &netv1.HTTPIngressRuleValue{
+							Paths: []netv1.HTTPIngressPath{
+								{
+									Path: "/api",
+									Backend: netv1.IngressBackend{
+										ServiceName: env,
+										ServicePort: intstr.IntOrString{IntVal: 8080},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		Status: netv1.IngressStatus{},
+	}
+}

build/ci/.build-container.yml

@@ -21,6 +21,7 @@ variables:
         --build-arg BUILDARGS=$BUILDARGS \
         -t $DOCKER_IMAGE_SHA .
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+    - docker push $DOCKER_IMAGE_SHA
     - docker tag $DOCKER_IMAGE_SHA $TAG
     - docker push $TAG
 

build/ci/.deploy-k8s.yml

+build:k8s-bot:
+  stage: build
+  image: golang:1.16
+  rules:
+    - if: $CI_COMMIT_BRANCH == "develop"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  script:
+    - cd build/cd
+    - go build -o k8s-bot
+  artifacts:
+    name: binary
+    paths:
+      - build/cd/k8s-bot
+
+.deploy: &deploy
+  image: bitnami/kubectl:latest
+  before_script:
+    - echo "override global before script"
+  variables:
+    K8S_OP: "create"
+  script:
+    - ./build/cd/k8s-bot
+
+deploy:integration-test:
+  <<: *deploy
+  stage: apply
+  needs:
+    - job: "build:merge-request"
+    - job: "build:k8s-bot"
+      artifacts: true
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+
+deploy:develop:
+  <<: *deploy
+  stage: deploy
+  needs:
+    - job: "build:develop"
+    - job: "build:k8s-bot"
+      artifacts: true
+  rules:
+    - if: $CI_COMMIT_BRANCH == "develop"
+
+deploy:latest:
+  <<: *deploy
+  stage: deploy
+  needs:
+    - job: "build:latest"
+    - job: "build:k8s-bot"
+      artifacts: true
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+k8s:destroy:
+  image: bitnami/kubectl:latest
+  rules:
+    - if: $CI_COMMIT_BRANCH == "develop"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  before_script:
+    - echo "override global before script"
+  stage: .post
+  variables:
+    K8S_OP: "delete"
+  script:
+    - ./build/cd/k8s-bot
+  dependencies:
+    - build:k8s-bot
\ No newline at end of file

build/ci/.terraform-ci.yml

@@ -64,7 +64,7 @@ apply:
     - plan
   <<: *tf
 
-destroy:
+tf:destroy:
   stage: .post
   script:
     - gitlab-terraform destroy

build/ci/.test.yml

 integration-test:
   image: golang:1.14
   stage: integration-test
-  needs: [ "apply" ]
+  needs:
+    - job: "apply"
+    - job: "deploy:integration-test"
   variables:
     GOSDN_LOG: "nolog"
+    GOSDN_TEST_API_ENDPOINT: http://gosdn-$CI_COMMIT_SHA.apps.ocp.fbi.h-da.de/api
   rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME != $CI_DEFAULT_BRANCH
       allow_failure: true
-    - if: $CI_COMMIT_BRANCH == "integration-test"
-      allow_failure: true
   script:
-    - sleep 1m
+    - sleep 2m
     - go test -race ./test/integration -v -coverprofile=coverage.out
 
 .test: &test

cli/http.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 )
 
-const apiRoot = "/api?"
+const apiRoot = "?"
 
 var builder *strings.Builder
 

cmd/cli.go

@@ -53,6 +53,6 @@ func init() {
 	rootCmd.AddCommand(cliCmd)
 
 	cliCmd.PersistentFlags().StringVar(&uuid, "uuid", "", "uuid of the requested device")
-	cliCmd.PersistentFlags().StringVar(&apiEndpoint, "api-endpoint", "http://localhost:8080", "address of the target")
+	cliCmd.PersistentFlags().StringVar(&apiEndpoint, "api-endpoint", "http://gosdn-develop.apps.ocp.fbi.h-da.de/api", "address of the target")
 
 }

configs/gosdn.toml

-CliSocket = ":55055"

go.mod

@@ -19,4 +19,7 @@ require (
 	golang.org/x/net v0.0.0-20201216054612-986b41b23924
 	google.golang.org/grpc v1.34.0
 	google.golang.org/protobuf v1.26.0
+	k8s.io/api v0.20.5
+	k8s.io/apimachinery v0.20.5
+	k8s.io/client-go v0.20.5
 )

nucleus/controller_test.go

@@ -47,7 +47,7 @@ func TestRun(t *testing.T) {
 			return
 		}
 		if !reflect.DeepEqual(got.StatusCode, tests[0].want) {
-			t.Errorf("Run() got: %v, want %v", got.StatusCode, tests[0].want)
+			t.Errorf("livez got: %v, want %v", got.StatusCode, tests[0].want)
 		}
 		got, err = http.Get(tests[0].args.request)
 		if err != nil {
@@ -55,7 +55,7 @@ func TestRun(t *testing.T) {
 			return
 		}
 		if !reflect.DeepEqual(got.StatusCode, tests[1].want) {
-			t.Errorf("Run() got: %v, want %v", got.StatusCode, tests[1].want)
+			t.Errorf("readyz got: %v, want %v", got.StatusCode, tests[1].want)
 		}
 		got, err = http.Get(tests[0].args.request)
 		if err != nil {
@@ -63,7 +63,7 @@ func TestRun(t *testing.T) {
 			return
 		}
 		if !reflect.DeepEqual(got.StatusCode, tests[2].want) {
-			t.Errorf("Run() got: %v, want %v", got.StatusCode, tests[2].want)
+			t.Errorf("api init got: %v, want %v", got.StatusCode, tests[2].want)
 		}
 	})
 

test/integration/cliIntegration_test.go

@@ -2,15 +2,9 @@ package integration
 
 import (
 	"code.fbi.h-da.de/cocsn/gosdn/cli"
-	"os"
 	"testing"
 )
 
-func TestMain(m *testing.M) {
-	testSetupIntegration()
-	os.Exit(m.Run())
-}
-
 func TestCapabilities(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")

test/integration/cmdIntegration_test.go

@@ -17,12 +17,18 @@ import (
 const unreachable = "203.0.113.10:6030"
 
 var testAddress = "141.100.70.171:6030"
-var testAPIEndpoint = "http://141.100.70.171:8080"
+var testAPIEndpoint = "http://141.100.70.171:8080/api"
 var testUsername = "admin"
 var testPassword = "arista"
 var defaultPath = []string{"/system/config/hostname"}
 var opt *nucleus.GnmiTransportOptions
 
+func TestMain(m *testing.M) {
+	testSetupIntegration()
+	os.Exit(m.Run())
+}
+
+
 func testSetupIntegration() {
 	if os.Getenv("GOSDN_LOG") == "nolog" {
 		log.SetLevel(log.PanicLevel)
@@ -31,18 +37,22 @@ func testSetupIntegration() {
 	a := os.Getenv("GOSDN_TEST_ENDPOINT")
 	if a != "" {
 		testAddress = a
+		log.Infof("GOSDN_TEST_ENDPOINT set to %v", testAddress)
 	}
 	api := os.Getenv("GOSDN_TEST_API_ENDPOINT")
 	if api != "" {
 		testAPIEndpoint = api
+		log.Infof("GOSDN_TEST_API_ENDPOINT set to %v", testAddress)
 	}
 	u := os.Getenv("GOSDN_TEST_USER")
 	if u != "" {
 		testUsername = u
+		log.Infof("GOSDN_TEST_USER set to %v", testAddress)
 	}
 	p := os.Getenv("GOSDN_TEST_PASSWORD")
 	if p != "" {
 		testPassword = p
+		log.Infof("GOSDN_TEST_PASSWORD set to %v", testAddress)
 	}
 
 	gnmiMessages = map[string]pb.Message{