Merge branch 'prepare-k8s-deployment' into 'master'

Prepare k8s deployment

See merge request cocsn/gosdn!128

.gitlab-ci.yml

@@ -6,9 +6,9 @@ stages:
   - .pre
   - test
   - build
-  - deploy
   - apply
   - integration-test
+  - deploy
   - .post
 
 default:
@@ -21,4 +21,5 @@ include:
   - local: '/build/ci/.security-and-compliance-ci.yml'
   - local: '/build/ci/.build-container.yml'
   - local: '/build/ci/.test.yml'
-  - local: '/build/ci/.terraform-ci.yml'
+  - local: '/build/ci/.terraform-ci.yml'
\ No newline at end of file
+  - local: '/build/ci/.deploy-k8s.yml'
\ No newline at end of file

build/cd/deploy.go

+package main
+
+import (
+	"code.fbi.h-da.de/cocsn/gosdn/nucleus"
+	"context"
+	appv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"log"
+	"os"
+)
+
+func main() {
+	kubeconfig, err := clientcmd.BuildConfigFromFlags("https://api.ocp.fbi.h-da.de:6443", "")
+	if err != nil {
+		log.Fatal(err)
+	}
+	kubeconfig.Username = os.Getenv("K8S_DEPLOY_USER")
+	kubeconfig.BearerToken = os.Getenv("K8S_DEPLOY_TOKEN")
+	clientset, err := kubernetes.NewForConfig(kubeconfig)
+	if err != nil {
+		log.Fatal(err)
+	}
+	var tag string
+	switch os.Getenv("CI_COMMIT_BRANCH") {
+	case "master":
+		tag = "latest"
+	case "develop":
+		tag = "develop"
+	default:
+		tag = os.Getenv("CI_COMMIT_SHA")
+	}
+	env := "gosdn-" + tag
+	service := createService(env)
+	config := createConfigMap(env)
+	deployment := createDeployment(env, tag)
+	switch os.Getenv("K8S_OP") {
+	case "create":
+		if err := create(clientset, service, config, deployment, env); err != nil {
+			log.Println(err)
+		}
+	case "delete":
+		if err := remove(clientset, env); err != nil {
+			log.Println(err)
+		}
+	case "update":
+		if err := update(clientset, service, env); err != nil {
+			log.Println(err)
+		}
+		if err := update(clientset, config, env); err != nil {
+			log.Println(err)
+		}
+		if err := update(clientset, deployment, env); err != nil {
+			log.Println(err)
+		}
+	default:
+		log.Fatal("invalid option")
+	}
+}
+
+func create(clientset *kubernetes.Clientset, service *corev1.Service, config *corev1.ConfigMap, deployment *appv1.Deployment, env string) error {
+	opts := metav1.CreateOptions{}
+	ctx := context.Background()
+	_, err := clientset.CoreV1().Services("cocsn").Create(ctx, service, opts)
+	if err != nil {
+		switch err.(type) {
+		case *errors.StatusError:
+			if err.(*errors.StatusError).ErrStatus.Code == 409 {
+				err = update(clientset, service, env)
+			}
+		default:
+			return err
+		}
+	}
+	_, err = clientset.CoreV1().ConfigMaps("cocsn").Create(ctx, config, opts)
+	if err != nil {
+		switch err.(type) {
+		case *errors.StatusError:
+			if err.(*errors.StatusError).ErrStatus.Code == 409 {
+				err = update(clientset, config, env)
+			}
+		default:
+			return err
+		}
+	}
+	_, err = clientset.AppsV1().Deployments("cocsn").Create(ctx, deployment, opts)
+	if err != nil {
+		switch err.(type) {
+		case *errors.StatusError:
+			if err.(*errors.StatusError).ErrStatus.Code == 409 {
+				err = update(clientset, deployment, env)
+			}
+		default:
+			return err
+		}
+	}
+	return err
+}
+
+func update(clientset *kubernetes.Clientset, resource metav1.Common, env string) error {
+	opts := metav1.UpdateOptions{}
+	getOpts := metav1.GetOptions{}
+	ctx := context.Background()
+	switch resource.(type) {
+	case *corev1.Service:
+		service := resource.(*corev1.Service)
+		s, err := clientset.CoreV1().Services("cocsn").Get(ctx, env, getOpts)
+		if err != nil {
+			return err
+		}
+		s.DeepCopyInto(service)
+		_, err = clientset.CoreV1().Services("cocsn").Update(ctx, service, opts)
+		if err != nil {
+			return err
+		}
+	case *corev1.ConfigMap:
+		config := resource.(*corev1.ConfigMap)
+		c, err := clientset.CoreV1().ConfigMaps("cocsn").Get(ctx, env+"-config", getOpts)
+		if err != nil {
+			return err
+		}
+		c.DeepCopyInto(config)
+		_, err = clientset.CoreV1().ConfigMaps("cocsn").Update(ctx, config, opts)
+		if err != nil {
+			return err
+		}
+	case *appv1.Deployment:
+		deployment := resource.(*appv1.Deployment)
+		d, err := clientset.AppsV1().Deployments("cocsn").Get(ctx, env, getOpts)
+		if err != nil {
+			return err
+		}
+		d.DeepCopyInto(deployment)
+		_, err = clientset.AppsV1().Deployments("cocsn").Update(ctx, deployment, opts)
+		if err != nil {
+			return err
+		}
+	default:
+		return &nucleus.ErrInvalidParameters{}
+	}
+	return nil
+}
+
+func remove(clientset *kubernetes.Clientset, env string) error {
+	opts := metav1.DeleteOptions{}
+	ctx := context.Background()
+	err := clientset.CoreV1().Services("cocsn").Delete(ctx, env, opts)
+	if err != nil {
+		log.Println(err)
+	}
+	err = clientset.CoreV1().ConfigMaps("cocsn").Delete(ctx, env+"-config", opts)
+	if err != nil {
+		log.Println(err)
+	}
+	err = clientset.AppsV1().Deployments("cocsn").Delete(ctx, env, opts)
+	if err != nil {
+		log.Println(err)
+	}
+	return err
+}
+
+func createService(environment string) *corev1.Service {
+	return &corev1.Service{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Service",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      environment,
+			Namespace: "cocsn",
+			Labels:    map[string]string{"run": environment},
+		},
+		Spec: corev1.ServiceSpec{
+			Ports: []corev1.ServicePort{
+				{
+					Name:       "http",
+					Port:       8080,
+					TargetPort: intstr.IntOrString{IntVal: 8080},
+				},
+				{
+					Name:       "grpc",
+					Port:       55055,
+					TargetPort: intstr.IntOrString{IntVal: 55055},
+				},
+			},
+			Selector:     map[string]string{"run": environment},
+			Type:         "NodePort",
+			ExternalName: environment + ".apps.ocp.fbi.h-da.de",
+		},
+	}
+}
+
+func createDeployment(environment, hash string) *appv1.Deployment {
+	return &appv1.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Deployment",
+			APIVersion: "apps/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: environment,
+		},
+		Spec: appv1.DeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"run": environment},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"run": environment},
+				},
+				Spec: corev1.PodSpec{
+					Volumes: []corev1.Volume{
+						{
+							Name: "gosdn-config-volume",
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: "gosdn-develop-config",
+									},
+								},
+							},
+						},
+					},
+					Containers: []corev1.Container{
+						{
+							Name:       "gosdn",
+							Image:      "registry.code.fbi.h-da.de/cocsn/gosdn:" + hash,
+							Command:    nil,
+							Args:       nil,
+							WorkingDir: "",
+							Ports: []corev1.ContainerPort{
+								{
+									Name:          "grpc",
+									ContainerPort: 55055,
+								},
+								{
+									Name:          "http",
+									ContainerPort: 8080,
+								},
+							},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "gosdn-config-volume",
+									MountPath: "/usr/local/etc/gosdn/gosdn.toml",
+								},
+							},
+							LivenessProbe: &corev1.Probe{
+								Handler: corev1.Handler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/livez",
+										Port: intstr.IntOrString{IntVal: 8080},
+									},
+								},
+								InitialDelaySeconds: 20,
+								PeriodSeconds:       2,
+							},
+							ReadinessProbe: &corev1.Probe{
+								Handler: corev1.Handler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/readyz",
+										Port: intstr.IntOrString{IntVal: 8080},
+									},
+								},
+								InitialDelaySeconds: 120,
+								PeriodSeconds:       2,
+							},
+							ImagePullPolicy: "Always",
+						},
+					},
+					ImagePullSecrets: []corev1.LocalObjectReference{
+						{Name: "k8s-gosdn-test"},
+					},
+				},
+			},
+			Strategy: appv1.DeploymentStrategy{
+				Type:          "RollingUpdate",
+				RollingUpdate: &appv1.RollingUpdateDeployment{},
+			},
+		},
+	}
+}
+
+func createConfigMap(env string) *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ConfigMap",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: env + "-config",
+		},
+		Data: map[string]string{"gosdn.toml": "#empty"},
+	}
+}

build/ci/.deploy-k8s.yml

+build-k8s-bot:
+  stage: build
+  image: golang:1.16
+  rules:
+    - if: $CI
+  script:
+    - cd build/cd
+    - go build -o k8s-bot
+  artifacts:
+    name: binary
+    paths:
+      - build/cd/k8s-bot
+
+.deploy: &deploy
+  image: bitnami/kubectl:latest
+  before_script:
+    - echo "override global before script"
+  variables:
+    K8S_OP: "create"
+  script:
+    - ./k8s-bot
+  needs:
+    - job: "build-k8s-bot"
+      artifacts: true
+
+deploy:integration-test:
+  <<: *deploy
+  stage: apply
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+
+deploy:develop:
+  <<: *deploy
+  stage: deploy
+  rules:
+    - if: $CI_COMMIT_BRANCH == "develop"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+k8s:destroy:
+  image: bitnami/kubectl:latest
+  rules:
+    - if: $CI_COMMIT_BRANCH == "develop"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+  before_script:
+    - echo "override global before script"
+  stage: .post
+  variables:
+    K8S_OP: "delete"
+  script:
+    - ./k8s-bot
+  needs:
+    - job: "deploy:integration-test"
+      artifacts: true
\ No newline at end of file

build/ci/.terraform-ci.yml

@@ -64,7 +64,7 @@ apply:
     - plan
   <<: *tf
 
-destroy:
+tf:destroy:
   stage: .post
   script:
     - gitlab-terraform destroy

cli/http.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 )
 
-const apiRoot = "/api?"
+const apiRoot = "?"
 
 var builder *strings.Builder
 

cmd/cli.go

@@ -53,6 +53,6 @@ func init() {
 	rootCmd.AddCommand(cliCmd)
 
 	cliCmd.PersistentFlags().StringVar(&uuid, "uuid", "", "uuid of the requested device")
-	cliCmd.PersistentFlags().StringVar(&apiEndpoint, "api-endpoint", "http://localhost:8080", "address of the target")
+	cliCmd.PersistentFlags().StringVar(&apiEndpoint, "api-endpoint", "http://gosdn-develop.apps.ocp.fbi.h-da.de/api", "address of the target")
 
 }

configs/gosdn.toml

-CliSocket = ":55055"

go.mod

@@ -19,4 +19,7 @@ require (
 	golang.org/x/net v0.0.0-20201216054612-986b41b23924
 	google.golang.org/grpc v1.34.0
 	google.golang.org/protobuf v1.26.0
+	k8s.io/api v0.20.5
+	k8s.io/apimachinery v0.20.5
+	k8s.io/client-go v0.20.5
 )