Commit 3b4a6531 authored by Manuel Kieweg's avatar Manuel Kieweg 🤷
Browse files

Merge branch 'prepare-k8s-deployment' into 'master'

Prepare k8s deployment

See merge request cocsn/gosdn!128
parents 8dc01bac 6318c0fc
......@@ -6,9 +6,9 @@ stages:
- .pre
- test
- build
- deploy
- apply
- integration-test
- deploy
- .post
default:
......@@ -21,4 +21,5 @@ include:
- local: '/build/ci/.security-and-compliance-ci.yml'
- local: '/build/ci/.build-container.yml'
- local: '/build/ci/.test.yml'
- local: '/build/ci/.terraform-ci.yml'
\ No newline at end of file
- local: '/build/ci/.terraform-ci.yml'
- local: '/build/ci/.deploy-k8s.yml'
\ No newline at end of file
package main
import (
"code.fbi.h-da.de/cocsn/gosdn/nucleus"
"context"
log "github.com/sirupsen/logrus"
appv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
netv1 "k8s.io/api/networking/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"os"
)
func main() {
log.SetFormatter(&log.JSONFormatter{})
kubeconfig, err := clientcmd.BuildConfigFromFlags("https://api.ocp.fbi.h-da.de:6443", "")
if err != nil {
log.Fatal(err)
}
kubeconfig.BearerToken = os.Getenv("K8S_DEPLOY_TOKEN")
clientset, err := kubernetes.NewForConfig(kubeconfig)
if err != nil {
log.Fatal(err)
}
var tag string
switch os.Getenv("CI_COMMIT_BRANCH") {
case "master":
tag = "latest"
case "develop":
tag = "develop"
default:
tag = os.Getenv("CI_COMMIT_SHA")
}
switch os.Getenv("K8S_OP") {
case "create":
if err := create(clientset, tag); err != nil {
log.Fatal(err)
}
case "delete":
if err := remove(clientset, tag); err != nil {
log.Fatal(err)
}
default:
log.Fatal("invalid option")
}
}
// nolint
func create(clientset *kubernetes.Clientset, tag string) error {
env := "gosdn-" + tag
service := createService(env)
ingress := createIngress(env)
config := createConfigMap(env)
deployment := createDeployment(env, tag)
opts := metav1.CreateOptions{}
ctx := context.Background()
_, err := clientset.CoreV1().Services("cocsn").Create(ctx, service, opts)
if err != nil {
switch err.(type) {
case *errors.StatusError:
if err.(*errors.StatusError).ErrStatus.Code == 409 {
if err := update(clientset, service, env); err != nil {
return err
}
} else {
log.Error(err)
}
default:
log.Error(err)
}
} else {
log.Printf("service %v created", service.Name)
}
_, err = clientset.NetworkingV1beta1().Ingresses("cocsn").Create(ctx, ingress, opts)
if err != nil {
switch err.(type) {
case *errors.StatusError:
if err.(*errors.StatusError).ErrStatus.Code == 409 {
if err := update(clientset, ingress, env); err != nil {
log.Error(err)
}
} else {
log.Error(err)
}
default:
log.Error(err)
}
} else {
log.Printf("ingress %v created", ingress.Name)
}
_, err = clientset.CoreV1().ConfigMaps("cocsn").Create(ctx, config, opts)
if err != nil {
switch err.(type) {
case *errors.StatusError:
if err.(*errors.StatusError).ErrStatus.Code == 409 {
if err := update(clientset, config, env); err != nil {
log.Error(err)
}
} else {
log.Error(err)
}
default:
log.Error(err)
}
} else {
log.Printf("configMap %v created", config.Name)
}
_, err = clientset.AppsV1().Deployments("cocsn").Create(ctx, deployment, opts)
if err != nil {
switch err.(type) {
case *errors.StatusError:
if err.(*errors.StatusError).ErrStatus.Code == 409 {
if err := update(clientset, deployment, env); err != nil {
log.Error(err)
}
} else {
log.Error(err)
}
default:
log.Error(err)
}
} else {
log.Printf("deployment %v created", deployment.Name)
}
return nil
}
func update(clientset *kubernetes.Clientset, resource metav1.Common, env string) error {
opts := metav1.UpdateOptions{}
getOpts := metav1.GetOptions{}
ctx := context.Background()
switch resource.(type) {
case *corev1.Service:
service := resource.(*corev1.Service)
s, err := clientset.CoreV1().Services("cocsn").Get(ctx, env, getOpts)
if err != nil {
return err
}
s.DeepCopyInto(service)
_, err = clientset.CoreV1().Services("cocsn").Update(ctx, service, opts)
if err != nil {
return err
}
log.Printf("service %v updated", service.Name)
case *netv1.Ingress:
ingress := resource.(*netv1.Ingress)
i, err := clientset.NetworkingV1beta1().Ingresses("cocsn").Get(ctx, env, getOpts)
if err != nil {
return err
}
i.DeepCopyInto(ingress)
_, err = clientset.NetworkingV1beta1().Ingresses("cocsn").Update(ctx, ingress, opts)
if err != nil {
return err
}
log.Printf("ingress %v updated", ingress.Name)
case *corev1.ConfigMap:
config := resource.(*corev1.ConfigMap)
c, err := clientset.CoreV1().ConfigMaps("cocsn").Get(ctx, env+"-config", getOpts)
if err != nil {
return err
}
c.DeepCopyInto(config)
_, err = clientset.CoreV1().ConfigMaps("cocsn").Update(ctx, config, opts)
if err != nil {
return err
}
log.Printf("configMap %v updated", config.Name)
case *appv1.Deployment:
deployment := resource.(*appv1.Deployment)
d, err := clientset.AppsV1().Deployments("cocsn").Get(ctx, env, getOpts)
if err != nil {
return err
}
d.DeepCopyInto(deployment)
_, err = clientset.AppsV1().Deployments("cocsn").Update(ctx, deployment, opts)
if err != nil {
return err
}
log.Printf("deployment %v updated", deployment.Name)
default:
return &nucleus.ErrInvalidParameters{}
}
return nil
}
func remove(clientset *kubernetes.Clientset, tag string) error {
env := "gosdn-" + tag
opts := metav1.DeleteOptions{}
ctx := context.Background()
err := clientset.CoreV1().Services("cocsn").Delete(ctx, env, opts)
if err != nil {
log.Error(err)
} else {
log.Printf("service %v deleted", env)
}
err = clientset.CoreV1().ConfigMaps("cocsn").Delete(ctx, env+"-config", opts)
if err != nil {
log.Error(err)
} else {
log.Printf("config %v deleted", env+"-config")
}
err = clientset.AppsV1().Deployments("cocsn").Delete(ctx, env, opts)
if err != nil {
log.Error(err)
} else {
log.Printf("deployment %v deleted", env)
}
err = clientset.NetworkingV1beta1().Ingresses("cocsn").Delete(ctx, env, opts)
if err != nil {
log.Error(err)
} else {
log.Printf("ingress %v deleted", env)
}
return err
}
func createService(environment string) *corev1.Service {
return &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: environment,
Namespace: "cocsn",
Labels: map[string]string{"run": environment},
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Name: "http",
Port: 8080,
TargetPort: intstr.IntOrString{IntVal: 8080},
},
{
Name: "grpc",
Port: 55055,
TargetPort: intstr.IntOrString{IntVal: 55055},
},
},
Selector: map[string]string{"run": environment},
Type: "NodePort",
ExternalName: environment + ".apps.ocp.fbi.h-da.de",
},
}
}
func createDeployment(environment, hash string) *appv1.Deployment {
return &appv1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: environment,
},
Spec: appv1.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"run": environment},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{"run": environment},
},
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: "gosdn-config-volume",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: "gosdn-develop-config",
},
},
},
},
},
Containers: []corev1.Container{
{
Name: "gosdn",
Image: "registry.code.fbi.h-da.de/cocsn/gosdn:" + hash,
Command: nil,
Args: nil,
WorkingDir: "",
Ports: []corev1.ContainerPort{
{
Name: "grpc",
ContainerPort: 55055,
},
{
Name: "http",
ContainerPort: 8080,
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "gosdn-config-volume",
MountPath: "/usr/local/etc/gosdn/gosdn.toml",
},
},
LivenessProbe: &corev1.Probe{
Handler: corev1.Handler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/livez",
Port: intstr.IntOrString{IntVal: 8080},
},
},
InitialDelaySeconds: 5,
PeriodSeconds: 2,
},
ReadinessProbe: &corev1.Probe{
Handler: corev1.Handler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/readyz",
Port: intstr.IntOrString{IntVal: 8080},
},
},
InitialDelaySeconds: 10,
PeriodSeconds: 2,
},
ImagePullPolicy: "Always",
},
},
ImagePullSecrets: []corev1.LocalObjectReference{
{Name: "k8s-gosdn-test"},
},
},
},
Strategy: appv1.DeploymentStrategy{
Type: "RollingUpdate",
RollingUpdate: &appv1.RollingUpdateDeployment{},
},
},
}
}
func createConfigMap(env string) *corev1.ConfigMap {
return &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
Kind: "ConfigMap",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: env + "-config",
},
Data: map[string]string{"gosdn.toml": "#empty"},
}
}
func createIngress(env string) *netv1.Ingress {
return &netv1.Ingress{
TypeMeta: metav1.TypeMeta{
Kind: "Ingress",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: env,
Namespace: "cocsn",
},
Spec: netv1.IngressSpec{
Rules: []netv1.IngressRule{
{
Host: env + ".apps.ocp.fbi.h-da.de",
IngressRuleValue: netv1.IngressRuleValue{
HTTP: &netv1.HTTPIngressRuleValue{
Paths: []netv1.HTTPIngressPath{
{
Path: "/api",
Backend: netv1.IngressBackend{
ServiceName: env,
ServicePort: intstr.IntOrString{IntVal: 8080},
},
},
},
},
},
},
},
},
Status: netv1.IngressStatus{},
}
}
......@@ -21,6 +21,7 @@ variables:
--build-arg BUILDARGS=$BUILDARGS \
-t $DOCKER_IMAGE_SHA .
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker push $DOCKER_IMAGE_SHA
- docker tag $DOCKER_IMAGE_SHA $TAG
- docker push $TAG
......
build:k8s-bot:
stage: build
image: golang:1.16
rules:
- if: $CI_COMMIT_BRANCH == "develop"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
script:
- cd build/cd
- go build -o k8s-bot
artifacts:
name: binary
paths:
- build/cd/k8s-bot
.deploy: &deploy
image: bitnami/kubectl:latest
before_script:
- echo "override global before script"
variables:
K8S_OP: "create"
script:
- ./build/cd/k8s-bot
deploy:integration-test:
<<: *deploy
stage: apply
needs:
- job: "build:merge-request"
- job: "build:k8s-bot"
artifacts: true
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
deploy:develop:
<<: *deploy
stage: deploy
needs:
- job: "build:develop"
- job: "build:k8s-bot"
artifacts: true
rules:
- if: $CI_COMMIT_BRANCH == "develop"
deploy:latest:
<<: *deploy
stage: deploy
needs:
- job: "build:latest"
- job: "build:k8s-bot"
artifacts: true
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
k8s:destroy:
image: bitnami/kubectl:latest
rules:
- if: $CI_COMMIT_BRANCH == "develop"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
before_script:
- echo "override global before script"
stage: .post
variables:
K8S_OP: "delete"
script:
- ./build/cd/k8s-bot
dependencies:
- build:k8s-bot
\ No newline at end of file
......@@ -64,7 +64,7 @@ apply:
- plan
<<: *tf
destroy:
tf:destroy:
stage: .post
script:
- gitlab-terraform destroy
......
integration-test:
image: golang:1.14
stage: integration-test
needs: [ "apply" ]
needs:
- job: "apply"
- job: "deploy:integration-test"
variables:
GOSDN_LOG: "nolog"
GOSDN_TEST_API_ENDPOINT: http://gosdn-$CI_COMMIT_SHA.apps.ocp.fbi.h-da.de/api
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME != $CI_DEFAULT_BRANCH
allow_failure: true
- if: $CI_COMMIT_BRANCH == "integration-test"
allow_failure: true
script:
- sleep 1m
- sleep 2m
- go test -race ./test/integration -v -coverprofile=coverage.out
.test: &test
......
......@@ -10,7 +10,7 @@ import (
"strings"
)
const apiRoot = "/api?"
const apiRoot = "?"
var builder *strings.Builder
......
......@@ -53,6 +53,6 @@ func init() {
rootCmd.AddCommand(cliCmd)
cliCmd.PersistentFlags().StringVar(&uuid, "uuid", "", "uuid of the requested device")
cliCmd.PersistentFlags().StringVar(&apiEndpoint, "api-endpoint", "http://localhost:8080", "address of the target")
cliCmd.PersistentFlags().StringVar(&apiEndpoint, "api-endpoint", "http://gosdn-develop.apps.ocp.fbi.h-da.de/api", "address of the target")
}
CliSocket = ":55055"
......@@ -19,4 +19,7 @@ require (
golang.org/x/net v0.0.0-20201216054612-986b41b23924
google.golang.org/grpc v1.34.0
google.golang.org/protobuf v1.26.0
k8s.io/api v0.20.5
k8s.io/apimachinery v0.20.5
k8s.io/client-go v0.20.5
)
This diff is collapsed.
......@@ -47,7 +47,7 @@ func TestRun(t *testing.T) {
return
}
if !reflect.DeepEqual(got.StatusCode, tests[0].want) {
t.Errorf("Run() got: %v, want %v", got.StatusCode, tests[0].want)
t.Errorf("livez got: %v, want %v", got.StatusCode, tests[0].want)
}
got, err = http.Get(tests[0].args.request)
if err != nil {
......@@ -55,7 +55,7 @@ func TestRun(t *testing.T) {
return
}
if !reflect.DeepEqual(got.StatusCode, tests[1].want) {
t.Errorf("Run() got: %v, want %v", got.StatusCode, tests[1].want)
t.Errorf("readyz got: %v, want %v", got.StatusCode, tests[1].want)
}
got, err = http.Get(tests[0].args.request)
if err != nil {
......@@ -63,7 +63,7 @@ func TestRun(t *testing.T) {
return
}
if !reflect.DeepEqual(got.StatusCode, tests[2].want) {
t.Errorf("Run() got: %v, want %v", got.StatusCode, tests[2].want)
t.Errorf("api init got: %v, want %v", got.StatusCode, tests[2].want)
}
})
......
......@@ -2,15 +2,9 @@ package integration
import (
"code.fbi.h-da.de/cocsn/gosdn/cli"
"os"
"testing"
)
func TestMain(m *testing.M) {
testSetupIntegration()
os.Exit(m.Run())
}
func TestCapabilities(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
......
......@@ -17,12 +17,18 @@ import (
const unreachable = "203.0.113.10:6030"
var testAddress = "141.100.70.171:6030"
var testAPIEndpoint = "http://141.100.70.171:8080"
var testAPIEndpoint = "http://141.100.70.171:8080/api"
var testUsername = "admin"
var testPassword = "arista"
var defaultPath = []string{"/system/config/hostname"}
var opt *nucleus.GnmiTransportOptions
func TestMain(m *testing.M) {
testSetupIntegration()
os.Exit(m.Run())
}
func testSetupIntegration() {
if os.Getenv("GOSDN_LOG") == "nolog" {
log.SetLevel(log.PanicLevel)
......@@ -31,18 +37,22 @@ func testSetupIntegration() {
a := os.Getenv("GOSDN_TEST_ENDPOINT")
if a != "" {
testAddress = a
log.Infof("GOSDN_TEST_ENDPOIN