Resolve "Integration Tests Fail"

Description

Related Issue

Motivation and Context

How Has This Been Tested?

Screenshots (if appropriate):

Types of changes

• Bug fix (non-breaking change which fixes an issue)
• New feature (non-breaking change which adds functionality)
• Breaking change (fix or feature that would cause existing functionality to change)

Checklist:

• My code follows the code style of this project.
• My change requires a change to the documentation.
• I have updated the documentation accordingly.
• I have read the CONTRIBUTING document.
• I have added tests to cover my changes.
• All new and existing tests passed.

Closes #120 (closed)

build/cd/deploy.go

@@ -3,12 +3,13 @@ package main
 import (
 	"context"
 	"os"
+	"strconv"
 
 	nucleus "code.fbi.h-da.de/cocsn/gosdn/nucleus/errors"
 	log "github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
 	appv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	netv1 "k8s.io/api/networking/v1beta1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -16,6 +17,16 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 )
 
+// GRPC defines the grpc port used by the deployment
+const GRPC = 55055
+
+// HTTP defines the http port used by the deployment
+const HTTP = 8080
+
+var service *corev1.Service
+var deployment *appv1.Deployment
+var config *corev1.ConfigMap
+
 func main() {
 	log.SetFormatter(&log.JSONFormatter{})
 	kubeconfig, err := clientcmd.BuildConfigFromFlags("https://api.ocp.fbi.h-da.de:6443", "")
@@ -37,15 +48,6 @@ func main() {
 		tag = os.Getenv("CI_COMMIT_SHA")
 	}
 
-	switch os.Getenv("CI_NIGHTLY") {
-	case "mainline":
-		tag = "nightly"
-	case "develop":
-		tag = "nightly-develop"
-	default:
-		tag = os.Getenv("CI_COMMIT_SHA")
-	}
-
 	switch os.Getenv("K8S_OP") {
 	case "create":
 		if err := create(clientset, tag); err != nil {
@@ -55,27 +57,56 @@ func main() {
 		if err := remove(clientset, tag); err != nil {
 			log.Fatal(err)
 		}
+	case "getenv":
+		if err := getenv(clientset, tag); err != nil {
+			log.Fatal(err)
+		}
 	default:
 		log.Fatal("invalid option")
 	}
 }
 
+func getenv(clientset *kubernetes.Clientset, tag string) error {
+	env := "gosdn-" + tag
+	opts := metav1.GetOptions{}
+	ctx := context.Background()
+	service, err := clientset.CoreV1().Services("cocsn").Get(ctx, env, opts)
+	if err != nil {
+		return err
+	}
+	var ip string
+	for _, ingress := range service.Status.LoadBalancer.Ingress {
+		ip = ingress.IP
+	}
+
+	var port string
+	for _, p := range service.Spec.Ports {
+		if p.Name == "grpc" {
+			port = strconv.Itoa(int(p.NodePort))
+		}
+	}
+	log.WithFields(log.Fields{
+		"ip":   ip,
+		"port": port,
+	}).Info(env)
+	viper.Set("GOSDN_TEST_API_ENDPOINT", ip+":"+port)
+	return viper.SafeWriteConfigAs(".k8s.toml")
+}
+
 // nolint
 func create(clientset *kubernetes.Clientset, tag string) error {
 	env := "gosdn-" + tag
-	service := createService(env)
-	ingress := createIngress(env)
-	config := createConfigMap(env)
-	deployment := createDeployment(env, tag)
+	deployment = createDeployment(env, tag)
 	opts := metav1.CreateOptions{}
 	ctx := context.Background()
-	_, err := clientset.CoreV1().Services("cocsn").Create(ctx, service, opts)
+
+	_, err := clientset.AppsV1().Deployments("cocsn").Create(ctx, deployment, opts)
 	if err != nil {
 		switch err.(type) {
 		case *errors.StatusError:
 			if err.(*errors.StatusError).ErrStatus.Code == 409 {
-				if err := update(clientset, service, env); err != nil {
-					return err
+				if err := update(clientset, deployment, env); err != nil {
+					log.Error(err)
 				}
 			} else {
 				log.Error(err)
@@ -84,25 +115,14 @@ func create(clientset *kubernetes.Clientset, tag string) error {
 			log.Error(err)
 		}
 	} else {
-		log.Printf("service %v created", service.Name)
+		log.Printf("deployment %v created", deployment.Name)
 	}
-	_, err = clientset.NetworkingV1beta1().Ingresses("cocsn").Create(ctx, ingress, opts)
+	deployment, err = clientset.AppsV1().Deployments("cocsn").Get(ctx, env, metav1.GetOptions{})
 	if err != nil {
-		switch err.(type) {
-		case *errors.StatusError:
-			if err.(*errors.StatusError).ErrStatus.Code == 409 {
-				if err := update(clientset, ingress, env); err != nil {
-					log.Error(err)
-				}
-			} else {
-				log.Error(err)
-			}
-		default:
-			log.Error(err)
-		}
-	} else {
-		log.Printf("ingress %v created", ingress.Name)
+		log.Error(err)
 	}
+
+	config = createConfigMap(env)
 	_, err = clientset.CoreV1().ConfigMaps("cocsn").Create(ctx, config, opts)
 	if err != nil {
 		switch err.(type) {
@@ -120,13 +140,15 @@ func create(clientset *kubernetes.Clientset, tag string) error {
 	} else {
 		log.Printf("configMap %v created", config.Name)
 	}
-	_, err = clientset.AppsV1().Deployments("cocsn").Create(ctx, deployment, opts)
+
+	service = createService(env)
+	_, err = clientset.CoreV1().Services("cocsn").Create(ctx, service, opts)
 	if err != nil {
 		switch err.(type) {
 		case *errors.StatusError:
 			if err.(*errors.StatusError).ErrStatus.Code == 409 {
-				if err := update(clientset, deployment, env); err != nil {
-					log.Error(err)
+				if err := update(clientset, service, env); err != nil {
+					return err
 				}
 			} else {
 				log.Error(err)
@@ -135,7 +157,7 @@ func create(clientset *kubernetes.Clientset, tag string) error {
 			log.Error(err)
 		}
 	} else {
-		log.Printf("deployment %v created", deployment.Name)
+		log.Printf("service %v created", service.Name)
 	}
 	return nil
 }
@@ -157,18 +179,6 @@ func update(clientset *kubernetes.Clientset, resource metav1.Common, env string)
 			return err
 		}
 		log.Printf("service %v updated", service.Name)
-	case *netv1.Ingress:
-		ingress := resource
-		i, err := clientset.NetworkingV1beta1().Ingresses("cocsn").Get(ctx, env, getOpts)
-		if err != nil {
-			return err
-		}
-		i.DeepCopyInto(ingress)
-		_, err = clientset.NetworkingV1beta1().Ingresses("cocsn").Update(ctx, ingress, opts)
-		if err != nil {
-			return err
-		}
-		log.Printf("ingress %v updated", ingress.Name)
 	case *corev1.ConfigMap:
 		config := resource
 		c, err := clientset.CoreV1().ConfigMaps("cocsn").Get(ctx, env+"-config", getOpts)
@@ -221,12 +231,6 @@ func remove(clientset *kubernetes.Clientset, tag string) error {
 	} else {
 		log.Printf("deployment %v deleted", env)
 	}
-	err = clientset.NetworkingV1beta1().Ingresses("cocsn").Delete(ctx, env, opts)
-	if err != nil {
-		log.Error(err)
-	} else {
-		log.Printf("ingress %v deleted", env)
-	}
 	return err
 }
 
@@ -240,23 +244,33 @@ func createService(environment string) *corev1.Service {
 			Name:      environment,
 			Namespace: "cocsn",
 			Labels:    map[string]string{"run": environment},
+			Annotations: map[string]string{
+				"metallb.universe.tf/address-pool": "hdanet",
+			},
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion: "v1",
+					Kind:       "Deployment",
+					Name:       environment,
+					UID:        deployment.GetUID(),
+				},
+			},
 		},
 		Spec: corev1.ServiceSpec{
 			Ports: []corev1.ServicePort{
 				{
 					Name:       "http",
-					Port:       8080,
-					TargetPort: intstr.IntOrString{IntVal: 8080},
+					Port:       HTTP,
+					TargetPort: intstr.IntOrString{IntVal: HTTP},
 				},
 				{
 					Name:       "grpc",
-					Port:       55055,
-					TargetPort: intstr.IntOrString{IntVal: 55055},
+					Port:       GRPC,
+					TargetPort: intstr.IntOrString{IntVal: GRPC},
 				},
 			},
-			Selector:     map[string]string{"run": environment},
-			Type:         "NodePort",
-			ExternalName: environment + ".apps.ocp.fbi.h-da.de",
+			Selector: map[string]string{"run": environment},
+			Type:     "LoadBalancer",
 		},
 	}
 }
@@ -301,11 +315,11 @@ func createDeployment(environment, hash string) *appv1.Deployment {
 							Ports: []corev1.ContainerPort{
 								{
 									Name:          "grpc",
-									ContainerPort: 55055,
+									ContainerPort: GRPC,
 								},
 								{
 									Name:          "http",
-									ContainerPort: 8080,
+									ContainerPort: HTTP,
 								},
 							},
 							VolumeMounts: []corev1.VolumeMount{
@@ -318,7 +332,7 @@ func createDeployment(environment, hash string) *appv1.Deployment {
 								Handler: corev1.Handler{
 									HTTPGet: &corev1.HTTPGetAction{
 										Path: "/livez",
-										Port: intstr.IntOrString{IntVal: 8080},
+										Port: intstr.IntOrString{IntVal: HTTP},
 									},
 								},
 								InitialDelaySeconds: 5,
@@ -328,7 +342,7 @@ func createDeployment(environment, hash string) *appv1.Deployment {
 								Handler: corev1.Handler{
 									HTTPGet: &corev1.HTTPGetAction{
 										Path: "/readyz",
-										Port: intstr.IntOrString{IntVal: 8080},
+										Port: intstr.IntOrString{IntVal: HTTP},
 									},
 								},
 								InitialDelaySeconds: 10,
@@ -358,41 +372,15 @@ func createConfigMap(env string) *corev1.ConfigMap {
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: env + "-config",
-		},
-		Data: map[string]string{"gosdn.toml": "#empty"},
-	}
-}
-
-func createIngress(env string) *netv1.Ingress {
-	return &netv1.Ingress{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Ingress",
-			APIVersion: "v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      env,
-			Namespace: "cocsn",
-		},
-		Spec: netv1.IngressSpec{
-			Rules: []netv1.IngressRule{
+			OwnerReferences: []metav1.OwnerReference{
 				{
-					Host: env + ".apps.ocp.fbi.h-da.de",
-					IngressRuleValue: netv1.IngressRuleValue{
-						HTTP: &netv1.HTTPIngressRuleValue{
-							Paths: []netv1.HTTPIngressPath{
-								{
-									Path: "/api",
-									Backend: netv1.IngressBackend{
-										ServiceName: env,
-										ServicePort: intstr.IntOrString{IntVal: 8080},
-									},
-								},
-							},
-						},
-					},
+					APIVersion: "v1",
+					Kind:       "Deployment",
+					Name:       env,
+					UID:        deployment.GetUID(),
 				},
 			},
 		},
-		Status: netv1.IngressStatus{},
+		Data: map[string]string{"gosdn.toml": "#empty"},
 	}
 }