Merge branch '120-integration-tests-fail' into 'develop'

Resolve "Integration Tests Fail"

See merge request cocsn/gosdn!159

build/cd/deploy.go

@@ -3,12 +3,14 @@ package main
 import (
 	"context"
 	"os"
+	"strconv"
 
 	nucleus "code.fbi.h-da.de/cocsn/gosdn/nucleus/errors"
 	log "github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
 	appv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	netv1 "k8s.io/api/networking/v1beta1"
+	netv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -16,6 +18,12 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 )
 
+// GRPC defines the grpc port used by the deployment
+const GRPC = 55055
+
+// HTTP defines the http port used by the deployment
+const HTTP = 8080
+
 func main() {
 	log.SetFormatter(&log.JSONFormatter{})
 	kubeconfig, err := clientcmd.BuildConfigFromFlags("https://api.ocp.fbi.h-da.de:6443", "")
@@ -37,15 +45,6 @@ func main() {
 		tag = os.Getenv("CI_COMMIT_SHA")
 	}
 
-	switch os.Getenv("CI_NIGHTLY") {
-	case "mainline":
-		tag = "nightly"
-	case "develop":
-		tag = "nightly-develop"
-	default:
-		tag = os.Getenv("CI_COMMIT_SHA")
-	}
-
 	switch os.Getenv("K8S_OP") {
 	case "create":
 		if err := create(clientset, tag); err != nil {
@@ -55,16 +54,46 @@ func main() {
 		if err := remove(clientset, tag); err != nil {
 			log.Fatal(err)
 		}
+	case "getenv":
+		if err := getenv(clientset, tag); err != nil {
+			log.Fatal(err)
+		}
 	default:
 		log.Fatal("invalid option")
 	}
 }
 
+func getenv(clientset *kubernetes.Clientset, tag string) error {
+	env := "gosdn-" + tag
+	opts := metav1.GetOptions{}
+	ctx := context.Background()
+	service, err := clientset.CoreV1().Services("cocsn").Get(ctx, env, opts)
+	if err != nil {
+		return err
+	}
+	var ip string
+	for _, ingress := range service.Status.LoadBalancer.Ingress {
+		ip = ingress.IP
+	}
+
+	var port string
+	for _, p := range service.Spec.Ports {
+		if p.Name == "grpc" {
+			port = strconv.Itoa(int(p.NodePort))
+		}
+	}
+	log.WithFields(log.Fields{
+		"ip":   ip,
+		"port": port,
+	}).Info(env)
+	viper.Set("GOSDN_TEST_API_ENDPOINT", ip+":"+port)
+	return viper.SafeWriteConfigAs(".k8s.toml")
+}
+
 // nolint
 func create(clientset *kubernetes.Clientset, tag string) error {
 	env := "gosdn-" + tag
 	service := createService(env)
-	ingress := createIngress(env)
 	config := createConfigMap(env)
 	deployment := createDeployment(env, tag)
 	opts := metav1.CreateOptions{}
@@ -86,23 +115,7 @@ func create(clientset *kubernetes.Clientset, tag string) error {
 	} else {
 		log.Printf("service %v created", service.Name)
 	}
-	_, err = clientset.NetworkingV1beta1().Ingresses("cocsn").Create(ctx, ingress, opts)
-	if err != nil {
-		switch err.(type) {
-		case *errors.StatusError:
-			if err.(*errors.StatusError).ErrStatus.Code == 409 {
-				if err := update(clientset, ingress, env); err != nil {
-					log.Error(err)
-				}
-			} else {
-				log.Error(err)
-			}
-		default:
-			log.Error(err)
-		}
-	} else {
-		log.Printf("ingress %v created", ingress.Name)
-	}
+
 	_, err = clientset.CoreV1().ConfigMaps("cocsn").Create(ctx, config, opts)
 	if err != nil {
 		switch err.(type) {
@@ -159,12 +172,12 @@ func update(clientset *kubernetes.Clientset, resource metav1.Common, env string)
 		log.Printf("service %v updated", service.Name)
 	case *netv1.Ingress:
 		ingress := resource
-		i, err := clientset.NetworkingV1beta1().Ingresses("cocsn").Get(ctx, env, getOpts)
+		i, err := clientset.NetworkingV1().Ingresses("cocsn").Get(ctx, env, getOpts)
 		if err != nil {
 			return err
 		}
 		i.DeepCopyInto(ingress)
-		_, err = clientset.NetworkingV1beta1().Ingresses("cocsn").Update(ctx, ingress, opts)
+		_, err = clientset.NetworkingV1().Ingresses("cocsn").Update(ctx, ingress, opts)
 		if err != nil {
 			return err
 		}
@@ -240,23 +253,25 @@ func createService(environment string) *corev1.Service {
 			Name:      environment,
 			Namespace: "cocsn",
 			Labels:    map[string]string{"run": environment},
+			Annotations: map[string]string{
+				"metallb.universe.tf/address-pool": "hdanet",
+			},
 		},
 		Spec: corev1.ServiceSpec{
 			Ports: []corev1.ServicePort{
 				{
 					Name:       "http",
-					Port:       8080,
-					TargetPort: intstr.IntOrString{IntVal: 8080},
+					Port:       HTTP,
+					TargetPort: intstr.IntOrString{IntVal: HTTP},
 				},
 				{
 					Name:       "grpc",
-					Port:       55055,
-					TargetPort: intstr.IntOrString{IntVal: 55055},
+					Port:       GRPC,
+					TargetPort: intstr.IntOrString{IntVal: GRPC},
 				},
 			},
-			Selector:     map[string]string{"run": environment},
-			Type:         "NodePort",
-			ExternalName: environment + ".apps.ocp.fbi.h-da.de",
+			Selector: map[string]string{"run": environment},
+			Type:     "LoadBalancer",
 		},
 	}
 }
@@ -301,11 +316,11 @@ func createDeployment(environment, hash string) *appv1.Deployment {
 							Ports: []corev1.ContainerPort{
 								{
 									Name:          "grpc",
-									ContainerPort: 55055,
+									ContainerPort: GRPC,
 								},
 								{
 									Name:          "http",
-									ContainerPort: 8080,
+									ContainerPort: HTTP,
 								},
 							},
 							VolumeMounts: []corev1.VolumeMount{
@@ -318,7 +333,7 @@ func createDeployment(environment, hash string) *appv1.Deployment {
 								Handler: corev1.Handler{
 									HTTPGet: &corev1.HTTPGetAction{
 										Path: "/livez",
-										Port: intstr.IntOrString{IntVal: 8080},
+										Port: intstr.IntOrString{IntVal: HTTP},
 									},
 								},
 								InitialDelaySeconds: 5,
@@ -328,7 +343,7 @@ func createDeployment(environment, hash string) *appv1.Deployment {
 								Handler: corev1.Handler{
 									HTTPGet: &corev1.HTTPGetAction{
 										Path: "/readyz",
-										Port: intstr.IntOrString{IntVal: 8080},
+										Port: intstr.IntOrString{IntVal: HTTP},
 									},
 								},
 								InitialDelaySeconds: 10,
@@ -362,37 +377,3 @@ func createConfigMap(env string) *corev1.ConfigMap {
 		Data: map[string]string{"gosdn.toml": "#empty"},
 	}
 }
-
-func createIngress(env string) *netv1.Ingress {
-	return &netv1.Ingress{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Ingress",
-			APIVersion: "v1",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      env,
-			Namespace: "cocsn",
-		},
-		Spec: netv1.IngressSpec{
-			Rules: []netv1.IngressRule{
-				{
-					Host: env + ".apps.ocp.fbi.h-da.de",
-					IngressRuleValue: netv1.IngressRuleValue{
-						HTTP: &netv1.HTTPIngressRuleValue{
-							Paths: []netv1.HTTPIngressPath{
-								{
-									Path: "/api",
-									Backend: netv1.IngressBackend{
-										ServiceName: env,
-										ServicePort: intstr.IntOrString{IntVal: 8080},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-		Status: netv1.IngressStatus{},
-	}
-}

build/ci/.build-container.yml

 
 services:
-  - docker:19.03.12-dind
+  - name: docker:19.03.12-dind
+    command: ["--registry-mirror", "http://141.100.70.170:6000"]
 
 variables:
   DOCKER_TLS_CERTDIR: "/certs"
@@ -14,6 +15,7 @@ variables:
   tags:
     - dind
   script:
+    - docker info
     - >
       docker build \
         --build-arg GITLAB_USER=$GO_MODULES_USER \

build/ci/.deploy-k8s.yml

@@ -14,7 +14,9 @@ build:k8s-bot:
       - build/cd/k8s-bot
 
 .deploy: &deploy
-  image: bitnami/kubectl:latest
+  image: 
+    name: bitnami/kubectl:latest
+    entrypoint: [""]
   before_script:
     - echo "override global before script"
   variables:
@@ -73,7 +75,9 @@ deploy:nightly:develop:
     - if: $CI_COMMIT_BRANCH == "develop" && $CI_NIGHTLY == "mainline"
 
 destroy:k8s:
-  image: bitnami/kubectl:latest
+  image: 
+    name: bitnami/kubectl:latest
+    entrypoint: [""]
   rules:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: always

build/ci/.test.yml

-integration-test:
+.integration-test: &integration-test
   image: golang:1.16
   stage: integration-test
   needs:
     - job: "apply"
     - job: "deploy:integration-test"
-  variables:
-    GOSDN_LOG: "nolog"
-    GOSDN_TEST_API_ENDPOINT: http://gosdn-$CI_COMMIT_SHA.apps.ocp.fbi.h-da.de/api
   rules:
     - if: $CI_NIGHTLY
       when: delayed
@@ -18,8 +15,36 @@ integration-test:
       allow_failure: true
       when: delayed
       start_in: 2 minutes
+
+integration-test:nucleus:
+  <<: *integration-test
+  script:
+    - cd ./test/integration
+    - go test -race -v -run TestGnmi_SetIntegration
+    - go test -race -v -run TestGnmi_GetIntegration
+    - go test -race -v -run TestGnmi_SubscribeIntegration
+    - go test -race -v -run TestGnmi_CapabilitiesIntegration
+
+integration-test:cmd:
+  <<: *integration-test
+  variables:
+    K8S_OP: "getenv"
+  needs:
+    - job: "build:merge-request"
+    - job: "apply"
+    - job: "deploy:integration-test"
+    - job: "build:k8s-bot"
+      artifacts: true
   script:
-    - go test -race ./test/integration -v -coverprofile=coverage.out
+    - cd ./test/integration
+    - ./../build/cd/k8s-bot
+    - go test -race -v -run TestCmdIntegration
+
+integration-test:cli:
+  <<: *integration-test
+  script:
+    - cd ./test/integration
+    - go test -race -v -run TestCapabilities
 
 .test: &test
   image: golang:1.16
@@ -45,4 +70,4 @@ unit-test:
 controller-test:
   script:
     - go test -race -v -run TestRun
-  <<: *test
\ No newline at end of file
+  <<: *test

cli/capabilities.go

@@ -19,6 +19,9 @@ func Capabilities(a, u, p string) error {
 		Address:  a,
 		Username: u,
 		Password: p,
+		TransportOption: &tpb.TransportOption_GnmiTransportOption{
+			GnmiTransportOption: &tpb.GnmiTransportOption{},
+		},
 	}
 	transport, err := nucleus.NewGnmiTransport(opts, nucleus.NewSBI(spb.Type_OPENCONFIG))
 	if err != nil {

controller.go

@@ -69,6 +69,7 @@ func startGrpcServer() error {
 	if err != nil {
 		return err
 	}
+	log.Infof("listening to %v", lis.Addr())
 	c.grpcServer = grpc.NewServer()
 	c.nbi = nbi.NewNBI(c.pndc)
 	pb.RegisterCoreServer(c.grpcServer, c.nbi.Core)

go.mod

@@ -6,8 +6,8 @@ require (
 	code.fbi.h-da.de/cocsn/api/go v0.0.0-20210528163424-f37e91dc5895
 	code.fbi.h-da.de/cocsn/yang-models v0.0.7
 	github.com/aristanetworks/goarista v0.0.0-20201120222254-94a892eb0c6a
-	github.com/golang/protobuf v1.5.2
 	github.com/docker/docker v1.13.1
+	github.com/golang/protobuf v1.5.2
 	github.com/google/gnxi v0.0.0-20201221102247-c26672548161
 	github.com/google/uuid v1.1.2
 	github.com/neo4j/neo4j-go-driver v1.8.3

nucleus/gnmi_transport.go

@@ -43,20 +43,18 @@ type Gnmi struct {
 // transport based on the values of it.
 func NewGnmiTransport(opts *tpb.TransportOption, sbi SouthboundInterface) (*Gnmi, error) {
 	gnmiConfig := &gnmi.Config{
-		Addr:        opts.Address,
-		Password:    opts.Password,
-		Username:    opts.Username,
-		TLS:         opts.Tls,
-		Compression: opts.GetGnmiTransportOption().Compression,
+		Addr:     opts.Address,
+		Password: opts.Password,
+		Username: opts.Username,
+		TLS:      opts.Tls,
 	}
 	c, err := gnmi.Dial(gnmiConfig)
 	if err != nil {
 		return nil, err
 	}
 	log.WithFields(log.Fields{
-		"target":   opts.Address,
-		"tls":      opts.Tls,
-		"encoding": opts.GetGnmiTransportOption().Encoding,
+		"target": opts.Address,
+		"tls":    opts.Tls,
 	}).Info("building new gNMI transport")
 	return &Gnmi{
 		SetNode:  sbi.SetNode(),

test/integration/cmdIntegration_test.go

@@ -19,7 +19,7 @@ const unreachable = "203.0.113.10:6030"
 const testPath = "/system/config/hostname"
 
 var testAddress = "141.100.70.171:6030"
-var testAPIEndpoint = "http://gosdn-latest.apps.ocp.fbi.h-da.de/api"
+var testAPIEndpoint = "gosdn-latest.apps.ocp.fbi.h-da.de:55055"
 var testUsername = "admin"
 var testPassword = "arista"
 var opt *tpb.TransportOption
@@ -30,6 +30,11 @@ func TestMain(m *testing.M) {
 }
 
 func testSetupIntegration() {
+	viper.SetConfigFile(".k8s.toml")
+	if err := viper.ReadInConfig(); err != nil {
+		log.Fatal(err)
+	}
+
 	if os.Getenv("GOSDN_LOG") == "nolog" {
 		log.SetLevel(log.PanicLevel)
 	}
@@ -39,7 +44,7 @@ func testSetupIntegration() {
 		testAddress = a
 		log.Infof("GOSDN_TEST_ENDPOINT set to %v", testAddress)
 	}
-	api := os.Getenv("GOSDN_TEST_API_ENDPOINT")
+	api := viper.GetString("gosdn_test_api_endpoint")
 	if api != "" {
 		testAPIEndpoint = api
 		log.Infof("GOSDN_TEST_API_ENDPOINT set to %v", testAPIEndpoint)

test/integration/nucleusIntegration_test.go

@@ -172,9 +172,9 @@ func TestGnmi_SubscribeIntegration(t *testing.T) {
 			name: "default",
 			fields: fields{
 				opt: &tpb.TransportOption{
-					Address:  "",
-					Username: "",
-					Password: "",
+					Address:  testAddress,
+					Username: testUsername,
+					Password: testPassword,
 					Tls:      false,
 					TransportOption: &tpb.TransportOption_GnmiTransportOption{
 						GnmiTransportOption: &tpb.GnmiTransportOption{
@@ -250,7 +250,7 @@ func TestGnmi_SubscribeIntegration(t *testing.T) {
 				if (subErr != nil) != wantErr {
 					if !wantErr && subErr != nil {
 						if subErr.Error() != "rpc error: code = Canceled desc = context canceled" {
-							t.Errorf("Subscribe() error = %v, wantErr %v", err, tt.wantErr)
+							t.Errorf("Subscribe() error = %v, wantErr %v", subErr, tt.wantErr)
 						}
 					}
 				}