Merge branch 'develop' into istaester/update-deps

.dockerignore

@@ -9,5 +9,4 @@ test
 .gitlab-ci.yaml
 ARCHITECTURE.md
 CONTRIBUTING.md
-Dockerfile.alpine
 README.md

.gitignore

@@ -21,3 +21,6 @@ debug.test
 
 # Binary
 gosdn
+
+# Storage
+stores/

.gitlab/ci/.containerlab-ci.yml

@@ -40,7 +40,7 @@ variables:
 containerlab:template:integration:
   extends: .containerlab_template
   variables:
-    CLAB_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/complex-1.0.clab.tmpl.yml"
+    CLAB_TEMPLATE: "${CI_PROJECT_DIR}/test/containerlab/int01.clab.tmpl.yml"
 
 
 containerlab:deploy:integration:
@@ -52,12 +52,7 @@ containerlab:deploy:integration:
       echo -e "\
       GOSDN_HTTP_PORT=$(docker_host_port 8080 clab-${CLAB_NAME}-gosdn)\n\
       GOSDN_GRPC_PORT=$(docker_host_port 55055 clab-${CLAB_NAME}-gosdn)\n\
-      CEOS1_1_PORT=$(docker_host_port 6030 clab-${CLAB_NAME}-ceos1-1)\n\
-      CEOS1_2_PORT=$(docker_host_port 6030 clab-${CLAB_NAME}-ceos1-2)\n\
-      CEOS2_1_1_PORT=$(docker_host_port 6030 clab-${CLAB_NAME}-ceos2-1-1)\n\
-      CEOS2_1_2_PORT=$(docker_host_port 6030 clab-${CLAB_NAME}-ceos2-1-2)\n\
-      CEOS2_2_1_PORT=$(docker_host_port 6030 clab-${CLAB_NAME}-ceos2-2-1)\n\
-      CEOS2_2_2_PORT=$(docker_host_port 6030 clab-${CLAB_NAME}-ceos2-2-2)" \
+      CEOS1_PORT=$(docker_host_port 6030 clab-${CLAB_NAME}-ceos1)" \
       > ${CI_PROJECT_DIR}/build.env
     - cat ${CI_PROJECT_DIR}/build.env
   artifacts:

.gitlab/ci/.integration-test.yml

@@ -3,10 +3,12 @@
   stage: integration-test
   needs:
     - job: "containerlab:deploy:integration"
+  tags:
+    - integration-test-docker
   variables:
     GOSDN_LOG: "nolog"
     GOSDN_TEST_API_ENDPOINT: "141.100.70.178:${GOSDN_GRPC_PORT}"
-    GOSDN_TEST_ENDPOINT: "141.100.70.178:${CEOS1_1_PORT}"
+    CEOS_TEST_ENDPOINT: "141.100.70.178:${CEOS1_PORT}"
     GOSDN_TEST_USER: "admin"
     GOSDN_TEST_PASSWORD: "admin"
 
@@ -14,8 +16,7 @@
 integration-test:nucleus:
   <<: *integration-test
   script:
-    - ${CI_PROJECT_DIR}/.gitlab/ci/scripts/wait-for-it.sh ${GOSDN_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
-    - sleep 30
+    - ${CI_PROJECT_DIR}/.gitlab/ci/scripts/wait-for-it.sh ${CEOS_TEST_ENDPOINT} -s -t 180 -- echo "CEOS is up"
     - cd ./test/integration
     - go test -race -v -run TestGnmi_SetIntegration
     - go test -race -v -run TestGnmi_GetIntegration

.gitlab/ci/.test.yml

@@ -16,7 +16,6 @@
 unit-test:
   script:
     - gotestsum --junitfile report.xml --format testname -- -short -race $(go list ./... | grep -v /forks/ | grep -v /mocks ) -v -coverprofile=coverage.out 
-    # -short -race $(go list ./... | grep -v /forks/ | grep -v /mocks ) -v
   after_script:
     - go tool cover -func=coverage.out
   <<: *test

Dockerfile.alpine

-# syntax = docker/dockerfile:1.2
-
-FROM golang:1.16-alpine AS builder
-ARG GITLAB_USER
-ARG GITLAB_TOKEN
-ARG BUILDARGS
-WORKDIR /src/gosdn
-RUN apk add --no-cache git
-RUN git config --global url."https://$GITLAB_USER:$GITLAB_TOKEN@code.fbi.h-da.de".insteadOf "https://code.fbi.h-da.de"
-COPY go.* .
-RUN go mod download
-COPY . .
-RUN --mount=type=cache,target=/root/.cache/go-build \
-GOOS=linux go build -o gosdn ./cmd/gosdn
-
-FROM alpine
-EXPOSE 8080
-EXPOSE 55055
-COPY --from=builder /src/gosdn/gosdn .
-COPY --from=builder /src/gosdn/configs ./configs
-
-ENTRYPOINT [ "./gosdn" ]
-CMD [""]

api/initialise_test.go

@@ -5,11 +5,13 @@ import (
 	"net"
 	"os"
 	"testing"
+
 	"time"
 
 	cpb "code.fbi.h-da.de/danet/api/go/gosdn/core"
 	ppb "code.fbi.h-da.de/danet/api/go/gosdn/pnd"
 	tpb "code.fbi.h-da.de/danet/api/go/gosdn/transport"
+	"code.fbi.h-da.de/danet/gosdn/config"
 	"code.fbi.h-da.de/danet/gosdn/mocks"
 	nbi "code.fbi.h-da.de/danet/gosdn/northbound/server"
 	"code.fbi.h-da.de/danet/gosdn/nucleus"
@@ -128,14 +130,12 @@ func TestMain(m *testing.M) {
 }
 
 func bootstrapIntegrationTest() {
-	if os.Getenv("GOSDN_LOG") == "nolog" {
-		log.SetLevel(log.PanicLevel)
-	}
+	log.SetLevel(config.LogLevel)
 
-	addr := os.Getenv("GOSDN_TEST_ENDPOINT")
+	addr := os.Getenv("CEOS_TEST_ENDPOINT")
 	if addr != "" {
 		testAddress = addr
-		log.Infof("GOSDN_TEST_ENDPOINT set to %v", testAddress)
+		log.Infof("CEOS_TEST_ENDPOINT set to %v", testAddress)
 	}
 	api := os.Getenv("GOSDN_TEST_API_ENDPOINT")
 	if api != "" {

cmd/root.go

@@ -34,6 +34,7 @@ package cmd
 import (
 	"context"
 	"os"
+	"path/filepath"
 
 	"code.fbi.h-da.de/danet/gosdn"
 
@@ -80,16 +81,22 @@ func init() {
 	rootCmd.Flags().StringVar(&csbiOrchestrstor, "csbi-orchestrator", "localhost:55056", "csbi orchestrator address")
 }
 
+const (
+	configHome string = "./configs"
+	configName string = "gosdn"
+	configType string = "toml"
+)
+
 // initConfig reads in config file and ENV variables if set.
 func initConfig() {
 	if cfgFile != "" {
 		// Use config file from the flag.
 		viper.SetConfigFile(cfgFile)
 	} else {
-		viper.AddConfigPath("./configs")
+		viper.AddConfigPath(configHome)
 		viper.AddConfigPath("/usr/local/etc/gosdn/")
-		viper.SetConfigType("toml")
-		viper.SetConfigName("gosdn")
+		viper.SetConfigType(configType)
+		viper.SetConfigName(configName)
 	}
 
 	viper.AutomaticEnv() // read in environment variables that match
@@ -97,6 +104,8 @@ func initConfig() {
 	// If a config file is found, read it in.
 	if err := viper.ReadInConfig(); err == nil {
 		log.Debug("Using config file:", viper.ConfigFileUsed())
+	} else {
+		ensureViperConfigFileExists()
 	}
 
 	viper.SetDefault("socket", ":55055")
@@ -118,3 +127,28 @@ func initConfig() {
 		log.SetReportCaller(false)
 	}
 }
+
+func ensureFileSystemStoreExists(pathToStore string) error {
+	emptyString := []byte("")
+	err := os.WriteFile(pathToStore, emptyString, 0600)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func ensureViperConfigFileExists() {
+	// Viper will crash if you call 'WriteConfig()' and the file does
+	// not exists yet.
+	// Therefore we handle this case here.
+	// Inspired by //https://github.com/spf13/viper/issues/430#issuecomment-661945101
+	configPath := filepath.Join(configHome, configName+"."+configType)
+
+	if _, err := os.Stat(configPath); os.IsNotExist(err) {
+		err := ensureFileSystemStoreExists(configPath)
+		if err != nil {
+			panic(err)
+		}
+	}
+}

config.go

-package gosdn
-
-import (
-	"github.com/google/uuid"
-	"github.com/spf13/viper"
-)
-
-// Config represents the nucleus configuration
-type Config struct {
-	BasePndUUID        uuid.UUID
-	BaseSouthBoundType int32
-	BaseSouthBoundUUID uuid.UUID
-}
-
-func getUUIDFromViper(viperKey string) (uuid.UUID, error) {
-	UUIDAsString := viper.GetString(viperKey)
-	if UUIDAsString == "" {
-		newUUID := uuid.New()
-		viper.Set(viperKey, newUUID.String())
-		viper.WriteConfig()
-
-		return newUUID, nil
-	}
-
-	parsedUUID, err := uuid.Parse(UUIDAsString)
-	if err != nil {
-		return uuid.Nil, err
-	}
-
-	return parsedUUID, nil
-}
-
-// InitializeConfig loads the configuration
-func (c *Config) InitializeConfig() error {
-	var err error
-	basePNDUUIDKey := "basePNDUUID"
-	baseSouthBoundTypeKey := "baseSouthBoundType"
-	baseSouthBoundUUIDKey := "baseSouthBoundUUID"
-
-	basePNDUUID, err := getUUIDFromViper(basePNDUUIDKey)
-	if err != nil {
-		return err
-	}
-
-	c.BasePndUUID = basePNDUUID
-
-	baseSouthBoundUUID, err := getUUIDFromViper(baseSouthBoundUUIDKey)
-	if err != nil {
-		return err
-	}
-
-	c.BaseSouthBoundUUID = baseSouthBoundUUID
-
-	c.BaseSouthBoundType = viper.GetInt32("BaseSouthBoundType")
-	if c.BaseSouthBoundType != 0 {
-		viper.Set(baseSouthBoundTypeKey, 0)
-		viper.WriteConfig()
-	}
-
-	return nil
-}

config/config.go

+package config
+
+import (
+	"os"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/sirupsen/logrus"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+)
+
+const (
+	defaultTimeOutDuration10minutes = time.Minute * 10
+	basePNDUUIDKey                  = "basePNDUUID"
+	baseSouthBoundTypeKey           = "baseSouthBoundType"
+	baseSouthBoundUUIDKey           = "baseSouthBoundUUID"
+	changeTimeoutKey                = "GOSDN_CHANGE_TIMEOUT"
+)
+
+// BasePndUUID is an uuid for the base PND
+var BasePndUUID uuid.UUID
+
+// BaseSouthBoundType is the type of the base SBI
+var BaseSouthBoundType int32
+
+// BaseSouthBoundUUID is an uuid for the base SBI
+var BaseSouthBoundUUID uuid.UUID
+
+// ChangeTimeout is the default timeout for a change
+var ChangeTimeout time.Duration
+
+// LogLevel ist the default log level
+var LogLevel logrus.Level
+
+// Init gets called on module import
+func Init() {
+	InitializeConfig()
+}
+
+// InitializeConfig loads the configuration
+func InitializeConfig() error {
+	var err error
+
+	basePNDUUIDFromViper, err := getUUIDFromViper(basePNDUUIDKey)
+	if err != nil {
+		return err
+	}
+
+	BasePndUUID = basePNDUUIDFromViper
+
+	baseSouthBoundUUIDFromViper, err := getUUIDFromViper(baseSouthBoundUUIDKey)
+	if err != nil {
+		return err
+	}
+
+	BaseSouthBoundUUID = baseSouthBoundUUIDFromViper
+
+	BaseSouthBoundType = viper.GetInt32(baseSouthBoundTypeKey)
+	if BaseSouthBoundType != 0 {
+		viper.Set(baseSouthBoundTypeKey, 0)
+		viper.WriteConfig()
+	}
+
+	err = setChangeTimeout()
+	if err != nil {
+		return err
+	}
+
+	setLogLevel()
+
+	return nil
+}
+
+func getUUIDFromViper(viperKey string) (uuid.UUID, error) {
+	UUIDAsString := viper.GetString(viperKey)
+	if UUIDAsString == "" {
+		newUUID := uuid.New()
+		viper.Set(viperKey, newUUID.String())
+		viper.WriteConfig()
+
+		return newUUID, nil
+	}
+
+	parsedUUID, err := uuid.Parse(UUIDAsString)
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	return parsedUUID, nil
+}
+
+func setChangeTimeout() error {
+	e := os.Getenv(changeTimeoutKey)
+	if e != "" {
+		changeTimeout, err := time.ParseDuration(e)
+		if err != nil {
+			log.Fatal(err)
+		}
+		ChangeTimeout = changeTimeout
+	} else {
+		ChangeTimeout = time.Minute * 10
+	}
+
+	return nil
+}
+
+func setLogLevel() {
+	if os.Getenv("GOSDN_LOG") == "nolog" {
+		LogLevel = logrus.PanicLevel
+	} else {
+		LogLevel = logrus.InfoLevel
+	}
+}

config/config_test.go

+package config
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+)
+
+func TestInit(t *testing.T) {
+	viper.SetConfigFile("./config_test.toml")
+	viper.Set("baseSouthBoundType", 0)
+	viper.Set("baseSouthBoundUUID", "bf8160d4-4659-4a1b-98fd-f409a04111eb")
+	viper.Set("basePNDUUID", "bf8160d4-4659-4a1b-98fd-f409a04111ec")
+	viper.Set("GOSDN_CHANGE_TIMEOUT", "10m")
+}
+
+func TestUseExistingConfig(t *testing.T) {
+	TestInit(t)
+
+	err := InitializeConfig()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if BasePndUUID.String() != "bf8160d4-4659-4a1b-98fd-f409a04111ec" {
+		t.Fatalf("BasePndUUID.String() is not bf8160d4-4659-4a1b-98fd-f409a04111ec. got=%s",
+			BasePndUUID.String())
+	}
+
+	if BaseSouthBoundUUID.String() != "bf8160d4-4659-4a1b-98fd-f409a04111eb" {
+		t.Fatalf("BaseSouthBoundUUID.String() is not bf8160d4-4659-4a1b-98fd-f409a04111eb. got=%s",
+			BaseSouthBoundUUID.String())
+	}
+
+	if BaseSouthBoundType != 0 {
+		t.Fatalf("BaseSouthBoundType is not 0. got=%d",
+			BaseSouthBoundType)
+	}
+
+	testChangeTimeout, _ := time.ParseDuration("10m")
+	defaultChangeTimeout := defaultTimeOutDuration10minutes
+	if defaultChangeTimeout != testChangeTimeout {
+		t.Fatalf("ChangeTimeout is not 10ms. got=%v",
+			ChangeTimeout)
+	}
+
+	if os.Getenv("GOSDN_LOG") == "nolog" {
+		if LogLevel != logrus.PanicLevel {
+			t.Fatalf("LogLevel is not %v. got=%v",
+				logrus.PanicLevel, LogLevel)
+		}
+	} else {
+		if LogLevel != logrus.InfoLevel {
+			t.Fatalf("LogLevel is not %v. got=%v",
+				logrus.InfoLevel, LogLevel)
+		}
+	}
+
+}

config_test.go

-package gosdn
-
-import (
-	"testing"
-
-	"github.com/spf13/viper"
-)
-
-func Test_Init(t *testing.T) {
-	viper.SetConfigFile("./config_test.toml")
-	viper.Set("baseSouthBoundType", 0)
-	viper.Set("baseSouthBoundUUID", "bf8160d4-4659-4a1b-98fd-f409a04111eb")
-	viper.Set("basePNDUUID", "bf8160d4-4659-4a1b-98fd-f409a04111ec")
-}
-
-func Test_UseExistingConfig(t *testing.T) {
-	Test_Init(t)
-
-	testConfig := Config{}
-
-	err := testConfig.InitializeConfig()
-	if err != nil {
-		t.Error(err)
-		return
-	}
-
-	if testConfig.BasePndUUID.String() != "bf8160d4-4659-4a1b-98fd-f409a04111ec" {
-		t.Fatalf("testConfig.BasePndUUID.String() is not bf8160d4-4659-4a1b-98fd-f409a04111ec. got=%s",
-			testConfig.BasePndUUID.String())
-	}
-
-	if testConfig.BaseSouthBoundUUID.String() != "bf8160d4-4659-4a1b-98fd-f409a04111eb" {
-		t.Fatalf("testConfig.BaseSouthBoundUUID.String() is not bf8160d4-4659-4a1b-98fd-f409a04111eb. got=%s",
-			testConfig.BaseSouthBoundUUID.String())
-	}
-
-	if testConfig.BaseSouthBoundType != 0 {
-		t.Fatalf("testConfig.BaseSouthBoundType is not 0. got=%d",
-			testConfig.BaseSouthBoundType)
-	}
-}

controller.go

@@ -18,6 +18,7 @@ import (
 	cpb "code.fbi.h-da.de/danet/api/go/gosdn/csbi"
 	ppb "code.fbi.h-da.de/danet/api/go/gosdn/pnd"
 	spb "code.fbi.h-da.de/danet/api/go/gosdn/southbound"
+	"code.fbi.h-da.de/danet/gosdn/config"
 	"code.fbi.h-da.de/danet/gosdn/interfaces/southbound"
 	nbi "code.fbi.h-da.de/danet/gosdn/northbound/server"
 	"code.fbi.h-da.de/danet/gosdn/store"
@@ -61,13 +62,12 @@ func initialize() error {
 	startHttpServer()
 	coreLock.Unlock()
 
-	config := Config{}
 	err := config.InitializeConfig()
 	if err != nil {
 		return err
 	}
 
-	return createSouthboundInterfaces(config)
+	return createSouthboundInterfaces()
 }
 
 func startGrpc() error {
@@ -98,13 +98,13 @@ func startGrpc() error {
 }
 
 // createSouthboundInterfaces initializes the controller with its supported SBIs
-func createSouthboundInterfaces(config Config) error {
+func createSouthboundInterfaces() error {
 	sbi := nucleus.NewSBI(spb.Type(config.BaseSouthBoundType), config.BaseSouthBoundUUID)
-	return createPrincipalNetworkDomain(sbi, config)
+	return createPrincipalNetworkDomain(sbi)
 }
 
 // createPrincipalNetworkDomain initializes the controller with an initial PND
-func createPrincipalNetworkDomain(s southbound.SouthboundInterface, config Config) error {
+func createPrincipalNetworkDomain(s southbound.SouthboundInterface) error {
 	pnd, err := nucleus.NewPND("base", "gosdn base pnd", config.BasePndUUID, s, c.csbiClient, callback)
 	if err != nil {
 		return err

initialise_test.go

@@ -4,6 +4,8 @@ import (
 	"os"
 	"testing"
 
+	"code.fbi.h-da.de/danet/gosdn/config"
+
 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 )
@@ -17,10 +19,8 @@ var cuid uuid.UUID
 
 func TestMain(m *testing.M) {
 	log.SetReportCaller(true)
+	log.SetLevel(config.LogLevel)
 
-	if os.Getenv("GOSDN_LOG") == "nolog" {
-		log.SetLevel(log.PanicLevel)
-	}
 	readTestUUIDs()
 	os.Exit(m.Run())
 }

nucleus/change_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	ppb "code.fbi.h-da.de/danet/api/go/gosdn/pnd"
+	"code.fbi.h-da.de/danet/gosdn/config"
 	"github.com/google/uuid"
 	"github.com/openconfig/ygot/exampleoc"
 	"github.com/openconfig/ygot/ygot"
@@ -61,7 +62,7 @@ func TestChange_CommitRollback(t *testing.T) {
 		if err := c.Commit(); (err != nil) != wantErr {
 			t.Errorf("Commit() error = %v, wantErr %v", err, wantErr)
 		}
-		time.Sleep(time.Millisecond * 200)
+		time.Sleep(config.ChangeTimeout)
 	}()
 	got := <-callback
 	if !reflect.DeepEqual(got, want) {
@@ -103,7 +104,7 @@ func TestChange_CommitRollbackError(t *testing.T) {
 		if err := c.Commit(); (err != nil) != wantErr {
 			t.Errorf("Commit() error = %v, wantErr %v", err, wantErr)
 		}
-		time.Sleep(time.Millisecond * 200)
+		time.Sleep(config.ChangeTimeout)
 	}()
 	got := <-c.errChan
 	if !reflect.DeepEqual(got, want) {

nucleus/southbound.go

@@ -114,7 +114,10 @@ func unmarshal(schema *ytypes.Schema, bytes []byte, fields []string, goStruct yg
 		return err
 	}
 	ygot.PruneEmptyBranches(validatedDeepCopy)
-	return ygot.MergeStructInto(goStruct, validatedDeepCopy)
+
+	opts := []ygot.MergeOpt{&ygot.MergeOverwriteExistingFields{}}
+
+	return ygot.MergeStructInto(goStruct, validatedDeepCopy, opts...)
 }
 
 // getField traverses the GoStruct and returns the field that represents the

test/integration/nucleusIntegration_test.go

@@ -45,10 +45,10 @@ func testSetupIntegration() {
 		log.SetLevel(log.PanicLevel)
 	}
 
-	addr := os.Getenv("GOSDN_TEST_ENDPOINT")
+	addr := os.Getenv("CEOS_TEST_ENDPOINT")
 	if addr != "" {
 		testAddress = addr
-		log.Infof("GOSDN_TEST_ENDPOINT set to %v", testAddress)
+		log.Infof("CEOS_TEST_ENDPOINT set to %v", testAddress)
 	}
 	u := os.Getenv("GOSDN_TEST_USER")
 	if u != "" {