clab-config.go: derive config with different ssl folder for twin, don't rename endpoint names

remove generate flag, always generate the realnet clab environment in manager
don't start event system for testing purposes of twin environment
enhance mneadd/main.go with more outputs and better testing
rtdt-manager.go: change event system to try to apply gnmiPaths to twin

applications/rtdt-manager/clab-config/clab-config.go

@@ -81,7 +81,7 @@ func LoadConfig(filename string) (*ClabConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("Failed to unmarshal YAML: %w", err)
 	}
-
+	fmt.Println("--- Successfully loaded file:", absFilepath, "----")
 	return &clabconfig, nil
 }
 
@@ -98,9 +98,9 @@ func DeriveConfig(clabconfig *ClabConfig, newIPv4Subnet, newIPv6Subnet string, p
 	derivedConfig := *clabconfig
 	derivedConfig.Topology.Nodes = make(map[string]Node)
 	derivedConfig.Topology.Links = append([]Link{}, clabconfig.Topology.Links...) // Copy links
-	portOffset := 5
+	portOffset := 5 // TODO set dynamically in some way
 
-	derivedConfig.Name = fmt.Sprintf("%s_%s", clabconfig.Name, postfix)
+	derivedConfig.Name = fmt.Sprintf("%s-%s", clabconfig.Name, postfix)
 	subnetParts := strings.Split(newIPv4Subnet, ".")
 	derivedConfig.Mgmt.IPv4Subnet = newIPv4Subnet
 	derivedConfig.Mgmt.IPv6Subnet = newIPv6Subnet
@@ -112,6 +112,15 @@ func DeriveConfig(clabconfig *ClabConfig, newIPv4Subnet, newIPv6Subnet string, p
 		splitIPv4[0], splitIPv4[1], splitIPv4[2] = subnetParts[0], subnetParts[1], subnetParts[2]
 		node.MgmtIPv4 = strings.Join(splitIPv4, ".")
 
+		if strings.HasPrefix(name, "gosdn") {
+			node.Cmd = "--config /app/configs/containerlab-gosdn-twin.toml"
+			node.Binds = []string{"../../../applications/rtdt-manager/data/ssl/gosdn:/app/ssl"}
+		}
+
+		//use separate ssl folders, testing
+		if strings.HasPrefix(name, "gnmi-target") {
+			node.Binds = []string{"../../../applications/rtdt-manager/data/ssl/gnmi-target:/etc/gnmi-target/ssl"}
+		}
 		// Ports: host side needs to be incremented or there will be conflicts
 		// for now just use 5 as increment
 		for i, portBinding := range node.Ports {
@@ -137,21 +146,19 @@ func DeriveConfig(clabconfig *ClabConfig, newIPv4Subnet, newIPv6Subnet string, p
 			}
 			node.Ports[i] = strings.Join(parts, ":")
 		}
-		newName := fmt.Sprintf("%s-%s", name, postfix)
-		derivedConfig.Topology.Nodes[newName] = node
+		derivedConfig.Topology.Nodes[name] = node
 	}
 
 	// Update Links so they reference the correct nodes
-	for i, link := range clabconfig.Topology.Links {
-		for j, endpoint := range link.Endpoints {
-			parts := strings.Split(endpoint, ":")
-			if len(parts) != 2 {
-				return nil, fmt.Errorf("invalid link endpoint: %s", endpoint)
-			}
-			parts[0] = fmt.Sprintf("%s-%s", parts[0], postfix) // Update the node name
-			clabconfig.Topology.Links[i].Endpoints[j] = strings.Join(parts, ":")
-		}
-	}
+	// for i, link := range clabconfig.Topology.Links {
+	// 	for j, endpoint := range link.Endpoints {
+	// 		parts := strings.Split(endpoint, ":")
+	// 		if len(parts) != 2 {
+	// 			return nil, fmt.Errorf("invalid link endpoint: %s", endpoint)
+	// 		}
+	// 		clabconfig.Topology.Links[i].Endpoints[j] = strings.Join(parts, ":")
+	// 	}
+	// }
 
 	return &derivedConfig, nil
 }

applications/rtdt-manager/data/containerlab-gosdn-twin.toml.example

+basepnduuid = "5f20f34b-cbd0-4511-9ddc-c50cf6a3b49e"
+plugin-registry = "clab-gosdn_csbi_arista_base-TEST-TWIN-plugin-registry:55057"
+help = false
+plugin-folder = "plugins"
+log-level = "debug"
+socket = ":55055"
+databaseConnection = "mongodb://root:example@clab-gosdn_csbi_arista_base-TEST-TWIN-mongodb:27017"
+filesystemPathToStores = "stores"
+gNMISubscriptionsPath = "configs/gNMISubscriptions.txt"
+tlscertfile = '/ssl/certs/gosdn-selfsigned.crt'
+tlskeyfile = '/ssl/private/gosdn-selfsigned.key'
+tlscacertfile = '/ssl/ca.crt'
+maxTokensPerUser = 100
+
+amqpPrefix = "amqp://"
+amqpUser = "guest"
+amqpPassword = "guest"
+amqpHost = "rabbitmq"
+amqpPort = "5672"

applications/rtdt-manager/gosdnutil/gosdnutil.go

@@ -15,7 +15,7 @@ func FetchPnd(conn *grpc.ClientConn, auth *rtdt_auth.RtdtAuth) (*pnd.PrincipalNe
 	ctx := auth.CreateContextWithAuthorization()
 	pndResponse, err := pndService.GetPndList(ctx, &pnd.GetPndListRequest{Timestamp: util.Now()})
 	if err != nil {
-		return nil, fmt.Errorf("Failed to retrieve PND information: %w", err)
+        return nil, fmt.Errorf("Failed to retrieve PND information:\npndresponse: %v\nerror: %w\n", pndResponse, err)
 	}
 	if pndList := pndResponse.GetPnd(); pndList != nil {
 		fmt.Println("pndUUID is now: ", pndList[0].Id)

applications/rtdt-manager/main.go

@@ -23,61 +23,48 @@ func main() {
 	var pass string
 	var user string
 	var topology_file string
-	var generate bool
 	var withTwin bool
-	var testMode bool
 	flag.StringVar(&address, "address", "172.100.0.5:55055", "Address of the gosdn controller")
 	flag.StringVar(&address, "a", "172.100.0.5:55055", "Address of the gosdn controller (shorthand)")
 	flag.StringVar(&pass, "password", "TestPassword", "Password for admin user")
 	flag.StringVar(&pass, "p", "TestPassword", "Password for admin user (shorthand)")
 	flag.StringVar(&user, "user", "admin", "Username")
 	flag.StringVar(&user, "u", "admin", "Username (shorthand)")
-	flag.BoolVar(&generate, "generate", false, "Whether to start the realnet containerlab environment ourselves")
-	flag.BoolVar(&generate, "g", false, "Whether to start the realnet containerlab environment ourselves (shorthand)")
 	flag.BoolVar(&withTwin, "with-twin", false, "Whether to start a twin")
-
 	flag.StringVar(&topology_file, "topology", "data/clab.yaml", "Containerlab file on the basis of which to create topo")
 	flag.StringVar(&topology_file, "t", "data/clab.yaml", "Containerlab file on the basis of which to create topo (shorthand)")
 
-	flag.BoolVar(&testMode, "test", false, "Whether to execute the test")
-
 	flag.Usage = func() {
 		fmt.Println("--address, -a: Address of the gosdn controller (realnet)")
 		fmt.Println("--user, -u: User to log into realnet as")
 		fmt.Println("--password, -p: Password for the user to log into realnet as")
 		fmt.Println("--topology, -t: Topology .yaml file to use to generate realnet and twins")
-		fmt.Println("--generate, -g: Whether to start the realnet containerlab environment ourselves")
-		fmt.Println("--with-twin: Whether to start the realnet containerlab environment for the twin")
-		fmt.Println("--test: Whether to execute a test of the event system")
+		fmt.Println("--with-twin: Whether to start the containerlab virtual environment for the twin")
 	}
 	flag.Parse()
 	fmt.Println("Topology file path: ", topology_file)
+	// Start virtual environment for realnet
 	var realnet *venv.VEnv
-	if generate {
-		fmt.Println("Generate flag is set, starting clab environment for realnet")
 	realnet = venv.NewVEnv("REALNET", topology_file, user, pass, &wg)
 	if realnet == nil {
 		fmt.Println("ERROR: Couldn't deploy the physical network")
+		return
 	} else {
 		fmt.Println("Successfully deployed physical network")
 	}
-	} else {
-		fmt.Println("Generate flag not set!")
-	}
-
 	// Register realnet with rtdt-manager
 	rtdtMan := RtdtMan.NewRtdtManager(realnet, &wg, &stopChan)
 	if rtdtMan == nil {
 		fmt.Println("Couldn't initialize rtdt-manager, quitting!")
 		return
 	}
-	err := rtdtMan.InitEventSystem()
-	if err != nil {
-		fmt.Printf("Error occured while initializing event system: %v", err)
-		return
-	}
+	// err := rtdtMan.InitEventSystem()
+	// if err != nil {
+	// 	fmt.Printf("Error occured while initializing event system: %v", err)
+	// 	return
+	// }
 	if withTwin {
-		rtdtMan.LaunchTwin("172.101.0.0/16", "2001:db9::/64", "test-twin")
+		rtdtMan.LaunchTwin("172.101.0.0/16", "2001:db9::/64", "TEST-TWIN")
 	}
 	if err := rtdtMan.Run(); err != nil {
 		fmt.Println("Program exited with errors: %w", err)

applications/rtdt-manager/mneadd/main.go

@@ -15,44 +15,47 @@ import (
 )
 
 func main() {
-	gosdn_addr := "172.100.0.5:55055"
-	addr := "172.100.0.11:7030"
-	var sessionToken string
+	gosdn_addr1 := "172.100.0.5:55055"
+	gosdn_addr2 := "172.101.0.5:55055"
+	mneAddr1 := "172.100.0.11:7030"
+	mneAddr2 := "172.101.0.11:7030"
+	var sessionToken1 string
+	var sessionToken2 string
 
 	// Create Connection
 	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
-	conn, err := grpc.NewClient(gosdn_addr, dialOption, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(100*1024*1024)))
+	conn1, err := grpc.NewClient(gosdn_addr1, dialOption, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(100*1024*1024)))
 	// Log in to controller
 	// Register
-	loginResponse, err := api.Login(context.Background(), gosdn_addr, "admin", "TestPassword")
+	loginResponse1, err := api.Login(context.Background(), gosdn_addr1, "admin", "TestPassword")
 	if err != nil {
 		fmt.Println("Couldn't log in to gosdn controller:", err)
 		return
 	} else {
-		fmt.Println("Successully logged in to gosdn controller")
+		fmt.Println("Successully logged in to gosdn controller at", gosdn_addr1)
 		time.Sleep(time.Second)
 	}
-	sessionToken = loginResponse.GetToken()
+	sessionToken1 = loginResponse1.GetToken()
 
 	// Create Context with Authorization
-	md := metadata.Pairs("authorize", sessionToken)
-	ctx := metadata.NewOutgoingContext(context.Background(), md)
+	md := metadata.Pairs("authorize", sessionToken1)
+	ctx1 := metadata.NewOutgoingContext(context.Background(), md)
 
 	// PND
-	var p *pnd.PrincipalNetworkDomain
-	pndService := pnd.NewPndServiceClient(conn)
-	pndResponse, err := pndService.GetPndList(ctx, &pnd.GetPndListRequest{Timestamp: int64(time.Now().Nanosecond())})
+	var p1 *pnd.PrincipalNetworkDomain
+	pndService := pnd.NewPndServiceClient(conn1)
+	pndResponse, err := pndService.GetPndList(ctx1, &pnd.GetPndListRequest{Timestamp: int64(time.Now().UnixNano())})
 	if err != nil {
 		fmt.Println("Failed to fetch PND")
 		return
 	} else {
-		fmt.Println("Successfully got PND")
+		fmt.Println("Successfully got PND:", pndResponse.GetPnd())
 		time.Sleep(time.Second)
 	}
 
 	// Add MNE
-	p = pndResponse.GetPnd()[0]
-	pndID, _ := uuid.Parse(p.GetId())
+	p1 = pndResponse.GetPnd()[0]
+	pndID1, _ := uuid.Parse(p1.GetId())
 	//fmt.Println("pndID: ", pndID)
 	//pluginUUID, _ := uuid.Parse("823aad29-69be-42f0-b279-90f2c1b6a94d")
 	//pluginID0, _ := uuid.Parse("d1c269a2-6482-4010-b0d8-679dff73153b") // TODO Get this dynamically
@@ -61,7 +64,7 @@ func main() {
 	//fmt.Println("pluginID: ", pluginID)
 
 	opt := &tpb.TransportOption{
-		Address:  addr,
+		Address:  mneAddr1,
 		Username: "admin",
 		Password: "admin",
 		Tls:      true,
@@ -69,7 +72,7 @@ func main() {
 			GnmiTransportOption: &tpb.GnmiTransportOption{},
 		},
 	}
-	listResponse, err := api.AddNetworkElement(ctx, gosdn_addr, "1234", "", opt, pluginID, pndID, nil)
+	listResponse, err := api.AddNetworkElement(ctx1, gosdn_addr1, "elem1", "", opt, pluginID, pndID1, nil)
 	if err != nil {
 		fmt.Println("Failed to add network element: ", err)
 		return
@@ -77,4 +80,70 @@ func main() {
 		fmt.Println("Successfylly added network element!")
 	}
 	_ = listResponse
+
+	conn1.Close()
+	// More! -------------------------------------------------------------------------------------------
+
+	// Create Connection 2
+	dialOption2 := grpc.WithTransportCredentials(insecure.NewCredentials())
+	conn2, err := grpc.NewClient(gosdn_addr2, dialOption2, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(100*1024*1024)))
+    conn2.Connect()
+	// Log in to controller
+	// Register
+	loginResponse2, err := api.Login(context.Background(), gosdn_addr2, "admin", "TestPassword")
+	if err != nil {
+		fmt.Println("Couldn't log in to gosdn controller:", err)
+		return
+	} else {
+		fmt.Println("Successully logged in to gosdn controller at", gosdn_addr2)
+		time.Sleep(time.Second)
+	}
+	sessionToken2 = loginResponse2.GetToken()
+
+	// Create Context with Authorization
+	md2 := metadata.Pairs("authorize", sessionToken2)
+	ctx2 := metadata.NewOutgoingContext(context.Background(), md2)
+
+	// PND
+	var p2 *pnd.PrincipalNetworkDomain
+	pndService2 := pnd.NewPndServiceClient(conn2)
+	pndResponse2, err := pndService2.GetPndList(ctx2, &pnd.GetPndListRequest{Timestamp: int64(time.Now().UnixNano())})
+	if err != nil {
+		fmt.Println("Failed to fetch PND2, err: ", err)
+	} else {
+		fmt.Println("Successfully got PND2:", pndResponse2.GetPnd())
+		time.Sleep(time.Second)
+	}
+
+	// Add MNE
+	if len(pndResponse2.GetPnd()) > 0 {
+		p2 = pndResponse2.GetPnd()[0]
+	}
+	pndID2, _ := uuid.Parse(p2.GetId())
+	//fmt.Println("pndID: ", pndID)
+	//pluginUUID, _ := uuid.Parse("823aad29-69be-42f0-b279-90f2c1b6a94d")
+	//pluginID0, _ := uuid.Parse("d1c269a2-6482-4010-b0d8-679dff73153b") // TODO Get this dynamically
+
+	pluginID2, _ := uuid.Parse("e2c358b3-6482-4010-b0d8-679dff73153b") // TODO Get this dynamically
+	//fmt.Println("pluginID: ", pluginID)
+
+	opt2 := &tpb.TransportOption{
+		Address:  mneAddr2,
+		Username: "admin",
+		Password: "admin",
+		Tls:      true,
+		TransportOption: &tpb.TransportOption_GnmiTransportOption{
+			GnmiTransportOption: &tpb.GnmiTransportOption{},
+		},
+	}
+	fmt.Println("Using pnd:", pndID2)
+	listResponse2, err := api.AddNetworkElement(ctx2, gosdn_addr2, "elem2", "", opt2, pluginID2, pndID2, nil)
+	if err != nil {
+		fmt.Println("Failed to add network element: ", err)
+		return
+	} else {
+		fmt.Println("Successfylly added network element!")
+	}
+	_ = listResponse2
+	conn2.Close()
 }

applications/rtdt-manager/rtdt-auth/rtdt-auth.go

@@ -23,8 +23,6 @@ func NewRtdtAuth(userName, url, password string, conn *grpc.ClientConn) *RtdtAut
 	if err != nil {
 		fmt.Println("Encountered error while trying to log in: ", err)
 		return nil
-	} else {
-		fmt.Println("Successfully logged in to gosdn controller at ", url, " as user ", userName)
 	}
 	return &RtdtAuth{
 		userName:   userName,
@@ -36,6 +34,7 @@ func NewRtdtAuth(userName, url, password string, conn *grpc.ClientConn) *RtdtAut
 
 // createContextWithAuthorization creates a context with the token received after login.
 func (r *RtdtAuth) CreateContextWithAuthorization() context.Context {
+    fmt.Println("Creating Context with session token: ", r.sessionTok)
 	md := metadata.Pairs("authorize", r.sessionTok)
 	return metadata.NewOutgoingContext(context.Background(), md)
 }

applications/rtdt-manager/rtdt-manager/rtdt-manager.go

 package rtdtmanager
 
 import (
+	"code.fbi.h-da.de/danet/gosdn/application-framework/event"
 	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
 	"strings"
 	"sync"
 
-	"code.fbi.h-da.de/danet/gosdn/api/go/gosdn/networkelement"
-	"code.fbi.h-da.de/danet/gosdn/application-framework/event"
-
-	//"code.fbi.h-da.de/danet/gosdn/csbi/cmd"
-
 	"code.fbi.h-da.de/danet/gosdn/application-framework/registration"
 	clabconfig "code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/clab-config"
-	"code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/gosdnutil"
-	"code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/util"
 	"code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/venv"
 )
 
@@ -26,8 +19,7 @@ type RtdtManager struct {
 	rtdt_twins   []*venv.VEnv
 	eventService event.ServiceInterface // Receive events from realnet gosdn (not used yet)
 	waitGroup    *sync.WaitGroup
-	stopChan     *chan os.Signal // Global stop channel TODO Can I use that like this?
-	// TODO auth and conn for virtual net?
+	stopChan     *chan os.Signal
 }
 
 // needs to be passed a running realnet VEnv
@@ -41,14 +33,11 @@ func NewRtdtManager(realnet *venv.VEnv, wg *sync.WaitGroup, stopChan *chan os.Si
 	return &rMan
 }
 
-//
-
 // To launch a new twin, this runs through the following steps:
 // - Load the clab config for current realnet gosdn (passed on cli)
 // - Derive the config we need for twin
 // - Write the config to disk
 // - Use that config to call "containerlab deploy"
-// Takes
 // Launch a second gosdn instance which will manage the virtual network
 func (r *RtdtManager) LaunchTwin(twinSubnetIPv4, twinSubnetIPv6, twinName string) error {
 	var derivedConfig *clabconfig.ClabConfig
@@ -67,17 +56,15 @@ func (r *RtdtManager) LaunchTwin(twinSubnetIPv4, twinSubnetIPv6, twinName string
 	}
 
 	// Now run deploy with new config file
-	clabconfig.ClabDeploy(clabConfigFullPath)
-	user := r.realnet.GetAuth().GetUsername()
-	password := r.realnet.GetAuth().GetPassword()
-	twin := venv.NewVEnv(twinName, clabConfigFullPath, user, password, r.waitGroup)
+	// user := r.realnet.GetAuth().GetUsername()
+	// password := r.realnet.GetAuth().GetPassword()
+	twin := venv.NewVEnv(twinName, clabConfigFullPath, "admin", "TestPassword", r.waitGroup)
 	r.rtdt_twins = append(r.rtdt_twins, twin)
 	return nil
 }
 
-// Start the eventsystem for each venv, manage clean up
+// Run realnet and twins (if present) until signal received, manage clean up
 func (r *RtdtManager) Run() error {
-	fmt.Println("Starting twin..")
 	if r.realnet == nil {
 		fmt.Println("You did not start the physical network, please do so before executing App.Run()")
 		return nil
@@ -118,8 +105,8 @@ func (r *RtdtManager) Run() error {
 	return nil
 }
 
+// Receive events from realnet VEnv
 func (r *RtdtManager) InitEventSystem() error {
-
 	realnet_auth := r.realnet.GetAuth()
 	ctx := realnet_auth.CreateContextWithAuthorization()
 	queueCredentials, err := registration.Register(ctx, realnet_auth.GetAddress(), "basic-interface-monitoring", "SecurePresharedToken")
@@ -168,42 +155,42 @@ func (r *RtdtManager) updateMNECallback(event *event.Event) {
 	fmt.Println("EventID: ", event.ID.ID())
 	fmt.Println("Event Type: ", event.Type)
 	fmt.Println("PathsAndValuesMap: ", event.PathsAndValuesMap)
+	fmt.Println("EntityID", event.EntityID)
 	prefix := "/interfaces/interface[name="
-	//suffix := "]/state/oper-status"
-	re := regexp.MustCompile(`/interfaces/interface\[name=([^]]+)]/state/oper-status`)
+	prefixHostname := "/system/config/hostname"
+	//realnetClabData := r.realnet.GetClabData()
+	// for i, node := range realnetClabData.Topology.Nodes {
+	// }
+	suffix := "]/config/mtu"
+	//re := regexp.MustCompile(`/interfaces/interface\[name=([^]]+)]/state/oper-status`)
 
+	// Apply event to a twin
+	// No twins to apply changes to:
+	if len(r.rtdt_twins) == 0 {
+		fmt.Println("--- NO TWINS TO APPLY CHANGE TO ---")
+		return
+	}
+	// Test if the path is supported first, only set Gnmi Paths I actually want to be able to set for now
 	for path, value := range event.PathsAndValuesMap {
-		if strings.HasPrefix(path, prefix) {
-			matches := re.FindStringSubmatch(path)
-			if len(matches) > 1 {
-				fmt.Printf("matches: %v", matches)
-				interfaceName := matches[1] // Extracted interface name (eth0, eth1, etc.)
-				state := value              // UP or DOWN
-				_ = state
-				mneService := networkelement.NewNetworkElementServiceClient(r.realnet.GetConn())
-				_ = mneService
-
-				// Construct the request to twin?
-				request := &networkelement.GetRequest{
-					Timestamp: util.Now(),
-					Mneid:     event.EntityID.String(),
+		if strings.HasPrefix(path, prefix) && strings.HasSuffix(path, suffix) {
+			fmt.Println("--- CHANGE MTU TRIGGERED ---")
+			fmt.Println("Value of new MTU: ", value)
+			for _, twin := range r.rtdt_twins {
+				twin.SetGnmiPath(path, value, event.EntityID.String())
 			}
-				_ = request
-
-				fmt.Printf("Interface: %s, State: %s\n", interfaceName, state)
 		}
-
+		if strings.HasPrefix(path, prefixHostname) {
+			fmt.Println("--- CHANGE HOSTNAME TRIGGERED ---")
+			for _, twin := range r.rtdt_twins {
+				if twin == nil {
+					fmt.Println("Encountered nil twin, skipping")
+					continue
+				}
+				fmt.Println("ENTERING SETGNMIPATH, value: ", value, "path:", path)
+				twin.SetGnmiPath(path, value, event.EntityID.String())
 			}
 		}
-	ctx := r.realnet.GetAuth().CreateContextWithAuthorization()
-	gosdn_pnd, err := gosdnutil.FetchPnd(r.realnet.GetConn(), r.realnet.GetAuth())
-	if err != nil {
-		fmt.Printf("Error encountered in callback: %v", err)
-	} else {
-		fmt.Println("Retrieved pnd in callback: ", gosdn_pnd)
 	}
-	//api.UpdateNetworkElement(ctx, "172.100.0.5:55055")
-	_ = ctx
 }
 func (r *RtdtManager) userEventCallback(event *event.Event) {
 	fmt.Println("--------------------------------")

applications/rtdt-manager/rtdt-topology/rtdt-topology.go

 package rtdt_topology
 
 import (
-	"fmt"
-	"os"
-	"path/filepath"
-
-	"code.fbi.h-da.de/danet/gosdn/applications/venv-manager/containerlab"
-	"gopkg.in/yaml.v3"
+	"regexp"
+	"code.fbi.h-da.de/danet/gosdn/models/generated/openconfig"
 )
 
-// Probably don't need a package, just use api inside rtdt-manager.go
-// import (
-// 	"code.fbi.h-da.de/danet/gosdn/api/go/gosdn/topology"
-// 	"code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/rtdt-auth"
-// 	"google.golang.org/grpc"
-// )
-//
-// type Link struct {
-// 	Id   string
-// 	name string
-// }
-// type Port struct {
-// }
-//
-// type Node struct {
-// }
-//
-// type Topology struct {
-// 	Links *[]Link
-// 	Ports *[]Port
-// 	Nodes *[]Node
-// }
-//
-// func NewTopology() {
-//
-// 	return &Topology{}
-// }
-//
-// func ApplyTopology(conn grpc.ClientConnInterface, auth rtdt_auth.RtdtAuth) {
-// 	ctx := auth.CreateContextWithAuthorization()
-// 	topoService := topology.NewTopologyServiceClient(conn)
-//     addLink := topology.AddLinkRequest{}
-//     topoService.AddLink(ctx, addLink)
-// }
-//
-// func LoadTopologyFromFile()
+// Struct definitions were taken from:
+// "code.fbi.h-da.de/danet/gosdn/applications/venv-manager/topology/port.go"
+// "code.fbi.h-da.de/danet/gosdn/applications/venv-manager/topology/link.go"
+// "code.fbi.h-da.de/danet/gosdn/applications/venv-manager/topology/node.go"
+// commit: 0264b698286b6cbb965d743078c681f8af55edf6
+type Link struct {
+	ID         string
+	Name       string
+	SourceNode *Node
+	TargetNode *Node
+	SourcePort *Port
+	TargetPort *Port
+}
+type Port struct {
+	ID   string
+	Name string
+}
 
-func ParseTopology(filename string) (*containerlab.YamlStruct, error) {
-	fmt.Println("Parsing file: ", filename)
-	var absFilepath string
-	var err error
-	if absFilepath, err = filepath.Abs(filename); err != nil {
-		return nil, fmt.Errorf("Failed to convert filename %v to absolute path", filename)
+type Node struct {
+	ID       string
+	Name     string
+	Kind     string
+	Image    string
+	MgmtIpv4 string
+	YangData openconfig.Device
 }
-	fmt.Printf("absolute filepath: %v\n", absFilepath)
 
-	file, err := os.Open(absFilepath)
-	if err != nil {
-		return nil, fmt.Errorf("Encountered error while trying to parse clab file into topology: %v", err)
+type Topology struct {
+	Links []Link
+	Ports []Port
+	Nodes []Node
 }
-	decoder := yaml.NewDecoder(file)
-	var topoData containerlab.YamlStruct
-	if err := decoder.Decode(&topoData); err != nil {
-		return nil, fmt.Errorf("Failed to decode YAML: %v", err)
+
+func NewTopology() *Topology {
+
+	return &Topology{}
 }
 
-	fmt.Println("Successfully parsed given clab file into topology, nice!")
+// Source for function: "code.fbi.h-da.de/danet/gosdn/applications/venv-manager/link.go"
+// commit: 0264b698286b6cbb965d743078c681f8af55edf6
+// GetLinkAsSliceOfStrings returns the link as a slice of strings for yaml representation.
+func (l *Link) GetLinkAsSliceOfStrings() [2]string {
+	return [2]string{l.SourceNode.Name + ":" + l.SourcePort.Name, l.TargetNode.Name + ":" + l.TargetPort.Name}
+}
+
+// Source for function: "code.fbi.h-da.de/danet/gosdn/applications/venv-manager/node.go"
+// commit: 0264b698286b6cbb965d743078c681f8af55edf6
+// FillAllFields fills all remaining fields of object with data from YangData.
+func (n *Node) FillAllFields(containerRegistryURL string) {
+	// Works if linux and our gnmi target is used.
+	softwareVersion := n.YangData.System.State.SoftwareVersion
+	if softwareVersion != nil {
+		// Checks if software version is in compatible format.
+		result, _ := regexp.MatchString(`^([A-Za-z0-9\.\/])*:([A-Za-z0-9\.])*`, *softwareVersion)
+		if result {
+			n.Kind = "linux"
+			n.Image = containerRegistryURL + *softwareVersion
+			return
+		}
 
-	return &topoData, nil
+		n.Kind = "couldn't detect kind"
+		n.Image = "couldn't detect image"
+		return
 	}
 
+	// Specific to arista
+	regex := regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9][A-Z]`)
+	dockerTag := string(regex.FindAll([]byte(*n.YangData.Lldp.Config.SystemDescription), 1)[0])
 
-// Needs to be in rtdt-manager
-func ApplyTopology(topoYaml *containerlab.YamlStruct) error {
+	// If it's not linux with our gnmi target and not arista, we don't support it.
+	if len(dockerTag) == 0 {
+		n.Kind = "couldn't detect kind"
+		n.Image = "couldn't detect image"
+		return
+	}
 
-	return nil
+	n.Kind = "ceos"
+	n.Image = containerRegistryURL + n.Kind + ":" + dockerTag
 }

applications/rtdt-manager/venv/venv.go

 package venv
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"sync"
 	"time"
 
+	"code.fbi.h-da.de/danet/gosdn/api/go/gosdn/networkelement"
 	"code.fbi.h-da.de/danet/gosdn/api/go/gosdn/pnd"
 	topoPb "code.fbi.h-da.de/danet/gosdn/api/go/gosdn/topology"
 	tpb "code.fbi.h-da.de/danet/gosdn/api/go/gosdn/transport"
@@ -13,9 +15,14 @@ import (
 	clabconfig "code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/clab-config"
 	"code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/gosdnutil"
 	rtdt_auth "code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/rtdt-auth"
+	rtdt_topology "code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/rtdt-topology"
 	"code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/util"
+	yangparser "code.fbi.h-da.de/danet/gosdn/applications/venv-manager/yang-parser"
 	"code.fbi.h-da.de/danet/gosdn/controller/api"
+	"code.fbi.h-da.de/danet/gosdn/models/generated/openconfig"
 	uuid "github.com/google/uuid"
+	gnmi "github.com/openconfig/gnmi/proto/gnmi"
+	"github.com/openconfig/ygot/ygot"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 )
@@ -29,15 +36,17 @@ type VEnv struct {
 	clabFilename         string                 // This is the name of the yaml file clabData is based on
 	StopChan             <-chan struct{}
 	waitGroup            *sync.WaitGroup
+	topology             rtdt_topology.Topology
+	containerRegistryURL string
 }
 
 // Accepts a yaml filename to deploy a container lab environment
 func NewVEnv(name, topoYamlFile, user, pass string, wg *sync.WaitGroup) *VEnv {
-	fmt.Println("Creating new virtual environment with name: ", name)
+	fmt.Printf("[%s] - Creating new virtual environment\n", name)
 	wg.Add(1) // Register the venv and run atleast until it calls wg.Done()
 	var err error
 	if err = clab.ClabDeploy(topoYamlFile); err != nil {
-		fmt.Println("Failed to deploy the physical network")
+		fmt.Printf("[%s] - Failed to deploy the physical network\n", name)
 		return nil
 	}
 
@@ -45,7 +54,7 @@ func NewVEnv(name, topoYamlFile, user, pass string, wg *sync.WaitGroup) *VEnv {
 	var clabData *clab.ClabConfig
 	clabData, err = clab.LoadConfig(topoYamlFile)
 	if err != nil {
-		fmt.Println("Failed to load config from yaml file")
+		fmt.Printf("[%s] - Failed to load config from yaml file\n", name)
 		return nil
 	}
 
@@ -53,46 +62,68 @@ func NewVEnv(name, topoYamlFile, user, pass string, wg *sync.WaitGroup) *VEnv {
 	var gosdnAddress string
 	for nodename, val := range clabData.Topology.Nodes {
 		if strings.HasPrefix(nodename, "gosdn") {
-			fmt.Println("Found gosdn ipv4: ", val.MgmtIPv4)
 			gosdnAddress = val.MgmtIPv4 + ":55055"
+			fmt.Printf("[%s} - Found gosdn ipv4: %s\n", name, gosdnAddress)
 		}
 	}
 
-	fmt.Println("Sleep for 10 seconds to give things time to settle..")
-	time.Sleep(time.Second * 10)
+	fmt.Printf("[%s] - Sleep for 5 seconds to give containers time to settle..\n", name)
+	time.Sleep(time.Second * 5)
 	// Now log into gosdn physical network
 	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
-	conn, err := grpc.NewClient(gosdnAddress, dialOption, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(100*1024*1024)))
+	gosdnconn, err := grpc.NewClient(gosdnAddress, dialOption, grpc.WithDefaultCallOptions())
 	if err != nil {
-		fmt.Println("Failed to create connection to network")
+		fmt.Printf("[%s] - Failed to create connection to gosdn\n", name)
 		return nil
 	} else {
-		fmt.Println("Successfully created connection to gosdn")
+		fmt.Printf("[%s] - Successfully created connection to gosdn\n", name)
+		fmt.Printf("[%s] - State of conn: \n%v\n\n", name, gosdnconn)
 	}
-	auth := rtdt_auth.NewRtdtAuth(user, gosdnAddress, pass, conn) // logs in and stores token
-	if auth == nil {
-		fmt.Println("Couldn't log in to gosdn physical network, quitting!")
+	time.Sleep(time.Second * 2)
+	gosdnauth := rtdt_auth.NewRtdtAuth(user, gosdnAddress, pass, gosdnconn) // logs in and stores token
+	if gosdnauth == nil {
+		fmt.Printf("[%s] - Couldn't log in to gosdn, quitting!\n", name)
 		return nil
 	} else {
-		fmt.Println("Successfully logged into gosdn")
+		fmt.Printf("[%s] - Successfully logged into gosdn as user: %s, with password: %s, session token: %v\n", name, gosdnauth.GetUsername(), gosdnauth.GetPassword(), gosdnauth.GetSessionToken())
 	}
 	// Get PND of gosdn in created venv
-	gosdn_pnd, err := gosdnutil.FetchPnd(conn, auth)
+	var gosdn_pnd *pnd.PrincipalNetworkDomain
+	for {
+		gosdn_pnd, err = gosdnutil.FetchPnd(gosdnconn, gosdnauth)
+		if err == nil {
+			break
+		}
+		gosdnconn.Close()
+		gosdnconn, err = grpc.NewClient(gosdnAddress, dialOption, grpc.WithDefaultCallOptions())
 		if err != nil {
-		fmt.Println("Failed to fetch pnd from venv")
+			fmt.Println("All is over")
 			return nil
 		}
+		gosdnauth = rtdt_auth.NewRtdtAuth("admin", "172.101.0.5:55055", "TestPassword", gosdnconn)
+
+		fmt.Printf("[%s] - Failed to fetch pnd from venv: %v, retrying\n", name, err)
+		time.Sleep(time.Second * 150)
+	}
 
 	// Sleep before adding devices because otherwise it won't work
 	// Apply the topoYamlFile: we need to register the switches with the controller
-	fmt.Println("Sleep for 6 seconds to give things time to settle before creating mne..")
-	time.Sleep(time.Second * 6)
+	fmt.Printf("[%s] - Sleep for 7 more seconds to give containers time to settle before creating mne..\n", name)
+	time.Sleep(time.Second * 7)
+	var topo rtdt_topology.Topology
+	var topoNodes []rtdt_topology.Node
 	for node, val := range clabData.Topology.Nodes {
+		var topoNode rtdt_topology.Node
+		topoNode.Name = node
+		topoNode.MgmtIpv4 = val.MgmtIPv4
+		topoNode.Image = val.Image
+		topoNode.Kind = val.Kind
+
 		if strings.HasPrefix(node, "gnmi-target-") {
+			fmt.Printf("[%s] - Creating Network Element for node: %s\n", name, node)
 			ports := strings.Split(val.Ports[0], ":")
 			port := ports[1]
 			addr := val.MgmtIPv4 + ":" + port
-
 			opt := &tpb.TransportOption{
 				Address:  addr,
 				Username: "admin",
@@ -106,29 +137,133 @@ func NewVEnv(name, topoYamlFile, user, pass string, wg *sync.WaitGroup) *VEnv {
 			pluginID, _ := uuid.Parse("d1c269a2-6482-4010-b0d8-679dff73153b") // TODO Get this dynamically
 			pndID, _ := uuid.Parse(gosdn_pnd.GetId())
 
-			fmt.Println("Found target: ", node, " with addr: ", addr)
-			fmt.Println("Gosdn controller at ", gosdnAddress)
-			ctx := auth.CreateContextWithAuthorization()
+			fmt.Printf("[%s] - Found target: %s with addr: %s\n", name, node, addr)
+			fmt.Printf("[%s] - Gosdn controller at %s\n", name, gosdnAddress)
+			ctx := gosdnauth.CreateContextWithAuthorization()
 			//listResponse, err := api.AddNetworkElement(ctx, val.MgmtIPv4+":7030", node, "", opt, mneUUID, pid, []string{"/"})
-			listResponse, err := api.AddNetworkElement(ctx, gosdnAddress, "TEST", "", opt, pluginID, pndID, []string{"/"})
+			listResponse, err := api.AddNetworkElement(ctx, gosdnAddress, node, "", opt, pluginID, pndID, []string{"/interfaces"})
 			if err != nil {
-				fmt.Println("Failed to add network elements: ", err)
+				fmt.Printf("[%s] - Failed to add network elements: %v\n", name, err)
 				return nil
 			} else {
-				fmt.Println("Successfully created network element!!")
+				fmt.Printf("[%s] - Successfully created network element\n", name)
 			}
-			fmt.Printf("Got response from AddNetworkElement: %v\n", listResponse)
-			_ = listResponse // TODO: Might need the id's (are they the UUIDs?)
-			fmt.Println("Success: registered mne with gosdn controller")
+			fmt.Printf("[%s] - Got response from AddNetworkElement: %v\n", name, listResponse)
+			topoNode.ID = listResponse.GetResponses()[0].GetId()
+
+			fmt.Printf("[%s] - Success: registered mne with gosdn controller\n", name)
+			fmt.Printf("[%s] - Also created gosdn topo node: %v\n", name, topoNode)
 		}
+		topoNodes = append(topoNodes, topoNode)
 	}
-	return &VEnv{auth: auth,
+	topo.Nodes = topoNodes
+	return &VEnv{auth: gosdnauth,
 		pnd:                  gosdn_pnd,
-		conn:         conn,
+		conn:                 gosdnconn,
 		clabData:             clabData,
 		clabFilename:         topoYamlFile,
 		waitGroup:            wg,
+		topology:             topo,
+		containerRegistryURL: "registry.code.fbi.h-da.de/danet/gnmi-target/", // TODO: Could let user choose
+	}
+}
+
+// Source: "code.fbi.h-da.de/danet/gosdn/applications/venv-manager/venv-manager/venv-manager.go"
+// commit: 0264b698286b6cbb965d743078c681f8af55edf6
+func (v *VEnv) loadNetworkElementModelPathsIntoGosdn(ctx context.Context, conn *grpc.ClientConn, nodes *[]rtdt_topology.Node) error {
+	networkElementService := networkelement.NewNetworkElementServiceClient(conn)
+
+	paths := [2]string{"/lldp/config/system-description", "/system/state/"}
+	for _, path := range paths {
+		for _, node := range *nodes {
+			_, err := networkElementService.GetPath(ctx, &networkelement.GetPathRequest{Mneid: node.ID, Pid: v.pnd.Id, Path: path})
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// Based on getAndAddMoreData() in: "code.fbi.h-da.de/danet/gosdn/applications/venv-manager/venv-manager/venv-manager.go"
+// commit: 0264b698286b6cbb965d743078c681f8af55edf6
+// What this does: Load yang paths into topo, then iterate over
+// nodes and fill data in
+func (v *VEnv) ConstructTopology() error {
+	ctx := v.auth.CreateContextWithAuthorization()
+	conn := v.conn
+	var path = "/"
+	var ygotPath *gnmi.Path
+
+	// Create 'root' path to be able to load the whole model from the store.
+	var err error
+	ygotPath, err = ygot.StringToPath(path, ygot.StructuredPath)
+	if err != nil {
+		return err
+	}
+
+	// just to load model data into goSDN to guaranteed have new data available for get request
+	err = v.loadNetworkElementModelPathsIntoGosdn(ctx, conn, &v.topology.Nodes)
+	if err != nil {
+		return err
 	}
+
+	networkElementService := networkelement.NewNetworkElementServiceClient(conn)
+
+	// This code relies on Nodes already being filled, so what does this achieve??
+	for iterator, node := range v.topology.Nodes {
+		getNetworkElementResponse, _ := networkElementService.Get(ctx, &networkelement.GetRequest{Mneid: node.ID})
+		if err != nil {
+			return err
+		}
+
+		var marshalledYangData openconfig.Device
+
+		err = yangparser.Unmarshal([]byte(getNetworkElementResponse.Mne.Model), ygotPath, &marshalledYangData)
+		if err != nil {
+			return err
+		}
+
+		mgmntAddress := strings.Split(getNetworkElementResponse.Mne.TransportAddress, ":")
+		v.topology.Nodes[iterator].MgmtIpv4 = mgmntAddress[0]
+		v.topology.Nodes[iterator].YangData = marshalledYangData
+		v.topology.Nodes[iterator].FillAllFields(v.containerRegistryURL)
+	}
+
+	return nil
+}
+
+func (v *VEnv) SetGnmiPath(path, value, mneid string) error {
+	ctx := v.auth.CreateContextWithAuthorization()
+	fmt.Println("--IN SETGNMIPATH-----------------------")
+	mneService := networkelement.NewNetworkElementServiceClient(v.conn)
+	gnmiPath, err := ygot.StringToStructuredPath(path)
+	if err != nil {
+		return fmt.Errorf("Encountered error while trying to parse string path into gnmi path: %w", err)
+	}
+	//gosdnAddr := v.auth.GetAddress()
+	fmt.Println("Got to setting pid from v.pnd.Id")
+	pid := v.pnd.Id
+	gnmiVal := gnmi.TypedValue{}
+	req := &networkelement.ChangeRequest{
+		Mneid: mneid,
+		Path:  gnmiPath,
+		Value: &gnmiVal,
+		ApiOp: networkelement.ApiOperation_API_OPERATION_UPDATE,
+	}
+
+	changeRequests := []*networkelement.ChangeRequest{req}
+
+	setPathResponse, err := mneService.SetPathList(ctx, &networkelement.SetPathListRequest{
+		Timestamp:     util.Now(),
+		Pid:           pid,
+		ChangeRequest: changeRequests,
+	})
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println("setPathResponse: ", setPathResponse.String())
+	return nil
 }
 
 // For later, topology stuff
@@ -140,9 +275,8 @@ func NewVEnv(name, topoYamlFile, user, pass string, wg *sync.WaitGroup) *VEnv {
 // 	return nil
 // }
 
-// This retrieves the topology from the running realnet gosdn instance
+// This retrieves the topology from the running gosdn instance
 // This is needed to generate the clab file to be used with the virtual net
-// TODO Solve this not returning anything
 func (v *VEnv) fetchTopology() error {
 	topoService := topoPb.NewTopologyServiceClient(v.conn)
 	ctx := v.auth.CreateContextWithAuthorization()
@@ -182,3 +316,6 @@ func (v VEnv) GetAuth() *rtdt_auth.RtdtAuth {
 func (v *VEnv) GetWaitgroup() *sync.WaitGroup {
 	return v.waitGroup
 }
+func (v *VEnv) GetTopology() *rtdt_topology.Topology {
+	return &v.topology
+}

controller/controller.Dockerfile

@@ -19,6 +19,7 @@ COPY --from=builder /gosdn/controller/configs/development-gosdn.toml.example ./c
 COPY --from=builder /gosdn/controller/configs/integration-test-gosdn.toml ./configs/integration-test-gosdn.toml
 COPY --from=builder /gosdn/controller/configs/containerlab-gosdn.toml.example ./configs/containerlab-gosdn.toml
 COPY --from=builder /gosdn/controller/configs/gNMISubscriptions.txt.example ./configs/gNMISubscriptions.txt
+COPY --from=builder /gosdn/applications/rtdt-manager/data/containerlab-gosdn-twin.toml.example ./configs/containerlab-gosdn-twin.toml
 
 EXPOSE 55055 8080 40000
 ENTRYPOINT ["./gosdn"]

scripts/simple-dev-setup.sh

-#!/bin/bash
+#!/usr/bin/env bash
 
 POSITIONAL_ARGS=()
 KEEPDB='false'