Try to implement topology, so that clabconfig is initially only used to

parse the passed-in configuration file and then upload to the DB. Twin
should then be derived from DB topology data. So far, implemented
everything up to UploadTopology(). Also split up NewVEnv() function into
multiple functions.
- venv.go: CreateDevices(), replaces code in NewVEnv(), uses rtdt_topology
instead of clabconfig.topology, add DeriveTopologyFromClabData(), add
UploadTopology()
- rtdt-topology.go: Add GettNodeByName() function, change Topology members
to references, implement converter functions to return api structs
- clab-config.go: Add GetNodeByName() function
- main.go: Use DeriveTopologyFromClabData() and UploadTopology()

applications/rtdt-manager/clab-config/clab-config.go

@@ -227,3 +227,13 @@ func ClabDeploy(fullPath string) error {
 		return fmt.Errorf("Deployment interrupted by signal")
 	}
 }
+
+func (c *ClabConfig) GetNodeByName(name string) *Node {
+	for nodename, node := range c.Topology.Nodes {
+		if nodename == name {
+			return &node
+		}
+	}
+    fmt.Printf("Couldn't find a node with name %s!\n", name)
+	return nil
+}

applications/rtdt-manager/main.go

@@ -14,6 +14,7 @@ import (
 
 func main() {
 
+	var err error
 	// Global stop channel, should be passed to all venvs and App
 	stopChan := make(chan os.Signal, 1)
 	signal.Notify(stopChan, os.Interrupt, syscall.SIGTERM)
@@ -56,15 +57,25 @@ func main() {
 	} else {
 		fmt.Println("Successfully deployed physical network")
 	}
+	err = realnet.DeriveTopologyFromClabData()
+	if err != nil {
+		fmt.Printf("Error occured while trying to construct topology in realnet: %v\n", err)
+		return
+	}
+	err = realnet.UploadTopology()
+	if err != nil {
+		fmt.Printf("Error occured while trying to upload realnet topology to DB: %v\n", err)
+		return
+	}
 	// Register realnet with rtdt-manager
 	rtdtMan := RtdtMan.NewRtdtManager(realnet, &wg, &stopChan)
 	if rtdtMan == nil {
 		fmt.Println("Couldn't initialize rtdt-manager, quitting!")
 		return
 	}
-	err := rtdtMan.InitEventSystem()
+	err = rtdtMan.InitEventSystem()
 	if err != nil {
-		fmt.Printf("Error occured while initializing event system: %v", err)
+		fmt.Printf("Error occured while initializing event system: %v\n", err)
 		return
 	}
 	// Do performance tests of realnet before starting twin

applications/rtdt-manager/rtdt-topology/rtdt-topology.go

 package rtdt_topology
 
 import (
+	topoPb "code.fbi.h-da.de/danet/gosdn/api/go/gosdn/topology"
 	"code.fbi.h-da.de/danet/gosdn/models/generated/openconfig"
 	"regexp"
 )
@@ -33,9 +34,9 @@ type Node struct {
 }
 
 type Topology struct {
-	Links []Link
-	Ports []Port
-	Nodes []Node
+	Links []*Link
+	Ports []*Port
+	Nodes []*Node
 }
 
 func NewTopology() *Topology {
@@ -45,7 +46,16 @@ func NewTopology() *Topology {
 func (t *Topology) GetNodeByUUID(UUID string) *Node {
 	for _, node := range t.Nodes {
 		if node.ID == UUID {
-			return &node
+			return node
+		}
+	}
+	return nil
+}
+
+func (t *Topology) GetNodeByName(name string) *Node {
+	for _, node := range t.Nodes {
+		if node.Name == name {
+			return node
 		}
 	}
 	return nil
@@ -92,3 +102,29 @@ func (n *Node) FillAllFields(containerRegistryURL string) {
 	n.Kind = "ceos"
 	n.Image = containerRegistryURL + n.Kind + ":" + dockerTag
 }
+
+// Converter functions to convert between this and topoPb version of topology
+func (n *Node) Convert() *topoPb.Node {
+	return &topoPb.Node{
+		Id:   n.ID,
+		Name: n.Name,
+	}
+}
+func (p *Port) Convert() *topoPb.Port {
+	return &topoPb.Port{
+		Id:   p.ID,
+		Name: p.Name,
+	}
+}
+func (l *Link) Convert() *topoPb.Link {
+    return &topoPb.Link{
+        Id: l.ID,
+        Name: l.Name,
+        SourceNode: l.SourceNode.Convert(),
+        SourcePort: l.SourcePort.Convert(),
+        TargetNode: l.TargetNode.Convert(),
+        TargetPort: l.TargetPort.Convert(),
+
+    }
+}
+

applications/rtdt-manager/venv/venv.go

@@ -36,12 +36,13 @@ type VEnv struct {
 	clabFilename         string                 // This is the name of the yaml file clabData is based on
 	StopChan             <-chan struct{}
 	waitGroup            *sync.WaitGroup
-	topology             rtdt_topology.Topology
+	topology             *rtdt_topology.Topology
 	containerRegistryURL string
 	savedChanges         []*networkelement.ChangeRequest
 }
 
 // Accepts a yaml filename to deploy a container lab environment
+// TODO Split up into sub-functions
 func NewVEnv(name, topoYamlFile, user, pass string, wg *sync.WaitGroup) *VEnv {
 	fmt.Printf("[%s] - Creating new virtual environment\n", name)
 	wg.Add(1) // Register the venv and run atleast until it calls wg.Done()
@@ -58,7 +59,6 @@ func NewVEnv(name, topoYamlFile, user, pass string, wg *sync.WaitGroup) *VEnv {
 		fmt.Printf("[%s] - Failed to load config from yaml file\n", name)
 		return nil
 	}
-
 	// get gosdn address inside clab environment
 	var gosdnAddress string
 	for nodename, val := range clabData.Topology.Nodes {
@@ -97,22 +97,26 @@ func NewVEnv(name, topoYamlFile, user, pass string, wg *sync.WaitGroup) *VEnv {
 		time.Sleep(time.Second * 2)
 	}
 
-	// Sleep before adding devices because otherwise it won't work
-	// Apply the topoYamlFile: we need to register the switches with the controller
-	var topo rtdt_topology.Topology
-	var topoNodes []rtdt_topology.Node
-	for node, val := range clabData.Topology.Nodes {
-		var topoNode rtdt_topology.Node
-		topoNode.Name = node
-		topoNode.MgmtIpv4 = val.MgmtIPv4
-		topoNode.Image = val.Image
-		topoNode.Kind = val.Kind
-
-		if strings.HasPrefix(node, "gnmi-target-") {
-			fmt.Printf("[%s] - Creating Network Element for node: %s\n", name, node)
-			ports := strings.Split(val.Ports[0], ":")
-			port := ports[1]
-			addr := val.MgmtIPv4 + ":" + port
+	// load topo into DB via API
+	return &VEnv{
+		auth:                 gosdnauth,
+		pnd:                  gosdn_pnd,
+		conn:                 gosdnconn,
+		clabData:             clabData,
+		clabFilename:         topoYamlFile,
+		waitGroup:            wg,
+		topology:             nil,                                            // set this later
+		containerRegistryURL: "registry.code.fbi.h-da.de/danet/gnmi-target/", // TODO: Could let user choose
+	}
+}
+
+func (v *VEnv) CreateDevices() error {
+	for _, node := range v.topology.Nodes {
+		if strings.HasPrefix(node.Name, "gnmi-target-") {
+			fmt.Printf("[%s] - Creating Network Element for node: %s\n", v.Name, node.Name)
+			//ports := strings.Split(node.Ports[0], ":")
+			//port := ports[1]
+			addr := node.MgmtIpv4 + ":7030" // gnmi targets will always listen on 7030
 			opt := &tpb.TransportOption{
 				Address:  addr,
 				Username: "admin",
@@ -124,40 +128,29 @@ func NewVEnv(name, topoYamlFile, user, pass string, wg *sync.WaitGroup) *VEnv {
 			}
 			// Openconfig pluginid? TODO decide
 			pluginID, _ := uuid.Parse("d1c269a2-6482-4010-b0d8-679dff73153b") // TODO Get this dynamically
-			pndID, _ := uuid.Parse(gosdn_pnd.GetId())
+			pndID, _ := uuid.Parse(v.pnd.Id)
 
-			fmt.Printf("[%s] - Found target: %s with addr: %s\n", name, node, addr)
-			fmt.Printf("[%s] - Gosdn controller at %s\n", name, gosdnAddress)
-			listResponse, err := gosdnutil.AddNetworkElement(gosdnauth, gosdnAddress, node, "", opt, pluginID, pndID, []string{"/"})
+			fmt.Printf("[%s] - Found target: %s with addr: %s\n", v.Name, node.Name, addr)
+			fmt.Printf("[%s] - Gosdn controller at %s\n", v.Name, v.auth.GetAddress())
+			listResponse, err := gosdnutil.AddNetworkElement(v.auth, v.auth.GetAddress(), node.Name, node.ID, opt, pluginID, pndID, []string{"/"})
 			if err != nil {
-				fmt.Printf("[%s] - Failed to add network elements: %v\n", name, err)
+				fmt.Printf("[%s] - Failed to add network elements: %v\n", v.Name, err)
 				return nil
 			} else {
-				fmt.Printf("[%s] - Successfully created network element\n", name)
+				fmt.Printf("[%s] - Successfully created network element\n", v.Name)
 			}
-			fmt.Printf("[%s] - Got response from AddNetworkElement: %v\n", name, listResponse)
-			topoNode.ID = listResponse.GetResponses()[0].GetId()
+			fmt.Printf("[%s] - Got response from AddNetworkElement: %v\n", v.Name, listResponse)
 
-			fmt.Printf("[%s] - Success: registered mne with gosdn controller\n", name)
-			fmt.Printf("[%s] - Also created gosdn topo node: %v\n", name, topoNode)
+			fmt.Printf("[%s] - Success: registered mne with gosdn controller\n", v.Name)
 		}
-		topoNodes = append(topoNodes, topoNode)
-	}
-	topo.Nodes = topoNodes
-	return &VEnv{auth: gosdnauth,
-		pnd:                  gosdn_pnd,
-		conn:                 gosdnconn,
-		clabData:             clabData,
-		clabFilename:         topoYamlFile,
-		waitGroup:            wg,
-		topology:             topo,
-		containerRegistryURL: "registry.code.fbi.h-da.de/danet/gnmi-target/", // TODO: Could let user choose
 	}
+
+	return nil
 }
 
 // Source: "code.fbi.h-da.de/danet/gosdn/applications/venv-manager/venv-manager/venv-manager.go"
 // commit: 0264b698286b6cbb965d743078c681f8af55edf6
-func (v *VEnv) loadNetworkElementModelPathsIntoGosdn(ctx context.Context, conn *grpc.ClientConn, nodes *[]rtdt_topology.Node) error {
+func (v *VEnv) loadNetworkElementModelPathsIntoGosdn(ctx context.Context, conn *grpc.ClientConn, nodes *[]*rtdt_topology.Node) error {
 	networkElementService := networkelement.NewNetworkElementServiceClient(conn)
 
 	paths := [2]string{"/lldp/config/system-description", "/system/state/"}
@@ -177,6 +170,10 @@ func (v *VEnv) loadNetworkElementModelPathsIntoGosdn(ctx context.Context, conn *
 // What this does: Load yang paths into topo, then iterate over
 // nodes and fill data in
 func (v *VEnv) ConstructTopology() error {
+	if v.clabData == nil {
+		return fmt.Errorf("Error: Trying to construct topology without containerlab file being loaded first")
+	}
+	// Either fill topology from clabData or from database?
 	ctx := v.auth.CreateContextWithAuthorization()
 	conn := v.conn
 	var path = "/"
@@ -197,7 +194,6 @@ func (v *VEnv) ConstructTopology() error {
 
 	networkElementService := networkelement.NewNetworkElementServiceClient(conn)
 
-	// This code relies on Nodes already being filled, so what does this achieve??
 	for iterator, node := range v.topology.Nodes {
 		getNetworkElementResponse, _ := networkElementService.Get(ctx, &networkelement.GetRequest{Mneid: node.ID})
 		if err != nil {
@@ -220,6 +216,71 @@ func (v *VEnv) ConstructTopology() error {
 	return nil
 }
 
+// Create rtdt_topology.Topology from clabconfig.Topology
+func (v *VEnv) DeriveTopologyFromClabData() error {
+	if v.clabData == nil {
+		return fmt.Errorf("Can't derive topology without clabData\n")
+	}
+	v.topology = &rtdt_topology.Topology{}
+	var topoNodes []*rtdt_topology.Node
+	// Get all the nodes from clab structure
+	for nodeName, node := range v.clabData.Topology.Nodes {
+		topoNode := rtdt_topology.Node{
+			ID:       uuid.NewString(),
+			Name:     nodeName,
+			Kind:     node.Kind,
+			Image:    node.Image,
+			MgmtIpv4: node.MgmtIPv4,
+		}
+		topoNodes = append(topoNodes, &topoNode)
+	}
+	v.topology.Nodes = topoNodes
+	for _, link := range v.clabData.Topology.Links {
+		if len(link.Endpoints) != 2 {
+			return fmt.Errorf("Couldn't parse clabData into topology\n")
+		}
+		var topoLink = rtdt_topology.Link{}
+		// determine Node name and port
+		node0Full := strings.Split(link.Endpoints[0], ":")
+		node0Name := node0Full[0]
+		node0Port := node0Full[1]
+		node1Full := strings.Split(link.Endpoints[1], ":")
+		node1Name := node1Full[0]
+		node1Port := node1Full[1]
+
+		// find the node that has right name, add links and ports
+		topoLink.SourceNode = v.topology.GetNodeByName(node0Name)
+		topoLink.SourcePort = &rtdt_topology.Port{Name: node0Port, ID: uuid.NewString()}
+		v.topology.Ports = append(v.topology.Ports, topoLink.SourcePort)
+		topoLink.TargetNode = v.topology.GetNodeByName(node1Name)
+		topoLink.TargetPort = &rtdt_topology.Port{Name: node1Port, ID: uuid.NewString()}
+		v.topology.Ports = append(v.topology.Ports, topoLink.TargetPort)
+		v.topology.Links = append(v.topology.Links, &topoLink)
+		v.topology.Nodes = topoNodes
+	}
+	return nil
+}
+
+func (v *VEnv) UploadTopology() error {
+	conn := v.auth.GetConn()
+	topoService := topoPb.NewTopologyServiceClient(conn)
+	for _, link := range v.topology.Links {
+
+		ctx := v.auth.CreateContextWithAuthorization()
+		addLinkRequest := &topoPb.AddLinkRequest{
+			Timestamp: util.Now(),
+			Link:      link.Convert(),
+		}
+		fmt.Println("AddLink is:", addLinkRequest.String())
+		topoResponse, err := topoService.AddLink(ctx, addLinkRequest)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("Successfully uploaded Link to DB: %s\n", topoResponse.String())
+	}
+	return nil
+}
+
 func getTypedValue(value string) *gnmi.TypedValue {
 	if boolVal, err := strconv.ParseBool(value); err == nil {
 		return &gnmi.TypedValue{Value: &gnmi.TypedValue_BoolVal{BoolVal: boolVal}}
@@ -282,9 +343,9 @@ func (v *VEnv) SetGnmiPath(path, value, mneid string, save bool) error {
 	} else {
 		fmt.Println("Successfully applied changes:", clResponse)
 	}
-    if save {
-        v.savedChanges = append(v.savedChanges, changeRequest)
-    }
+	if save {
+		v.savedChanges = append(v.savedChanges, changeRequest)
+	}
 	return nil
 }
 
@@ -339,8 +400,8 @@ func (v *VEnv) GetWaitgroup() *sync.WaitGroup {
 	return v.waitGroup
 }
 func (v *VEnv) GetTopology() *rtdt_topology.Topology {
-	return &v.topology
+	return v.topology
 }
 func (v *VEnv) GetSavedChanges() *[]*networkelement.ChangeRequest {
-    return &v.savedChanges
+	return &v.savedChanges
 }