Move RetrieveTopology to topology.go from venv.go for more flexibility.

Add base-clab.yaml for a clab config file that only contains the base
containers without topology.
rtdt-manager.go: Enhance LaunchTwin to load the base-clab.yaml file and
retrieve the mnes, then get topology data too.

applications/rtdt-manager/clab-config/clab-config.go

@@ -12,6 +12,7 @@ import (
 
 	"gopkg.in/yaml.v3"
 
+	"code.fbi.h-da.de/danet/gosdn/api/go/gosdn/networkelement"
 	util "code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/util"
 )
 
@@ -66,6 +67,7 @@ func ClabConfigPath() (string, error) {
 }
 
 // Read file and parse into ClabConfig struct
+// Only load gosdn environment: rabbitmq, mongodb,
 func LoadConfig(filename string) (*ClabConfig, error) {
 	absFilepath, err := filepath.Abs(filename)
 	if err != nil {
@@ -93,6 +95,18 @@ func incrementPort(port string, offset int) (string, error) {
 	}
 	return strconv.Itoa(portNum + offset), nil
 }
+
+// TODO
+func (c *ClabConfig) InsertMNE(mnes []*networkelement.ManagedNetworkElement) {
+	for _, mne := range mnes {
+		c.Topology.Nodes[mne.Name] = Node{
+			Kind:  "TBD",
+			Image: "TBD",
+		}
+	}
+}
+
+// Take a clab yaml config file and derive another clab config from it
 func DeriveConfig(clabconfig *ClabConfig, newIPv4Subnet, newIPv6Subnet string, postfix string) (*ClabConfig, error) {
 	// Create deep copy
 	derivedConfig := *clabconfig

applications/rtdt-manager/data/base-clab.yaml

+name: gosdn_realnet
+
+mgmt:
+  network: gosdn-realnet-net
+  ipv4-subnet: 172.100.0.0/16
+  ipv6-subnet: 2001:db8::/64
+  mtu: 1500
+
+topology:
+  nodes:
+    plugin-registry:
+      kind: linux
+      image: plugin-registry
+      mgmt-ipv4: 172.100.0.16
+
+    gosdn:
+      kind: linux
+      image: gosdn
+      ports:
+        - 55055:55055
+        - 8080:8080
+        - 40000:40000
+      cmd: --config /app/configs/containerlab-gosdn.toml
+      mgmt-ipv4: 172.100.0.5
+      env:
+        GOSDN_ADMIN_PASSWORD: TestPassword
+      binds:
+        - ../../../artifacts/ssl/gosdn:/app/ssl
+
+    gnmi-target-switch0:
+      kind: linux
+      image: registry.code.fbi.h-da.de/danet/gnmi-target/debian:interface-enabled-test
+      #only for local use
+      #image: gnmi-target:latest
+      binds:
+        - ../../../artifacts/ssl/gnmi-target:/etc/gnmi-target/ssl
+      ports:
+        - 7030:7030
+      cmd: start --ca_file /etc/gnmi-target/ssl/ca.crt --cert /etc/gnmi-target/ssl/certs/gnmi-target-selfsigned.crt --key /etc/gnmi-target/ssl/private/gnmi-target-selfsigned.key
+      mgmt-ipv4: 172.100.0.11
+      startup-delay: 5
+
+    gnmi-target-switch1:
+      kind: linux
+      image: registry.code.fbi.h-da.de/danet/gnmi-target/debian:interface-enabled-test
+      #only for local use
+      #image: gnmi-target:latest
+      binds:
+        - ../../../artifacts/ssl/gnmi-target:/etc/gnmi-target/ssl
+      ports:
+        - 7031:7030
+      cmd: start --ca_file /etc/gnmi-target/ssl/ca.crt --cert /etc/gnmi-target/ssl/certs/gnmi-target-selfsigned.crt --key /etc/gnmi-target/ssl/private/gnmi-target-selfsigned.key
+      mgmt-ipv4: 172.100.0.12
+      startup-delay: 5
+
+    centos0:
+      kind: linux
+      image: centos:8
+      mgmt-ipv4: 172.100.0.3
+      group: server
+
+    centos1:
+      kind: linux
+      image: centos:8
+      mgmt-ipv4: 172.100.0.4
+      group: server
+
+    mongodb:
+      kind: linux
+      image: mongo:7
+      ports:
+        - 27017:27017
+      env:
+        MONGO_INITDB_ROOT_USERNAME: root
+        MONGO_INITDB_ROOT_PASSWORD: example
+      mgmt-ipv4: 172.100.0.13
+
+    mongodb-express:
+      kind: linux
+      image: mongo-express:1.0.2
+      ports:
+        - 8081:8081
+      env:
+        ME_CONFIG_MONGODB_AUTH_USERNAME: root
+        ME_CONFIG_MONGODB_AUTH_PASSWORD: example
+        ME_CONFIG_MONGODB_SERVER: mongodb
+        ME_CONFIG_BASICAUTH: "false"
+      mgmt-ipv4: 172.100.0.14
+
+    rabbitmq:
+      kind: linux
+      image: rabbitmq:3-management
+      ports:
+        - 127.0.0.1:5672:5672
+        - 127.0.0.1:15672:15672
+      mgmt-ipv4: 172.100.0.15
+
+  links:
+    - endpoints: ["gnmi-target-switch0:eth1", "gnmi-target-switch1:eth1"]
+    - endpoints: ["gnmi-target-switch0:eth2", "centos0:eth1"]
+    - endpoints: ["gnmi-target-switch1:eth2", "centos1:eth1"]

applications/rtdt-manager/data/realnet-clab.yaml

-name: gosdn_csbi_arista_base
+name: gosdn_realnet
 
 mgmt:
-  network: gosdn-csbi-arista-base-net
+  network: gosdn-realnet-net
   ipv4-subnet: 172.100.0.0/16
   ipv6-subnet: 2001:db8::/64
   mtu: 1500

applications/rtdt-manager/main.go

@@ -66,11 +66,10 @@ func main() {
 	if err != nil {
 		fmt.Printf("Error: Couldnt upload clab config: %v\n", err)
 	}
-    err = realnet.RetrieveClabConfig()
-    if err != nil {
-        fmt.Printf("Error: Couldn't retrieve clab config: %v\n", err)
-    }
-	return
+	err = realnet.RetrieveClabConfig()
+	if err != nil {
+		fmt.Printf("Error: Couldn't retrieve clab config: %v\n", err)
+	}
 	err = realnet.CreateDevices()
 	if err != nil {
 		fmt.Printf("Error: Couldn't create devices!")

applications/rtdt-manager/rtdt-manager/rtdt-manager.go

@@ -10,11 +10,14 @@ import (
 	"sync"
 	"time"
 
+	"code.fbi.h-da.de/danet/gosdn/api/go/gosdn/networkelement"
+	topoPb "code.fbi.h-da.de/danet/gosdn/api/go/gosdn/topology"
 	"code.fbi.h-da.de/danet/gosdn/application-framework/event"
 
 	"code.fbi.h-da.de/danet/gosdn/application-framework/registration"
 	clabconfig "code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/clab-config"
 	rtdt_topology "code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/rtdt-topology"
+	"code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/util"
 	"code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/venv"
 )
 
@@ -46,21 +49,36 @@ func NewRtdtManager(realnet *venv.VEnv, wg *sync.WaitGroup, stopChan *chan os.Si
 // Launch a second gosdn instance which will manage the virtual network
 func (r *RtdtManager) LaunchTwin(twinSubnetIPv4, twinSubnetIPv6, twinName string) error {
 	var derivedConfig *clabconfig.ClabConfig
+	var baseConfig *clabconfig.ClabConfig
 	var clabConfigPath string
 	var err error
 	var topo *rtdt_topology.Topology
+	conn := r.realnet.GetConn()
 
 	topo = r.realnet.GetTopology()
 	if topo == nil {
 		return fmt.Errorf("Error: Couldn't get topology since it hasn't been retrieved from DB yet")
 	}
-	baseConfig := r.realnet.GetClabData()
 
-	// Instead of deriving config do the following:
-	// - retrieve topology for realnet from DB
-	// -
+	// Get the basic containerlab config with gosdn, mongodb, rabbitmq
+	baseConfig, err = clabconfig.LoadConfig("base-clab.yaml")
+	// Now retrieve the nodes (gnmi-targets and hosts + topology) via MNE Api
+	// reminder: Topology might not include all targets?
+	mneServiceRealnet := networkelement.NewNetworkElementServiceClient(conn)
+	getRequest := &networkelement.GetAllRequest{
+		Timestamp: util.Now(),
+		Pid:       r.realnet.GetPnd().Id,
+	}
+	mneCtx := r.realnet.GetAuth().CreateContextWithAuthorization()
+	getAllResponse, err := mneServiceRealnet.GetAll(mneCtx, getRequest)
+	elements := getAllResponse.GetMne()
+	// These now need to go into the new VEnv's clabData so clab yaml config can be created
+	if len(elements) > 0 {
+		baseConfig.InsertMNE(elements)
+	}
+	// Also get topology from db
+	rtdt_topology.NewTopology().RetrieveTopology(r.realnet.GetPnd().Id, r.realnet.GetAuth())
 
-	/// OOOOOLLLLDDD
 	if derivedConfig, err = clabconfig.DeriveConfig(baseConfig, twinSubnetIPv4, twinSubnetIPv6, twinName); err != nil {
 		return fmt.Errorf("Failed to derive config for twin: %w", err)
 	}
@@ -282,4 +300,3 @@ func (r *RtdtManager) userEventCallback(event *event.Event) {
 	fmt.Println("Event Type: ", event.Type)
 	fmt.Println("PathsAndValuesMap: ", event.PathsAndValuesMap)
 }
-

applications/rtdt-manager/rtdt-topology/rtdt-topology.go

 package rtdt_topology
 
 import (
+	"fmt"
 	"regexp"
 	"strings"
 
 	topoPb "code.fbi.h-da.de/danet/gosdn/api/go/gosdn/topology"
 	clabconfig "code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/clab-config"
+	rtdt_auth "code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/rtdt-auth"
+	"code.fbi.h-da.de/danet/gosdn/applications/rtdt-manager/util"
 	"code.fbi.h-da.de/danet/gosdn/models/generated/openconfig"
 )
 
@@ -123,6 +126,77 @@ func (n *Node) FillAllFields(containerRegistryURL string) {
 	n.Image = containerRegistryURL + n.Kind + ":" + dockerTag
 }
 
+// Based on the given auth and a pnd, get a topology from the DB store
+func (t *Topology) RetrieveTopology(pnd string, auth *rtdt_auth.RtdtAuth) error {
+	conn := auth.GetConn()
+	ctx := auth.CreateContextWithAuthorization()
+	topoService := topoPb.NewTopologyServiceClient(conn)
+	topoResponse, err := topoService.GetTopology(ctx, &topoPb.GetTopologyRequest{Timestamp: util.Now()})
+	if err != nil {
+		return fmt.Errorf("Couldn't retrieve topology from DB: %v\n", err)
+	}
+	sourceTopo := topoResponse.GetToplogy()
+	targetTopo := NewTopology()
+	for _, link := range sourceTopo.Links {
+		var n0 Node
+		var n1 Node
+		if targetTopo.GetNodeByUUID(link.SourceNode.Id) == nil {
+			snode := link.SourceNode
+			n0 = Node{
+				ID:       snode.Id,
+				Name:     snode.Name,
+				Kind:     "todo",
+				Image:    "todo", // How to do this?
+				MgmtIpv4: "todo",
+			}
+			// n0.FillAllFields(containerRegistryURL) // TODO get this later
+			t.Nodes = append(t.Nodes, &n0)
+		}
+		if targetTopo.GetNodeByUUID(link.TargetNode.Id) == nil {
+			tnode := link.TargetNode
+			n1 = Node{
+				ID:       tnode.Id,
+				Name:     tnode.Name,
+				Kind:     "todo",
+				Image:    "todo", // How to do this?
+				MgmtIpv4: "todo",
+			}
+			//n1.FillAllFields(v.containerRegistryURL)
+			t.Nodes = append(t.Nodes, &n1)
+		}
+		if targetTopo.GetLinkByUUID(link.Id) == nil {
+			var p0 Port
+			var p1 Port
+
+			if targetTopo.GetPortByUUID(link.SourcePort.Id) == nil {
+				p0 = Port{
+					Name: link.SourcePort.Name,
+					ID:   link.SourcePort.Id,
+				}
+			}
+			if targetTopo.GetPortByUUID(link.SourcePort.Id) == nil {
+				p1 = Port{
+					Name: link.TargetPort.Name,
+					ID:   link.TargetPort.Id,
+				}
+			}
+
+			var newLink = Link{
+				ID:         link.Id,
+				Name:       link.Name,
+				SourceNode: &n0,
+				TargetNode: &n1,
+				SourcePort: &p0,
+				TargetPort: &p1,
+			}
+			t.Links = append(t.Links, &newLink)
+		}
+	}
+
+	return nil
+
+}
+
 func (t *Topology) ToYAML() *clabconfig.ClabConfig {
 	var yamlMgmt clabconfig.Mgmt
 	var yamlNodes map[string]clabconfig.Node
@@ -159,7 +233,7 @@ func (t *Topology) ToYAML() *clabconfig.ClabConfig {
 		yamlNodes[n.Name] = node
 	}
 
-    // Now: Get configuration for environment (controller, rabbitmq, mongodb) somehow..
+	// Now: Get configuration for environment (controller, rabbitmq, mongodb) somehow..
 
 	return &clabconfig.ClabConfig{
 		Name: "",

applications/rtdt-manager/venv/venv.go

@@ -369,6 +369,7 @@ func (v *VEnv) UploadClabConfig() error {
 	return nil
 }
 
+// TODO This doesn't really work this way..
 func (v *VEnv) RetrieveClabConfig() error {
 	ctx := v.auth.CreateContextWithAuthorization()
 	conn := v.auth.GetConn()
@@ -388,7 +389,7 @@ func (v *VEnv) RetrieveClabConfig() error {
 	if err != nil {
 		return fmt.Errorf("Failed to unmarshal SDN config data: %w", err)
 	}
-    // Pretty-print the JSON data
+	// Pretty-print the JSON data
 	formattedJSON, err := json.MarshalIndent(jsonData, "", "  ")
 	if err != nil {
 		return fmt.Errorf("failed to format JSON data: %w", err)
@@ -397,85 +398,10 @@ func (v *VEnv) RetrieveClabConfig() error {
 	fmt.Println("Retrieved SDN Config Data:")
 	fmt.Println(string(formattedJSON)) // Print the formatted JSON
 
-
 	return nil
 
 }
 
-// Topology doesn't give Kind, Image and MgmtIpv4
-// For Kind: could just set linux since we don't use arista switches
-// For Image: möp
-// For MgmtIpv4: Find some way to get it from realnet
-// Also TODO Should be possible to retrieve topology from realnet, so maybe put this in rtdt-manager
-func (v *VEnv) RetrieveTopology() error {
-	conn := v.auth.GetConn()
-	ctx := v.auth.CreateContextWithAuthorization()
-	topoService := topoPb.NewTopologyServiceClient(conn)
-	topoResponse, err := topoService.GetTopology(ctx, &topoPb.GetTopologyRequest{Timestamp: util.Now()})
-	if err != nil {
-		return fmt.Errorf("Couldn't retrieve topology from DB: %v\n", err)
-	}
-	sourceTopo := topoResponse.GetToplogy()
-	targetTopo := rtdt_topology.NewTopology()
-	for _, link := range sourceTopo.Links {
-		var n0 rtdt_topology.Node
-		var n1 rtdt_topology.Node
-		if targetTopo.GetNodeByUUID(link.SourceNode.Id) == nil {
-			snode := link.SourceNode
-			n0 = rtdt_topology.Node{
-				ID:       snode.Id,
-				Name:     snode.Name,
-				Kind:     "todo",
-				Image:    "todo", // How to do this?
-				MgmtIpv4: "todo",
-			}
-			n0.FillAllFields(v.containerRegistryURL)
-			v.topology.Nodes = append(v.topology.Nodes, &n0)
-		}
-		if targetTopo.GetNodeByUUID(link.TargetNode.Id) == nil {
-			tnode := link.TargetNode
-			n1 = rtdt_topology.Node{
-				ID:       tnode.Id,
-				Name:     tnode.Name,
-				Kind:     "todo",
-				Image:    "todo", // How to do this?
-				MgmtIpv4: "todo",
-			}
-			n1.FillAllFields(v.containerRegistryURL)
-			v.topology.Nodes = append(v.topology.Nodes, &n1)
-		}
-		if targetTopo.GetLinkByUUID(link.Id) == nil {
-			var p0 rtdt_topology.Port
-			var p1 rtdt_topology.Port
-
-			if targetTopo.GetPortByUUID(link.SourcePort.Id) == nil {
-				p0 = rtdt_topology.Port{
-					Name: link.SourcePort.Name,
-					ID:   link.SourcePort.Id,
-				}
-			}
-			if targetTopo.GetPortByUUID(link.SourcePort.Id) == nil {
-				p1 = rtdt_topology.Port{
-					Name: link.TargetPort.Name,
-					ID:   link.TargetPort.Id,
-				}
-			}
-
-			var newLink = rtdt_topology.Link{
-				ID:         link.Id,
-				Name:       link.Name,
-				SourceNode: &n0,
-				TargetNode: &n1,
-				SourcePort: &p0,
-				TargetPort: &p1,
-			}
-			v.topology.Links = append(v.topology.Links, &newLink)
-		}
-	}
-
-	return nil
-}
-
 func getTypedValue(value string) *gnmi.TypedValue {
 	if boolVal, err := strconv.ParseBool(value); err == nil {
 		return &gnmi.TypedValue{Value: &gnmi.TypedValue_BoolVal{BoolVal: boolVal}}