json unmarshal for nodes, connections and links

added function for preformatting

sadly the json we get from ciena for node-edge-point doesnt fit to anything
from ygot.

configs/clients.toml

 [[client]]
 identifier = "ciena-mcp"
 endpoint = "172.17.0.3:8080"
+gjsonDefaultPath = "data.#.object_data.tapi-object-data"
+gjsonConnectionsPath = "data.#.object_data"

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/onsi/ginkgo v1.13.0 // indirect
 	github.com/openconfig/ygot v0.8.7
 	github.com/rivo/tview v0.0.0-20201018122409-d551c850a743
+	github.com/tidwall/gjson v1.6.3
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73 // indirect
 	google.golang.org/grpc v1.29.1
 	google.golang.org/protobuf v1.23.0

go.sum

@@ -257,8 +257,14 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/gjson v1.6.3 h1:aHoiiem0dr7GHkW001T1SMTJ7X5PvyekH5WX0whWGnI=
+github.com/tidwall/gjson v1.6.3/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
+github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
+github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=

nucleus/interfaces/clientConfig.go

@@ -3,8 +3,10 @@ package interfaces
 // ClientConfig contains SBI client
 // configuration parameters
 type ClientConfig struct {
-	Identifier string `toml:"identifier"`
-	Endpoint   string `toml:"endpoint"`
-	Username   string `toml:"username"`
-	Password   string `toml:"password"`
+	Identifier           string `toml:"identifier"`
+	Endpoint             string `toml:"endpoint"`
+	Username             string `toml:"username"`
+	Password             string `toml:"password"`
+	GjsonDefaultPath     string `toml:"gjsonDefaultPath"`
+	GjsonConnectionsPath string `toml:"gjsonConnectionsPath"`
 }

sbi/restconf/client/ciena/client.go

@@ -9,11 +9,14 @@ import (
 	apiclient "code.fbi.h-da.de/cocsn/swagger/apis/mcp/client"
 	"code.fbi.h-da.de/cocsn/yang-modules/generated/tapi"
 	"crypto/tls"
+	"encoding/json"
 	"github.com/go-openapi/runtime"
 	httptransport "github.com/go-openapi/runtime/client"
 	"github.com/go-openapi/strfmt"
 	"github.com/openconfig/ygot/ygot"
+	"github.com/tidwall/gjson"
 	"net/http"
+	"strings"
 )
 
 //MCPClient handles requests to a Ciena MCP RESTCONF endpoint
@@ -61,11 +64,25 @@ func NewMCPClient(endpoint, username, password string, database *database.Databa
 func (c *MCPClient) GetConnections() error {
 	defer c.buffer.Reset()
 	_, err := c.client.TapiConnectivityCore.GetTapiCoreContextConnection(nil)
-	if err != nil { return err}
-	dest := &tapi.TapiCommon_Context_ConnectivityContext_Connection{}
-	if err := tapi.Unmarshal(c.buffer.Bytes(), dest); err != nil { return err }
-	c.database.StoreConnections(c.buffer.String())
-	log.Debug(c.buffer.Next(25))
+	if err != nil {
+		return err
+	}
+
+	json := preformatJSON(c.buffer.String(), c.config.GjsonConnectionsPath)
+
+	for _, jsonEntry := range json.Array() {
+		dest := &tapi.TapiCommon_Context_ConnectivityContext_Connection{}
+		if err := tapi.Unmarshal([]byte(jsonEntry.String()), dest); err != nil {
+			//TODO: think about a way how to handle this.
+			//logging every error is kinda ugly, since ciena tapi throws a
+			//lot of them.
+			log.Info(err)
+		}
+		log.Info(*dest.Uuid)
+	}
+
+	// c.database.StoreConnections(c.buffer.String())
+	// log.Debug(c.buffer.Next(25))
 	return err
 }
 
@@ -74,11 +91,25 @@ func (c *MCPClient) GetConnections() error {
 func (c *MCPClient) GetLinks() error {
 	defer c.buffer.Reset()
 	_, err := c.client.TapiTopologyCore.GetTapiCoreContextTopologyMcpBaseTopologyLink(nil)
-	if err != nil { return err}
-	dest := &tapi.TapiCommon_Context_TopologyContext_Topology_Link{}
-	if err := tapi.Unmarshal(c.buffer.Bytes(), dest); err != nil { return err }
-	c.database.StoreLinks(c.buffer.String())
-	log.Debug(c.buffer.Next(25))
+	if err != nil {
+		return err
+	}
+
+	json := preformatJSON(c.buffer.String(), c.config.GjsonDefaultPath)
+
+	for _, jsonEntry := range json.Array() {
+		dest := &tapi.TapiCommon_Context_TopologyContext_Topology_Link{}
+		if err := tapi.Unmarshal([]byte(jsonEntry.String()), dest); err != nil {
+			//TODO: think about a way how to handle this.
+			//logging every error is kinda ugly, since ciena tapi throws a
+			//lot of them.
+			log.Info(err)
+		}
+		log.Info(*dest.Uuid)
+	}
+
+	// c.database.StoreLinks(c.buffer.String())
+	// log.Debug(c.buffer.Next(25))
 	return err
 }
 
@@ -87,11 +118,25 @@ func (c *MCPClient) GetLinks() error {
 func (c *MCPClient) GetNodes() error {
 	defer c.buffer.Reset()
 	_, err := c.client.TapiTopologyCore.GetTapiCoreContextTopologyMcpBaseTopologyNode(nil)
-	if err != nil { return err}
-	dest := &tapi.TapiCommon_Context_TopologyContext_Topology_Node{}
-	if err := tapi.Unmarshal(c.buffer.Bytes(), dest); err != nil { return err }
-	c.database.StoreNodes(c.buffer.String())
-	log.Debug(c.buffer.Next(25))
+	if err != nil {
+		return err
+	}
+
+	json := preformatJSON(c.buffer.String(), c.config.GjsonDefaultPath)
+
+	for _, jsonEntry := range json.Array() {
+		dest := &tapi.TapiCommon_Context_TopologyContext_Topology_Node{}
+		if err := tapi.Unmarshal([]byte(jsonEntry.String()), dest); err != nil {
+			//TODO: think about a way how to handle this.
+			//logging every error is kinda ugly, since ciena tapi throws a
+			//lot of them.
+			log.Info(err)
+		}
+		log.Info(*dest.Uuid)
+	}
+
+	// c.database.StoreNodes(c.buffer.String())
+	// log.Debug(c.buffer.Next(25))
 	return err
 }
 
@@ -100,10 +145,75 @@ func (c *MCPClient) GetNodes() error {
 func (c *MCPClient) GetNodeEdgePoints() error {
 	defer c.buffer.Reset()
 	_, err := c.client.TapiTopologyCore.GetTapiCoreContextTopologyMcpBaseTopologyNodeEdgePoint(nil)
-	if err != nil { return err}
+	if err != nil {
+		return err
+	}
+	//TODO: there is no tapi ygot struct that fits the ciena node-edge-point
 	dest := &tapi.TapiCommon_Context_TopologyContext_Topology_Link_NodeEdgePoint{}
-	if err := tapi.Unmarshal(c.buffer.Bytes(), dest); err != nil { return err }
+	if err := tapi.Unmarshal(c.buffer.Bytes(), dest); err != nil {
+		return err
+	}
 	c.database.StoreNodeEdgePoints(c.buffer.String())
 	log.Debug(c.buffer.Next(25))
 	return err
-}
+}
\ No newline at end of file
+
+//preformatJSON preformats the recieved JSON for further processing
+func preformatJSON(jsn string, path string) *gjson.Result {
+	//TODO: move this!
+	modifierName := "uppercase"
+	gjson.AddModifier(modifierName, func(jsonString, arg string) string {
+		var jsonMap interface{}
+		err := json.Unmarshal([]byte(jsonString), &jsonMap)
+		if err != nil {
+			log.Info("failed unmarshal for JSON")
+		}
+
+		jsonMap = uppercaseJSONValues(jsonMap)
+		result, err := json.Marshal(jsonMap)
+		if err != nil {
+			log.Info("failed marshal for JSON")
+		}
+		return string(result)
+	})
+
+	formattedJSON := gjson.Get(jsn, path+"|@"+modifierName)
+	return &formattedJSON
+}
+
+//uppercaseJSONValues takes a interface{} created with json.Unmarshal() and changes the containing
+//string values to uppercase
+//returns a interface{} which can be changed back into a JSON string via
+//json.Marshal()
+func uppercaseJSONValues(jsonMap interface{}) interface{} {
+	switch jsonMap := jsonMap.(type) {
+	//check if []interface{} and go through every entry recursively
+	case []interface{}:
+		for i := range jsonMap {
+			jsonMap[i] = uppercaseJSONValues(jsonMap[i])
+		}
+		return jsonMap
+	//check if map[string]interface, handle ciena tapi json specific fixes
+	//and go on recursively
+	case map[string]interface{}:
+		tmpInterfaceMap := make(map[string]interface{}, len(jsonMap))
+		for k, v := range jsonMap {
+			//TODO: maybe we can uppercase them too, but for now:
+			//DO NOT uppercase uuid's
+			if strings.Contains(k, "uuid") {
+				tmpInterfaceMap[k] = v
+			} else {
+				tmpInterfaceMap[k] = uppercaseJSONValues(v)
+			}
+		}
+		return tmpInterfaceMap
+	//ygot: requires enums in uppercase and since CIENA TAPI does sometimes
+	//provide faulty JSON values we need to uppercase them before we process
+	//them further
+	case string:
+		return strings.ToUpper(jsonMap)
+	//default: do nothing (like for bool or other stuff)
+	default:
+		return jsonMap
+	}
+}