build plugins within orchestrator

build.go

@@ -2,11 +2,13 @@ package csbi
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"os/exec"
 	"path/filepath"
 	"time"
 
@@ -15,7 +17,9 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
 )
 
 // nolint
@@ -78,3 +82,57 @@ func print(rd io.Reader) error {
 
 	return scanner.Err()
 }
+
+// buildPlugin builds a go plugin from ygot generated go code provided within
+// a plugin folder.
+func buildPlugin(id uuid.UUID) error {
+	labels := prometheus.Labels{"type": spb.Type_PLUGIN.String()}
+	start := promStartHook(labels, buildsTotal)
+
+	var stderr bytes.Buffer
+	buildDir := id.String()
+
+	if err := executeGoCommand(buildDir, &stderr, []string{"go", "mod", "tidy"}); err != nil {
+		log.Error(stderr.String())
+		return err
+	}
+
+	stderr.Reset()
+
+	buildCommand := []string{
+		"go",
+		"build",
+		"-buildmode=plugin",
+		"-o",
+		"./plugin.so",
+		"./gostructs.go",
+	}
+	if err := executeGoCommand(buildDir, &stderr, buildCommand); err != nil {
+		log.Error(stderr.String())
+		return err
+	}
+	promEndHook(labels, start, buildDurationSecondsTotal, buildDurationSeconds)
+	return nil
+}
+
+/*
+executeGoCommand runs a go command. Therefore it creates a new *exec.Cmd and
+adds the provided build directory as string, a byte buffer and the build
+commands as string slice.
+
+Example for a build command slice:
+
+buildCommand := []string{
+    "build",
+    "-o",
+    outputPath,
+    sourcePath,
+}
+*/
+func executeGoCommand(dir string, stderr *bytes.Buffer, buildCommand []string) error {
+	cmd := exec.Command(buildCommand[0], buildCommand[1:]...)
+	cmd.Dir = dir
+	cmd.Stderr = stderr
+
+	return cmd.Run()
+}

go.mod

@@ -3,8 +3,8 @@ module code.fbi.h-da.de/danet/csbi
 go 1.17
 
 require (
-	code.fbi.h-da.de/danet/api v0.2.5-0.20220120151437-a3719e95faf2
-	code.fbi.h-da.de/danet/gosdn v0.0.3-0.20220120152434-de2a634e03ba
+	code.fbi.h-da.de/danet/api v0.2.5-0.20220125160614-789e7e1c26f0
+	code.fbi.h-da.de/danet/gosdn v0.0.3-0.20220127123058-90495a469857
 	github.com/docker/docker v20.10.11+incompatible // as per https://github.com/moby/moby/issues/41191#issuecomment-656342401
 	github.com/google/uuid v1.2.0
 	github.com/mitchellh/go-homedir v1.1.0

go.sum

@@ -45,12 +45,18 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 code.fbi.h-da.de/danet/api v0.2.5-0.20220120151437-a3719e95faf2 h1:YABbazS13g70cftlTqZ1zzmAJr0kHmHIzMD32NXlaFY=
 code.fbi.h-da.de/danet/api v0.2.5-0.20220120151437-a3719e95faf2/go.mod h1:kjazkgCFLje+z4BBNBLlyozhQUnkJd0sqlZz1Axe0wM=
+code.fbi.h-da.de/danet/api v0.2.5-0.20220125160614-789e7e1c26f0 h1:QdTE7B6ScMWtPpwmqKvvwGTWsKyWXTR8AWTYu6AWLRA=
+code.fbi.h-da.de/danet/api v0.2.5-0.20220125160614-789e7e1c26f0/go.mod h1:kjazkgCFLje+z4BBNBLlyozhQUnkJd0sqlZz1Axe0wM=
 code.fbi.h-da.de/danet/forks/goarista v0.0.0-20210709163519-47ee8958ef40 h1:x7rVYGqfJSMWuYBp+JE6JVMcFP03Gx0mnR2ftsgqjVI=
 code.fbi.h-da.de/danet/forks/goarista v0.0.0-20210709163519-47ee8958ef40/go.mod h1:uVe3gCeF2DcIho8K9CIO46uAkHW/lUF+fAaUX1vHrF0=
 code.fbi.h-da.de/danet/forks/google v0.0.0-20210709163519-47ee8958ef40 h1:B45k5tGEdjjdsKK4f+0dQoyReFmsWdwYEzHofA7DPM8=
 code.fbi.h-da.de/danet/forks/google v0.0.0-20210709163519-47ee8958ef40/go.mod h1:Uutdj5aA3jpzfNm3C8gt2wctYE6cRrdyZsILUgJ+tMY=
 code.fbi.h-da.de/danet/gosdn v0.0.3-0.20220120152434-de2a634e03ba h1:e7SXJ+cf04cHaOC8+HAU9xO47vn9KfdCyAUMp5G1X3E=
 code.fbi.h-da.de/danet/gosdn v0.0.3-0.20220120152434-de2a634e03ba/go.mod h1:jlFu92Dx/AIuhERvZDKHX3ipmOVqON6g7I1gBt9RwF4=
+code.fbi.h-da.de/danet/gosdn v0.0.3-0.20220125173344-b64ac9efc3b9 h1:xCZQvil6G6PJEMV/tg2y5KTSVafVU19n2N1loQvvf40=
+code.fbi.h-da.de/danet/gosdn v0.0.3-0.20220125173344-b64ac9efc3b9/go.mod h1:/JfwV+FUs/bZZD3P1gvQ3EuwzvFSWxjtmf0UoVU/JmM=
+code.fbi.h-da.de/danet/gosdn v0.0.3-0.20220127123058-90495a469857 h1:iRFhTyTajxhn8QtPnlMYzRGqHwdsEAnKr7jNpZ36pUk=
+code.fbi.h-da.de/danet/gosdn v0.0.3-0.20220127123058-90495a469857/go.mod h1:/JfwV+FUs/bZZD3P1gvQ3EuwzvFSWxjtmf0UoVU/JmM=
 code.fbi.h-da.de/danet/yang-models v0.1.0 h1:C658HkGYZSV5Eq5nY2NnC/PQPKp3BaTXwGZICCr0sqk=
 code.fbi.h-da.de/danet/yang-models v0.1.0/go.mod h1:0TNkzPA1OW9lF9ey18GQWcMd4ORvOfhhFOA/t0SjenM=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=

grpc.go

@@ -12,6 +12,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
 	codes "google.golang.org/grpc/codes"
 	status "google.golang.org/grpc/status"
 )
@@ -31,6 +32,11 @@ const (
 	YB
 )
 
+type pluginStream interface {
+	Send(*pb.Payload) error
+	grpc.ServerStream
+}
+
 type server struct {
 	pb.UnimplementedCsbiServer
 	orchestrator Orchestrator
@@ -89,8 +95,9 @@ func (s server) Create(ctx context.Context, req *pb.CreateRequest) (*pb.CreateRe
 	}, nil
 }
 
-func (s server) GetGoStruct(req *pb.GetRequest, stream pb.Csbi_GetGoStructServer) error {
-	log.Info("started GetGoStruct")
+// TODO(maba): add description and consider to allow requesting
+func (s server) GetPlugin(req *pb.GetRequest, stream pb.Csbi_GetPluginServer) error {
+	log.Info("started GetPlugin")
 	labels := prometheus.Labels{"rpc": "get_go_struct"}
 	start := promStartHook(labels, grpcRequestsTotal)
 	defer promEndHook(labels, start, grpcRequestDurationSecondsTotal, grpcRequestDurationSeconds)
@@ -101,33 +108,11 @@ func (s server) GetGoStruct(req *pb.GetRequest, stream pb.Csbi_GetGoStructServer
 		return handleRPCError(labels, err)
 	}
 
-	file, err := os.Open(filepath.Join(dep.ID.String(), "gostructs.go"))
-	if err != nil {
-		return handleRPCError(labels, err)
-	}
-	defer file.Close()
-
-	buffer := make([]byte, int(MB))
-
-	for {
-		n, err := file.Read(buffer)
-		if err != nil {
-			if err != io.EOF {
-				fmt.Println(err)
-			}
-			break
-		}
-		log.WithField("n", n).Trace("read bytes")
-		payload := &pb.Payload{Chunk: buffer[:n]}
-		err = stream.Send(payload)
-		if err != nil {
-			return handleRPCError(labels, err)
-		}
-	}
-	return nil
+	return sendPlugin(dep.ID, labels, stream)
 }
-func (s server) CreateGoStruct(req *pb.CreateRequest, stream pb.Csbi_CreateGoStructServer) error {
-	log.Info("started CreateGoStruct")
+
+func (s server) CreatePlugin(req *pb.CreateRequest, stream pb.Csbi_CreatePluginServer) error {
+	log.Info("started CreatePlugin")
 	labels := prometheus.Labels{"rpc": "create_plugin"}
 	start := promStartHook(labels, grpcRequestsTotal)
 	defer promEndHook(labels, start, grpcRequestDurationSecondsTotal, grpcRequestDurationSeconds)
@@ -142,30 +127,44 @@ func (s server) CreateGoStruct(req *pb.CreateRequest, stream pb.Csbi_CreateGoStr
 		if err != nil {
 			return handleRPCError(labels, err)
 		}
-		file, err := os.Open(filepath.Join(d.ID.String(), "gostructs.go"))
+		buildPlugin(d.ID)
 		if err != nil {
 			return handleRPCError(labels, err)
 		}
-		defer file.Close()
+		err = sendPlugin(d.ID, labels, stream)
+		if err != nil {
+			return handleRPCError(labels, err)
+		}
+	}
+	return nil
+}
 
-		buffer := make([]byte, int(MB))
+// sendPlugin takes a
+func sendPlugin(id uuid.UUID, labels prometheus.Labels, stream pluginStream) error {
+	file, err := os.Open(filepath.Join(id.String(), "plugin.so"))
+	if err != nil {
+		return handleRPCError(labels, err)
+	}
+	defer file.Close()
 
-		for {
-			n, err := file.Read(buffer)
-			if err != nil {
-				if err != io.EOF {
-					fmt.Println(err)
-				}
-				break
-			}
-			log.WithField("n", n).Trace("read bytes")
-			payload := &pb.Payload{Chunk: buffer[:n]}
-			err = stream.Send(payload)
-			if err != nil {
-				return handleRPCError(labels, err)
+	buffer := make([]byte, int(MB))
+
+	for {
+		n, err := file.Read(buffer)
+		if err != nil {
+			if err != io.EOF {
+				fmt.Println(err)
 			}
+			break
+		}
+		log.WithField("n", n).Trace("read bytes")
+		payload := &pb.Payload{Chunk: buffer[:n]}
+		err = stream.Send(payload)
+		if err != nil {
+			return handleRPCError(labels, err)
 		}
 	}
+
 	return nil
 }
 

resources/plugin_deps.json

+{
+	"Module": {
+		"Path": "code.fbi.h-da.de/danet/plugin-sbi"
+	},
+	"Go": "1.17",
+	"Require": [
+	 		{
+			"Path": "code.fbi.h-da.de/danet/gosdn",
+			"Version": "v0.0.3-0.20220127123058-90495a469857"
+		},
+		{
+			"Path": "code.fbi.h-da.de/danet/api",
+			"Version": "v0.2.5-0.20220125160614-789e7e1c26f0"
+		},
+		{
+			"Path": "code.fbi.h-da.de/danet/forks/goarista",
+			"Version": "v0.0.0-20210709163519-47ee8958ef40"
+		},
+		{
+			"Path": "code.fbi.h-da.de/danet/forks/google",
+			"Version": "v0.0.0-20210709163519-47ee8958ef40"
+		},
+		{
+			"Path": "code.fbi.h-da.de/danet/yang-models",
+			"Version": "v0.1.0"
+		},
+		{
+			"Path": "github.com/google/uuid",
+			"Version": "v1.2.0"
+		},
+		{
+			"Path": "github.com/openconfig/gnmi",
+			"Version": "v0.0.0-20210914185457-51254b657b7d"
+		},
+		{
+			"Path": "github.com/openconfig/goyang",
+			"Version": "v0.3.1"
+		},
+		{
+			"Path": "github.com/openconfig/ygot",
+			"Version": "v0.12.5"
+		},
+		{
+			"Path": "github.com/prometheus/client_golang",
+			"Version": "v1.9.0"
+		},
+		{
+			"Path": "github.com/sirupsen/logrus",
+			"Version": "v1.8.1"
+		},
+		{
+			"Path": "github.com/spf13/cobra",
+			"Version": "v1.1.3"
+		},
+		{
+			"Path": "github.com/spf13/viper",
+			"Version": "v1.9.0"
+		},
+		{
+			"Path": "github.com/stretchr/objx",
+			"Version": "v0.2.0",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/stretchr/testify",
+			"Version": "v1.7.0"
+		},
+		{
+			"Path": "google.golang.org/grpc",
+			"Version": "v1.43.0"
+		},
+		{
+			"Path": "google.golang.org/protobuf",
+			"Version": "v1.27.1"
+		},
+		{
+			"Path": "github.com/beorn7/perks",
+			"Version": "v1.0.1",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/cespare/xxhash/v2",
+			"Version": "v2.1.1",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/davecgh/go-spew",
+			"Version": "v1.1.1",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/fsnotify/fsnotify",
+			"Version": "v1.5.1",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/golang/glog",
+			"Version": "v1.0.0",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/golang/protobuf",
+			"Version": "v1.5.2",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/google/go-cmp",
+			"Version": "v0.5.6",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/hashicorp/hcl",
+			"Version": "v1.0.0",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/inconshreveable/mousetrap",
+			"Version": "v1.0.0",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/kylelemons/godebug",
+			"Version": "v1.1.0",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/magiconair/properties",
+			"Version": "v1.8.5",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/matttproud/golang_protobuf_extensions",
+			"Version": "v1.0.1",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/mitchellh/mapstructure",
+			"Version": "v1.4.2",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/pelletier/go-toml",
+			"Version": "v1.9.4",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/pmezard/go-difflib",
+			"Version": "v1.0.0",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/prometheus/client_model",
+			"Version": "v0.2.0",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/prometheus/common",
+			"Version": "v0.18.0",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/prometheus/procfs",
+			"Version": "v0.6.0",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/spf13/afero",
+			"Version": "v1.6.0",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/spf13/cast",
+			"Version": "v1.4.1",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/spf13/jwalterweatherman",
+			"Version": "v1.1.0",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/spf13/pflag",
+			"Version": "v1.0.5",
+			"Indirect": true
+		},
+		{
+			"Path": "github.com/subosito/gotenv",
+			"Version": "v1.2.0",
+			"Indirect": true
+		},
+		{
+			"Path": "golang.org/x/net",
+			"Version": "v0.0.0-20211123203042-d83791d6bcd9",
+			"Indirect": true
+		},
+		{
+			"Path": "golang.org/x/sys",
+			"Version": "v0.0.0-20211123173158-ef496fb156ab",
+			"Indirect": true
+		},
+		{
+			"Path": "golang.org/x/text",
+			"Version": "v0.3.7",
+			"Indirect": true
+		},
+		{
+			"Path": "google.golang.org/genproto",
+			"Version": "v0.0.0-20211208223120-3a66f561d7aa",
+			"Indirect": true
+		},
+		{
+			"Path": "gopkg.in/ini.v1",
+			"Version": "v1.64.0",
+			"Indirect": true
+		},
+		{
+			"Path": "gopkg.in/yaml.v2",
+			"Version": "v2.4.0",
+			"Indirect": true
+		},
+		{
+			"Path": "gopkg.in/yaml.v3",
+			"Version": "v3.0.0-20210107192922-496545a6307b",
+			"Indirect": true
+		}
+	],
+	"Exclude": null,
+	"Replace": null,
+	"Retract": null
+}

templates.go

 package csbi
 
-import "github.com/openconfig/ygot/ygen"
+import (
+	"html/template"
+
+	"github.com/openconfig/ygot/ygen"
+)
 
 var pluginStruct = ygen.GoStructCodeSnippet{
 	StructName: "Csbi",
@@ -100,3 +104,31 @@ const pluginImportAmendmend = `
 
 var PluginSymbol Csbi
 `
+
+var templater *template.Template
+
+const pluginGoModTemplate = `module << .Module.Path >>
+
+go << .GoVersion >>
+
+require (<<range $element := .Dependencies>>
+	<<$element.Path>> <<$element.Version>>
+<<end>>)
+`
+
+type Module struct {
+	Path string `json:"Path"`
+}
+
+type Dependency struct {
+	Path    string `json:"Path"`
+	Version string `json:"Version"`
+	//Indirect bool   `json:"Indirect,omitempty"`
+}
+
+// GoMod represents a go.mod file used for templates.
+type GoMod struct {
+	Module       Module       `json:"Module"`
+	GoVersion    string       `json:"Go"`
+	Dependencies []Dependency `json:"Require"`
+}

write.go

@@ -3,7 +3,9 @@ package csbi
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
+	"html/template"
 	"io/fs"
 	"net"
 	"os"
@@ -91,7 +93,11 @@ func writeCsbi(ctx context.Context, code *ygen.GeneratedGoCode, path string) err
 }
 
 func writePlugin(code *ygen.GeneratedGoCode, path string) error {
-	return writeCode(path, code)
+	err := writeCode(path, code)
+	if err != nil {
+		return err
+	}
+	return writeGoMod(path)
 }
 
 func copyFile(path, filename string) error {
@@ -147,3 +153,30 @@ func writeCode(path string, code *ygen.GeneratedGoCode) error {
 	}
 	return nil
 }
+
+func writeGoMod(path string) error {
+	// Read dependencies from JSON file
+	deps, err := os.ReadFile(filepath.Join("resources", "plugin_deps.json"))
+	if err != nil {
+		return err
+	}
+	module := GoMod{}
+	if err := json.Unmarshal(deps, &module); err != nil {
+		return err
+	}
+
+	// Create go.mod in destination directory and write template
+	file := filepath.Join(path, "go.mod")
+	goMod, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY, 0755)
+	if err != nil {
+		return err
+	}
+	defer goMod.Sync()
+
+	templater := template.New("goMod").Delims("<<", ">>")
+	_, err = templater.Parse(pluginGoModTemplate)
+	if err != nil {
+		return err
+	}
+	return templater.Execute(goMod, module)
+}