Commit f81b9b38 authored by Manuel Kieweg's avatar Manuel Kieweg 🤷
Browse files

Develop

parent 967e5141
.idea/
.vscode/
/00000000-0000-0000-0000-000000000000
testdata/00000000-0000-0000-0000-000000000000/go.sum
testdata/00000000-0000-0000-0000-000000000000/plugin.so
coverage.out
\ No newline at end of file
variables:
SECURE_ANALYZERS_PREFIX: registry.gitlab.com/gitlab-org/security-products/analyzers
DOCKER_IMAGE_SHA: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_PIPELINE_SOURCE == "schedule"'
- if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH != "develop" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
when: never
- if: '$CI_COMMIT_BRANCH'
stages:
- .pre
- build
- test
- apply
- integration-test
- deploy
- .post
include:
- local: '/build/ci/.code-quality-ci.yml'
- local: '/build/ci/.security-and-compliance-ci.yml'
- local: '/build/ci/.build-container.yml'
- local: '/build/ci/.test.yml'
[submodule "public"]
path = public
url = https://github.com/openconfig/public.git
[submodule "models/yang"]
path = models/yang
url = https://github.com/YangModels/yang.git
# syntax = docker/dockerfile:1.2
FROM golang:1.16-alpine AS installer
WORKDIR /build
RUN apk add --no-cache git make build-base
RUN apk add --update --no-cache alpine-sdk
COPY go.mod .
COPY go.sum .
RUN go mod download
FROM installer as builder
COPY . .
RUN --mount=type=cache,target=/root/.cache/go-build \
GOOS=linux go build -o orchestrator ./cmd/csbi/main.go
FROM golang:1.16-alpine
COPY --from=builder /build/orchestrator .
COPY --from=builder /build/models ./models
COPY --from=builder /build/resources ./resources
EXPOSE 55056
ENTRYPOINT [ "./orchestrator" ]
CMD [""]
\ No newline at end of file
Copyright © 2021 Manuel Kieweg <mail@manuelkieweg.de>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package csbi
import "github.com/openconfig/ygot/ygen"
var pluginBoilerplate = ygen.GoStructCodeSnippet{
StructName: "Csbi",
StructDef: `type Csbi struct {
schema *ytypes.Schema
id uuid.UUID
}`,
Methods: `
func (sbi *Csbi) SbiIdentifier() string {
return "plugin sbi"
}
func (sbi *Csbi) SetNode(schema *yang.Entry, root interface{}, path *gpb.Path, val interface{}, opts ...ytypes.SetNodeOpt) error {
log.WithFields(log.Fields{
"schema": schema.Name,
"path": path,
"val": val,
"opts": opts,
}).Trace("entering SetNode()")
return ytypes.SetNode(schema, root.(*Device), path, val, opts...)
}
func (sbi *Csbi) Schema() *ytypes.Schema {
schema, err := Schema()
sbi.schema = schema
if err != nil {
log.Fatal(err)
}
return schema
}
func (sbi *Csbi) ID() uuid.UUID {
return sbi.id
}
func (sbi *Csbi) Type() spb.Type {
return spb.Type_CONTAINERISED
}
func (sbi *Csbi) Unmarshal(bytes []byte, fields []string, goStruct ygot.ValidatedGoStruct, opt ...ytypes.UnmarshalOpt) error {
log.WithFields(log.Fields{}).Trace("entering Unmarshal()")
return unmarshal(sbi.Schema(), bytes, fields, goStruct, opt...)
}
// unmarshal parses gNMI response to a go struct.
func unmarshal(schema *ytypes.Schema, bytes []byte, fields []string, goStruct ygot.ValidatedGoStruct, opt ...ytypes.UnmarshalOpt) error {
defer func() {
if r := recover(); r != nil {
log.Error(r.(error))
}
}()
log.WithFields(log.Fields{
"schema": schema.RootSchema().Name,
"fields": fields,
"bytes": len(bytes),
"opts": opt,
}).Trace("entering unmarshal()")
// Load SBI definition
root, err := ygot.DeepCopy(schema.Root)
if err != nil {
return err
}
validatedDeepCopy, ok := root.(ygot.ValidatedGoStruct)
if !ok {
return fmt.Errorf("no validated go struct")
}
ygot.BuildEmptyTree(validatedDeepCopy)
log.Trace("built empty tree")
var fieldStruct ygot.ValidatedGoStruct
if len(fields) != 0 {
log.Trace("fetching fields")
fieldStruct, err = getField(validatedDeepCopy, fields)
if err != nil {
return err
}
} else {
log.Trace("using root struct")
fieldStruct = validatedDeepCopy
}
if err := Unmarshal(bytes, fieldStruct, opt...); err != nil {
return err
}
ygot.PruneEmptyBranches(validatedDeepCopy)
log.Trace("pruned empty branches")
log.Trace("merging structs...")
return ygot.MergeStructInto(goStruct, validatedDeepCopy)
}
// getField traverses the GoStruct and returns the field that represents the
// tail of the path
func getField(inStruct ygot.ValidatedGoStruct, fields []string) (ygot.ValidatedGoStruct, error) {
defer func() {
if r := recover(); r != nil {
log.Error(r.(error))
}
}()
log.WithFields(log.Fields{
"fields": fields,
}).Trace("getting field")
f := fields[0]
log.Tracef("field name: %v", f)
s := reflect.ValueOf(inStruct)
h := reflect.Indirect(s).FieldByName(f).Interface()
outStruct, ok := h.(ygot.ValidatedGoStruct)
if !ok {
t := reflect.TypeOf(h)
if !(util.IsTypeStruct(t) || util.IsTypeStructPtr(t)) {
return nil, fmt.Errorf("cannot process entry of type %v, request longer or shorter path", t)
}
return nil, fmt.Errorf("expected ValidatedGoStruct got %v", t)
}
if len(fields) > 1 {
log.Trace("fetching fields")
return getField(outStruct, fields[1:])
}
return outStruct, nil
}`,
}
var pluginImports = `
spb "code.fbi.h-da.de/danet/api/go/gosdn/southbound"
"github.com/google/uuid"
"github.com/openconfig/ygot/util"
log "github.com/sirupsen/logrus"
)
var PluginSymbol Csbi
`
var pluginGoMod = `module code.fbi.h-da.de/danet/plugin-autogen
go 1.16
require (
code.fbi.h-da.de/danet/api thesis-mk
github.com/google/uuid v1.2.0
github.com/openconfig/gnmi v0.0.0-20210707145734-c69a5df04b53
github.com/openconfig/goyang v0.2.7
github.com/openconfig/ygot v0.11.2
github.com/sirupsen/logrus v1.8.1
)
`
package csbi
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os/exec"
"path/filepath"
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/docker/docker/pkg/archive"
log "github.com/sirupsen/logrus"
)
// nolint
type ErrorLine struct {
Error string `json:"error"`
ErrorDetail ErrorDetail `json:"errorDetail"`
}
// nolint
type ErrorDetail struct {
Message string `json:"message"`
}
func buildImage(d Deployment, dockerClient *client.Client) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*120)
defer cancel()
p := filepath.Join(d.ID.String(), "/")
tar, err := archive.TarWithOptions(p, &archive.TarOptions{})
if err != nil {
return err
}
opts := types.ImageBuildOptions{
Dockerfile: "Dockerfile",
Tags: []string{d.Name},
Remove: true,
}
res, err := dockerClient.ImageBuild(ctx, tar, opts)
if err != nil {
return err
}
defer res.Body.Close()
err = print(res.Body)
if err != nil {
return err
}
return nil
}
func print(rd io.Reader) error {
var lastLine string
scanner := bufio.NewScanner(rd)
for scanner.Scan() {
lastLine = scanner.Text()
fmt.Println(scanner.Text())
}
errLine := &ErrorLine{}
json.Unmarshal([]byte(lastLine), errLine)
if errLine.Error != "" {
return errors.New(errLine.Error)
}
return scanner.Err()
}
func buildPlugin(id string) error {
var stderr bytes.Buffer
buildDir := id
goModDownload := exec.Command("go", "mod", "tidy")
goModDownload.Dir = buildDir
goModDownload.Stderr = &stderr
err := goModDownload.Run()
if err != nil {
log.Error(stderr.String())
return err
}
stderr.Reset()
buildCommand := []string{
"go",
"build",
"-buildmode=plugin",
"-o",
"./plugin.so",
"./gostructs.go",
}
cmd := exec.Command(buildCommand[0], buildCommand[1:]...)
cmd.Dir = buildDir
cmd.Stderr = &stderr
err = cmd.Run()
if err != nil {
log.Error(stderr.String())
return err
}
return nil
}
variables:
DOCKER_TLS_CERTDIR: "/certs"
build-docker:
before_script:
- echo "override global before script"
stage: build
allow_failure: false
needs: []
tags:
- shell-builder
rules:
- if: $CI_COMMIT_BRANCH == "develop" && $CI_NIGHTLY == null
variables:
TAG: $CI_REGISTRY_IMAGE:develop
BUILDARGS: -race
- if: $CI_NIGHTLY == "develop"
variables:
TAG: $CI_REGISTRY_IMAGE:nightly-develop
BUILDARGS: -race
- if: $CI_NIGHTLY == "mainline"
variables:
TAG: $CI_REGISTRY_IMAGE:nightly
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
variables:
TAG: $CI_REGISTRY_IMAGE:merge-request
BUILDARGS: -race
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_NIGHTLY == null
variables:
TAG: $CI_REGISTRY_IMAGE:latest
- if: '$CI_COMMIT_BRANCH'
variables:
TAG: $CI_REGISTRY_IMAGE:branch
script:
- echo "$CI_REGISTRY_PASSWORD" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
- docker build -t $DOCKER_IMAGE_SHA .
- docker push $DOCKER_IMAGE_SHA
- docker tag $DOCKER_IMAGE_SHA $TAG
- docker push $TAG
- docker build --target installer -t ${CI_REGISTRY_IMAGE}:testing_${CI_PIPELINE_ID} .
- docker push ${CI_REGISTRY_IMAGE}:testing_${CI_PIPELINE_ID}
code-quality:
image: golangci/golangci-lint:latest-alpine
stage: test
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_COMMIT_BRANCH == "develop"
script:
# writes golangci-lint output to gl-code-quality-report.json
- golangci-lint run --config build/ci/.golangci-config/.golangci.yml --out-format code-climate | tee gl-code-quality-report.json
artifacts:
reports:
codequality: gl-code-quality-report.json
paths:
- gl-code-quality-report.json
run:
timeout: 10m
issues-exit-code: 1
# directories to be ignored by linters
skip-dirs:
- forks
- test
skip-dirs-default: true
skip-files:
- http.go
# output settings -> code-climate for GitLab
output:
format: code-climate
print-issued-lines: true
print-linter-name: true
uniq-by-line: true
path-prefix: ""
# custom settings for linters
linters-settings:
gocyclo:
min-complexity: 15
golint:
min-confidence: 0.8
# enable the specific needed linters
linters:
disable-all: true
enable:
- gofmt
- goimports
- revive
- gocyclo
- govet
issues:
exclude-use-default: false
max-issues-per-linter: 0
max-same-issues: 0
sast:
variables:
SAST_ANALYZER_IMAGE_TAG: '2'
SAST_EXCLUDED_PATHS: spec, test, tests, tmp
SEARCH_MAX_DEPTH: '4'
include:
- template: Security/SAST.gitlab-ci.yml
- template: Dependency-Scanning.gitlab-ci.yml
- template: Security/License-Scanning.gitlab-ci.yml
.test: &test
image: ${CI_REGISTRY_IMAGE}:testing_${CI_PIPELINE_ID}
stage: test
variables:
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH
- if: $CI_NIGHTLY
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
allow_failure: true
unit-test:
script:
- go test -short $(go list ./...) -v -coverprofile=coverage.out
after_script:
- go tool cover -func=coverage.out
<<: *test
\ No newline at end of file
package csbi
import (
"fmt"
"io"
"plugin"
"testing"
"code.fbi.h-da.de/danet/api/go/gosdn/southbound"
"github.com/docker/docker/client"
"github.com/google/uuid"
)
func Test_buildImage(t *testing.T) {
type args struct {
d Deployment
dockerClient *client.Client
}
tests := []struct {
name string
args args
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := buildImage(tt.args.d, tt.args.dockerClient); (err != nil) != tt.wantErr {
t.Errorf("buildImage() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func Test_print(t *testing.T) {
type args struct {
rd io.Reader
}
tests := []struct {
name string
args args
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := print(tt.args.rd); (err != nil) != tt.wantErr {
t.Errorf("print() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func Test_buildPlugin(t *testing.T) {
t.Skip("skipped due to package version inconsistencies")
type args struct {
d Deployment
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "default",
args: args{
d: Deployment{
ID: uuid.Nil,
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := buildPlugin("testdata/00000000-0000-0000-0000-000000000000"); (err != nil) != tt.wantErr {
t.Errorf("buildPlugin() error = %v, wantErr %v", err, tt.wantErr)
return
}
if err := loadPlugin(tt.args.d.ID.String()); (err != nil) != tt.wantErr {
t.Errorf("buildPlugin() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func loadPlugin(path string) error {
p, err := plugin.Open("testdata/00000000-0000-0000-0000-000000000000/plugin.so")
if err != nil {
return err
}
symbol, err := p.Lookup("PluginSymbol")
if err != nil {
return err
}
_, ok := symbol.(southbound.SouthboundInterface)
if !ok {
return fmt.Errorf("invalid plugin type")
}
return nil
}
/*
Copyright © 2021 Manuel Kieweg <mail@manuelkieweg.de>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
*/
package main
import "code.fbi.h-da.de/danet/csbi/cmd"
func main() {
cmd.Execute()
}
package main
import (
spb "code.fbi.h-da.de/danet/api/go/gosdn/southbound"
"code.fbi.h-da.de/danet/csbi"
log "github.com/sirupsen/logrus"
)
func main() {
_, err := csbi.Generate(nil, csbi.NewRepository(), spb.Type_CONTAINERISED)
if err != nil {