Skip to content
Snippets Groups Projects
Commit c38be1f5 authored by Lars Seipel's avatar Lars Seipel
Browse files

runner/libvirt: add test verifying the created topology 

After starting up the topology, use LLDP info (through ptmd)
to verify that what was built matches the original DOT
description.
parent 2073b123
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 598 additions and 148 deletions
runner/libvirt/helper.go
+ 5
− 1
View file @ c38be1f5
...@@ -262,9 +262,13 @@ func isASCIIDigit(c rune) bool { ...@@ -262,9 +262,13 @@ func isASCIIDigit(c rune) bool {
return '0' <= c && c <= '9' return '0' <= c && c <= '9'
} }
func hasFunction(d *device, fs ...topology.DeviceFunction) bool {
return topology.HasFunction(&d.topoDev, fs...)
}
// Returns whether d defaults to Cumulus Linux. // Returns whether d defaults to Cumulus Linux.
func hasCumulusFunction(d *device) bool { func hasCumulusFunction(d *device) bool {
return topology.HasFunction(&d.topoDev, return hasFunction(d,
topology.OOBSwitch, topology.OOBSwitch,
topology.Exit, topology.Exit,
topology.SuperSpine, topology.SuperSpine,
......
runner/libvirt/runner_test.go
+ 136
− 147
View file @ c38be1f5
...@@ -3,22 +3,27 @@ package libvirt ...@@ -3,22 +3,27 @@ package libvirt
import ( import (
"bytes" "bytes"
"context" "context"
"crypto/ed25519"
"crypto/rand" "crypto/rand"
"encoding/json"
"errors"
"fmt" "fmt"
"io"
"net" "net"
"os" "os"
"strings" "path/filepath"
"testing" "testing"
"time" "time"
"github.com/pkg/sftp"
"go4.org/writerutil"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
"slrz.net/runtopo/topology" "slrz.net/runtopo/topology"
) )
type ptmDetail struct {
Port string `json:"port"`
Status string `json:"cbl status"`
ActualNeighbor string `json:"act nbr"`
ExpectedNeighbor string `json:"exp nbr"`
}
func TestRuntopo(t *testing.T) { func TestRuntopo(t *testing.T) {
if testing.Short() { if testing.Short() {
t.SkipNow() t.SkipNow()
...@@ -86,36 +91,147 @@ func TestRuntopo(t *testing.T) { ...@@ -86,36 +91,147 @@ func TestRuntopo(t *testing.T) {
} }
defer oob.Close() defer oob.Close()
for hostname, d := range r.devices { // Upload configuration for network devices (frr.conf and interfaces)
if d.topoDev.Function() != topology.Host { for hostname := range r.devices {
var files [2][]byte
sources := []string{
filepath.Join("testdata/configs/interfaces", hostname),
filepath.Join("testdata/configs/frr", hostname),
}
for i, src := range sources {
p, err := os.ReadFile(src)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
continue
}
t.Fatal(err)
}
files[i] = p
}
if files[0] == nil && files[1] == nil {
continue continue
} }
var fileData []byte
err := withBackoff(nretries, func() error { interfaces, frrConf := files[0], files[1]
err = withBackoff(nretries, func() error {
c, err := proxyJump(oob, hostname, sshConfig) c, err := proxyJump(oob, hostname, sshConfig)
if err != nil { if err != nil {
return err return err
} }
defer c.Close() defer c.Close()
p, err := sftpGet(c, "/kilroywashere") if len(interfaces) > 0 {
if err != nil { err := sftpPut(c, "/etc/network/interfaces",
return err interfaces)
if err != nil {
return err
}
}
if len(frrConf) > 0 {
return sftpPut(c, "/etc/frr/frr.conf", frrConf)
} }
fileData = p
return nil return nil
}) })
if err != nil { if err != nil {
t.Errorf("%s: %v (giving up after %d retries)", t.Fatal(err)
hostname, err, nretries)
continue
} }
if !bytes.Equal(fileData, []byte("abcdef\n")) {
t.Errorf("%s: unexpected file content: got %q, want %q", err = withBackoff(nretries, func() error {
hostname, fileData, "abcdef\n") c, err := proxyJump(oob, hostname, sshConfig)
if err != nil {
return err
}
defer c.Close()
commands := [][]string{
{"sed", "-i", "s/^bgpd=no/bgpd=yes/", "/etc/frr/daemons"},
{"ifreload", "-a"},
{"systemctl", "restart", "frr.service"},
}
for _, argv := range commands {
_, err := runCommand(c, argv[0], argv[1:]...)
if err != nil {
return err
}
}
t.Logf("=== %s ===", hostname)
p, err := runCommand(c, "net", "show", "int")
t.Logf("%s\n===", p)
return err
})
if err != nil {
t.Fatal(err)
} }
} }
t.Run("config-nodeattr", func(t *testing.T) {
for hostname, d := range r.devices {
if !hasFunction(d, topology.Host) {
continue
}
var fileData []byte
err := withBackoff(nretries, func() error {
c, err := proxyJump(oob, hostname, sshConfig)
if err != nil {
return err
}
defer c.Close()
p, err := sftpGet(c, "/kilroywashere")
if err != nil {
return err
}
fileData = p
return nil
})
if err != nil {
t.Errorf("%s: %v (giving up after %d retries)",
hostname, err, nretries)
continue
}
if !bytes.Equal(fileData, []byte("abcdef\n")) {
t.Errorf("%s: unexpected file content: got %q, want %q",
hostname, fileData, "abcdef\n")
}
}
})
t.Run("ptm-topology", func(t *testing.T) {
for hostname, d := range r.devices {
if !hasFunction(d, topology.Spine, topology.Leaf) {
continue
}
err := withBackoff(nretries, func() error {
c, err := proxyJump(oob, hostname, sshConfig)
if err != nil {
return err
}
defer c.Close()
p, err := runCommand(c, "ptmctl", "--json", "--detail")
if err != nil {
return err
}
// Ptmctl gives us a JSON object with numeric
// string indices: {"0": {}, "1": {}, ...}.
ptm := make(map[string]*ptmDetail)
if err := json.Unmarshal(p, &ptm); err != nil {
return err
}
for _, v := range ptm {
if v.Status != "pass" {
return fmt.Errorf("%s: got %s, want %s",
v.Port, v.ActualNeighbor, v.ExpectedNeighbor)
}
}
return nil
})
if err != nil {
t.Fatalf("%s: %v", hostname, err)
}
}
})
} }
func withBackoff(attempts int, f func() error) (err error) { func withBackoff(attempts int, f func() error) (err error) {
...@@ -147,130 +263,3 @@ func minInt64(a, b int64) int64 { ...@@ -147,130 +263,3 @@ func minInt64(a, b int64) int64 {
} }
return a return a
} }
func proxyJump(c *ssh.Client, addr string, config *ssh.ClientConfig) (cc *ssh.Client, err error) {
defer func() {
if err != nil {
err = fmt.Errorf("proxyJump %s: %w", addr, err)
}
}()
conn, err := c.Dial("tcp", net.JoinHostPort(addr, "22"))
if err != nil {
return nil, err
}
sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config)
if err != nil {
conn.Close()
return nil, err
}
return ssh.NewClient(sshConn, chans, reqs), nil
}
func runCommand(c *ssh.Client, name string, args ...string) ([]byte, error) {
var b strings.Builder
b.WriteString(shellQuote(name))
for _, a := range args {
b.WriteByte(' ')
b.WriteString(shellQuote(a))
}
cmd := b.String()
sess, err := c.NewSession()
if err != nil {
return nil, err
}
defer sess.Close()
var stdout bytes.Buffer
stderr := &writerutil.PrefixSuffixSaver{N: 1024}
sess.Stdout = &stdout
sess.Stderr = stderr
if err := sess.Run(cmd); err != nil {
if msg := stderr.Bytes(); len(msg) > 0 {
return nil, fmt.Errorf("runCommand: %w | %s |", err, msg)
}
return nil, fmt.Errorf("runCommand: %w", err)
}
return stdout.Bytes(), nil
}
func sftpGet(conn *ssh.Client, path string) (content []byte, err error) {
c, err := sftp.NewClient(conn)
if err != nil {
return nil, err
}
defer c.Close()
fd, err := c.Open(path)
if err != nil {
return nil, err
}
defer fd.Close()
return io.ReadAll(fd)
}
func sftpPutReader(conn *ssh.Client, dstPath string, src io.Reader) (err error) {
c, err := sftp.NewClient(conn)
if err != nil {
return err
}
defer c.Close()
fd, err := c.Create(dstPath)
if err != nil {
return err
}
defer func() {
if cerr := fd.Close(); err == nil {
err = cerr
}
}()
_, err = io.Copy(fd, src)
return err
}
func sftpPut(conn *ssh.Client, dstPath string, content []byte) (err error) {
return sftpPutReader(conn, dstPath, bytes.NewReader(content))
}
func sshKeygen(rand io.Reader) (ssh.Signer, []byte, error) {
_, sk, err := ed25519.GenerateKey(rand)
if err != nil {
return nil, nil, err
}
signer, err := ssh.NewSignerFromSigner(sk)
if err != nil {
return nil, nil, err
}
sshPubKey := ssh.MarshalAuthorizedKey(signer.PublicKey())
return signer, sshPubKey, nil
}
func mustReadFile(path string) []byte {
p, err := os.ReadFile(path)
if err != nil {
panic(err)
}
return p
}
// ShellQuote returns s in a form suitable to pass it to the shell as an
// argument. Obviously, it works for Bourne-like shells only. The way this
// works is that first the whole string is enclosed in single quotes. Now the
// only character that needs special handling is the single quote itself. We
// replace it by '\'' (the outer quotes are part of the replacement) and make
// use of the fact that the shell concatenates adjacent strings.
func shellQuote(s string) string {
t := strings.Replace(s, "'", `'\''`, -1)
return "'" + t + "'"
}
runner/libvirt/sshutil_test.go 0 → 100644
+ 135
− 0
View file @ c38be1f5
package libvirt
// SSH helpers used in tests.
import (
"bytes"
"crypto/ed25519"
"fmt"
"io"
"net"
"strings"
"github.com/pkg/sftp"
"go4.org/writerutil"
"golang.org/x/crypto/ssh"
)
func proxyJump(c *ssh.Client, addr string, config *ssh.ClientConfig) (cc *ssh.Client, err error) {
defer func() {
if err != nil {
err = fmt.Errorf("proxyJump %s: %w", addr, err)
}
}()
conn, err := c.Dial("tcp", net.JoinHostPort(addr, "22"))
if err != nil {
return nil, err
}
sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config)
if err != nil {
conn.Close()
return nil, err
}
return ssh.NewClient(sshConn, chans, reqs), nil
}
func runCommand(c *ssh.Client, name string, args ...string) ([]byte, error) {
var b strings.Builder
b.WriteString(shellQuote(name))
for _, a := range args {
b.WriteByte(' ')
b.WriteString(shellQuote(a))
}
cmd := b.String()
sess, err := c.NewSession()
if err != nil {
return nil, err
}
defer sess.Close()
var stdout bytes.Buffer
stderr := &writerutil.PrefixSuffixSaver{N: 1024}
sess.Stdout = &stdout
sess.Stderr = stderr
if err := sess.Run(cmd); err != nil {
if msg := stderr.Bytes(); len(msg) > 0 {
return nil, fmt.Errorf("runCommand: %w | %s |", err, msg)
}
return nil, fmt.Errorf("runCommand: %w", err)
}
return stdout.Bytes(), nil
}
func sftpGet(conn *ssh.Client, path string) (content []byte, err error) {
c, err := sftp.NewClient(conn)
if err != nil {
return nil, err
}
defer c.Close()
fd, err := c.Open(path)
if err != nil {
return nil, err
}
defer fd.Close()
return io.ReadAll(fd)
}
func sftpPutReader(conn *ssh.Client, dstPath string, src io.Reader) (err error) {
c, err := sftp.NewClient(conn)
if err != nil {
return err
}
defer c.Close()
fd, err := c.Create(dstPath)
if err != nil {
return err
}
defer func() {
if cerr := fd.Close(); err == nil {
err = cerr
}
}()
_, err = io.Copy(fd, src)
return err
}
func sftpPut(conn *ssh.Client, dstPath string, content []byte) (err error) {
return sftpPutReader(conn, dstPath, bytes.NewReader(content))
}
func sshKeygen(rand io.Reader) (ssh.Signer, []byte, error) {
_, sk, err := ed25519.GenerateKey(rand)
if err != nil {
return nil, nil, err
}
signer, err := ssh.NewSignerFromSigner(sk)
if err != nil {
return nil, nil, err
}
sshPubKey := ssh.MarshalAuthorizedKey(signer.PublicKey())
return signer, sshPubKey, nil
}
// ShellQuote returns s in a form suitable to pass it to the shell as an
// argument. Obviously, it works for Bourne-like shells only. The way this
// works is that first the whole string is enclosed in single quotes. Now the
// only character that needs special handling is the single quote itself. We
// replace it by '\'' (the outer quotes are part of the replacement) and make
// use of the fact that the shell concatenates adjacent strings.
func shellQuote(s string) string {
t := strings.Replace(s, "'", `'\''`, -1)
return "'" + t + "'"
}
runner/libvirt/testdata/configs/frr/leaf0 0 → 100644
+ 33
− 0
View file @ c38be1f5
frr defaults datacenter
hostname spine0
username cumulus nopassword
!
service integrated-vtysh-config
!
log syslog informational
!
line vty
!
router bgp 65100
bgp router-id 10.42.42.100
bgp bestpath as-path multipath-relax
neighbor fabric peer-group
neighbor fabric remote-as external
neighbor fabric bfd
neighbor swp1 interface peer-group fabric
neighbor swp2 interface peer-group fabric
!
address-family ipv4 unicast
redistribute connected
exit-address-family
!
address-family ipv6 unicast
redistribute connected
neighbor fabric activate
exit-address-family
!
interface vlan100
no ipv6 nd suppress-ra
!
runner/libvirt/testdata/configs/frr/leaf1 0 → 100644
+ 33
− 0
View file @ c38be1f5
frr defaults datacenter
hostname spine0
username cumulus nopassword
!
service integrated-vtysh-config
!
log syslog informational
!
line vty
!
router bgp 65110
bgp router-id 10.42.42.110
bgp bestpath as-path multipath-relax
neighbor fabric peer-group
neighbor fabric remote-as external
neighbor fabric bfd
neighbor swp1 interface peer-group fabric
neighbor swp2 interface peer-group fabric
!
address-family ipv4 unicast
redistribute connected
exit-address-family
!
address-family ipv6 unicast
redistribute connected
neighbor fabric activate
exit-address-family
!
interface vlan101
no ipv6 nd suppress-ra
!
runner/libvirt/testdata/configs/frr/leaf2 0 → 100644
+ 33
− 0
View file @ c38be1f5
frr defaults datacenter
hostname spine0
username cumulus nopassword
!
service integrated-vtysh-config
!
log syslog informational
!
line vty
!
router bgp 65120
bgp router-id 10.42.42.120
bgp bestpath as-path multipath-relax
neighbor fabric peer-group
neighbor fabric remote-as external
neighbor fabric bfd
neighbor swp1 interface peer-group fabric
neighbor swp2 interface peer-group fabric
!
address-family ipv4 unicast
redistribute connected
exit-address-family
!
address-family ipv6 unicast
redistribute connected
neighbor fabric activate
exit-address-family
!
interface vlan102
no ipv6 nd suppress-ra
!
runner/libvirt/testdata/configs/frr/spine0 0 → 100644
+ 30
− 0
View file @ c38be1f5
frr defaults datacenter
hostname spine0
username cumulus nopassword
!
service integrated-vtysh-config
!
log syslog informational
!
line vty
!
router bgp 65200
bgp router-id 10.42.42.200
bgp bestpath as-path multipath-relax
neighbor fabric peer-group
neighbor fabric remote-as external
neighbor fabric bfd
neighbor swp1 interface peer-group fabric
neighbor swp2 interface peer-group fabric
neighbor swp3 interface peer-group fabric
!
address-family ipv4 unicast
redistribute connected
exit-address-family
!
address-family ipv6 unicast
redistribute connected
neighbor fabric activate
exit-address-family
!
runner/libvirt/testdata/configs/frr/spine1 0 → 100644
+ 30
− 0
View file @ c38be1f5
frr defaults datacenter
hostname spine0
username cumulus nopassword
!
service integrated-vtysh-config
!
log syslog informational
!
line vty
!
router bgp 65200
bgp router-id 10.42.42.201
bgp bestpath as-path multipath-relax
neighbor fabric peer-group
neighbor fabric remote-as external
neighbor fabric bfd
neighbor swp1 interface peer-group fabric
neighbor swp2 interface peer-group fabric
neighbor swp3 interface peer-group fabric
!
address-family ipv4 unicast
redistribute connected
exit-address-family
!
address-family ipv6 unicast
redistribute connected
neighbor fabric activate
exit-address-family
!
runner/libvirt/testdata/configs/interfaces/leaf0 0 → 100644
+ 39
− 0
View file @ c38be1f5
auto lo
iface lo inet loopback
address 10.42.42.100/32
address fd4c:3138:80cc::100/128
auto mgmt
iface mgmt
vrf-table auto
address 127.0.0.1/8
address ::1/128
auto eth0
iface eth0 inet dhcp
vrf mgmt
auto swp1
iface swp1
auto swp2
iface swp2
auto swp3
iface swp3
bridge-access 100
mstpctl-bpduguard yes
mstpctl-portadminedge yes
auto bridge
iface bridge
bridge-ports swp3
bridge-vids 100
bridge-vlan-aware yes
auto vlan100
iface vlan100
address 10.42.100.1/24
address fd4c:3138:80cc:64::1/64
vlan-id 100
vlan-raw-device bridge
runner/libvirt/testdata/configs/interfaces/leaf1 0 → 100644
+ 39
− 0
View file @ c38be1f5
auto lo
iface lo inet loopback
address 10.42.42.110/32
address fd4c:3138:80cc::110/128
auto mgmt
iface mgmt
vrf-table auto
address 127.0.0.1/8
address ::1/128
auto eth0
iface eth0 inet dhcp
vrf mgmt
auto swp1
iface swp1
auto swp2
iface swp2
auto swp3
iface swp3
bridge-access 101
mstpctl-bpduguard yes
mstpctl-portadminedge yes
auto bridge
iface bridge
bridge-ports swp3
bridge-vids 101
bridge-vlan-aware yes
auto vlan101
iface vlan101
address 10.42.101.1/24
address fd4c:3138:80cc:65::1/64
vlan-id 101
vlan-raw-device bridge
runner/libvirt/testdata/configs/interfaces/leaf2 0 → 100644
+ 39
− 0
View file @ c38be1f5
auto lo
iface lo inet loopback
address 10.42.42.120/32
address fd4c:3138:80cc::120/128
auto mgmt
iface mgmt
vrf-table auto
address 127.0.0.1/8
address ::1/128
auto eth0
iface eth0 inet dhcp
vrf mgmt
auto swp1
iface swp1
auto swp2
iface swp2
auto swp3
iface swp3
bridge-access 102
mstpctl-bpduguard yes
mstpctl-portadminedge yes
auto bridge
iface bridge
bridge-ports swp3
bridge-vids 102
bridge-vlan-aware yes
auto vlan102
iface vlan102
address 10.42.102.1/24
address fd4c:3138:80cc:66::1/64
vlan-id 102
vlan-raw-device bridge
runner/libvirt/testdata/configs/interfaces/spine0 0 → 100644
+ 23
− 0
View file @ c38be1f5
auto lo
iface lo inet loopback
address 10.42.42.200/32
address fd4c:3138:80cc::200/128
auto mgmt
iface mgmt
vrf-table auto
address 127.0.0.1/8
address ::1/128
auto eth0
iface eth0 inet dhcp
vrf mgmt
auto swp1
iface swp1
auto swp2
iface swp2
auto swp3
iface swp3
runner/libvirt/testdata/configs/interfaces/spine1 0 → 100644
+ 23
− 0
View file @ c38be1f5
auto lo
iface lo inet loopback
address 10.42.42.201/32
address fd4c:3138:80cc::201/128
auto mgmt
iface mgmt
vrf-table auto
address 127.0.0.1/8
address ::1/128
auto eth0
iface eth0 inet dhcp
vrf mgmt
auto swp1
iface swp1
auto swp2
iface swp2
auto swp3
iface swp3
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment