diff --git a/runner/libvirt/helper.go b/runner/libvirt/helper.go index a6ca134caf433fa67bd975e00de1b5d2c203b372..8295e8a40570079ad27399e177b56572052395e0 100644 --- a/runner/libvirt/helper.go +++ b/runner/libvirt/helper.go @@ -262,9 +262,13 @@ func isASCIIDigit(c rune) bool { return '0' <= c && c <= '9' } +func hasFunction(d *device, fs ...topology.DeviceFunction) bool { + return topology.HasFunction(&d.topoDev, fs...) +} + // Returns whether d defaults to Cumulus Linux. func hasCumulusFunction(d *device) bool { - return topology.HasFunction(&d.topoDev, + return hasFunction(d, topology.OOBSwitch, topology.Exit, topology.SuperSpine, diff --git a/runner/libvirt/runner_test.go b/runner/libvirt/runner_test.go index 1ded3a4588d47f85a751948a13710631aafafd76..3a5669b5fd95e19b730af3e0e54847938f7d36fc 100644 --- a/runner/libvirt/runner_test.go +++ b/runner/libvirt/runner_test.go @@ -3,22 +3,27 @@ package libvirt import ( "bytes" "context" - "crypto/ed25519" "crypto/rand" + "encoding/json" + "errors" "fmt" - "io" "net" "os" - "strings" + "path/filepath" "testing" "time" - "github.com/pkg/sftp" - "go4.org/writerutil" "golang.org/x/crypto/ssh" "slrz.net/runtopo/topology" ) +type ptmDetail struct { + Port string `json:"port"` + Status string `json:"cbl status"` + ActualNeighbor string `json:"act nbr"` + ExpectedNeighbor string `json:"exp nbr"` +} + func TestRuntopo(t *testing.T) { if testing.Short() { t.SkipNow() @@ -86,36 +91,147 @@ func TestRuntopo(t *testing.T) { } defer oob.Close() - for hostname, d := range r.devices { - if d.topoDev.Function() != topology.Host { + // Upload configuration for network devices (frr.conf and interfaces) + for hostname := range r.devices { + var files [2][]byte + sources := []string{ + filepath.Join("testdata/configs/interfaces", hostname), + filepath.Join("testdata/configs/frr", hostname), + } + + for i, src := range sources { + p, err := os.ReadFile(src) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + t.Fatal(err) + } + files[i] = p + } + + if files[0] == nil && files[1] == nil { continue } - var fileData []byte - err := withBackoff(nretries, func() error { + + interfaces, frrConf := files[0], files[1] + err = withBackoff(nretries, func() error { c, err := proxyJump(oob, hostname, sshConfig) if err != nil { return err } defer c.Close() - p, err := sftpGet(c, "/kilroywashere") - if err != nil { - return err + if len(interfaces) > 0 { + err := sftpPut(c, "/etc/network/interfaces", + interfaces) + if err != nil { + return err + } + } + if len(frrConf) > 0 { + return sftpPut(c, "/etc/frr/frr.conf", frrConf) } - - fileData = p return nil }) if err != nil { - t.Errorf("%s: %v (giving up after %d retries)", - hostname, err, nretries) - continue + t.Fatal(err) } - if !bytes.Equal(fileData, []byte("abcdef\n")) { - t.Errorf("%s: unexpected file content: got %q, want %q", - hostname, fileData, "abcdef\n") + + err = withBackoff(nretries, func() error { + c, err := proxyJump(oob, hostname, sshConfig) + if err != nil { + return err + } + defer c.Close() + commands := [][]string{ + {"sed", "-i", "s/^bgpd=no/bgpd=yes/", "/etc/frr/daemons"}, + {"ifreload", "-a"}, + {"systemctl", "restart", "frr.service"}, + } + for _, argv := range commands { + _, err := runCommand(c, argv[0], argv[1:]...) + if err != nil { + return err + } + } + t.Logf("=== %s ===", hostname) + p, err := runCommand(c, "net", "show", "int") + t.Logf("%s\n===", p) + return err + }) + if err != nil { + t.Fatal(err) } } + + t.Run("config-nodeattr", func(t *testing.T) { + for hostname, d := range r.devices { + if !hasFunction(d, topology.Host) { + continue + } + var fileData []byte + err := withBackoff(nretries, func() error { + c, err := proxyJump(oob, hostname, sshConfig) + if err != nil { + return err + } + defer c.Close() + + p, err := sftpGet(c, "/kilroywashere") + if err != nil { + return err + } + + fileData = p + return nil + }) + if err != nil { + t.Errorf("%s: %v (giving up after %d retries)", + hostname, err, nretries) + continue + } + if !bytes.Equal(fileData, []byte("abcdef\n")) { + t.Errorf("%s: unexpected file content: got %q, want %q", + hostname, fileData, "abcdef\n") + } + } + }) + t.Run("ptm-topology", func(t *testing.T) { + for hostname, d := range r.devices { + if !hasFunction(d, topology.Spine, topology.Leaf) { + continue + } + err := withBackoff(nretries, func() error { + c, err := proxyJump(oob, hostname, sshConfig) + if err != nil { + return err + } + defer c.Close() + + p, err := runCommand(c, "ptmctl", "--json", "--detail") + if err != nil { + return err + } + // Ptmctl gives us a JSON object with numeric + // string indices: {"0": {}, "1": {}, ...}. + ptm := make(map[string]*ptmDetail) + if err := json.Unmarshal(p, &ptm); err != nil { + return err + } + for _, v := range ptm { + if v.Status != "pass" { + return fmt.Errorf("%s: got %s, want %s", + v.Port, v.ActualNeighbor, v.ExpectedNeighbor) + } + } + return nil + }) + if err != nil { + t.Fatalf("%s: %v", hostname, err) + } + } + }) } func withBackoff(attempts int, f func() error) (err error) { @@ -147,130 +263,3 @@ func minInt64(a, b int64) int64 { } return a } - -func proxyJump(c *ssh.Client, addr string, config *ssh.ClientConfig) (cc *ssh.Client, err error) { - defer func() { - if err != nil { - err = fmt.Errorf("proxyJump %s: %w", addr, err) - } - }() - - conn, err := c.Dial("tcp", net.JoinHostPort(addr, "22")) - if err != nil { - return nil, err - } - sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config) - if err != nil { - conn.Close() - return nil, err - } - - return ssh.NewClient(sshConn, chans, reqs), nil -} - -func runCommand(c *ssh.Client, name string, args ...string) ([]byte, error) { - var b strings.Builder - - b.WriteString(shellQuote(name)) - for _, a := range args { - b.WriteByte(' ') - b.WriteString(shellQuote(a)) - } - cmd := b.String() - - sess, err := c.NewSession() - if err != nil { - return nil, err - } - defer sess.Close() - - var stdout bytes.Buffer - stderr := &writerutil.PrefixSuffixSaver{N: 1024} - sess.Stdout = &stdout - sess.Stderr = stderr - - if err := sess.Run(cmd); err != nil { - if msg := stderr.Bytes(); len(msg) > 0 { - return nil, fmt.Errorf("runCommand: %w | %s |", err, msg) - } - return nil, fmt.Errorf("runCommand: %w", err) - } - - return stdout.Bytes(), nil -} - -func sftpGet(conn *ssh.Client, path string) (content []byte, err error) { - c, err := sftp.NewClient(conn) - if err != nil { - return nil, err - } - defer c.Close() - - fd, err := c.Open(path) - if err != nil { - return nil, err - } - defer fd.Close() - - return io.ReadAll(fd) -} - -func sftpPutReader(conn *ssh.Client, dstPath string, src io.Reader) (err error) { - c, err := sftp.NewClient(conn) - if err != nil { - return err - } - defer c.Close() - - fd, err := c.Create(dstPath) - if err != nil { - return err - } - defer func() { - if cerr := fd.Close(); err == nil { - err = cerr - } - }() - - _, err = io.Copy(fd, src) - return err -} - -func sftpPut(conn *ssh.Client, dstPath string, content []byte) (err error) { - return sftpPutReader(conn, dstPath, bytes.NewReader(content)) -} - -func sshKeygen(rand io.Reader) (ssh.Signer, []byte, error) { - _, sk, err := ed25519.GenerateKey(rand) - if err != nil { - return nil, nil, err - } - - signer, err := ssh.NewSignerFromSigner(sk) - if err != nil { - return nil, nil, err - } - - sshPubKey := ssh.MarshalAuthorizedKey(signer.PublicKey()) - - return signer, sshPubKey, nil -} - -func mustReadFile(path string) []byte { - p, err := os.ReadFile(path) - if err != nil { - panic(err) - } - return p -} - -// ShellQuote returns s in a form suitable to pass it to the shell as an -// argument. Obviously, it works for Bourne-like shells only. The way this -// works is that first the whole string is enclosed in single quotes. Now the -// only character that needs special handling is the single quote itself. We -// replace it by '\'' (the outer quotes are part of the replacement) and make -// use of the fact that the shell concatenates adjacent strings. -func shellQuote(s string) string { - t := strings.Replace(s, "'", `'\''`, -1) - return "'" + t + "'" -} diff --git a/runner/libvirt/sshutil_test.go b/runner/libvirt/sshutil_test.go new file mode 100644 index 0000000000000000000000000000000000000000..c148a77f9661f18aa4a3c992a0f354811baac385 --- /dev/null +++ b/runner/libvirt/sshutil_test.go @@ -0,0 +1,135 @@ +package libvirt + +// SSH helpers used in tests. + +import ( + "bytes" + "crypto/ed25519" + "fmt" + "io" + "net" + "strings" + + "github.com/pkg/sftp" + "go4.org/writerutil" + "golang.org/x/crypto/ssh" +) + +func proxyJump(c *ssh.Client, addr string, config *ssh.ClientConfig) (cc *ssh.Client, err error) { + defer func() { + if err != nil { + err = fmt.Errorf("proxyJump %s: %w", addr, err) + } + }() + + conn, err := c.Dial("tcp", net.JoinHostPort(addr, "22")) + if err != nil { + return nil, err + } + sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config) + if err != nil { + conn.Close() + return nil, err + } + + return ssh.NewClient(sshConn, chans, reqs), nil +} + +func runCommand(c *ssh.Client, name string, args ...string) ([]byte, error) { + var b strings.Builder + + b.WriteString(shellQuote(name)) + for _, a := range args { + b.WriteByte(' ') + b.WriteString(shellQuote(a)) + } + cmd := b.String() + + sess, err := c.NewSession() + if err != nil { + return nil, err + } + defer sess.Close() + + var stdout bytes.Buffer + stderr := &writerutil.PrefixSuffixSaver{N: 1024} + sess.Stdout = &stdout + sess.Stderr = stderr + + if err := sess.Run(cmd); err != nil { + if msg := stderr.Bytes(); len(msg) > 0 { + return nil, fmt.Errorf("runCommand: %w | %s |", err, msg) + } + return nil, fmt.Errorf("runCommand: %w", err) + } + + return stdout.Bytes(), nil +} + +func sftpGet(conn *ssh.Client, path string) (content []byte, err error) { + c, err := sftp.NewClient(conn) + if err != nil { + return nil, err + } + defer c.Close() + + fd, err := c.Open(path) + if err != nil { + return nil, err + } + defer fd.Close() + + return io.ReadAll(fd) +} + +func sftpPutReader(conn *ssh.Client, dstPath string, src io.Reader) (err error) { + c, err := sftp.NewClient(conn) + if err != nil { + return err + } + defer c.Close() + + fd, err := c.Create(dstPath) + if err != nil { + return err + } + defer func() { + if cerr := fd.Close(); err == nil { + err = cerr + } + }() + + _, err = io.Copy(fd, src) + return err +} + +func sftpPut(conn *ssh.Client, dstPath string, content []byte) (err error) { + return sftpPutReader(conn, dstPath, bytes.NewReader(content)) +} + +func sshKeygen(rand io.Reader) (ssh.Signer, []byte, error) { + _, sk, err := ed25519.GenerateKey(rand) + if err != nil { + return nil, nil, err + } + + signer, err := ssh.NewSignerFromSigner(sk) + if err != nil { + return nil, nil, err + } + + sshPubKey := ssh.MarshalAuthorizedKey(signer.PublicKey()) + + return signer, sshPubKey, nil +} + +// ShellQuote returns s in a form suitable to pass it to the shell as an +// argument. Obviously, it works for Bourne-like shells only. The way this +// works is that first the whole string is enclosed in single quotes. Now the +// only character that needs special handling is the single quote itself. We +// replace it by '\'' (the outer quotes are part of the replacement) and make +// use of the fact that the shell concatenates adjacent strings. +func shellQuote(s string) string { + t := strings.Replace(s, "'", `'\''`, -1) + return "'" + t + "'" +} diff --git a/runner/libvirt/testdata/configs/frr/leaf0 b/runner/libvirt/testdata/configs/frr/leaf0 new file mode 100644 index 0000000000000000000000000000000000000000..03699a067921f5ca8698f156429e5310783058d4 --- /dev/null +++ b/runner/libvirt/testdata/configs/frr/leaf0 @@ -0,0 +1,33 @@ +frr defaults datacenter +hostname spine0 +username cumulus nopassword +! +service integrated-vtysh-config +! +log syslog informational +! +line vty +! + +router bgp 65100 + bgp router-id 10.42.42.100 + bgp bestpath as-path multipath-relax + neighbor fabric peer-group + neighbor fabric remote-as external + neighbor fabric bfd + neighbor swp1 interface peer-group fabric + neighbor swp2 interface peer-group fabric + ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + neighbor fabric activate + exit-address-family +! + +interface vlan100 + no ipv6 nd suppress-ra +! diff --git a/runner/libvirt/testdata/configs/frr/leaf1 b/runner/libvirt/testdata/configs/frr/leaf1 new file mode 100644 index 0000000000000000000000000000000000000000..34343e057ccae9aca37981e6c8fa07807f788a16 --- /dev/null +++ b/runner/libvirt/testdata/configs/frr/leaf1 @@ -0,0 +1,33 @@ +frr defaults datacenter +hostname spine0 +username cumulus nopassword +! +service integrated-vtysh-config +! +log syslog informational +! +line vty +! + +router bgp 65110 + bgp router-id 10.42.42.110 + bgp bestpath as-path multipath-relax + neighbor fabric peer-group + neighbor fabric remote-as external + neighbor fabric bfd + neighbor swp1 interface peer-group fabric + neighbor swp2 interface peer-group fabric + ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + neighbor fabric activate + exit-address-family +! + +interface vlan101 + no ipv6 nd suppress-ra +! diff --git a/runner/libvirt/testdata/configs/frr/leaf2 b/runner/libvirt/testdata/configs/frr/leaf2 new file mode 100644 index 0000000000000000000000000000000000000000..eb3148ee7299df3dacf3d72be295324aae6b48a0 --- /dev/null +++ b/runner/libvirt/testdata/configs/frr/leaf2 @@ -0,0 +1,33 @@ +frr defaults datacenter +hostname spine0 +username cumulus nopassword +! +service integrated-vtysh-config +! +log syslog informational +! +line vty +! + +router bgp 65120 + bgp router-id 10.42.42.120 + bgp bestpath as-path multipath-relax + neighbor fabric peer-group + neighbor fabric remote-as external + neighbor fabric bfd + neighbor swp1 interface peer-group fabric + neighbor swp2 interface peer-group fabric + ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + neighbor fabric activate + exit-address-family +! + +interface vlan102 + no ipv6 nd suppress-ra +! diff --git a/runner/libvirt/testdata/configs/frr/spine0 b/runner/libvirt/testdata/configs/frr/spine0 new file mode 100644 index 0000000000000000000000000000000000000000..bdb66c7359d63d56692da363621f7fea802ae492 --- /dev/null +++ b/runner/libvirt/testdata/configs/frr/spine0 @@ -0,0 +1,30 @@ +frr defaults datacenter +hostname spine0 +username cumulus nopassword +! +service integrated-vtysh-config +! +log syslog informational +! +line vty +! + +router bgp 65200 + bgp router-id 10.42.42.200 + bgp bestpath as-path multipath-relax + neighbor fabric peer-group + neighbor fabric remote-as external + neighbor fabric bfd + neighbor swp1 interface peer-group fabric + neighbor swp2 interface peer-group fabric + neighbor swp3 interface peer-group fabric + ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + neighbor fabric activate + exit-address-family +! diff --git a/runner/libvirt/testdata/configs/frr/spine1 b/runner/libvirt/testdata/configs/frr/spine1 new file mode 100644 index 0000000000000000000000000000000000000000..4ff7be239226cd9c711a295302d0f0c36d049baf --- /dev/null +++ b/runner/libvirt/testdata/configs/frr/spine1 @@ -0,0 +1,30 @@ +frr defaults datacenter +hostname spine0 +username cumulus nopassword +! +service integrated-vtysh-config +! +log syslog informational +! +line vty +! + +router bgp 65200 + bgp router-id 10.42.42.201 + bgp bestpath as-path multipath-relax + neighbor fabric peer-group + neighbor fabric remote-as external + neighbor fabric bfd + neighbor swp1 interface peer-group fabric + neighbor swp2 interface peer-group fabric + neighbor swp3 interface peer-group fabric + ! + address-family ipv4 unicast + redistribute connected + exit-address-family + ! + address-family ipv6 unicast + redistribute connected + neighbor fabric activate + exit-address-family +! diff --git a/runner/libvirt/testdata/configs/interfaces/leaf0 b/runner/libvirt/testdata/configs/interfaces/leaf0 new file mode 100644 index 0000000000000000000000000000000000000000..4a9a5774511b1575e8e23dbd4bfafc5b8c8b8687 --- /dev/null +++ b/runner/libvirt/testdata/configs/interfaces/leaf0 @@ -0,0 +1,39 @@ +auto lo +iface lo inet loopback + address 10.42.42.100/32 + address fd4c:3138:80cc::100/128 + +auto mgmt +iface mgmt + vrf-table auto + address 127.0.0.1/8 + address ::1/128 + +auto eth0 +iface eth0 inet dhcp + vrf mgmt + +auto swp1 +iface swp1 + +auto swp2 +iface swp2 + +auto swp3 +iface swp3 + bridge-access 100 + mstpctl-bpduguard yes + mstpctl-portadminedge yes + +auto bridge +iface bridge + bridge-ports swp3 + bridge-vids 100 + bridge-vlan-aware yes + +auto vlan100 +iface vlan100 + address 10.42.100.1/24 + address fd4c:3138:80cc:64::1/64 + vlan-id 100 + vlan-raw-device bridge diff --git a/runner/libvirt/testdata/configs/interfaces/leaf1 b/runner/libvirt/testdata/configs/interfaces/leaf1 new file mode 100644 index 0000000000000000000000000000000000000000..517289de2cdd4f5216e753c3524c799c1c339a3a --- /dev/null +++ b/runner/libvirt/testdata/configs/interfaces/leaf1 @@ -0,0 +1,39 @@ +auto lo +iface lo inet loopback + address 10.42.42.110/32 + address fd4c:3138:80cc::110/128 + +auto mgmt +iface mgmt + vrf-table auto + address 127.0.0.1/8 + address ::1/128 + +auto eth0 +iface eth0 inet dhcp + vrf mgmt + +auto swp1 +iface swp1 + +auto swp2 +iface swp2 + +auto swp3 +iface swp3 + bridge-access 101 + mstpctl-bpduguard yes + mstpctl-portadminedge yes + +auto bridge +iface bridge + bridge-ports swp3 + bridge-vids 101 + bridge-vlan-aware yes + +auto vlan101 +iface vlan101 + address 10.42.101.1/24 + address fd4c:3138:80cc:65::1/64 + vlan-id 101 + vlan-raw-device bridge diff --git a/runner/libvirt/testdata/configs/interfaces/leaf2 b/runner/libvirt/testdata/configs/interfaces/leaf2 new file mode 100644 index 0000000000000000000000000000000000000000..ce7ea41b8cc87ef8d035df6c450ec01bce7647f2 --- /dev/null +++ b/runner/libvirt/testdata/configs/interfaces/leaf2 @@ -0,0 +1,39 @@ +auto lo +iface lo inet loopback + address 10.42.42.120/32 + address fd4c:3138:80cc::120/128 + +auto mgmt +iface mgmt + vrf-table auto + address 127.0.0.1/8 + address ::1/128 + +auto eth0 +iface eth0 inet dhcp + vrf mgmt + +auto swp1 +iface swp1 + +auto swp2 +iface swp2 + +auto swp3 +iface swp3 + bridge-access 102 + mstpctl-bpduguard yes + mstpctl-portadminedge yes + +auto bridge +iface bridge + bridge-ports swp3 + bridge-vids 102 + bridge-vlan-aware yes + +auto vlan102 +iface vlan102 + address 10.42.102.1/24 + address fd4c:3138:80cc:66::1/64 + vlan-id 102 + vlan-raw-device bridge diff --git a/runner/libvirt/testdata/configs/interfaces/spine0 b/runner/libvirt/testdata/configs/interfaces/spine0 new file mode 100644 index 0000000000000000000000000000000000000000..878d8d9553b76067a9cefd26a6b12638f743cc3d --- /dev/null +++ b/runner/libvirt/testdata/configs/interfaces/spine0 @@ -0,0 +1,23 @@ +auto lo +iface lo inet loopback + address 10.42.42.200/32 + address fd4c:3138:80cc::200/128 + +auto mgmt +iface mgmt + vrf-table auto + address 127.0.0.1/8 + address ::1/128 + +auto eth0 +iface eth0 inet dhcp + vrf mgmt + +auto swp1 +iface swp1 + +auto swp2 +iface swp2 + +auto swp3 +iface swp3 diff --git a/runner/libvirt/testdata/configs/interfaces/spine1 b/runner/libvirt/testdata/configs/interfaces/spine1 new file mode 100644 index 0000000000000000000000000000000000000000..36e77081684d71c589564ba4bb5f9fb737d286cc --- /dev/null +++ b/runner/libvirt/testdata/configs/interfaces/spine1 @@ -0,0 +1,23 @@ +auto lo +iface lo inet loopback + address 10.42.42.201/32 + address fd4c:3138:80cc::201/128 + +auto mgmt +iface mgmt + vrf-table auto + address 127.0.0.1/8 + address ::1/128 + +auto eth0 +iface eth0 inet dhcp + vrf mgmt + +auto swp1 +iface swp1 + +auto swp2 +iface swp2 + +auto swp3 +iface swp3