package libvirt import ( "bytes" "context" "fmt" "io" "net" "os" "os/exec" "strings" "golang.org/x/crypto/bcrypt" "inet.af/netaddr" "slrz.net/runtopo/topology" ) func customizeDomain(ctx context.Context, uri string, d *device, extraCommands io.Reader) (err error) { defer func() { if err != nil { err = fmt.Errorf("customizeDomain %s: %w", d.name, err) } }() if extraCommands == nil { extraCommands = eofReader{} } rules, err := renderUdevRules(d) if err != nil { return err } cmd := exec.CommandContext(ctx, "virt-customize", "-q", "-d", d.name, "-c", uri, "--hostname", d.Name, "--timezone", "Etc/UTC", // This rename script basically does s/eth/swp/ and breaks // proper interface naming using udev rules. Delete it. "--delete", "/etc/hw_init.d/S10rename_eth_swp.sh", "--write", "/etc/udev/rules.d/70-persistent-net.rules:"+string(rules), "--commands-from-file", "/dev/stdin", ) commands := []io.Reader{extraCommands} if len(d.config) > 0 { file, err := writeTempFile("", d.name+"-config", d.config) if err != nil { return err } defer os.Remove(file) commands = append(commands, strings.NewReader("run "+file+"\n")) } cmd.Stdin = io.MultiReader(append(commands, // As the commands returned from commandsForFunction may // contain selinux-relabel, they need to come last, after any // other operation touching the guest file system. The commands // are basically required for proper functioning anyway, so // there's not much of an use case for overriding them. bytes.NewReader(commandsForFunction(d)))..., ) out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("%w (stderr: %s)", err, out) } return nil } func commandsForFunction(d *device) []byte { var buf bytes.Buffer if hasCumulusFunction(d) { // These eat enough memory to summon the OOM killer in 512MiB // VMs. buf.WriteString("run-command systemctl disable netq-agent.service\n") buf.WriteString("run-command systemctl disable netqd@mgmt.service\n") buf.WriteString("run-command passwd -x 99999 cumulus\n") // CL4+ buf.WriteString("write /etc/sudoers.d/no-passwd:%sudo ALL=(ALL:ALL) NOPASSWD: ALL\n") // Set password for user cumulus to some random string. // Otherwise, CL4+ forces a password change on first login. cryptPW, err := bcrypt.GenerateFromPassword([]byte(randomString(16)), -1) if err != nil { panic(err) // something is very wrong if this happens } fmt.Fprintf(&buf, "run-command usermod -p %s cumulus\n", cryptPW) // libguestfs (1.44) thinks it doesn't know how to set // hostnames for CL. Work around by directly writing to // /etc/hostname. fmt.Fprintf(&buf, "write /etc/hostname:%s\\\n\n", d.Name) if d.Function() == topology.OOBSwitch { writeExtraMgmtSwitchCommands(&buf, d) } return buf.Bytes() } var cloudInitUnits = []string{ "cloud-init.service", "cloud-init-local.service", "cloud-config.service", "cloud-final.service", } // We use cloud images but don't provide the VMs with any cloud init // configuration source. Disable cloud-init or it will block the boot. for _, u := range cloudInitUnits { buf.WriteString("run-command systemctl disable " + u + "\n") } buf.WriteString("install lldpd\n") buf.WriteString("run-command systemctl enable lldpd.service\n") // Make lldpd emit the interface name instead of the MAC address. It's // what we have in the topology file. buf.WriteString("write /etc/lldpd.d/ifname.conf:configure lldp portidsubtype ifname\\\n\n") if d.Function() == topology.OOBServer { writeExtraMgmtServerCommands(&buf, d) } // Only required for SELinux-enabled systems (mostly Fedora/EL) buf.WriteString("selinux-relabel\n") return buf.Bytes() } func writeExtraMgmtSwitchCommands(w io.Writer, d *device) { var bridgePorts []string for _, intf := range d.interfaces { if intf.name == "eth0" { // skip mgmt interface continue } bridgePorts = append(bridgePorts, intf.name) } bridgeConf := "auto bridge\niface bridge\n bridge-ports " + strings.Join(bridgePorts, " ") + "\n" // From virt-customize(1): […] arguments can be spread across multiple // lines, by adding a "\" (continuation character) at the of a line […] io.WriteString(w, "write /etc/network/interfaces.d/bridge.intf:"+ strings.Replace(bridgeConf, "\n", "\\\n", -1)+"\n") } const ( nftablesRuleset = ` table ip nat { chain postrouting { type nat hook postrouting priority srcnat; policy accept; masquerade } } ` dnsmasqConf = ` strict-order interface=eth1 dhcp-range=%s,static dhcp-no-override dhcp-authoritative dhcp-hostsfile=/etc/dnsmasq.hostsfile ` ifcfgEth1 = ` TYPE=Ethernet DEVICE=eth1 ONBOOT=yes BOOTPROTO=none IPADDR=%s PREFIX=%d ` ) func writeExtraMgmtServerCommands(w io.Writer, d *device) { io.WriteString(w, "install nftables,dnsmasq\n") // We assume that the prefix has already been validated. p := netaddr.MustParseIPPrefix(d.Attr("mgmt_ip")) io.WriteString(w, "write /etc/sysconfig/network-scripts/ifcfg-eth0:"+ "TYPE=Ethernet\\\nDEVICE=eth0\\\nPEERDNS=yes\\\nBOOTPROTO=dhcp\\\nONBOOT=yes\n") io.WriteString(w, "write /etc/sysconfig/network-scripts/ifcfg-eth1:"+ strings.Replace( fmt.Sprintf(ifcfgEth1, p.IP, p.Bits), "\n", "\\\n", -1, )+"\n", ) io.WriteString(w, "write /etc/sysconfig/nftables.conf:"+ strings.Replace(nftablesRuleset, "\n", "\\\n", -1)+"\n") io.WriteString(w, "run-command systemctl enable nftables.service\n") io.WriteString(w, "write /etc/sysctl.d/98-ipfwd.conf:net.ipv4.ip_forward=1\n") io.WriteString(w, "write /etc/dnsmasq.conf:"+ strings.Replace(fmt.Sprintf(dnsmasqConf, p.Masked().IP), "\n", "\\\n", -1)+"\n") io.WriteString(w, "run-command systemctl disable systemd-resolved.service\n") // Ensure /etc/resolv.conf is a regular file (and not a symlink to // systemd-resolved's stub-resolv.conf). Dnsmasq reads its upstream // resolvers from resolv.conf and we need NM to write the ones received // from DHCP there. io.WriteString(w, "delete /etc/resolv.conf\n") io.WriteString(w, "write /etc/resolv.conf:#placeholder\n") io.WriteString(w, "run-command systemctl enable dnsmasq.service\n") } type etherHost struct { name string ip *net.IPAddr mac net.HardwareAddr } func gatherHosts(ctx context.Context, r *Runner, t *topology.T) []etherHost { var hosts []etherHost for name, d := range r.devices { if name == "oob-mgmt-server" || name == "oob-mgmt-switch" { continue } eth0 := d.interfaces[0] if eth0.name != "eth0" { // most likely, device does not have a mgmt interface continue } mgmtIP := d.MgmtIP() if mgmtIP == nil { continue } hosts = append(hosts, etherHost{ name: name, ip: mgmtIP, mac: eth0.mac, }) } return hosts } func generateDnsmasqHostsFile(hosts []etherHost) []byte { var buf bytes.Buffer for _, h := range hosts { fmt.Fprintf(&buf, "%s,%s,%s\n", h.mac, h.ip, h.name) } return buf.Bytes() }