test: add support for PXE nodes in qemu provision library

This isn't supposed to be used ever in Talos directly, but rather only
in integration tests for Sidero.

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

cmd/talosctl/cmd/mgmt/config.go

@@ -115,7 +115,7 @@ func genV1Alpha1Config(args []string) error {
 
 		var configString string
 
-		switch t {
+		switch t { //nolint: exhaustive
 		case machine.TypeInit:
 			configString, err = configBundle.Init().String()
 			if err != nil {

cmd/talosctl/cmd/talos/health.go

@@ -41,6 +41,8 @@ func (cluster *clusterNodes) NodesByType(t machine.Type) []string {
 		return cluster.ControlPlaneNodes
 	case machine.TypeJoin:
 		return cluster.WorkerNodes
+	case machine.TypeUnknown:
+		return nil
 	default:
 		panic("unsupported machine type")
 	}

internal/app/machined/internal/server/v1alpha1/v1alpha1_cluster.go

@@ -92,6 +92,8 @@ func (cluster *clusterState) NodesByType(t machine.Type) []string {
 		return cluster.controlPlaneNodes
 	case machine.TypeJoin:
 		return cluster.workerNodes
+	case machine.TypeUnknown:
+		return nil
 	default:
 		panic("unsupported machine type")
 	}

internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go

@@ -641,6 +641,8 @@ func StartAllServices(seq runtime.Sequence, data interface{}) (runtime.TaskExecu
 				&services.Etcd{},
 			)
 		case machine.TypeJoin:
+		case machine.TypeUnknown:
+			return fmt.Errorf("unexpected machine type: %s", r.Config().Machine().Type())
 		}
 
 		system.Services(r).StartAll()

pkg/machinery/config/types/v1alpha1/bundle/bundle.go

@@ -49,7 +49,7 @@ func NewConfigBundle(opts ...Option) (*v1alpha1.ConfigBundle, error) {
 				return bundle, err
 			}
 
-			switch configType {
+			switch configType { //nolint: exhaustive
 			case machine.TypeInit:
 				bundle.InitCfg = unmarshalledConfig
 			case machine.TypeControlPlane:
@@ -96,7 +96,7 @@ func NewConfigBundle(opts ...Option) (*v1alpha1.ConfigBundle, error) {
 			return bundle, err
 		}
 
-		switch configType {
+		switch configType { //nolint: exhaustive
 		case machine.TypeInit:
 			bundle.InitCfg = generatedConfig
 		case machine.TypeControlPlane:

pkg/machinery/config/types/v1alpha1/generate/generate.go

@@ -41,6 +41,8 @@ func Config(t machine.Type, in *Input) (c *v1alpha1.Config, err error) {
 		if c, err = workerUd(in); err != nil {
 			return c, err
 		}
+	case machine.TypeUnknown:
+		fallthrough
 	default:
 		return c, errors.New("failed to determine config type to generate")
 	}

pkg/machinery/config/types/v1alpha1/machine/machine.go

@@ -10,8 +10,10 @@ import "fmt"
 type Type int
 
 const (
+	// TypeUnknown represents undefined node type.
+	TypeUnknown Type = iota
 	// TypeInit represents a bootstrap node.
-	TypeInit Type = iota
+	TypeInit
 	// TypeControlPlane represents a control plane node.
 	TypeControlPlane
 	// TypeJoin represents a worker node.
@@ -19,6 +21,7 @@ const (
 )
 
 const (
+	typeUnknown      = "unknown"
 	typeInit         = "init"
 	typeControlPlane = "controlplane"
 	typeJoin         = "join"
@@ -26,12 +29,14 @@ const (
 
 // String returns the string representation of Type.
 func (t Type) String() string {
-	return [...]string{typeInit, typeControlPlane, typeJoin}[t]
+	return [...]string{typeUnknown, typeInit, typeControlPlane, typeJoin}[t]
 }
 
 // ParseType parses string constant as Type.
 func ParseType(t string) (Type, error) {
 	switch t {
+	case typeUnknown:
+		return TypeUnknown, nil
 	case typeInit:
 		return TypeInit, nil
 	case typeControlPlane:

pkg/provision/providers/qemu/create.go

@@ -85,6 +85,17 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque
 		return nil, err
 	}
 
+	var pxeNodeInfo []provision.NodeInfo
+
+	pxeNodes := request.Nodes.PXENodes()
+	if len(pxeNodes) > 0 {
+		fmt.Fprintln(options.LogWriter, "creating PXE nodes")
+
+		if pxeNodeInfo, err = p.createNodes(state, request, pxeNodes, &options); err != nil {
+			return nil, err
+		}
+	}
+
 	nodeInfo = append(nodeInfo, workerNodeInfo...)
 
 	state.ClusterInfo = provision.ClusterInfo{
@@ -95,7 +106,8 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque
 			GatewayAddr: request.Network.GatewayAddr,
 			MTU:         request.Network.MTU,
 		},
-		Nodes: nodeInfo,
+		Nodes:      nodeInfo,
+		ExtraNodes: pxeNodeInfo,
 	}
 
 	err = state.Save()

pkg/provision/providers/qemu/launch.go

@@ -42,6 +42,7 @@ type LaunchConfig struct {
 	MachineType       string
 	EnableKVM         bool
 	BootloaderEnabled bool
+	NodeUUID          uuid.UUID
 
 	// Talos config
 	Config string
@@ -56,6 +57,11 @@ type LaunchConfig struct {
 	MTU           int
 	Nameservers   []net.IP
 
+	// PXE
+	TFTPServer       string
+	BootFilename     string
+	IPXEBootFileName string
+
 	// filled by CNI invocation
 	tapName string
 	vmMAC   string
@@ -131,13 +137,15 @@ func withCNI(ctx context.Context, config *LaunchConfig, f func(config *LaunchCon
 
 	// dump node IP/mac/hostname for dhcp
 	if err = vm.DumpIPAMRecord(config.StatePath, vm.IPAMRecord{
-		IP:          config.IP,
+		IP:               config.IP,
-		Netmask:     config.CIDR.Mask,
+		Netmask:          config.CIDR.Mask,
-		MAC:         vmIface.Mac,
+		MAC:              vmIface.Mac,
-		Hostname:    config.Hostname,
+		Hostname:         config.Hostname,
-		Gateway:     config.GatewayAddr,
+		Gateway:          config.GatewayAddr,
-		MTU:         config.MTU,
+		MTU:              config.MTU,
-		Nameservers: config.Nameservers,
+		Nameservers:      config.Nameservers,
+		TFTPServer:       config.TFTPServer,
+		IPXEBootFilename: config.IPXEBootFileName,
 	}); err != nil {
 		return err
 	}
@@ -179,6 +187,8 @@ func launchVM(config *LaunchConfig) error {
 		"-device", fmt.Sprintf("virtio-net-pci,netdev=net0,mac=%s", config.vmMAC),
 		"-device", "virtio-rng-pci",
 		"-no-reboot",
+		"-boot", "order=cn,reboot-timeout=5000",
+		"-smbios", fmt.Sprintf("type=1,uuid=%s", config.NodeUUID),
 	}
 
 	machineArg := config.MachineType
@@ -203,7 +213,7 @@ func launchVM(config *LaunchConfig) error {
 		return err
 	}
 
-	if !diskBootable || !config.BootloaderEnabled {
+	if (!diskBootable || !config.BootloaderEnabled) && config.KernelImagePath != "" {
 		args = append(args,
 			"-kernel", config.KernelImagePath,
 			"-initrd", config.InitrdPath,

pkg/provision/providers/qemu/node.go

@@ -16,8 +16,10 @@ import (
 	"strconv"
 	"syscall"
 
+	"github.com/google/uuid"
 	multierror "github.com/hashicorp/go-multierror"
 
+	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
 	"github.com/talos-systems/talos/pkg/provision"
 	"github.com/talos-systems/talos/pkg/provision/providers/vm"
 
@@ -61,23 +63,28 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe
 	cmdline.Append("talos.platform", "metal")
 	cmdline.Append("talos.config", "{TALOS_CONFIG_URL}") // to be patched by launcher
 
-	nodeConfig, err := nodeReq.Config.String()
+	var nodeConfig string
-	if err != nil {
+
-		return provision.NodeInfo{}, err
+	if nodeReq.Config != nil {
+		nodeConfig, err = nodeReq.Config.String()
+		if err != nil {
+			return provision.NodeInfo{}, err
+		}
 	}
 
+	nodeUUID := uuid.New()
+
 	launchConfig := LaunchConfig{
 		QemuExecutable:    fmt.Sprintf("qemu-system-%s", arch.QemuArch()),
 		DiskPath:          diskPath,
 		VCPUCount:         vcpuCount,
 		MemSize:           memSize,
-		KernelImagePath:   clusterReq.KernelPath,
 		KernelArgs:        cmdline.String(),
-		InitrdPath:        clusterReq.InitramfsPath,
 		MachineType:       arch.QemuMachine(),
 		PFlashImages:      state.PFlashImages,
 		EnableKVM:         opts.TargetArch == runtime.GOARCH,
 		BootloaderEnabled: opts.BootloaderEnabled,
+		NodeUUID:          nodeUUID,
 		Config:            nodeConfig,
 		NetworkConfig:     state.VMCNIConfig,
 		CNI:               clusterReq.Network.CNI,
@@ -87,6 +94,13 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe
 		GatewayAddr:       clusterReq.Network.GatewayAddr,
 		MTU:               clusterReq.Network.MTU,
 		Nameservers:       clusterReq.Network.Nameservers,
+		TFTPServer:        nodeReq.TFTPServer,
+		IPXEBootFileName:  nodeReq.IPXEBootFilename,
+	}
+
+	if !nodeReq.PXEBooted {
+		launchConfig.KernelImagePath = clusterReq.KernelPath
+		launchConfig.InitrdPath = clusterReq.InitramfsPath
 	}
 
 	launchConfig.StatePath, err = state.StatePath()
@@ -127,10 +141,16 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe
 
 	// no need to wait here, as cmd has all the Stdin/out/err via *os.File
 
+	nodeType := machine.TypeUnknown
+	if nodeReq.Config != nil {
+		nodeType = nodeReq.Config.Machine().Type()
+	}
+
 	nodeInfo := provision.NodeInfo{
 		ID:   pidPath,
+		UUID: nodeUUID,
 		Name: nodeReq.Name,
-		Type: nodeReq.Config.Machine().Type(),
+		Type: nodeType,
 
 		NanoCPUs: nodeReq.NanoCPUs,
 		Memory:   nodeReq.Memory,

pkg/provision/providers/vm/dhcpd.go

@@ -21,6 +21,7 @@ import (
 	"github.com/talos-systems/talos/pkg/provision"
 )
 
+//nolint: gocyclo
 func handler(serverIP net.IP, statePath string) server4.Handler {
 	return func(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) {
 		if m.OpCode != dhcpv4.OpcodeBootRequest {
@@ -45,7 +46,6 @@ func handler(serverIP net.IP, statePath string) server4.Handler {
 
 		resp, err := dhcpv4.NewReplyFromRequest(m,
 			dhcpv4.WithNetmask(match.Netmask),
-			dhcpv4.WithServerIP(serverIP),
 			dhcpv4.WithYourIP(match.IP),
 			dhcpv4.WithOption(dhcpv4.OptHostName(match.Hostname)),
 			dhcpv4.WithOption(dhcpv4.OptDNS(match.Nameservers...)),
@@ -58,6 +58,18 @@ func handler(serverIP net.IP, statePath string) server4.Handler {
 			return
 		}
 
+		if m.IsOptionRequested(dhcpv4.OptionBootfileName) {
+			log.Printf("received PXE boot request from %s", m.ClientHWAddr)
+
+			if match.TFTPServer != "" {
+				log.Printf("sending PXE response to %s: %s/%s", m.ClientHWAddr, match.TFTPServer, match.IPXEBootFilename)
+
+				resp.ServerIPAddr = net.ParseIP(match.TFTPServer)
+				resp.UpdateOption(dhcpv4.OptTFTPServerName(match.TFTPServer))
+				resp.UpdateOption(dhcpv4.OptBootFileName(match.IPXEBootFilename))
+			}
+		}
+
 		resp.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionInterfaceMTU, dhcpv4.Uint16(match.MTU).ToBytes()))
 
 		switch mt := m.MessageType(); mt { //nolint: exhaustive

pkg/provision/providers/vm/ipam.go

@@ -22,6 +22,9 @@ type IPAMRecord struct {
 	Gateway     net.IP
 	MTU         int
 	Nameservers []net.IP
+
+	TFTPServer       string
+	IPXEBootFilename string
 }
 
 // IPAMDatabase is a mapping from MAC address to records.

pkg/provision/providers/vm/node.go

@@ -16,7 +16,9 @@ import (
 func (p *Provisioner) DestroyNodes(cluster provision.ClusterInfo, options *provision.Options) error {
 	errCh := make(chan error)
 
-	for _, node := range cluster.Nodes {
+	nodes := append(cluster.Nodes, cluster.ExtraNodes...)
+
+	for _, node := range nodes {
 		go func(node provision.NodeInfo) {
 			fmt.Fprintln(options.LogWriter, "stopping VM", node.Name)
 
@@ -26,7 +28,7 @@ func (p *Provisioner) DestroyNodes(cluster provision.ClusterInfo, options *provi
 
 	var multiErr *multierror.Error
 
-	for range cluster.Nodes {
+	for range nodes {
 		multiErr = multierror.Append(multiErr, <-errCh)
 	}
 

pkg/provision/request.go

@@ -57,6 +57,10 @@ func (reqs NodeRequests) FindInitNode() (req NodeRequest, err error) {
 	found := false
 
 	for i := range reqs {
+		if reqs[i].Config == nil {
+			continue
+		}
+
 		if reqs[i].Config.Machine().Type() == machine.TypeInit {
 			if found {
 				err = fmt.Errorf("duplicate init node in requests")
@@ -78,6 +82,10 @@ func (reqs NodeRequests) FindInitNode() (req NodeRequest, err error) {
 // MasterNodes returns subset of nodes which are Init/ControlPlane type.
 func (reqs NodeRequests) MasterNodes() (nodes []NodeRequest) {
 	for i := range reqs {
+		if reqs[i].Config == nil {
+			continue
+		}
+
 		if reqs[i].Config.Machine().Type() == machine.TypeInit || reqs[i].Config.Machine().Type() == machine.TypeControlPlane {
 			nodes = append(nodes, reqs[i])
 		}
@@ -89,6 +97,10 @@ func (reqs NodeRequests) MasterNodes() (nodes []NodeRequest) {
 // WorkerNodes returns subset of nodes which are Init/ControlPlane type.
 func (reqs NodeRequests) WorkerNodes() (nodes []NodeRequest) {
 	for i := range reqs {
+		if reqs[i].Config == nil {
+			continue
+		}
+
 		if reqs[i].Config.Machine().Type() == machine.TypeJoin {
 			nodes = append(nodes, reqs[i])
 		}
@@ -97,6 +109,17 @@ func (reqs NodeRequests) WorkerNodes() (nodes []NodeRequest) {
 	return
 }
 
+// PXENodes returns subset of nodes which are PXE booted.
+func (reqs NodeRequests) PXENodes() (nodes []NodeRequest) {
+	for i := range reqs {
+		if reqs[i].PXEBooted {
+			nodes = append(nodes, reqs[i])
+		}
+	}
+
+	return
+}
+
 // NodeRequest describes a request for a node.
 type NodeRequest struct {
 	Name   string
@@ -111,4 +134,9 @@ type NodeRequest struct {
 	DiskSize int64
 	// Ports
 	Ports []string
+
+	// PXE-booted VMs
+	PXEBooted        bool
+	TFTPServer       string
+	IPXEBootFilename string
 }

pkg/provision/result.go

@@ -7,6 +7,8 @@ package provision
 import (
 	"net"
 
+	"github.com/google/uuid"
+
 	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
 )
 
@@ -26,6 +28,9 @@ type ClusterInfo struct {
 
 	Network NetworkInfo
 	Nodes   []NodeInfo
+
+	// ExtraNodes are not part of the cluster.
+	ExtraNodes []NodeInfo
 }
 
 // NetworkInfo describes cluster network.
@@ -39,6 +44,7 @@ type NetworkInfo struct {
 // NodeInfo describes a node.
 type NodeInfo struct {
 	ID   string
+	UUID uuid.UUID
 	Name string
 	Type machine.Type