diff --git a/cmd/talosctl/cmd/mgmt/config.go b/cmd/talosctl/cmd/mgmt/config.go index 4fbd6fcf7..37c32d45c 100644 --- a/cmd/talosctl/cmd/mgmt/config.go +++ b/cmd/talosctl/cmd/mgmt/config.go @@ -115,7 +115,7 @@ func genV1Alpha1Config(args []string) error { var configString string - switch t { + switch t { //nolint: exhaustive case machine.TypeInit: configString, err = configBundle.Init().String() if err != nil { diff --git a/cmd/talosctl/cmd/talos/health.go b/cmd/talosctl/cmd/talos/health.go index f8c123869..96be7efd4 100644 --- a/cmd/talosctl/cmd/talos/health.go +++ b/cmd/talosctl/cmd/talos/health.go @@ -41,6 +41,8 @@ func (cluster *clusterNodes) NodesByType(t machine.Type) []string { return cluster.ControlPlaneNodes case machine.TypeJoin: return cluster.WorkerNodes + case machine.TypeUnknown: + return nil default: panic("unsupported machine type") } diff --git a/internal/app/machined/internal/server/v1alpha1/v1alpha1_cluster.go b/internal/app/machined/internal/server/v1alpha1/v1alpha1_cluster.go index 2fc740d65..23e6e3168 100644 --- a/internal/app/machined/internal/server/v1alpha1/v1alpha1_cluster.go +++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_cluster.go @@ -92,6 +92,8 @@ func (cluster *clusterState) NodesByType(t machine.Type) []string { return cluster.controlPlaneNodes case machine.TypeJoin: return cluster.workerNodes + case machine.TypeUnknown: + return nil default: panic("unsupported machine type") } diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go index a21085921..efc24f62c 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go @@ -641,6 +641,8 @@ func StartAllServices(seq runtime.Sequence, data interface{}) (runtime.TaskExecu &services.Etcd{}, ) case machine.TypeJoin: + case machine.TypeUnknown: + return fmt.Errorf("unexpected machine type: %s", r.Config().Machine().Type()) } system.Services(r).StartAll() diff --git a/pkg/machinery/config/types/v1alpha1/bundle/bundle.go b/pkg/machinery/config/types/v1alpha1/bundle/bundle.go index c79114d57..ea60c0643 100644 --- a/pkg/machinery/config/types/v1alpha1/bundle/bundle.go +++ b/pkg/machinery/config/types/v1alpha1/bundle/bundle.go @@ -49,7 +49,7 @@ func NewConfigBundle(opts ...Option) (*v1alpha1.ConfigBundle, error) { return bundle, err } - switch configType { + switch configType { //nolint: exhaustive case machine.TypeInit: bundle.InitCfg = unmarshalledConfig case machine.TypeControlPlane: @@ -96,7 +96,7 @@ func NewConfigBundle(opts ...Option) (*v1alpha1.ConfigBundle, error) { return bundle, err } - switch configType { + switch configType { //nolint: exhaustive case machine.TypeInit: bundle.InitCfg = generatedConfig case machine.TypeControlPlane: diff --git a/pkg/machinery/config/types/v1alpha1/generate/generate.go b/pkg/machinery/config/types/v1alpha1/generate/generate.go index 2d873f3d0..3bb1e346a 100644 --- a/pkg/machinery/config/types/v1alpha1/generate/generate.go +++ b/pkg/machinery/config/types/v1alpha1/generate/generate.go @@ -41,6 +41,8 @@ func Config(t machine.Type, in *Input) (c *v1alpha1.Config, err error) { if c, err = workerUd(in); err != nil { return c, err } + case machine.TypeUnknown: + fallthrough default: return c, errors.New("failed to determine config type to generate") } diff --git a/pkg/machinery/config/types/v1alpha1/machine/machine.go b/pkg/machinery/config/types/v1alpha1/machine/machine.go index 7e659f6cc..2d0cd633e 100644 --- a/pkg/machinery/config/types/v1alpha1/machine/machine.go +++ b/pkg/machinery/config/types/v1alpha1/machine/machine.go @@ -10,8 +10,10 @@ import "fmt" type Type int const ( + // TypeUnknown represents undefined node type. + TypeUnknown Type = iota // TypeInit represents a bootstrap node. - TypeInit Type = iota + TypeInit // TypeControlPlane represents a control plane node. TypeControlPlane // TypeJoin represents a worker node. @@ -19,6 +21,7 @@ const ( ) const ( + typeUnknown = "unknown" typeInit = "init" typeControlPlane = "controlplane" typeJoin = "join" @@ -26,12 +29,14 @@ const ( // String returns the string representation of Type. func (t Type) String() string { - return [...]string{typeInit, typeControlPlane, typeJoin}[t] + return [...]string{typeUnknown, typeInit, typeControlPlane, typeJoin}[t] } // ParseType parses string constant as Type. func ParseType(t string) (Type, error) { switch t { + case typeUnknown: + return TypeUnknown, nil case typeInit: return TypeInit, nil case typeControlPlane: diff --git a/pkg/provision/providers/qemu/create.go b/pkg/provision/providers/qemu/create.go index c803a8be4..b7beff22a 100644 --- a/pkg/provision/providers/qemu/create.go +++ b/pkg/provision/providers/qemu/create.go @@ -85,6 +85,17 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque return nil, err } + var pxeNodeInfo []provision.NodeInfo + + pxeNodes := request.Nodes.PXENodes() + if len(pxeNodes) > 0 { + fmt.Fprintln(options.LogWriter, "creating PXE nodes") + + if pxeNodeInfo, err = p.createNodes(state, request, pxeNodes, &options); err != nil { + return nil, err + } + } + nodeInfo = append(nodeInfo, workerNodeInfo...) state.ClusterInfo = provision.ClusterInfo{ @@ -95,7 +106,8 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque GatewayAddr: request.Network.GatewayAddr, MTU: request.Network.MTU, }, - Nodes: nodeInfo, + Nodes: nodeInfo, + ExtraNodes: pxeNodeInfo, } err = state.Save() diff --git a/pkg/provision/providers/qemu/launch.go b/pkg/provision/providers/qemu/launch.go index 9b3f3b070..a43af7166 100644 --- a/pkg/provision/providers/qemu/launch.go +++ b/pkg/provision/providers/qemu/launch.go @@ -42,6 +42,7 @@ type LaunchConfig struct { MachineType string EnableKVM bool BootloaderEnabled bool + NodeUUID uuid.UUID // Talos config Config string @@ -56,6 +57,11 @@ type LaunchConfig struct { MTU int Nameservers []net.IP + // PXE + TFTPServer string + BootFilename string + IPXEBootFileName string + // filled by CNI invocation tapName string vmMAC string @@ -131,13 +137,15 @@ func withCNI(ctx context.Context, config *LaunchConfig, f func(config *LaunchCon // dump node IP/mac/hostname for dhcp if err = vm.DumpIPAMRecord(config.StatePath, vm.IPAMRecord{ - IP: config.IP, - Netmask: config.CIDR.Mask, - MAC: vmIface.Mac, - Hostname: config.Hostname, - Gateway: config.GatewayAddr, - MTU: config.MTU, - Nameservers: config.Nameservers, + IP: config.IP, + Netmask: config.CIDR.Mask, + MAC: vmIface.Mac, + Hostname: config.Hostname, + Gateway: config.GatewayAddr, + MTU: config.MTU, + Nameservers: config.Nameservers, + TFTPServer: config.TFTPServer, + IPXEBootFilename: config.IPXEBootFileName, }); err != nil { return err } @@ -179,6 +187,8 @@ func launchVM(config *LaunchConfig) error { "-device", fmt.Sprintf("virtio-net-pci,netdev=net0,mac=%s", config.vmMAC), "-device", "virtio-rng-pci", "-no-reboot", + "-boot", "order=cn,reboot-timeout=5000", + "-smbios", fmt.Sprintf("type=1,uuid=%s", config.NodeUUID), } machineArg := config.MachineType @@ -203,7 +213,7 @@ func launchVM(config *LaunchConfig) error { return err } - if !diskBootable || !config.BootloaderEnabled { + if (!diskBootable || !config.BootloaderEnabled) && config.KernelImagePath != "" { args = append(args, "-kernel", config.KernelImagePath, "-initrd", config.InitrdPath, diff --git a/pkg/provision/providers/qemu/node.go b/pkg/provision/providers/qemu/node.go index b86d1a2c9..4c74a5298 100644 --- a/pkg/provision/providers/qemu/node.go +++ b/pkg/provision/providers/qemu/node.go @@ -16,8 +16,10 @@ import ( "strconv" "syscall" + "github.com/google/uuid" multierror "github.com/hashicorp/go-multierror" + "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" "github.com/talos-systems/talos/pkg/provision" "github.com/talos-systems/talos/pkg/provision/providers/vm" @@ -61,23 +63,28 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe cmdline.Append("talos.platform", "metal") cmdline.Append("talos.config", "{TALOS_CONFIG_URL}") // to be patched by launcher - nodeConfig, err := nodeReq.Config.String() - if err != nil { - return provision.NodeInfo{}, err + var nodeConfig string + + if nodeReq.Config != nil { + nodeConfig, err = nodeReq.Config.String() + if err != nil { + return provision.NodeInfo{}, err + } } + nodeUUID := uuid.New() + launchConfig := LaunchConfig{ QemuExecutable: fmt.Sprintf("qemu-system-%s", arch.QemuArch()), DiskPath: diskPath, VCPUCount: vcpuCount, MemSize: memSize, - KernelImagePath: clusterReq.KernelPath, KernelArgs: cmdline.String(), - InitrdPath: clusterReq.InitramfsPath, MachineType: arch.QemuMachine(), PFlashImages: state.PFlashImages, EnableKVM: opts.TargetArch == runtime.GOARCH, BootloaderEnabled: opts.BootloaderEnabled, + NodeUUID: nodeUUID, Config: nodeConfig, NetworkConfig: state.VMCNIConfig, CNI: clusterReq.Network.CNI, @@ -87,6 +94,13 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe GatewayAddr: clusterReq.Network.GatewayAddr, MTU: clusterReq.Network.MTU, Nameservers: clusterReq.Network.Nameservers, + TFTPServer: nodeReq.TFTPServer, + IPXEBootFileName: nodeReq.IPXEBootFilename, + } + + if !nodeReq.PXEBooted { + launchConfig.KernelImagePath = clusterReq.KernelPath + launchConfig.InitrdPath = clusterReq.InitramfsPath } launchConfig.StatePath, err = state.StatePath() @@ -127,10 +141,16 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe // no need to wait here, as cmd has all the Stdin/out/err via *os.File + nodeType := machine.TypeUnknown + if nodeReq.Config != nil { + nodeType = nodeReq.Config.Machine().Type() + } + nodeInfo := provision.NodeInfo{ ID: pidPath, + UUID: nodeUUID, Name: nodeReq.Name, - Type: nodeReq.Config.Machine().Type(), + Type: nodeType, NanoCPUs: nodeReq.NanoCPUs, Memory: nodeReq.Memory, diff --git a/pkg/provision/providers/vm/dhcpd.go b/pkg/provision/providers/vm/dhcpd.go index 44cb5882b..b8456ad2d 100644 --- a/pkg/provision/providers/vm/dhcpd.go +++ b/pkg/provision/providers/vm/dhcpd.go @@ -21,6 +21,7 @@ import ( "github.com/talos-systems/talos/pkg/provision" ) +//nolint: gocyclo func handler(serverIP net.IP, statePath string) server4.Handler { return func(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { if m.OpCode != dhcpv4.OpcodeBootRequest { @@ -45,7 +46,6 @@ func handler(serverIP net.IP, statePath string) server4.Handler { resp, err := dhcpv4.NewReplyFromRequest(m, dhcpv4.WithNetmask(match.Netmask), - dhcpv4.WithServerIP(serverIP), dhcpv4.WithYourIP(match.IP), dhcpv4.WithOption(dhcpv4.OptHostName(match.Hostname)), dhcpv4.WithOption(dhcpv4.OptDNS(match.Nameservers...)), @@ -58,6 +58,18 @@ func handler(serverIP net.IP, statePath string) server4.Handler { return } + if m.IsOptionRequested(dhcpv4.OptionBootfileName) { + log.Printf("received PXE boot request from %s", m.ClientHWAddr) + + if match.TFTPServer != "" { + log.Printf("sending PXE response to %s: %s/%s", m.ClientHWAddr, match.TFTPServer, match.IPXEBootFilename) + + resp.ServerIPAddr = net.ParseIP(match.TFTPServer) + resp.UpdateOption(dhcpv4.OptTFTPServerName(match.TFTPServer)) + resp.UpdateOption(dhcpv4.OptBootFileName(match.IPXEBootFilename)) + } + } + resp.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionInterfaceMTU, dhcpv4.Uint16(match.MTU).ToBytes())) switch mt := m.MessageType(); mt { //nolint: exhaustive diff --git a/pkg/provision/providers/vm/ipam.go b/pkg/provision/providers/vm/ipam.go index cdd2d53cb..f7108832c 100644 --- a/pkg/provision/providers/vm/ipam.go +++ b/pkg/provision/providers/vm/ipam.go @@ -22,6 +22,9 @@ type IPAMRecord struct { Gateway net.IP MTU int Nameservers []net.IP + + TFTPServer string + IPXEBootFilename string } // IPAMDatabase is a mapping from MAC address to records. diff --git a/pkg/provision/providers/vm/node.go b/pkg/provision/providers/vm/node.go index ee45393ee..04088c703 100644 --- a/pkg/provision/providers/vm/node.go +++ b/pkg/provision/providers/vm/node.go @@ -16,7 +16,9 @@ import ( func (p *Provisioner) DestroyNodes(cluster provision.ClusterInfo, options *provision.Options) error { errCh := make(chan error) - for _, node := range cluster.Nodes { + nodes := append(cluster.Nodes, cluster.ExtraNodes...) + + for _, node := range nodes { go func(node provision.NodeInfo) { fmt.Fprintln(options.LogWriter, "stopping VM", node.Name) @@ -26,7 +28,7 @@ func (p *Provisioner) DestroyNodes(cluster provision.ClusterInfo, options *provi var multiErr *multierror.Error - for range cluster.Nodes { + for range nodes { multiErr = multierror.Append(multiErr, <-errCh) } diff --git a/pkg/provision/request.go b/pkg/provision/request.go index 887c23b9c..60838c2ac 100644 --- a/pkg/provision/request.go +++ b/pkg/provision/request.go @@ -57,6 +57,10 @@ func (reqs NodeRequests) FindInitNode() (req NodeRequest, err error) { found := false for i := range reqs { + if reqs[i].Config == nil { + continue + } + if reqs[i].Config.Machine().Type() == machine.TypeInit { if found { err = fmt.Errorf("duplicate init node in requests") @@ -78,6 +82,10 @@ func (reqs NodeRequests) FindInitNode() (req NodeRequest, err error) { // MasterNodes returns subset of nodes which are Init/ControlPlane type. func (reqs NodeRequests) MasterNodes() (nodes []NodeRequest) { for i := range reqs { + if reqs[i].Config == nil { + continue + } + if reqs[i].Config.Machine().Type() == machine.TypeInit || reqs[i].Config.Machine().Type() == machine.TypeControlPlane { nodes = append(nodes, reqs[i]) } @@ -89,6 +97,10 @@ func (reqs NodeRequests) MasterNodes() (nodes []NodeRequest) { // WorkerNodes returns subset of nodes which are Init/ControlPlane type. func (reqs NodeRequests) WorkerNodes() (nodes []NodeRequest) { for i := range reqs { + if reqs[i].Config == nil { + continue + } + if reqs[i].Config.Machine().Type() == machine.TypeJoin { nodes = append(nodes, reqs[i]) } @@ -97,6 +109,17 @@ func (reqs NodeRequests) WorkerNodes() (nodes []NodeRequest) { return } +// PXENodes returns subset of nodes which are PXE booted. +func (reqs NodeRequests) PXENodes() (nodes []NodeRequest) { + for i := range reqs { + if reqs[i].PXEBooted { + nodes = append(nodes, reqs[i]) + } + } + + return +} + // NodeRequest describes a request for a node. type NodeRequest struct { Name string @@ -111,4 +134,9 @@ type NodeRequest struct { DiskSize int64 // Ports Ports []string + + // PXE-booted VMs + PXEBooted bool + TFTPServer string + IPXEBootFilename string } diff --git a/pkg/provision/result.go b/pkg/provision/result.go index be05114d3..8cd035add 100644 --- a/pkg/provision/result.go +++ b/pkg/provision/result.go @@ -7,6 +7,8 @@ package provision import ( "net" + "github.com/google/uuid" + "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" ) @@ -26,6 +28,9 @@ type ClusterInfo struct { Network NetworkInfo Nodes []NodeInfo + + // ExtraNodes are not part of the cluster. + ExtraNodes []NodeInfo } // NetworkInfo describes cluster network. @@ -39,6 +44,7 @@ type NetworkInfo struct { // NodeInfo describes a node. type NodeInfo struct { ID string + UUID uuid.UUID Name string Type machine.Type