From 59adf7315dcfc11b37e477c4b2508453b455b7a3 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 27 Aug 2020 23:23:20 +0300 Subject: [PATCH] feat: provide option to run Talos under UEFI in QEMU This also adds integration pipeline tests for UEFI. Signed-off-by: Andrey Smirnov --- .drone.jsonnet | 14 +++++--- cmd/talosctl/cmd/mgmt/cluster/create.go | 3 ++ docs/talosctl/talosctl_cluster_create.md | 1 + hack/test/e2e-qemu.sh | 6 ++++ pkg/provision/options.go | 12 +++++++ pkg/provision/providers/qemu/arch.go | 21 +++++++---- pkg/provision/providers/qemu/create.go | 8 ----- pkg/provision/providers/qemu/node.go | 12 ++++++- pkg/provision/providers/qemu/pflash.go | 45 ++++++++++++++++-------- pkg/provision/providers/vm/state.go | 2 -- 10 files changed, 88 insertions(+), 36 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index e214a958c..5834b9321 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -227,7 +227,7 @@ local image_vmware = Step("image-vmware", depends_on=[image_gcp]); local unit_tests = Step("unit-tests", depends_on=[initramfs]); local unit_tests_race = Step("unit-tests-race", depends_on=[initramfs]); local e2e_docker = Step("e2e-docker-short", depends_on=[talos, talosctl_linux, unit_tests, unit_tests_race], target="e2e-docker", environment={"SHORT_INTEGRATION_TEST": "yes"}); -local e2e_qemu = Step("e2e-qemu-short", privileged=true, target="e2e-qemu", depends_on=[talosctl_linux, initramfs, kernel, installer, unit_tests, unit_tests_race], environment={"FIRECRACKER_GO_SDK_REQUEST_TIMEOUT_MILLISECONDS": "2000", "SHORT_INTEGRATION_TEST": "yes"}, when={event: ['pull_request']}); +local e2e_qemu = Step("e2e-qemu-short", privileged=true, target="e2e-qemu", depends_on=[talosctl_linux, initramfs, kernel, installer, unit_tests, unit_tests_race], environment={"SHORT_INTEGRATION_TEST": "yes"}, when={event: ['pull_request']}); local coverage = { name: 'coverage', @@ -334,15 +334,18 @@ local default_pipeline = Pipeline('default', default_steps) + default_trigger; // Full integration pipeline. -local integration_qemu = Step("e2e-qemu", privileged=true, depends_on=[initramfs, talosctl_linux, kernel, installer, unit_tests, unit_tests_race], environment={"FIRECRACKER_GO_SDK_REQUEST_TIMEOUT_MILLISECONDS": "2000"}); +local integration_qemu = Step("e2e-qemu", privileged=true, depends_on=[initramfs, talosctl_linux, kernel, installer, unit_tests, unit_tests_race]); local integration_provision_tests_prepare = Step("provision-tests-prepare", privileged=true, depends_on=[initramfs, talosctl_linux, kernel, installer, unit_tests, unit_tests_race, e2e_qemu, e2e_docker]); -local integration_provision_tests_track_0 = Step("provision-tests-track-0", privileged=true, depends_on=[integration_provision_tests_prepare], environment={"FIRECRACKER_GO_SDK_REQUEST_TIMEOUT_MILLISECONDS": "2000"}); -local integration_provision_tests_track_1 = Step("provision-tests-track-1", privileged=true, depends_on=[integration_provision_tests_prepare], environment={"FIRECRACKER_GO_SDK_REQUEST_TIMEOUT_MILLISECONDS": "2000"}); +local integration_provision_tests_track_0 = Step("provision-tests-track-0", privileged=true, depends_on=[integration_provision_tests_prepare]); +local integration_provision_tests_track_1 = Step("provision-tests-track-1", privileged=true, depends_on=[integration_provision_tests_prepare]); local integration_cilium = Step("e2e-cilium-1.8.0", target="e2e-qemu", privileged=true, depends_on=[integration_qemu], environment={ - "FIRECRACKER_GO_SDK_REQUEST_TIMEOUT_MILLISECONDS": "2000", "SHORT_INTEGRATION_TEST": "yes", "CUSTOM_CNI_URL": "https://raw.githubusercontent.com/cilium/cilium/v1.8.0/install/kubernetes/quick-install.yaml", }); +local integration_uefi = Step("e2e-uefi", target="e2e-qemu", privileged=true, depends_on=[integration_cilium], environment={ + "SHORT_INTEGRATION_TEST": "yes", + "WITH_UEFI": "true", +}); local integration_steps = default_steps + [ @@ -351,6 +354,7 @@ local integration_steps = default_steps + [ integration_provision_tests_track_0, integration_provision_tests_track_1, integration_cilium, + integration_uefi, ]; local integration_trigger = { diff --git a/cmd/talosctl/cmd/mgmt/cluster/create.go b/cmd/talosctl/cmd/mgmt/cluster/create.go index 6f24763d4..5c992d18d 100644 --- a/cmd/talosctl/cmd/mgmt/cluster/create.go +++ b/cmd/talosctl/cmd/mgmt/cluster/create.go @@ -42,6 +42,7 @@ var ( nodeVmlinuzPath string nodeInitramfsPath string bootloaderEnabled bool + uefiEnabled bool configDebug bool networkCIDR string networkMTU int @@ -164,6 +165,7 @@ func create(ctx context.Context) (err error) { provisionOptions := []provision.Option{ provision.WithDockerPortsHostIP(dockerHostIP), provision.WithBootlader(bootloaderEnabled), + provision.WithUEFI(uefiEnabled), provision.WithTargetArch(targetArch), } configBundleOpts := []bundle.Option{} @@ -368,6 +370,7 @@ func init() { createCmd.Flags().StringVar(&nodeVmlinuzPath, "vmlinuz-path", helpers.ArtifactPath(constants.KernelAsset), "the compressed kernel image to use") createCmd.Flags().StringVar(&nodeInitramfsPath, "initrd-path", helpers.ArtifactPath(constants.InitramfsAsset), "the uncompressed kernel image to use") createCmd.Flags().BoolVar(&bootloaderEnabled, "with-bootloader", true, "enable bootloader to load kernel and initramfs from disk image after install") + createCmd.Flags().BoolVar(&uefiEnabled, "with-uefi", false, "enable UEFI on x86_64 architecture (always enabled for arm64)") createCmd.Flags().StringSliceVar(®istryMirrors, "registry-mirror", []string{}, "list of registry mirrors to use in format: =") createCmd.Flags().BoolVar(&configDebug, "with-debug", false, "enable debug in Talos config to send service logs to the console") createCmd.Flags().IntVar(&networkMTU, "mtu", 1500, "MTU of the cluster network") diff --git a/docs/talosctl/talosctl_cluster_create.md b/docs/talosctl/talosctl_cluster_create.md index 786fd0a74..c5f9b3587 100644 --- a/docs/talosctl/talosctl_cluster_create.md +++ b/docs/talosctl/talosctl_cluster_create.md @@ -45,6 +45,7 @@ talosctl cluster create [flags] --with-bootloader enable bootloader to load kernel and initramfs from disk image after install (default true) --with-debug enable debug in Talos config to send service logs to the console --with-init-node create the cluster with an init node + --with-uefi enable UEFI on x86_64 architecture (always enabled for arm64) --workers int the number of workers to create (default 1) ``` diff --git a/hack/test/e2e-qemu.sh b/hack/test/e2e-qemu.sh index 820813655..49ee310ff 100755 --- a/hack/test/e2e-qemu.sh +++ b/hack/test/e2e-qemu.sh @@ -31,6 +31,12 @@ case "${CUSTOM_CNI_URL:-false}" in ;; esac +case "${WITH_UEFI:-false}" in + true) + QEMU_FLAGS="${QEMU_FLAGS} --with-uefi" + ;; +esac + function create_cluster { build_registry_mirrors diff --git a/pkg/provision/options.go b/pkg/provision/options.go index 5aa23e2b6..73f65feab 100644 --- a/pkg/provision/options.go +++ b/pkg/provision/options.go @@ -61,6 +61,15 @@ func WithBootlader(enabled bool) Option { } } +// WithUEFI enables or disables UEFI boot on amd64 (default for amd64 is BIOS boot). +func WithUEFI(enabled bool) Option { + return func(o *Options) error { + o.UEFIEnabled = enabled + + return nil + } +} + // WithTargetArch specifies target architecture for the cluster. func WithTargetArch(arch string) Option { return func(o *Options) error { @@ -99,6 +108,9 @@ type Options struct { // Enable bootloader by booting from disk image after install. BootloaderEnabled bool + // Enable UEFI (for amd64), arm64 can only boot UEFI + UEFIEnabled bool + // Expose ports to worker machines in docker provisioner DockerPorts []string DockerPortsHostIP string diff --git a/pkg/provision/providers/qemu/arch.go b/pkg/provision/providers/qemu/arch.go index 8991a8e74..c73719ab5 100644 --- a/pkg/provision/providers/qemu/arch.go +++ b/pkg/provision/providers/qemu/arch.go @@ -61,25 +61,34 @@ func (arch Arch) Console() string { // PFlash for UEFI boot. type PFlash struct { - Size int64 - SourcePath string + Size int64 + SourcePaths []string } // PFlash returns settings for parallel flash. -func (arch Arch) PFlash() []PFlash { +func (arch Arch) PFlash(uefiEnabled bool) []PFlash { switch arch { case ArchArm64: return []PFlash{ { - Size: 64 * 1024 * 1024, - SourcePath: "/usr/share/qemu-efi-aarch64/QEMU_EFI.fd", + Size: 64 * 1024 * 1024, + SourcePaths: []string{"/usr/share/qemu-efi-aarch64/QEMU_EFI.fd", "/usr/share/OVMF/QEMU_EFI.fd"}, }, { Size: 64 * 1024 * 1024, }, } case ArchAmd64: - fallthrough + if !uefiEnabled { + return nil + } + + return []PFlash{ + { + Size: 0, + SourcePaths: []string{"/usr/share/ovmf/OVMF.fd", "/usr/share/OVMF/OVMF.fd"}, + }, + } default: return nil } diff --git a/pkg/provision/providers/qemu/create.go b/pkg/provision/providers/qemu/create.go index b7beff22a..fff4ed564 100644 --- a/pkg/provision/providers/qemu/create.go +++ b/pkg/provision/providers/qemu/create.go @@ -43,14 +43,6 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque return nil, err } - if pflashSpec := arch.PFlash(); pflashSpec != nil { - fmt.Fprintln(options.LogWriter, "creating flash images") - - if err = p.createPFlashImages(state, pflashSpec); err != nil { - return nil, fmt.Errorf("error creating flash images: %w", err) - } - } - fmt.Fprintln(options.LogWriter, "creating network", request.Network.Name) if err = p.CreateNetwork(ctx, state, request.Network); err != nil { diff --git a/pkg/provision/providers/qemu/node.go b/pkg/provision/providers/qemu/node.go index 4c74a5298..7a2bb436c 100644 --- a/pkg/provision/providers/qemu/node.go +++ b/pkg/provision/providers/qemu/node.go @@ -31,6 +31,16 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe arch := Arch(opts.TargetArch) pidPath := state.GetRelativePath(fmt.Sprintf("%s.pid", nodeReq.Name)) + var pflashImages []string + + if pflashSpec := arch.PFlash(opts.UEFIEnabled); pflashSpec != nil { + var err error + + if pflashImages, err = p.createPFlashImages(state, nodeReq.Name, pflashSpec); err != nil { + return provision.NodeInfo{}, fmt.Errorf("error creating flash images: %w", err) + } + } + vcpuCount := int64(math.RoundToEven(float64(nodeReq.NanoCPUs) / 1000 / 1000 / 1000)) if vcpuCount < 2 { vcpuCount = 1 @@ -81,7 +91,7 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe MemSize: memSize, KernelArgs: cmdline.String(), MachineType: arch.QemuMachine(), - PFlashImages: state.PFlashImages, + PFlashImages: pflashImages, EnableKVM: opts.TargetArch == runtime.GOARCH, BootloaderEnabled: opts.BootloaderEnabled, NodeUUID: nodeUUID, diff --git a/pkg/provision/providers/qemu/pflash.go b/pkg/provision/providers/qemu/pflash.go index eea0f2b2b..e6470c826 100644 --- a/pkg/provision/providers/qemu/pflash.go +++ b/pkg/provision/providers/qemu/pflash.go @@ -12,42 +12,59 @@ import ( "github.com/talos-systems/talos/pkg/provision/providers/vm" ) -func (p *provisioner) createPFlashImages(state *vm.State, pflashSpec []PFlash) error { +//nolint: gocyclo +func (p *provisioner) createPFlashImages(state *vm.State, nodeName string, pflashSpec []PFlash) ([]string, error) { + var images []string + for i, pflash := range pflashSpec { if err := func(i int, pflash PFlash) error { - path := state.GetRelativePath(fmt.Sprintf("flash%d.img", i)) + path := state.GetRelativePath(fmt.Sprintf("%s-flash%d.img", nodeName, i)) f, err := os.Create(path) if err != nil { - return nil + return err } defer f.Close() //nolint: errcheck - if err := f.Truncate(pflash.Size); err != nil { + if err = f.Truncate(pflash.Size); err != nil { return err } - if pflash.SourcePath != "" { - src, err := os.Open(pflash.SourcePath) - if err != nil { - return nil + if pflash.SourcePaths != nil { + for _, sourcePath := range pflash.SourcePaths { + var src *os.File + + src, err = os.Open(sourcePath) + if err != nil { + if os.IsNotExist(err) { + continue + } + + return err + } + + defer src.Close() //nolint: errcheck + + if _, err = io.Copy(f, src); err != nil { + return err + } + + break } - defer src.Close() //nolint: errcheck - - if _, err := io.Copy(f, src); err != nil { + if err != nil { return err } } - state.PFlashImages = append(state.PFlashImages, path) + images = append(images, path) return nil }(i, pflash); err != nil { - return err + return nil, err } } - return nil + return images, nil } diff --git a/pkg/provision/providers/vm/state.go b/pkg/provision/providers/vm/state.go index 4b874f55d..4f41a72af 100644 --- a/pkg/provision/providers/vm/state.go +++ b/pkg/provision/providers/vm/state.go @@ -25,8 +25,6 @@ type State struct { VMCNIConfig *libcni.NetworkConfigList - PFlashImages []string - statePath string }