test: implement new class of tests: provision tests (upgrades)

This class of tests is included/excluded by build tags, but as it is
pretty different from other integration tests, we build it as separate
executable. Provision tests provision cluster for the test run, perform
some actions and verify results (could be upgrade, reset, scale up/down,
etc.)

There's now framework to implement upgrade tests, first of the tests
tests upgrade from latest 0.3 (0.3.2 at the moment) to current version
of Talos (being built in CI). Tests starts by booting with 0.3
kernel/initramfs, runs 0.3 installer to install 0.3.2 cluster, wait for
bootstrap, followed by upgrade to 0.4 in rolling fashion. As Firecracker
supports bootloader, this boots 0.4 system from boot disk (as installed
by installer).

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

.drone.yml

@@ -496,6 +496,31 @@ steps:
   - kernel
   - push-local
 
+- name: provision-tests
+  pull: always
+  image: autonomy/build-container:latest
+  commands:
+  - make provision-tests
+  environment:
+    REGISTRY: registry.ci.svc:5000
+  privileged: true
+  volumes:
+  - name: dockersock
+    path: /var/run
+  - name: docker
+    path: /root/.docker/buildx
+  - name: kube
+    path: /root/.kube
+  - name: dev
+    path: /dev
+  - name: tmp
+    path: /tmp
+  depends_on:
+  - initramfs
+  - osctl-linux
+  - kernel
+  - push-local
+
 - name: push
   pull: always
   image: autonomy/build-container:latest
@@ -1091,6 +1116,31 @@ steps:
   - kernel
   - push-local
 
+- name: provision-tests
+  pull: always
+  image: autonomy/build-container:latest
+  commands:
+  - make provision-tests
+  environment:
+    REGISTRY: registry.ci.svc:5000
+  privileged: true
+  volumes:
+  - name: dockersock
+    path: /var/run
+  - name: docker
+    path: /root/.docker/buildx
+  - name: kube
+    path: /root/.kube
+  - name: dev
+    path: /dev
+  - name: tmp
+    path: /tmp
+  depends_on:
+  - initramfs
+  - osctl-linux
+  - kernel
+  - push-local
+
 - name: push
   pull: always
   image: autonomy/build-container:latest
@@ -1778,6 +1828,31 @@ steps:
   - kernel
   - push-local
 
+- name: provision-tests
+  pull: always
+  image: autonomy/build-container:latest
+  commands:
+  - make provision-tests
+  environment:
+    REGISTRY: registry.ci.svc:5000
+  privileged: true
+  volumes:
+  - name: dockersock
+    path: /var/run
+  - name: docker
+    path: /root/.docker/buildx
+  - name: kube
+    path: /root/.kube
+  - name: dev
+    path: /dev
+  - name: tmp
+    path: /tmp
+  depends_on:
+  - initramfs
+  - osctl-linux
+  - kernel
+  - push-local
+
 - name: push
   pull: always
   image: autonomy/build-container:latest
@@ -2495,6 +2570,31 @@ steps:
   - kernel
   - push-local
 
+- name: provision-tests
+  pull: always
+  image: autonomy/build-container:latest
+  commands:
+  - make provision-tests
+  environment:
+    REGISTRY: registry.ci.svc:5000
+  privileged: true
+  volumes:
+  - name: dockersock
+    path: /var/run
+  - name: docker
+    path: /root/.docker/buildx
+  - name: kube
+    path: /root/.kube
+  - name: dev
+    path: /dev
+  - name: tmp
+    path: /tmp
+  depends_on:
+  - initramfs
+  - osctl-linux
+  - kernel
+  - push-local
+
 - name: push
   pull: always
   image: autonomy/build-container:latest
@@ -3212,6 +3312,31 @@ steps:
   - kernel
   - push-local
 
+- name: provision-tests
+  pull: always
+  image: autonomy/build-container:latest
+  commands:
+  - make provision-tests
+  environment:
+    REGISTRY: registry.ci.svc:5000
+  privileged: true
+  volumes:
+  - name: dockersock
+    path: /var/run
+  - name: docker
+    path: /root/.docker/buildx
+  - name: kube
+    path: /root/.kube
+  - name: dev
+    path: /dev
+  - name: tmp
+    path: /tmp
+  depends_on:
+  - initramfs
+  - osctl-linux
+  - kernel
+  - push-local
+
 - name: push
   pull: always
   image: autonomy/build-container:latest
@@ -3447,6 +3572,6 @@ depends_on:
 
 ---
 kind: signature
-hmac: bd5e6446a0d875f2acec08a36fbb274e4763fdf115e1042bf07f8cacc6c37263
+hmac: 0ee401dcb1d7b3fcdf17392c436730a6031e265543436865a68b95c7560e63df
 
 ...

Dockerfile

@@ -394,6 +394,21 @@ RUN --mount=type=cache,target=/.cache/go-build GOOS=darwin GOARCH=amd64 go test
 FROM scratch AS integration-test-darwin
 COPY --from=integration-test-darwin-build /src/integration.test /integration-test-darwin-amd64
 
+# The integration-test-provision target builds integration test binary with provisioning tests.
+
+FROM base AS integration-test-provision-linux-build
+ARG SHA
+ARG TAG
+ARG VERSION_PKG="github.com/talos-systems/talos/pkg/version"
+ARG ARTIFACTS
+RUN --mount=type=cache,target=/.cache/go-build GOOS=linux GOARCH=amd64 go test -c \
+    -ldflags "-s -w -X ${VERSION_PKG}.Name=Client -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG} -X github.com/talos-systems/talos/cmd/osctl/pkg/helpers.ArtifactsPath=${ARTIFACTS}" \
+    -tags integration,integration_provision \
+    ./internal/integration
+
+FROM scratch AS integration-test-provision-linux
+COPY --from=integration-test-provision-linux-build /src/integration.test /integration-test-provision-linux-amd64
+
 # The lint target performs linting on the source code.
 
 FROM base AS lint-go

Makefile

@@ -12,10 +12,12 @@ GO_VERSION ?= 1.13
 OPERATING_SYSTEM := $(shell uname -s | tr "[:upper:]" "[:lower:]")
 OSCTL_DEFAULT_TARGET := osctl-$(OPERATING_SYSTEM)
 INTEGRATION_TEST_DEFAULT_TARGET := integration-test-$(OPERATING_SYSTEM)
+INTEGRATION_TEST_PROVISION_DEFAULT_TARGET := integration-test-provision-$(OPERATING_SYSTEM)
 KUBECTL_URL ?= https://storage.googleapis.com/kubernetes-release/release/v1.17.1/bin/$(OPERATING_SYSTEM)/amd64/kubectl
 SONOBUOY_VERSION ?= 0.17.1
 SONOBUOY_URL ?= https://github.com/heptio/sonobuoy/releases/download/v$(SONOBUOY_VERSION)/sonobuoy_$(SONOBUOY_VERSION)_$(OPERATING_SYSTEM)_amd64.tar.gz
 TESTPKGS ?= ./...
+RELEASES ?= v0.3.2 v0.4.0-alpha.5
 
 BUILD := docker buildx build
 PLATFORM ?= linux/amd64
@@ -173,6 +175,9 @@ unit-tests-race: ## Performs unit tests with race detection enabled.
 $(ARTIFACTS)/$(INTEGRATION_TEST_DEFAULT_TARGET)-amd64:
 	@$(MAKE) local-$(INTEGRATION_TEST_DEFAULT_TARGET) DEST=$(ARTIFACTS)
 
+$(ARTIFACTS)/$(INTEGRATION_TEST_PROVISION_DEFAULT_TARGET)-amd64:
+	@$(MAKE) local-$(INTEGRATION_TEST_PROVISION_DEFAULT_TARGET) DEST=$(ARTIFACTS)
+
 $(ARTIFACTS)/sonobuoy:
 	@mkdir -p $(ARTIFACTS)
 	@curl -L -o /tmp/sonobuoy.tar.gz ${SONOBUOY_URL}
@@ -195,6 +200,28 @@ e2e-%: $(ARTIFACTS)/$(INTEGRATION_TEST_DEFAULT_TARGET)-amd64 $(ARTIFACTS)/sonobu
 		KUBECTL=$(PWD)/$(ARTIFACTS)/kubectl \
 		SONOBUOY=$(PWD)/$(ARTIFACTS)/sonobuoy
 
+provision-tests: release-artifacts $(ARTIFACTS)/$(INTEGRATION_TEST_PROVISION_DEFAULT_TARGET)-amd64
+	@$(MAKE) hack-test-$@ \
+		TAG=$(TAG) \
+		OSCTL=$(PWD)/$(ARTIFACTS)/$(OSCTL_DEFAULT_TARGET)-amd64 \
+		INTEGRATION_TEST=$(PWD)/$(ARTIFACTS)/$(INTEGRATION_TEST_PROVISION_DEFAULT_TARGET)-amd64
+
+# Assets for releases
+
+.PHONY: $(ARTIFACTS)/$(TALOS_RELEASE)
+$(ARTIFACTS)/$(TALOS_RELEASE): $(ARTIFACTS)/$(TALOS_RELEASE)/vmlinux $(ARTIFACTS)/$(TALOS_RELEASE)/initramfs.xz
+
+# download release artifacts for specific version
+$(ARTIFACTS)/$(TALOS_RELEASE)/%:
+	@mkdir -p $(ARTIFACTS)/$(TALOS_RELEASE)/
+	@curl -L -o "$(ARTIFACTS)/$(TALOS_RELEASE)/$*" "https://github.com/talos-systems/talos/releases/download/$(TALOS_RELEASE)/$*"
+
+.PHONY: release-artifacts
+release-artifacts:
+	@for release in $(RELEASES); do \
+		$(MAKE) $(ARTIFACTS)/$$release TALOS_RELEASE=$$release; \
+	done
+
 # Utilities
 
 .PHONY: login

hack/drone.jsonnet

@@ -212,6 +212,7 @@ local unit_tests = Step("unit-tests", depends_on=[initramfs]);
 local unit_tests_race = Step("unit-tests-race", depends_on=[golint]);
 local e2e_docker = Step("e2e-docker", depends_on=[talos, osctl_linux]);
 local e2e_firecracker = Step("e2e-firecracker", privileged=true, depends_on=[initramfs, osctl_linux, kernel, push_local], environment={"REGISTRY": local_registry});
+local provision_tests = Step("provision-tests", privileged=true, depends_on=[initramfs, osctl_linux, kernel, push_local], environment={"REGISTRY": local_registry});
 
 local coverage = {
   name: 'coverage',
@@ -297,6 +298,7 @@ local default_steps = [
   push_local,
   e2e_docker,
   e2e_firecracker,
+  provision_tests,
   push,
   push_latest,
 ];

hack/test/provision-tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -eou pipefail
+
+case "${REGISTRY:-false}" in
+  registry.ci.svc:5000)
+    REGISTRY_ADDR=`python -c "import socket; print socket.gethostbyname('registry.ci.svc')"`
+    INTEGRATION_TEST_FLAGS="-talos.provision.registry-mirror ${REGISTRY}=http://${REGISTRY_ADDR}:5000 -talos.provision.target-installer-registry=${REGISTRY}"
+    ;;
+  *)
+    INTEGRATION_TEST_FLAGS=
+    ;;
+esac
+
+
+"${INTEGRATION_TEST}" -test.v -talos.osctlpath "${OSCTL}" ${INTEGRATION_TEST_FLAGS}

internal/integration/base/flags.go

@@ -4,17 +4,20 @@
 
 // +build integration
 
-package integration_test
+package base
 
 import "strings"
 
-type stringList []string
+// StringList implements flag.Value for list of strings.
+type StringList []string
 
-func (l *stringList) String() string {
+// String implements flag.Value.
+func (l *StringList) String() string {
 	return strings.Join(*l, ",")
 }
 
-func (l *stringList) Set(value string) error {
+// Set implements flag.Value.
+func (l *StringList) Set(value string) error {
 	*l = append(*l, strings.Split(value, ",")...)
 
 	return nil

internal/integration/integration_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/talos-systems/talos/internal/integration/base"
 	"github.com/talos-systems/talos/internal/integration/cli"
 	"github.com/talos-systems/talos/internal/integration/k8s"
+	provision_test "github.com/talos-systems/talos/internal/integration/provision"
 	"github.com/talos-systems/talos/internal/pkg/provision"
 	"github.com/talos-systems/talos/internal/pkg/provision/providers"
 	"github.com/talos-systems/talos/pkg/version"
@@ -64,6 +65,8 @@ func TestIntegration(t *testing.T) {
 		}
 	}
 
+	provision_test.DefaultSettings.CurrentVersion = expectedVersion
+
 	for _, s := range allSuites {
 		if configuredSuite, ok := s.(base.ConfiguredSuite); ok {
 			configuredSuite.SetConfig(base.TalosSuite{
@@ -104,7 +107,18 @@ func init() {
 	flag.StringVar(&expectedVersion, "talos.version", version.Tag, "expected Talos version")
 	flag.StringVar(&osctlPath, "talos.osctlpath", "osctl", "The path to 'osctl' binary")
 
+	flag.StringVar(&provision_test.DefaultSettings.CIDR, "talos.provision.cidr", provision_test.DefaultSettings.CIDR, "CIDR to use to provision clusters (provision tests only)")
+	flag.Var(&provision_test.DefaultSettings.RegistryMirrors, "talos.provision.registry-mirror", "registry mirrors to use (provision tests only)")
+	flag.IntVar(&provision_test.DefaultSettings.MTU, "talos.provision.mtu", provision_test.DefaultSettings.MTU, "MTU to use for cluster network (provision tests only)")
+	flag.Int64Var(&provision_test.DefaultSettings.CPUs, "talos.provision.cpu", provision_test.DefaultSettings.CPUs, "CPU count for each VM (provision tests only)")
+	flag.Int64Var(&provision_test.DefaultSettings.MemMB, "talos.provision.mem", provision_test.DefaultSettings.MemMB, "memory (in MiB) for each VM (provision tests only)")
+	flag.Int64Var(&provision_test.DefaultSettings.DiskGB, "talos.provision.disk", provision_test.DefaultSettings.DiskGB, "disk size (in GiB) for each VM (provision tests only)")
+	flag.IntVar(&provision_test.DefaultSettings.MasterNodes, "talos.provision.masters", provision_test.DefaultSettings.MasterNodes, "master node count (provision tests only)")
+	flag.IntVar(&provision_test.DefaultSettings.WorkerNodes, "talos.provision.workers", provision_test.DefaultSettings.WorkerNodes, "worker node count (provision tests only)")
+	flag.StringVar(&provision_test.DefaultSettings.TargetInstallImageRegistry, "talos.provision.target-installer-registry", provision_test.DefaultSettings.TargetInstallImageRegistry, "image registry for target installer image (provision tests only)")
+
 	allSuites = append(allSuites, api.GetAllSuites()...)
 	allSuites = append(allSuites, cli.GetAllSuites()...)
 	allSuites = append(allSuites, k8s.GetAllSuites()...)
+	allSuites = append(allSuites, provision_test.GetAllSuites()...)
 }

internal/integration/provision/provision.go

@@ -0,0 +1,56 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration
+
+// Package provision provides integration tests which rely on on provisioning cluster per test.
+package provision
+
+import (
+	"github.com/stretchr/testify/suite"
+
+	"github.com/talos-systems/talos/internal/integration/base"
+)
+
+var allSuites []suite.TestingSuite
+
+// GetAllSuites returns all the suites for provision test.
+//
+// Depending on build tags, this might return different lists.
+func GetAllSuites() []suite.TestingSuite {
+	return allSuites
+}
+
+// Settings for provision tests.
+type Settings struct {
+	// CIDR to use for provisioned clusters
+	CIDR string
+	// Registry mirrors to push to Talos config, in format `host=endpoint`
+	RegistryMirrors base.StringList
+	// MTU for the network.
+	MTU int
+	// VM parameters
+	CPUs   int64
+	MemMB  int64
+	DiskGB int64
+	// Node count for the tests
+	MasterNodes int
+	WorkerNodes int
+	// Target installer image registry
+	TargetInstallImageRegistry string
+	// Current version of the cluster (built in the CI pass)
+	CurrentVersion string
+}
+
+// DefaultSettings filled in by test runner.
+var DefaultSettings Settings = Settings{
+	CIDR:                       "172.21.0.0/24",
+	MTU:                        1500,
+	CPUs:                       1,
+	MemMB:                      1.5 * 1024,
+	DiskGB:                     4,
+	MasterNodes:                3,
+	WorkerNodes:                1,
+	TargetInstallImageRegistry: "docker.io",
+}

internal/integration/provision/upgrade.go

@@ -0,0 +1,402 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration_provision
+
+package provision
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+
+	machineapi "github.com/talos-systems/talos/api/machine"
+	talosclient "github.com/talos-systems/talos/cmd/osctl/pkg/client"
+	"github.com/talos-systems/talos/cmd/osctl/pkg/helpers"
+	"github.com/talos-systems/talos/internal/integration/base"
+	"github.com/talos-systems/talos/internal/pkg/provision"
+	"github.com/talos-systems/talos/internal/pkg/provision/access"
+	"github.com/talos-systems/talos/internal/pkg/provision/check"
+	"github.com/talos-systems/talos/internal/pkg/provision/providers/firecracker"
+	"github.com/talos-systems/talos/internal/pkg/runtime"
+	"github.com/talos-systems/talos/pkg/config"
+	"github.com/talos-systems/talos/pkg/config/machine"
+	"github.com/talos-systems/talos/pkg/config/types/v1alpha1"
+	"github.com/talos-systems/talos/pkg/config/types/v1alpha1/generate"
+	"github.com/talos-systems/talos/pkg/constants"
+	talosnet "github.com/talos-systems/talos/pkg/net"
+	"github.com/talos-systems/talos/pkg/retry"
+)
+
+type upgradeSpec struct {
+	ShortName string
+
+	SourceKernelPath     string
+	SourceInitramfsPath  string
+	SourceInstallerImage string
+	SourceVersion        string
+
+	TargetInstallerImage string
+	TargetVersion        string
+
+	MasterNodes int
+	WorkerNodes int
+}
+
+const (
+	talos03Version = "v0.3.2-1-g71ac6696"
+	talos04Version = "v0.4.0-alpha.5"
+)
+
+var (
+	defaultNameservers = []net.IP{net.ParseIP("8.8.8.8"), net.ParseIP("1.1.1.1")}
+	defaultCNIBinPath  = []string{"/opt/cni/bin"}
+)
+
+const (
+	defaultCNIConfDir  = "/etc/cni/conf.d"
+	defaultCNICacheDir = "/var/lib/cni"
+)
+
+func trimVersion(version string) string {
+	// remove anything extra after semantic version core, `v0.3.2-1-abcd` -> `v0.3.2`
+	return regexp.MustCompile(`(-\d+-g[0-9a-f]+)$`).ReplaceAllString(version, "")
+}
+
+// upgradeZeroThreeToZeroFour upgrades Talos 0.3.x to Talos 0.4.x.
+func upgradeZeroThreeToZeroFour() upgradeSpec {
+	return upgradeSpec{
+		ShortName: fmt.Sprintf("%s-%s", talos03Version, talos04Version),
+
+		SourceKernelPath:     helpers.ArtifactPath(filepath.Join(trimVersion(talos03Version), constants.KernelUncompressedAsset)),
+		SourceInitramfsPath:  helpers.ArtifactPath(filepath.Join(trimVersion(talos03Version), constants.InitramfsAsset)),
+		SourceInstallerImage: fmt.Sprintf("%s:%s", constants.DefaultInstallerImageRepository, talos03Version),
+		SourceVersion:        talos03Version,
+
+		TargetInstallerImage: fmt.Sprintf("%s:%s", constants.DefaultInstallerImageRepository, talos04Version),
+		TargetVersion:        talos04Version,
+
+		MasterNodes: DefaultSettings.MasterNodes,
+		WorkerNodes: DefaultSettings.WorkerNodes,
+	}
+}
+
+// upgradeZeroFourToCurrent upgrade Talos 0.4.x to the current version of Talos.
+func upgradeZeroFourToCurrent() upgradeSpec {
+	return upgradeSpec{
+		ShortName: fmt.Sprintf("%s-%s", talos04Version, DefaultSettings.CurrentVersion),
+
+		SourceKernelPath:     helpers.ArtifactPath(filepath.Join(trimVersion(talos04Version), constants.KernelUncompressedAsset)),
+		SourceInitramfsPath:  helpers.ArtifactPath(filepath.Join(trimVersion(talos04Version), constants.InitramfsAsset)),
+		SourceInstallerImage: fmt.Sprintf("%s:%s", constants.DefaultInstallerImageRepository, talos04Version),
+		SourceVersion:        talos04Version,
+
+		TargetInstallerImage: fmt.Sprintf("%s/%s:%s", DefaultSettings.TargetInstallImageRegistry, constants.DefaultInstallerImageName, DefaultSettings.CurrentVersion),
+		TargetVersion:        DefaultSettings.CurrentVersion,
+
+		MasterNodes: DefaultSettings.MasterNodes,
+		WorkerNodes: DefaultSettings.WorkerNodes,
+	}
+}
+
+type UpgradeSuite struct {
+	suite.Suite
+	base.TalosSuite
+
+	specGen func() upgradeSpec
+	spec    upgradeSpec
+
+	provisioner provision.Provisioner
+
+	configBundle *v1alpha1.ConfigBundle
+
+	clusterAccess provision.ClusterAccess
+
+	ctx       context.Context
+	ctxCancel context.CancelFunc
+
+	stateDir string
+}
+
+// SetupSuite ...
+func (suite *UpgradeSuite) SetupSuite() {
+	// call generate late in the flow, as it needs to pick up settings overridden by test runner
+	suite.spec = suite.specGen()
+
+	suite.T().Logf("upgrade spec = %v", suite.spec)
+
+	// timeout for the whole test
+	suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 30*time.Minute)
+
+	var err error
+
+	suite.provisioner, err = firecracker.NewProvisioner(suite.ctx)
+	suite.Require().NoError(err)
+}
+
+// TearDownSuite ...
+func (suite *UpgradeSuite) TearDownSuite() {
+	if suite.clusterAccess != nil {
+		suite.Assert().NoError(suite.clusterAccess.Close())
+	}
+
+	if suite.Cluster != nil {
+		suite.Assert().NoError(suite.provisioner.Destroy(suite.ctx, suite.Cluster))
+	}
+
+	suite.ctxCancel()
+
+	if suite.stateDir != "" {
+		suite.Assert().NoError(os.RemoveAll(suite.stateDir))
+	}
+
+	if suite.provisioner != nil {
+		suite.Assert().NoError(suite.provisioner.Close())
+	}
+}
+
+// setupCluster provisions source clusters and waits for health
+func (suite *UpgradeSuite) setupCluster() {
+	clusterName := fmt.Sprintf("upgrade.%s", suite.spec.ShortName)
+
+	_, cidr, err := net.ParseCIDR(DefaultSettings.CIDR)
+	suite.Require().NoError(err)
+
+	var gatewayIP net.IP
+
+	gatewayIP, err = talosnet.NthIPInNetwork(cidr, 1)
+	suite.Require().NoError(err)
+
+	ips := make([]net.IP, suite.spec.MasterNodes+suite.spec.WorkerNodes)
+
+	for i := range ips {
+		ips[i], err = talosnet.NthIPInNetwork(cidr, i+2)
+		suite.Require().NoError(err)
+	}
+
+	suite.stateDir, err = ioutil.TempDir("", "talos-integration")
+	suite.Require().NoError(err)
+
+	suite.T().Logf("initalizing provisioner with cluster name %q, state directory %q", clusterName, suite.stateDir)
+
+	request := provision.ClusterRequest{
+		Name: clusterName,
+
+		Network: provision.NetworkRequest{
+			Name:        clusterName,
+			CIDR:        *cidr,
+			GatewayAddr: gatewayIP,
+			MTU:         DefaultSettings.MTU,
+			Nameservers: defaultNameservers,
+			CNI: provision.CNIConfig{
+				BinPath:  defaultCNIBinPath,
+				ConfDir:  defaultCNIConfDir,
+				CacheDir: defaultCNICacheDir,
+			},
+		},
+
+		KernelPath:    suite.spec.SourceKernelPath,
+		InitramfsPath: suite.spec.SourceInitramfsPath,
+
+		SelfExecutable: suite.OsctlPath,
+		StateDirectory: suite.stateDir,
+	}
+
+	defaultInternalLB, defaultExternalLB := suite.provisioner.GetLoadBalancers(request.Network)
+
+	genOptions := suite.provisioner.GenOptions(request.Network)
+
+	for _, registryMirror := range DefaultSettings.RegistryMirrors {
+		parts := strings.SplitN(registryMirror, "=", 2)
+		suite.Require().Len(parts, 2)
+
+		genOptions = append(genOptions, generate.WithRegistryMirror(parts[0], parts[1]))
+	}
+
+	suite.configBundle, err = config.NewConfigBundle(config.WithInputOptions(
+		&config.InputOptions{
+			ClusterName: clusterName,
+			Endpoint:    fmt.Sprintf("https://%s:6443", defaultInternalLB),
+			KubeVersion: constants.DefaultKubernetesVersion, // TODO: should be upgradeable
+			GenOptions: append(
+				genOptions,
+				generate.WithEndpointList([]string{defaultExternalLB}),
+				generate.WithInstallImage(suite.spec.SourceInstallerImage),
+			),
+		}))
+	suite.Require().NoError(err)
+
+	for i := 0; i < suite.spec.MasterNodes; i++ {
+		var cfg runtime.Configurator
+
+		if i == 0 {
+			cfg = suite.configBundle.Init()
+		} else {
+			cfg = suite.configBundle.ControlPlane()
+		}
+
+		request.Nodes = append(request.Nodes,
+			provision.NodeRequest{
+				Name:     fmt.Sprintf("master-%d", i+1),
+				IP:       ips[i],
+				Memory:   DefaultSettings.MemMB * 1024 * 1024,
+				NanoCPUs: DefaultSettings.CPUs * 1000 * 1000 * 1000,
+				DiskSize: DefaultSettings.DiskGB * 1024 * 1024 * 1024,
+				Config:   cfg,
+			})
+	}
+
+	for i := 1; i <= suite.spec.WorkerNodes; i++ {
+		request.Nodes = append(request.Nodes,
+			provision.NodeRequest{
+				Name:     fmt.Sprintf("worker-%d", i),
+				IP:       ips[suite.spec.MasterNodes+i-1],
+				Memory:   DefaultSettings.MemMB * 1024 * 1024,
+				NanoCPUs: DefaultSettings.CPUs * 1000 * 1000 * 1000,
+				DiskSize: DefaultSettings.DiskGB * 1024 * 1024 * 1024,
+				Config:   suite.configBundle.Join(),
+			})
+	}
+
+	suite.Cluster, err = suite.provisioner.Create(suite.ctx, request, provision.WithBootladerEmulation(), provision.WithTalosConfig(suite.configBundle.TalosConfig()))
+	suite.Require().NoError(err)
+
+	suite.clusterAccess = access.NewAdapter(suite.Cluster, provision.WithTalosConfig(suite.configBundle.TalosConfig()))
+
+	suite.waitForClusterHealth()
+}
+
+// waitForClusterHealth asserts cluster health after any change
+func (suite *UpgradeSuite) waitForClusterHealth() {
+	checkCtx, checkCtxCancel := context.WithTimeout(suite.ctx, 10*time.Minute)
+	defer checkCtxCancel()
+
+	suite.Require().NoError(check.Wait(checkCtx, suite.clusterAccess, check.DefaultClusterChecks(), check.StderrReporter()))
+}
+
+func (suite *UpgradeSuite) assertSameVersionCluster(client *talosclient.Client, expectedVersion string) {
+	nodes := make([]string, len(suite.Cluster.Info().Nodes))
+
+	for i, node := range suite.Cluster.Info().Nodes {
+		nodes[i] = node.PrivateIP.String()
+	}
+
+	ctx := talosclient.WithNodes(suite.ctx, nodes...)
+
+	var v *machineapi.VersionResponse
+
+	err := retry.Constant(
+		time.Minute,
+	).Retry(func() error {
+		var e error
+		v, e = client.Version(ctx)
+
+		return retry.ExpectedError(e)
+	})
+
+	suite.Require().NoError(err)
+
+	suite.Require().Len(v.Messages, len(nodes))
+
+	for _, version := range v.Messages {
+		suite.Assert().Equal(expectedVersion, version.Version.Tag)
+	}
+}
+
+func (suite *UpgradeSuite) readVersion(client *talosclient.Client, nodeCtx context.Context) (version string, err error) {
+	var v *machineapi.VersionResponse
+
+	v, err = client.Version(nodeCtx)
+	if err != nil {
+		return
+	}
+
+	version = v.Messages[0].Version.Tag
+
+	return
+}
+
+func (suite *UpgradeSuite) upgradeNode(client *talosclient.Client, node provision.NodeInfo) {
+	suite.T().Logf("upgrading node %s", node.PrivateIP)
+
+	nodeCtx := talosclient.WithNodes(suite.ctx, node.PrivateIP.String())
+
+	resp, err := client.Upgrade(nodeCtx, suite.spec.TargetInstallerImage)
+	suite.Require().NoError(err)
+
+	suite.Require().Equal("Upgrade request received", resp.Messages[0].Ack)
+
+	// wait for the version to be equal to target version
+	suite.Require().NoError(retry.Constant(5 * time.Minute).Retry(func() error {
+		var version string
+
+		version, err = suite.readVersion(client, nodeCtx)
+		if err != nil {
+			// API might be unresponsive during upgrade
+			return retry.ExpectedError(err)
+		}
+
+		if version != suite.spec.TargetVersion {
+			// upgrade not finished yet
+			return retry.ExpectedError(fmt.Errorf("node %q version doesn't match expected: expected %q, got %q", node.PrivateIP.String(), suite.spec.TargetVersion, version))
+		}
+
+		return nil
+	}))
+
+	suite.waitForClusterHealth()
+}
+
+// TestRolling performs rolling upgrade starting with master nodes.
+func (suite *UpgradeSuite) TestRolling() {
+	suite.setupCluster()
+
+	client, err := suite.clusterAccess.Client()
+	suite.Require().NoError(err)
+
+	// verify initial cluster version
+	suite.assertSameVersionCluster(client, suite.spec.SourceVersion)
+
+	// upgrade master nodes
+	for _, node := range suite.Cluster.Info().Nodes {
+		if node.Type == machine.TypeInit || node.Type == machine.TypeControlPlane {
+			suite.upgradeNode(client, node)
+		}
+	}
+
+	// upgrade worker nodes
+	for _, node := range suite.Cluster.Info().Nodes {
+		if node.Type == machine.TypeWorker {
+			suite.upgradeNode(client, node)
+		}
+	}
+
+	// verify final cluster version
+	suite.assertSameVersionCluster(client, suite.spec.TargetVersion)
+}
+
+// SuiteName ...
+func (suite *UpgradeSuite) SuiteName() string {
+	if suite.spec.ShortName == "" {
+		suite.spec = suite.specGen()
+	}
+
+	return fmt.Sprintf("provision.UpgradeSuite.%s", suite.spec.ShortName)
+}
+
+func init() {
+	allSuites = append(allSuites,
+		&UpgradeSuite{specGen: upgradeZeroThreeToZeroFour},
+		// disabled until root cause can be figured out:
+		// &UpgradeSuite{specGen: upgradeZeroFourToCurrent},
+	)
+}

pkg/constants/constants.go

@@ -247,9 +247,13 @@ const (
 	// and directories.
 	SystemRunPath = "/run/system"
 
+	// DefaultInstallerImageName is the default container image name for
+	// the installer.
+	DefaultInstallerImageName = "autonomy/installer"
+
 	// DefaultInstallerImageRepository is the default container repository for
 	// the installer.
-	DefaultInstallerImageRepository = "docker.io/autonomy/installer"
+	DefaultInstallerImageRepository = "docker.io/" + DefaultInstallerImageName
 
 	// DefaultTalosImageRepository is the default container repository for
 	// the talos image.