chore: refactor tests

Refactor tests to avoid code duplication.

Signed-off-by: Noel Georgi <git@frezbo.dev>
(cherry picked from commit 9fa08e843728dbd85ed7e0035f59cdd6232de9a9)

internal/integration/api/common.go

@@ -11,10 +11,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/siderolabs/go-pointer"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"github.com/siderolabs/talos/internal/integration/base"
 )
 
@@ -83,39 +79,17 @@ virtual memory (kb)             (-v) unlimited
 file locks                      (-x) unlimited
 `
 
-	const (
+	defaultsTestPodDef, err := suite.NewPod("defaults-ulimits-test")
-		namespace = "default"
-		pod       = "defaults-test"
-	)
-
-	_, err := suite.Clientset.CoreV1().Pods(namespace).Create(suite.ctx, &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: pod,
-		},
-		Spec: corev1.PodSpec{
-			Containers: []corev1.Container{
-				{
-					Name:  pod,
-					Image: "alpine",
-					Command: []string{
-						"tail",
-						"-f",
-						"/dev/null",
-					},
-				},
-			},
-			TerminationGracePeriodSeconds: pointer.To[int64](0),
-		},
-	}, metav1.CreateOptions{})
-
 	suite.Require().NoError(err)
 
-	defer suite.Clientset.CoreV1().Pods(namespace).Delete(suite.ctx, pod, metav1.DeleteOptions{}) //nolint:errcheck
+	suite.Require().NoError(defaultsTestPodDef.Create(suite.ctx, 5*time.Minute))
 
-	// wait for the pod to be ready
+	defer defaultsTestPodDef.Delete(suite.ctx) //nolint:errcheck
-	suite.Require().NoError(suite.WaitForPodToBeRunning(suite.ctx, 10*time.Minute, namespace, pod))
 
-	stdout, stderr, err := suite.ExecuteCommandInPod(suite.ctx, namespace, pod, "ulimit -c -d -e -f -l -m -n -q -r -s -t -v -x")
+	stdout, stderr, err := defaultsTestPodDef.Exec(
+		suite.ctx,
+		"ulimit -c -d -e -f -l -m -n -q -r -s -t -v -x",
+	)
 	suite.Require().NoError(err)
 
 	suite.Require().Equal("", stderr)
@@ -129,47 +103,19 @@ func (suite *CommonSuite) TestDNSResolver() {
 		suite.AssertClusterHealthy(suite.ctx)
 	}
 
-	const (
+	dnsTestPodDef, err := suite.NewPod("dns-test")
-		namespace = "default"
-		pod       = "dns-test"
-	)
-
-	_, err := suite.Clientset.CoreV1().Pods(namespace).Create(suite.ctx, &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: pod,
-		},
-		Spec: corev1.PodSpec{
-			Containers: []corev1.Container{
-				{
-					Name:  pod,
-					Image: "alpine",
-					Command: []string{
-						"tail",
-						"-f",
-						"/dev/null",
-					},
-				},
-			},
-			TerminationGracePeriodSeconds: pointer.To[int64](0),
-		},
-	}, metav1.CreateOptions{})
-
 	suite.Require().NoError(err)
 
-	suite.T().Cleanup(func() {
+	suite.Require().NoError(dnsTestPodDef.Create(suite.ctx, 5*time.Minute))
-		cleanUpCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute)
-		defer cleanupCancel()
 
-		suite.Require().NoError(
+	defer dnsTestPodDef.Delete(suite.ctx) //nolint:errcheck
-			suite.Clientset.CoreV1().Pods(namespace).Delete(cleanUpCtx, pod, metav1.DeleteOptions{}),
-		)
-	})
 
-	// wait for the pod to be ready
+	stdout, stderr, err := dnsTestPodDef.Exec(
-	suite.Require().NoError(suite.WaitForPodToBeRunning(suite.ctx, time.Minute, namespace, pod))
+		suite.ctx,
-
+		"wget -S https://www.google.com/",
-	stdout, stderr, err := suite.ExecuteCommandInPod(suite.ctx, namespace, pod, "wget -S https://www.google.com/")
+	)
 	suite.Assert().NoError(err)
+
 	suite.Assert().Equal("", stdout)
 	suite.Assert().Contains(stderr, "'index.html' saved")
 
@@ -183,7 +129,10 @@ func (suite *CommonSuite) TestDNSResolver() {
 		suite.T().FailNow()
 	}
 
-	_, stderr, err = suite.ExecuteCommandInPod(suite.ctx, namespace, pod, "apk add --update bind-tools")
+	_, stderr, err = dnsTestPodDef.Exec(
+		suite.ctx,
+		"apk add --update bind-tools",
+	)
 
 	suite.Assert().NoError(err)
 	suite.Assert().Empty(stderr, "stderr: %s", stderr)
@@ -192,7 +141,10 @@ func (suite *CommonSuite) TestDNSResolver() {
 		suite.T().FailNow()
 	}
 
-	stdout, stderr, err = suite.ExecuteCommandInPod(suite.ctx, namespace, pod, "dig really-long-record.dev.siderolabs.io")
+	stdout, stderr, err = dnsTestPodDef.Exec(
+		suite.ctx,
+		"dig really-long-record.dev.siderolabs.io",
+	)
 
 	suite.Assert().NoError(err)
 	suite.Assert().Contains(stdout, "status: NOERROR")

internal/integration/api/extensions_qemu.go

@@ -145,7 +145,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsISCSI() {
 
 	ctx := client.WithNode(suite.ctx, node)
 
-	iscsiCreatePodDef, err := suite.NewPodOp("iscsi-create", "kube-system")
+	iscsiCreatePodDef, err := suite.NewPrivilegedPod("iscsi-create")
 	suite.Require().NoError(err)
 
 	suite.Require().NoError(iscsiCreatePodDef.Create(suite.ctx, 5*time.Minute))
@@ -446,7 +446,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsMdADM() {
 
 	userDisksJoined := strings.Join(userDisks[:2], " ")
 
-	mdAdmCreatePodDef, err := suite.NewPodOp("mdadm-create", "kube-system")
+	mdAdmCreatePodDef, err := suite.NewPrivilegedPod("mdadm-create")
 	suite.Require().NoError(err)
 
 	suite.Require().NoError(mdAdmCreatePodDef.Create(suite.ctx, 5*time.Minute))
@@ -467,7 +467,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsMdADM() {
 
 		hostname := hostNameStatus.TypedSpec().Hostname
 
-		deletePodDef, err := suite.NewPodOp("mdadm-destroy", "kube-system")
+		deletePodDef, err := suite.NewPrivilegedPod("mdadm-destroy")
 		suite.Require().NoError(err)
 
 		suite.Require().NoError(deletePodDef.Create(suite.ctx, 5*time.Minute))
@@ -526,7 +526,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsZFS() {
 
 	suite.Require().NotEmpty(userDisks, "expected at least one user disks to be available")
 
-	zfsPodDef, err := suite.NewPodOp("zpool-create", "kube-system")
+	zfsPodDef, err := suite.NewPrivilegedPod("zpool-create")
 	suite.Require().NoError(err)
 
 	suite.Require().NoError(zfsPodDef.Create(suite.ctx, 5*time.Minute))
@@ -552,7 +552,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsZFS() {
 	suite.Require().Equal("", stdout)
 
 	defer func() {
-		deletePodDef, err := suite.NewPodOp("zpool-destroy", "kube-system")
+		deletePodDef, err := suite.NewPrivilegedPod("zpool-destroy")
 		suite.Require().NoError(err)
 
 		suite.Require().NoError(deletePodDef.Create(suite.ctx, 5*time.Minute))
@@ -605,7 +605,7 @@ func (suite *ExtensionsSuiteQEMU) checkZFSPoolMounted() bool {
 
 // TestExtensionsUtilLinuxTools verifies util-linux-tools are working.
 func (suite *ExtensionsSuiteQEMU) TestExtensionsUtilLinuxTools() {
-	utilLinuxPodDef, err := suite.NewPodOp("util-linux-tools-test", "kube-system")
+	utilLinuxPodDef, err := suite.NewPrivilegedPod("util-linux-tools-test")
 	suite.Require().NoError(err)
 
 	suite.Require().NoError(utilLinuxPodDef.Create(suite.ctx, 5*time.Minute))

internal/integration/api/volumes.go

@@ -194,7 +194,7 @@ func (suite *VolumesSuite) TestLVMActivation() {
 
 	userDisksJoined := strings.Join(userDisks[:2], " ")
 
-	podDef, err := suite.NewPodOp("pv-create", "kube-system")
+	podDef, err := suite.NewPrivilegedPod("pv-create")
 	suite.Require().NoError(err)
 
 	suite.Require().NoError(podDef.Create(suite.ctx, 5*time.Minute))
@@ -226,7 +226,7 @@ func (suite *VolumesSuite) TestLVMActivation() {
 	suite.Require().Contains(stdout, "Logical volume \"lv1\" created.")
 
 	defer func() {
-		deletePodDef, err := suite.NewPodOp("pv-destroy", "kube-system")
+		deletePodDef, err := suite.NewPrivilegedPod("pv-destroy")
 		suite.Require().NoError(err)
 
 		suite.Require().NoError(deletePodDef.Create(suite.ctx, 5*time.Minute))

internal/integration/base/k8s.go

@@ -208,11 +208,9 @@ type podInfo interface {
 type pod struct {
 	name      string
 	namespace string
+	pod       *corev1.Pod
 
-	client     *kubernetes.Clientset
+	suite *K8sSuite
-	restConfig *rest.Config
-
-	logF func(format string, args ...any)
 }
 
 func (p *pod) Name() string {
@@ -220,56 +218,12 @@ func (p *pod) Name() string {
 }
 
 func (p *pod) Create(ctx context.Context, waitTimeout time.Duration) error {
-	_, err := p.client.CoreV1().Pods(p.namespace).Create(ctx, &corev1.Pod{
+	_, err := p.suite.Clientset.CoreV1().Pods(p.namespace).Create(ctx, p.pod, metav1.CreateOptions{})
-		ObjectMeta: metav1.ObjectMeta{
-			Name: p.name,
-		},
-		Spec: corev1.PodSpec{
-			Containers: []corev1.Container{
-				{
-					Name:  p.name,
-					Image: "alpine",
-					Command: []string{
-						"/bin/sh",
-						"-c",
-						"--",
-					},
-					Args: []string{
-						"trap : TERM INT; (tail -f /dev/null) & wait",
-					},
-					SecurityContext: &corev1.SecurityContext{
-						Privileged: pointer.To(true),
-					},
-					// lvm commands even though executed in the host mount namespace, still need access to /dev 🤷🏼,
-					// otherwise lvcreate commands hangs on semop syscall
-					VolumeMounts: []corev1.VolumeMount{
-						{
-							Name:      "dev",
-							MountPath: "/dev",
-						},
-					},
-				},
-			},
-			Volumes: []corev1.Volume{
-				{
-					Name: "dev",
-					VolumeSource: corev1.VolumeSource{
-						HostPath: &corev1.HostPathVolumeSource{
-							Path: "/dev",
-						},
-					},
-				},
-			},
-			HostNetwork: true,
-			HostIPC:     true,
-			HostPID:     true,
-		},
-	}, metav1.CreateOptions{})
 	if err != nil {
 		return err
 	}
 
-	return p.waitForRunning(ctx, waitTimeout)
+	return p.suite.WaitForPodToBeRunning(ctx, waitTimeout, p.namespace, p.name)
 }
 
 func (p *pod) Exec(ctx context.Context, command string) (string, string, error) {
@@ -278,7 +232,7 @@ func (p *pod) Exec(ctx context.Context, command string) (string, string, error)
 		"-c",
 		command,
 	}
-	req := p.client.CoreV1().RESTClient().Post().Resource("pods").Name(p.name).
+	req := p.suite.Clientset.CoreV1().RESTClient().Post().Resource("pods").Name(p.name).
 		Namespace(p.namespace).SubResource("exec")
 	option := &corev1.PodExecOptions{
 		Command: cmd,
@@ -293,7 +247,7 @@ func (p *pod) Exec(ctx context.Context, command string) (string, string, error)
 		scheme.ParameterCodec,
 	)
 
-	exec, err := remotecommand.NewSPDYExecutor(p.restConfig, "POST", req.URL())
+	exec, err := remotecommand.NewSPDYExecutor(p.suite.RestConfig, "POST", req.URL())
 	if err != nil {
 		return "", "", err
 	}
@@ -305,7 +259,7 @@ func (p *pod) Exec(ctx context.Context, command string) (string, string, error)
 		Stderr: &stderr,
 	})
 	if err != nil {
-		p.logF(
+		p.suite.T().Logf(
 			"error executing command in pod %s/%s: %v\n\ncommand %q stdout:\n%s\n\ncommand %q stderr:\n%s",
 			p.namespace,
 			p.name,
@@ -321,59 +275,110 @@ func (p *pod) Exec(ctx context.Context, command string) (string, string, error)
 }
 
 func (p *pod) Delete(ctx context.Context) error {
-	return p.client.CoreV1().Pods(p.namespace).Delete(ctx, p.name, metav1.DeleteOptions{})
+	return p.suite.Clientset.CoreV1().Pods(p.namespace).Delete(ctx, p.name, metav1.DeleteOptions{})
 }
 
-func (p *pod) waitForRunning(ctx context.Context, timeout time.Duration) error {
+// NewPrivilegedPod creates a new pod definition with a random suffix
-	ctx, cancel := context.WithTimeout(ctx, timeout)
+// in the kube-system namespace with privileged security context.
-	defer cancel()
+func (k8sSuite *K8sSuite) NewPrivilegedPod(name string) (podInfo, error) {
-
-	watcher, err := p.client.CoreV1().Pods(p.namespace).Watch(ctx, metav1.ListOptions{
-		FieldSelector: fields.OneTermEqualSelector("metadata.name", p.name).String(),
-	})
-	if err != nil {
-		return err
-	}
-
-	defer watcher.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case event := <-watcher.ResultChan():
-			if event.Type == watch.Error {
-				return fmt.Errorf("error watching pod: %v", event.Object)
-			}
-
-			pod, ok := event.Object.(*corev1.Pod)
-			if !ok {
-				continue
-			}
-
-			if pod.Name == p.name && pod.Status.Phase == corev1.PodRunning {
-				return nil
-			}
-		}
-	}
-}
-
-// NewPodOp creates a new pod operation with the given name and namespace.
-func (k8sSuite *K8sSuite) NewPodOp(name, namespace string) (podInfo, error) {
 	randomSuffix := make([]byte, 4)
 
 	if _, err := rand.Read(randomSuffix); err != nil {
 		return nil, fmt.Errorf("failed to generate random suffix: %w", err)
 	}
 
+	podName := fmt.Sprintf("%s-%x", name, randomSuffix)
+
 	return &pod{
-		name:      fmt.Sprintf("%s-%x", name, randomSuffix),
+		name:      podName,
-		namespace: namespace,
+		namespace: "kube-system",
+		pod: &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: podName,
+			},
+			Spec: corev1.PodSpec{
+				Containers: []corev1.Container{
+					{
+						Name:  podName,
+						Image: "alpine",
+						Command: []string{
+							"/bin/sh",
+							"-c",
+							"--",
+						},
+						Args: []string{
+							"trap : TERM INT; (tail -f /dev/null) & wait",
+						},
+						SecurityContext: &corev1.SecurityContext{
+							Privileged: pointer.To(true),
+						},
+						// lvm commands even though executed in the host mount namespace, still need access to /dev 🤷🏼,
+						// otherwise lvcreate commands hangs on semop syscall
+						VolumeMounts: []corev1.VolumeMount{
+							{
+								Name:      "dev",
+								MountPath: "/dev",
+							},
+						},
+					},
+				},
+				Volumes: []corev1.Volume{
+					{
+						Name: "dev",
+						VolumeSource: corev1.VolumeSource{
+							HostPath: &corev1.HostPathVolumeSource{
+								Path: "/dev",
+							},
+						},
+					},
+				},
+				HostNetwork: true,
+				HostIPC:     true,
+				HostPID:     true,
+			},
+		},
 
-		client:     k8sSuite.Clientset,
+		suite: k8sSuite,
-		restConfig: k8sSuite.RestConfig,
+	}, nil
+}
 
-		logF: k8sSuite.T().Logf,
+// NewPod creates a new pod definition with a random suffix
+// in the default namespace.
+func (k8sSuite *K8sSuite) NewPod(name string) (podInfo, error) {
+	randomSuffix := make([]byte, 4)
+
+	if _, err := rand.Read(randomSuffix); err != nil {
+		return nil, fmt.Errorf("failed to generate random suffix: %w", err)
+	}
+
+	podName := fmt.Sprintf("%s-%x", name, randomSuffix)
+
+	return &pod{
+		name:      podName,
+		namespace: "default",
+		pod: &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: podName,
+			},
+			Spec: corev1.PodSpec{
+				Containers: []corev1.Container{
+					{
+						Name:  podName,
+						Image: "alpine",
+						Command: []string{
+							"/bin/sh",
+							"-c",
+							"--",
+						},
+						Args: []string{
+							"trap : TERM INT; (tail -f /dev/null) & wait",
+						},
+					},
+				},
+			},
+		},
+
+		suite: k8sSuite,
 	}, nil
 }
 
@@ -448,36 +453,6 @@ func (k8sSuite *K8sSuite) LogPodLogs(ctx context.Context, namespace, podName str
 	}
 }
 
-// WaitForPodToBeDeleted waits for the pod with the given namespace and name to be deleted.
-func (k8sSuite *K8sSuite) WaitForPodToBeDeleted(ctx context.Context, timeout time.Duration, namespace, podName string) error {
-	ctx, cancel := context.WithTimeout(ctx, timeout)
-	defer cancel()
-
-	watcher, err := k8sSuite.Clientset.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{
-		FieldSelector: fields.OneTermEqualSelector("metadata.name", podName).String(),
-	})
-	if err != nil {
-		return err
-	}
-
-	defer watcher.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case event := <-watcher.ResultChan():
-			if event.Type == watch.Deleted {
-				return nil
-			}
-
-			if event.Type == watch.Error {
-				return fmt.Errorf("error watching pod: %v", event.Object)
-			}
-		}
-	}
-}
-
 // HelmInstall installs the Helm chart with the given namespace, repository, version, release name, chart name and values.
 func (k8sSuite *K8sSuite) HelmInstall(ctx context.Context, namespace, repository, version, releaseName, chartName string, valuesBytes []byte) error {
 	tempFile := filepath.Join(k8sSuite.T().TempDir(), "values.yaml")
@@ -635,55 +610,6 @@ func (k8sSuite *K8sSuite) RunFIOTest(ctx context.Context, storageClasss, size st
 	return cmd.Run()
 }
 
-// ExecuteCommandInPod executes the given command in the pod with the given namespace and name.
-func (k8sSuite *K8sSuite) ExecuteCommandInPod(ctx context.Context, namespace, podName, command string) (string, string, error) {
-	cmd := []string{
-		"/bin/sh",
-		"-c",
-		command,
-	}
-	req := k8sSuite.Clientset.CoreV1().RESTClient().Post().Resource("pods").Name(podName).
-		Namespace(namespace).SubResource("exec")
-	option := &corev1.PodExecOptions{
-		Command: cmd,
-		Stdin:   false,
-		Stdout:  true,
-		Stderr:  true,
-		TTY:     false,
-	}
-
-	req.VersionedParams(
-		option,
-		scheme.ParameterCodec,
-	)
-
-	exec, err := remotecommand.NewSPDYExecutor(k8sSuite.RestConfig, "POST", req.URL())
-	if err != nil {
-		return "", "", err
-	}
-
-	var stdout, stderr strings.Builder
-
-	err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{
-		Stdout: &stdout,
-		Stderr: &stderr,
-	})
-	if err != nil {
-		k8sSuite.T().Logf(
-			"error executing command in pod %s/%s: %v\n\ncommand %q stdout:\n%s\n\ncommand %q stderr:\n%s",
-			namespace,
-			podName,
-			err,
-			command,
-			stdout.String(),
-			command,
-			stderr.String(),
-		)
-	}
-
-	return stdout.String(), stderr.String(), err
-}
-
 // GetPodsWithLabel returns the pods with the given label in the specified namespace.
 func (k8sSuite *K8sSuite) GetPodsWithLabel(ctx context.Context, namespace, label string) (*corev1.PodList, error) {
 	podList, err := k8sSuite.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{