From 67ba47825343cae52aa72d0442ee4ae232eb242f Mon Sep 17 00:00:00 2001 From: Noel Georgi Date: Fri, 20 Sep 2024 16:22:01 +0530 Subject: [PATCH] chore: refactor tests Refactor tests to avoid code duplication. Signed-off-by: Noel Georgi (cherry picked from commit 9fa08e843728dbd85ed7e0035f59cdd6232de9a9) --- internal/integration/api/common.go | 94 ++----- internal/integration/api/extensions_qemu.go | 12 +- internal/integration/api/volumes.go | 4 +- internal/integration/base/k8s.go | 274 +++++++------------- 4 files changed, 131 insertions(+), 253 deletions(-) diff --git a/internal/integration/api/common.go b/internal/integration/api/common.go index f67854530..ae1ed7f2c 100644 --- a/internal/integration/api/common.go +++ b/internal/integration/api/common.go @@ -11,10 +11,6 @@ import ( "strings" "time" - "github.com/siderolabs/go-pointer" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/siderolabs/talos/internal/integration/base" ) @@ -83,39 +79,17 @@ virtual memory (kb) (-v) unlimited file locks (-x) unlimited ` - const ( - namespace = "default" - pod = "defaults-test" - ) - - _, err := suite.Clientset.CoreV1().Pods(namespace).Create(suite.ctx, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: pod, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: pod, - Image: "alpine", - Command: []string{ - "tail", - "-f", - "/dev/null", - }, - }, - }, - TerminationGracePeriodSeconds: pointer.To[int64](0), - }, - }, metav1.CreateOptions{}) - + defaultsTestPodDef, err := suite.NewPod("defaults-ulimits-test") suite.Require().NoError(err) - defer suite.Clientset.CoreV1().Pods(namespace).Delete(suite.ctx, pod, metav1.DeleteOptions{}) //nolint:errcheck + suite.Require().NoError(defaultsTestPodDef.Create(suite.ctx, 5*time.Minute)) - // wait for the pod to be ready - suite.Require().NoError(suite.WaitForPodToBeRunning(suite.ctx, 10*time.Minute, namespace, pod)) + defer defaultsTestPodDef.Delete(suite.ctx) //nolint:errcheck - stdout, stderr, err := suite.ExecuteCommandInPod(suite.ctx, namespace, pod, "ulimit -c -d -e -f -l -m -n -q -r -s -t -v -x") + stdout, stderr, err := defaultsTestPodDef.Exec( + suite.ctx, + "ulimit -c -d -e -f -l -m -n -q -r -s -t -v -x", + ) suite.Require().NoError(err) suite.Require().Equal("", stderr) @@ -129,47 +103,19 @@ func (suite *CommonSuite) TestDNSResolver() { suite.AssertClusterHealthy(suite.ctx) } - const ( - namespace = "default" - pod = "dns-test" - ) - - _, err := suite.Clientset.CoreV1().Pods(namespace).Create(suite.ctx, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: pod, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: pod, - Image: "alpine", - Command: []string{ - "tail", - "-f", - "/dev/null", - }, - }, - }, - TerminationGracePeriodSeconds: pointer.To[int64](0), - }, - }, metav1.CreateOptions{}) - + dnsTestPodDef, err := suite.NewPod("dns-test") suite.Require().NoError(err) - suite.T().Cleanup(func() { - cleanUpCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute) - defer cleanupCancel() + suite.Require().NoError(dnsTestPodDef.Create(suite.ctx, 5*time.Minute)) - suite.Require().NoError( - suite.Clientset.CoreV1().Pods(namespace).Delete(cleanUpCtx, pod, metav1.DeleteOptions{}), - ) - }) + defer dnsTestPodDef.Delete(suite.ctx) //nolint:errcheck - // wait for the pod to be ready - suite.Require().NoError(suite.WaitForPodToBeRunning(suite.ctx, time.Minute, namespace, pod)) - - stdout, stderr, err := suite.ExecuteCommandInPod(suite.ctx, namespace, pod, "wget -S https://www.google.com/") + stdout, stderr, err := dnsTestPodDef.Exec( + suite.ctx, + "wget -S https://www.google.com/", + ) suite.Assert().NoError(err) + suite.Assert().Equal("", stdout) suite.Assert().Contains(stderr, "'index.html' saved") @@ -183,7 +129,10 @@ func (suite *CommonSuite) TestDNSResolver() { suite.T().FailNow() } - _, stderr, err = suite.ExecuteCommandInPod(suite.ctx, namespace, pod, "apk add --update bind-tools") + _, stderr, err = dnsTestPodDef.Exec( + suite.ctx, + "apk add --update bind-tools", + ) suite.Assert().NoError(err) suite.Assert().Empty(stderr, "stderr: %s", stderr) @@ -192,7 +141,10 @@ func (suite *CommonSuite) TestDNSResolver() { suite.T().FailNow() } - stdout, stderr, err = suite.ExecuteCommandInPod(suite.ctx, namespace, pod, "dig really-long-record.dev.siderolabs.io") + stdout, stderr, err = dnsTestPodDef.Exec( + suite.ctx, + "dig really-long-record.dev.siderolabs.io", + ) suite.Assert().NoError(err) suite.Assert().Contains(stdout, "status: NOERROR") diff --git a/internal/integration/api/extensions_qemu.go b/internal/integration/api/extensions_qemu.go index d4eb2be5b..1136971bd 100644 --- a/internal/integration/api/extensions_qemu.go +++ b/internal/integration/api/extensions_qemu.go @@ -145,7 +145,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsISCSI() { ctx := client.WithNode(suite.ctx, node) - iscsiCreatePodDef, err := suite.NewPodOp("iscsi-create", "kube-system") + iscsiCreatePodDef, err := suite.NewPrivilegedPod("iscsi-create") suite.Require().NoError(err) suite.Require().NoError(iscsiCreatePodDef.Create(suite.ctx, 5*time.Minute)) @@ -446,7 +446,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsMdADM() { userDisksJoined := strings.Join(userDisks[:2], " ") - mdAdmCreatePodDef, err := suite.NewPodOp("mdadm-create", "kube-system") + mdAdmCreatePodDef, err := suite.NewPrivilegedPod("mdadm-create") suite.Require().NoError(err) suite.Require().NoError(mdAdmCreatePodDef.Create(suite.ctx, 5*time.Minute)) @@ -467,7 +467,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsMdADM() { hostname := hostNameStatus.TypedSpec().Hostname - deletePodDef, err := suite.NewPodOp("mdadm-destroy", "kube-system") + deletePodDef, err := suite.NewPrivilegedPod("mdadm-destroy") suite.Require().NoError(err) suite.Require().NoError(deletePodDef.Create(suite.ctx, 5*time.Minute)) @@ -526,7 +526,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsZFS() { suite.Require().NotEmpty(userDisks, "expected at least one user disks to be available") - zfsPodDef, err := suite.NewPodOp("zpool-create", "kube-system") + zfsPodDef, err := suite.NewPrivilegedPod("zpool-create") suite.Require().NoError(err) suite.Require().NoError(zfsPodDef.Create(suite.ctx, 5*time.Minute)) @@ -552,7 +552,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsZFS() { suite.Require().Equal("", stdout) defer func() { - deletePodDef, err := suite.NewPodOp("zpool-destroy", "kube-system") + deletePodDef, err := suite.NewPrivilegedPod("zpool-destroy") suite.Require().NoError(err) suite.Require().NoError(deletePodDef.Create(suite.ctx, 5*time.Minute)) @@ -605,7 +605,7 @@ func (suite *ExtensionsSuiteQEMU) checkZFSPoolMounted() bool { // TestExtensionsUtilLinuxTools verifies util-linux-tools are working. func (suite *ExtensionsSuiteQEMU) TestExtensionsUtilLinuxTools() { - utilLinuxPodDef, err := suite.NewPodOp("util-linux-tools-test", "kube-system") + utilLinuxPodDef, err := suite.NewPrivilegedPod("util-linux-tools-test") suite.Require().NoError(err) suite.Require().NoError(utilLinuxPodDef.Create(suite.ctx, 5*time.Minute)) diff --git a/internal/integration/api/volumes.go b/internal/integration/api/volumes.go index 30920c2e3..305583921 100644 --- a/internal/integration/api/volumes.go +++ b/internal/integration/api/volumes.go @@ -194,7 +194,7 @@ func (suite *VolumesSuite) TestLVMActivation() { userDisksJoined := strings.Join(userDisks[:2], " ") - podDef, err := suite.NewPodOp("pv-create", "kube-system") + podDef, err := suite.NewPrivilegedPod("pv-create") suite.Require().NoError(err) suite.Require().NoError(podDef.Create(suite.ctx, 5*time.Minute)) @@ -226,7 +226,7 @@ func (suite *VolumesSuite) TestLVMActivation() { suite.Require().Contains(stdout, "Logical volume \"lv1\" created.") defer func() { - deletePodDef, err := suite.NewPodOp("pv-destroy", "kube-system") + deletePodDef, err := suite.NewPrivilegedPod("pv-destroy") suite.Require().NoError(err) suite.Require().NoError(deletePodDef.Create(suite.ctx, 5*time.Minute)) diff --git a/internal/integration/base/k8s.go b/internal/integration/base/k8s.go index 9a993ea22..43ef7a8cd 100644 --- a/internal/integration/base/k8s.go +++ b/internal/integration/base/k8s.go @@ -208,11 +208,9 @@ type podInfo interface { type pod struct { name string namespace string + pod *corev1.Pod - client *kubernetes.Clientset - restConfig *rest.Config - - logF func(format string, args ...any) + suite *K8sSuite } func (p *pod) Name() string { @@ -220,56 +218,12 @@ func (p *pod) Name() string { } func (p *pod) Create(ctx context.Context, waitTimeout time.Duration) error { - _, err := p.client.CoreV1().Pods(p.namespace).Create(ctx, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: p.name, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: p.name, - Image: "alpine", - Command: []string{ - "/bin/sh", - "-c", - "--", - }, - Args: []string{ - "trap : TERM INT; (tail -f /dev/null) & wait", - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: pointer.To(true), - }, - // lvm commands even though executed in the host mount namespace, still need access to /dev 🤷🏼, - // otherwise lvcreate commands hangs on semop syscall - VolumeMounts: []corev1.VolumeMount{ - { - Name: "dev", - MountPath: "/dev", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "dev", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/dev", - }, - }, - }, - }, - HostNetwork: true, - HostIPC: true, - HostPID: true, - }, - }, metav1.CreateOptions{}) + _, err := p.suite.Clientset.CoreV1().Pods(p.namespace).Create(ctx, p.pod, metav1.CreateOptions{}) if err != nil { return err } - return p.waitForRunning(ctx, waitTimeout) + return p.suite.WaitForPodToBeRunning(ctx, waitTimeout, p.namespace, p.name) } func (p *pod) Exec(ctx context.Context, command string) (string, string, error) { @@ -278,7 +232,7 @@ func (p *pod) Exec(ctx context.Context, command string) (string, string, error) "-c", command, } - req := p.client.CoreV1().RESTClient().Post().Resource("pods").Name(p.name). + req := p.suite.Clientset.CoreV1().RESTClient().Post().Resource("pods").Name(p.name). Namespace(p.namespace).SubResource("exec") option := &corev1.PodExecOptions{ Command: cmd, @@ -293,7 +247,7 @@ func (p *pod) Exec(ctx context.Context, command string) (string, string, error) scheme.ParameterCodec, ) - exec, err := remotecommand.NewSPDYExecutor(p.restConfig, "POST", req.URL()) + exec, err := remotecommand.NewSPDYExecutor(p.suite.RestConfig, "POST", req.URL()) if err != nil { return "", "", err } @@ -305,7 +259,7 @@ func (p *pod) Exec(ctx context.Context, command string) (string, string, error) Stderr: &stderr, }) if err != nil { - p.logF( + p.suite.T().Logf( "error executing command in pod %s/%s: %v\n\ncommand %q stdout:\n%s\n\ncommand %q stderr:\n%s", p.namespace, p.name, @@ -321,59 +275,110 @@ func (p *pod) Exec(ctx context.Context, command string) (string, string, error) } func (p *pod) Delete(ctx context.Context) error { - return p.client.CoreV1().Pods(p.namespace).Delete(ctx, p.name, metav1.DeleteOptions{}) + return p.suite.Clientset.CoreV1().Pods(p.namespace).Delete(ctx, p.name, metav1.DeleteOptions{}) } -func (p *pod) waitForRunning(ctx context.Context, timeout time.Duration) error { - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - watcher, err := p.client.CoreV1().Pods(p.namespace).Watch(ctx, metav1.ListOptions{ - FieldSelector: fields.OneTermEqualSelector("metadata.name", p.name).String(), - }) - if err != nil { - return err - } - - defer watcher.Stop() - - for { - select { - case <-ctx.Done(): - return ctx.Err() - case event := <-watcher.ResultChan(): - if event.Type == watch.Error { - return fmt.Errorf("error watching pod: %v", event.Object) - } - - pod, ok := event.Object.(*corev1.Pod) - if !ok { - continue - } - - if pod.Name == p.name && pod.Status.Phase == corev1.PodRunning { - return nil - } - } - } -} - -// NewPodOp creates a new pod operation with the given name and namespace. -func (k8sSuite *K8sSuite) NewPodOp(name, namespace string) (podInfo, error) { +// NewPrivilegedPod creates a new pod definition with a random suffix +// in the kube-system namespace with privileged security context. +func (k8sSuite *K8sSuite) NewPrivilegedPod(name string) (podInfo, error) { randomSuffix := make([]byte, 4) if _, err := rand.Read(randomSuffix); err != nil { return nil, fmt.Errorf("failed to generate random suffix: %w", err) } + podName := fmt.Sprintf("%s-%x", name, randomSuffix) + return &pod{ - name: fmt.Sprintf("%s-%x", name, randomSuffix), - namespace: namespace, + name: podName, + namespace: "kube-system", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: podName, + Image: "alpine", + Command: []string{ + "/bin/sh", + "-c", + "--", + }, + Args: []string{ + "trap : TERM INT; (tail -f /dev/null) & wait", + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: pointer.To(true), + }, + // lvm commands even though executed in the host mount namespace, still need access to /dev 🤷🏼, + // otherwise lvcreate commands hangs on semop syscall + VolumeMounts: []corev1.VolumeMount{ + { + Name: "dev", + MountPath: "/dev", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "dev", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev", + }, + }, + }, + }, + HostNetwork: true, + HostIPC: true, + HostPID: true, + }, + }, - client: k8sSuite.Clientset, - restConfig: k8sSuite.RestConfig, + suite: k8sSuite, + }, nil +} - logF: k8sSuite.T().Logf, +// NewPod creates a new pod definition with a random suffix +// in the default namespace. +func (k8sSuite *K8sSuite) NewPod(name string) (podInfo, error) { + randomSuffix := make([]byte, 4) + + if _, err := rand.Read(randomSuffix); err != nil { + return nil, fmt.Errorf("failed to generate random suffix: %w", err) + } + + podName := fmt.Sprintf("%s-%x", name, randomSuffix) + + return &pod{ + name: podName, + namespace: "default", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: podName, + Image: "alpine", + Command: []string{ + "/bin/sh", + "-c", + "--", + }, + Args: []string{ + "trap : TERM INT; (tail -f /dev/null) & wait", + }, + }, + }, + }, + }, + + suite: k8sSuite, }, nil } @@ -448,36 +453,6 @@ func (k8sSuite *K8sSuite) LogPodLogs(ctx context.Context, namespace, podName str } } -// WaitForPodToBeDeleted waits for the pod with the given namespace and name to be deleted. -func (k8sSuite *K8sSuite) WaitForPodToBeDeleted(ctx context.Context, timeout time.Duration, namespace, podName string) error { - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - watcher, err := k8sSuite.Clientset.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{ - FieldSelector: fields.OneTermEqualSelector("metadata.name", podName).String(), - }) - if err != nil { - return err - } - - defer watcher.Stop() - - for { - select { - case <-ctx.Done(): - return ctx.Err() - case event := <-watcher.ResultChan(): - if event.Type == watch.Deleted { - return nil - } - - if event.Type == watch.Error { - return fmt.Errorf("error watching pod: %v", event.Object) - } - } - } -} - // HelmInstall installs the Helm chart with the given namespace, repository, version, release name, chart name and values. func (k8sSuite *K8sSuite) HelmInstall(ctx context.Context, namespace, repository, version, releaseName, chartName string, valuesBytes []byte) error { tempFile := filepath.Join(k8sSuite.T().TempDir(), "values.yaml") @@ -635,55 +610,6 @@ func (k8sSuite *K8sSuite) RunFIOTest(ctx context.Context, storageClasss, size st return cmd.Run() } -// ExecuteCommandInPod executes the given command in the pod with the given namespace and name. -func (k8sSuite *K8sSuite) ExecuteCommandInPod(ctx context.Context, namespace, podName, command string) (string, string, error) { - cmd := []string{ - "/bin/sh", - "-c", - command, - } - req := k8sSuite.Clientset.CoreV1().RESTClient().Post().Resource("pods").Name(podName). - Namespace(namespace).SubResource("exec") - option := &corev1.PodExecOptions{ - Command: cmd, - Stdin: false, - Stdout: true, - Stderr: true, - TTY: false, - } - - req.VersionedParams( - option, - scheme.ParameterCodec, - ) - - exec, err := remotecommand.NewSPDYExecutor(k8sSuite.RestConfig, "POST", req.URL()) - if err != nil { - return "", "", err - } - - var stdout, stderr strings.Builder - - err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ - Stdout: &stdout, - Stderr: &stderr, - }) - if err != nil { - k8sSuite.T().Logf( - "error executing command in pod %s/%s: %v\n\ncommand %q stdout:\n%s\n\ncommand %q stderr:\n%s", - namespace, - podName, - err, - command, - stdout.String(), - command, - stderr.String(), - ) - } - - return stdout.String(), stderr.String(), err -} - // GetPodsWithLabel returns the pods with the given label in the specified namespace. func (k8sSuite *K8sSuite) GetPodsWithLabel(ctx context.Context, namespace, label string) (*corev1.PodList, error) { podList, err := k8sSuite.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{