alt-orchestra/talos
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: refactor tests

Browse Source 
Refactor tests to avoid code duplication.

Signed-off-by: Noel Georgi <git@frezbo.dev>
(cherry picked from commit 9fa08e843728dbd85ed7e0035f59cdd6232de9a9)
This commit is contained in:
Noel Georgi 2024-09-20 16:22:01 +05:30 committed by Andrey Smirnov
parent 920d8c8297
commit 67ba478253
No known key found for this signature in database
GPG Key ID: FE042E3D4085A811
4 changed files with 131 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
internal/integration/api/common.go
View File

@ -11,10 +11,6 @@ import (
"strings"
"time"
"github.com/siderolabs/go-pointer"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/siderolabs/talos/internal/integration/base"
)
@ -83,39 +79,17 @@ virtual memory (kb) (-v) unlimited
file locks (-x) unlimited
`
const (
namespace = "default"
pod = "defaults-test"
)
_, err := suite.Clientset.CoreV1().Pods(namespace).Create(suite.ctx, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: pod,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: pod,
Image: "alpine",
Command: []string{
"tail",
"-f",
"/dev/null",
},
},
},
TerminationGracePeriodSeconds: pointer.To[int64](0),
},
}, metav1.CreateOptions{})
defaultsTestPodDef, err := suite.NewPod("defaults-ulimits-test")
suite.Require().NoError(err)
defer suite.Clientset.CoreV1().Pods(namespace).Delete(suite.ctx, pod, metav1.DeleteOptions{}) //nolint:errcheck
suite.Require().NoError(defaultsTestPodDef.Create(suite.ctx, 5*time.Minute))
// wait for the pod to be ready
suite.Require().NoError(suite.WaitForPodToBeRunning(suite.ctx, 10*time.Minute, namespace, pod))
defer defaultsTestPodDef.Delete(suite.ctx) //nolint:errcheck
stdout, stderr, err := suite.ExecuteCommandInPod(suite.ctx, namespace, pod, "ulimit -c -d -e -f -l -m -n -q -r -s -t -v -x")
stdout, stderr, err := defaultsTestPodDef.Exec(
suite.ctx,
"ulimit -c -d -e -f -l -m -n -q -r -s -t -v -x",
)
suite.Require().NoError(err)
suite.Require().Equal("", stderr)
@ -129,47 +103,19 @@ func (suite *CommonSuite) TestDNSResolver() {
suite.AssertClusterHealthy(suite.ctx)
}
const (
namespace = "default"
pod = "dns-test"
)
_, err := suite.Clientset.CoreV1().Pods(namespace).Create(suite.ctx, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: pod,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: pod,
Image: "alpine",
Command: []string{
"tail",
"-f",
"/dev/null",
},
},
},
TerminationGracePeriodSeconds: pointer.To[int64](0),
},
}, metav1.CreateOptions{})
dnsTestPodDef, err := suite.NewPod("dns-test")
suite.Require().NoError(err)
suite.T().Cleanup(func() {
cleanUpCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute)
defer cleanupCancel()
suite.Require().NoError(dnsTestPodDef.Create(suite.ctx, 5*time.Minute))
suite.Require().NoError(
suite.Clientset.CoreV1().Pods(namespace).Delete(cleanUpCtx, pod, metav1.DeleteOptions{}),
)
})
defer dnsTestPodDef.Delete(suite.ctx) //nolint:errcheck
// wait for the pod to be ready
suite.Require().NoError(suite.WaitForPodToBeRunning(suite.ctx, time.Minute, namespace, pod))
stdout, stderr, err := suite.ExecuteCommandInPod(suite.ctx, namespace, pod, "wget -S https://www.google.com/")
stdout, stderr, err := dnsTestPodDef.Exec(
suite.ctx,
"wget -S https://www.google.com/",
)
suite.Assert().NoError(err)
suite.Assert().Equal("", stdout)
suite.Assert().Contains(stderr, "'index.html' saved")
@ -183,7 +129,10 @@ func (suite *CommonSuite) TestDNSResolver() {
suite.T().FailNow()
}
_, stderr, err = suite.ExecuteCommandInPod(suite.ctx, namespace, pod, "apk add --update bind-tools")
_, stderr, err = dnsTestPodDef.Exec(
suite.ctx,
"apk add --update bind-tools",
)
suite.Assert().NoError(err)
suite.Assert().Empty(stderr, "stderr: %s", stderr)
@ -192,7 +141,10 @@ func (suite *CommonSuite) TestDNSResolver() {
suite.T().FailNow()
}
stdout, stderr, err = suite.ExecuteCommandInPod(suite.ctx, namespace, pod, "dig really-long-record.dev.siderolabs.io")
stdout, stderr, err = dnsTestPodDef.Exec(
suite.ctx,
"dig really-long-record.dev.siderolabs.io",
)
suite.Assert().NoError(err)
suite.Assert().Contains(stdout, "status: NOERROR")

12
internal/integration/api/extensions_qemu.go
View File

@ -145,7 +145,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsISCSI() {
ctx := client.WithNode(suite.ctx, node)
iscsiCreatePodDef, err := suite.NewPodOp("iscsi-create", "kube-system")
iscsiCreatePodDef, err := suite.NewPrivilegedPod("iscsi-create")
suite.Require().NoError(err)
suite.Require().NoError(iscsiCreatePodDef.Create(suite.ctx, 5*time.Minute))
@ -446,7 +446,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsMdADM() {
userDisksJoined := strings.Join(userDisks[:2], " ")
mdAdmCreatePodDef, err := suite.NewPodOp("mdadm-create", "kube-system")
mdAdmCreatePodDef, err := suite.NewPrivilegedPod("mdadm-create")
suite.Require().NoError(err)
suite.Require().NoError(mdAdmCreatePodDef.Create(suite.ctx, 5*time.Minute))
@ -467,7 +467,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsMdADM() {
hostname := hostNameStatus.TypedSpec().Hostname
deletePodDef, err := suite.NewPodOp("mdadm-destroy", "kube-system")
deletePodDef, err := suite.NewPrivilegedPod("mdadm-destroy")
suite.Require().NoError(err)
suite.Require().NoError(deletePodDef.Create(suite.ctx, 5*time.Minute))
@ -526,7 +526,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsZFS() {
suite.Require().NotEmpty(userDisks, "expected at least one user disks to be available")
zfsPodDef, err := suite.NewPodOp("zpool-create", "kube-system")
zfsPodDef, err := suite.NewPrivilegedPod("zpool-create")
suite.Require().NoError(err)
suite.Require().NoError(zfsPodDef.Create(suite.ctx, 5*time.Minute))
@ -552,7 +552,7 @@ func (suite *ExtensionsSuiteQEMU) TestExtensionsZFS() {
suite.Require().Equal("", stdout)
defer func() {
deletePodDef, err := suite.NewPodOp("zpool-destroy", "kube-system")
deletePodDef, err := suite.NewPrivilegedPod("zpool-destroy")
suite.Require().NoError(err)
suite.Require().NoError(deletePodDef.Create(suite.ctx, 5*time.Minute))
@ -605,7 +605,7 @@ func (suite *ExtensionsSuiteQEMU) checkZFSPoolMounted() bool {
// TestExtensionsUtilLinuxTools verifies util-linux-tools are working.
func (suite *ExtensionsSuiteQEMU) TestExtensionsUtilLinuxTools() {
utilLinuxPodDef, err := suite.NewPodOp("util-linux-tools-test", "kube-system")
utilLinuxPodDef, err := suite.NewPrivilegedPod("util-linux-tools-test")
suite.Require().NoError(err)
suite.Require().NoError(utilLinuxPodDef.Create(suite.ctx, 5*time.Minute))

4
internal/integration/api/volumes.go
View File

@ -194,7 +194,7 @@ func (suite *VolumesSuite) TestLVMActivation() {
userDisksJoined := strings.Join(userDisks[:2], " ")
podDef, err := suite.NewPodOp("pv-create", "kube-system")
podDef, err := suite.NewPrivilegedPod("pv-create")
suite.Require().NoError(err)
suite.Require().NoError(podDef.Create(suite.ctx, 5*time.Minute))
@ -226,7 +226,7 @@ func (suite *VolumesSuite) TestLVMActivation() {
suite.Require().Contains(stdout, "Logical volume \"lv1\" created.")
defer func() {
deletePodDef, err := suite.NewPodOp("pv-destroy", "kube-system")
deletePodDef, err := suite.NewPrivilegedPod("pv-destroy")
suite.Require().NoError(err)
suite.Require().NoError(deletePodDef.Create(suite.ctx, 5*time.Minute))

274
internal/integration/base/k8s.go
View File

@ -208,11 +208,9 @@ type podInfo interface {
type pod struct {
name string
namespace string
pod *corev1.Pod
client *kubernetes.Clientset
restConfig *rest.Config
logF func(format string, args ...any)
suite *K8sSuite
}
func (p *pod) Name() string {
@ -220,56 +218,12 @@ func (p *pod) Name() string {
}
func (p *pod) Create(ctx context.Context, waitTimeout time.Duration) error {
_, err := p.client.CoreV1().Pods(p.namespace).Create(ctx, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: p.name,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: p.name,
Image: "alpine",
Command: []string{
"/bin/sh",
"-c",
"--",
},
Args: []string{
"trap : TERM INT; (tail -f /dev/null) & wait",
},
SecurityContext: &corev1.SecurityContext{
Privileged: pointer.To(true),
},
// lvm commands even though executed in the host mount namespace, still need access to /dev 🤷🏼,
// otherwise lvcreate commands hangs on semop syscall
VolumeMounts: []corev1.VolumeMount{
{
Name: "dev",
MountPath: "/dev",
},
},
},
},
Volumes: []corev1.Volume{
{
Name: "dev",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/dev",
},
},
},
},
HostNetwork: true,
HostIPC: true,
HostPID: true,
},
}, metav1.CreateOptions{})
_, err := p.suite.Clientset.CoreV1().Pods(p.namespace).Create(ctx, p.pod, metav1.CreateOptions{})
if err != nil {
return err
}
return p.waitForRunning(ctx, waitTimeout)
return p.suite.WaitForPodToBeRunning(ctx, waitTimeout, p.namespace, p.name)
}
func (p *pod) Exec(ctx context.Context, command string) (string, string, error) {
@ -278,7 +232,7 @@ func (p *pod) Exec(ctx context.Context, command string) (string, string, error)
"-c",
command,
}
req := p.client.CoreV1().RESTClient().Post().Resource("pods").Name(p.name).
req := p.suite.Clientset.CoreV1().RESTClient().Post().Resource("pods").Name(p.name).
Namespace(p.namespace).SubResource("exec")
option := &corev1.PodExecOptions{
Command: cmd,
@ -293,7 +247,7 @@ func (p *pod) Exec(ctx context.Context, command string) (string, string, error)
scheme.ParameterCodec,
)
exec, err := remotecommand.NewSPDYExecutor(p.restConfig, "POST", req.URL())
exec, err := remotecommand.NewSPDYExecutor(p.suite.RestConfig, "POST", req.URL())
if err != nil {
return "", "", err
}
@ -305,7 +259,7 @@ func (p *pod) Exec(ctx context.Context, command string) (string, string, error)
Stderr: &stderr,
})
if err != nil {
p.logF(
p.suite.T().Logf(
"error executing command in pod %s/%s: %v\n\ncommand %q stdout:\n%s\n\ncommand %q stderr:\n%s",
p.namespace,
p.name,
@ -321,59 +275,110 @@ func (p *pod) Exec(ctx context.Context, command string) (string, string, error)
}
func (p *pod) Delete(ctx context.Context) error {
return p.client.CoreV1().Pods(p.namespace).Delete(ctx, p.name, metav1.DeleteOptions{})
return p.suite.Clientset.CoreV1().Pods(p.namespace).Delete(ctx, p.name, metav1.DeleteOptions{})
}
func (p *pod) waitForRunning(ctx context.Context, timeout time.Duration) error {
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
watcher, err := p.client.CoreV1().Pods(p.namespace).Watch(ctx, metav1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", p.name).String(),
})
if err != nil {
return err
}
defer watcher.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case event := <-watcher.ResultChan():
if event.Type == watch.Error {
return fmt.Errorf("error watching pod: %v", event.Object)
}
pod, ok := event.Object.(*corev1.Pod)
if !ok {
continue
}
if pod.Name == p.name && pod.Status.Phase == corev1.PodRunning {
return nil
}
}
}
}
// NewPodOp creates a new pod operation with the given name and namespace.
func (k8sSuite *K8sSuite) NewPodOp(name, namespace string) (podInfo, error) {
// NewPrivilegedPod creates a new pod definition with a random suffix
// in the kube-system namespace with privileged security context.
func (k8sSuite *K8sSuite) NewPrivilegedPod(name string) (podInfo, error) {
randomSuffix := make([]byte, 4)
if _, err := rand.Read(randomSuffix); err != nil {
return nil, fmt.Errorf("failed to generate random suffix: %w", err)
}
podName := fmt.Sprintf("%s-%x", name, randomSuffix)
return &pod{
name: fmt.Sprintf("%s-%x", name, randomSuffix),
namespace: namespace,
name: podName,
namespace: "kube-system",
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: podName,
Image: "alpine",
Command: []string{
"/bin/sh",
"-c",
"--",
},
Args: []string{
"trap : TERM INT; (tail -f /dev/null) & wait",
},
SecurityContext: &corev1.SecurityContext{
Privileged: pointer.To(true),
},
// lvm commands even though executed in the host mount namespace, still need access to /dev 🤷🏼,
// otherwise lvcreate commands hangs on semop syscall
VolumeMounts: []corev1.VolumeMount{
{
Name: "dev",
MountPath: "/dev",
},
},
},
},
Volumes: []corev1.Volume{
{
Name: "dev",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/dev",
},
},
},
},
HostNetwork: true,
HostIPC: true,
HostPID: true,
},
},
client: k8sSuite.Clientset,
restConfig: k8sSuite.RestConfig,
suite: k8sSuite,
}, nil
}
logF: k8sSuite.T().Logf,
// NewPod creates a new pod definition with a random suffix
// in the default namespace.
func (k8sSuite *K8sSuite) NewPod(name string) (podInfo, error) {
randomSuffix := make([]byte, 4)
if _, err := rand.Read(randomSuffix); err != nil {
return nil, fmt.Errorf("failed to generate random suffix: %w", err)
}
podName := fmt.Sprintf("%s-%x", name, randomSuffix)
return &pod{
name: podName,
namespace: "default",
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: podName,
Image: "alpine",
Command: []string{
"/bin/sh",
"-c",
"--",
},
Args: []string{
"trap : TERM INT; (tail -f /dev/null) & wait",
},
},
},
},
},
suite: k8sSuite,
}, nil
}
@ -448,36 +453,6 @@ func (k8sSuite *K8sSuite) LogPodLogs(ctx context.Context, namespace, podName str
}
}
// WaitForPodToBeDeleted waits for the pod with the given namespace and name to be deleted.
func (k8sSuite *K8sSuite) WaitForPodToBeDeleted(ctx context.Context, timeout time.Duration, namespace, podName string) error {
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
watcher, err := k8sSuite.Clientset.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", podName).String(),
})
if err != nil {
return err
}
defer watcher.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case event := <-watcher.ResultChan():
if event.Type == watch.Deleted {
return nil
}
if event.Type == watch.Error {
return fmt.Errorf("error watching pod: %v", event.Object)
}
}
}
}
// HelmInstall installs the Helm chart with the given namespace, repository, version, release name, chart name and values.
func (k8sSuite *K8sSuite) HelmInstall(ctx context.Context, namespace, repository, version, releaseName, chartName string, valuesBytes []byte) error {
tempFile := filepath.Join(k8sSuite.T().TempDir(), "values.yaml")
@ -635,55 +610,6 @@ func (k8sSuite *K8sSuite) RunFIOTest(ctx context.Context, storageClasss, size st
return cmd.Run()
}
// ExecuteCommandInPod executes the given command in the pod with the given namespace and name.
func (k8sSuite *K8sSuite) ExecuteCommandInPod(ctx context.Context, namespace, podName, command string) (string, string, error) {
cmd := []string{
"/bin/sh",
"-c",
command,
}
req := k8sSuite.Clientset.CoreV1().RESTClient().Post().Resource("pods").Name(podName).
Namespace(namespace).SubResource("exec")
option := &corev1.PodExecOptions{
Command: cmd,
Stdin: false,
Stdout: true,
Stderr: true,
TTY: false,
}
req.VersionedParams(
option,
scheme.ParameterCodec,
)
exec, err := remotecommand.NewSPDYExecutor(k8sSuite.RestConfig, "POST", req.URL())
if err != nil {
return "", "", err
}
var stdout, stderr strings.Builder
err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{
Stdout: &stdout,
Stderr: &stderr,
})
if err != nil {
k8sSuite.T().Logf(
"error executing command in pod %s/%s: %v\n\ncommand %q stdout:\n%s\n\ncommand %q stderr:\n%s",
namespace,
podName,
err,
command,
stdout.String(),
command,
stderr.String(),
)
}
return stdout.String(), stderr.String(), err
}
// GetPodsWithLabel returns the pods with the given label in the specified namespace.
func (k8sSuite *K8sSuite) GetPodsWithLabel(ctx context.Context, namespace, label string) (*corev1.PodList, error) {
podList, err := k8sSuite.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{