traefik/integration/integration_test.go

// This is the main file that sets up integration tests using go-check.
package integration

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/fs"
	stdlog "log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"slices"
	"strings"
	"testing"
	"text/template"
	"time"

	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/mount"
	dockernetwork "github.com/docker/docker/api/types/network"
	"github.com/fatih/structs"
	"github.com/rs/zerolog/log"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
	"github.com/testcontainers/testcontainers-go"
	"github.com/testcontainers/testcontainers-go/network"
	"github.com/traefik/traefik/v3/integration/try"
	"gopkg.in/yaml.v3"
)

var (
	showLog               = flag.Bool("tlog", false, "always show Traefik logs")
	k8sConformance        = flag.Bool("k8sConformance", false, "run K8s Gateway API conformance test")
	k8sConformanceRunTest = flag.String("k8sConformanceRunTest", "", "run a specific K8s Gateway API conformance test")
)

const tailscaleSecretFilePath = "tailscale.secret"

type composeConfig struct {
	Services map[string]composeService `yaml:"services"`
}

type composeService struct {
	Image       string            `yaml:"image"`
	Labels      map[string]string `yaml:"labels"`
	Hostname    string            `yaml:"hostname"`
	Volumes     []string          `yaml:"volumes"`
	CapAdd      []string          `yaml:"cap_add"`
	Command     []string          `yaml:"command"`
	Environment map[string]string `yaml:"environment"`
	Privileged  bool              `yaml:"privileged"`
	Deploy      composeDeploy     `yaml:"deploy"`
}

type composeDeploy struct {
	Replicas int `yaml:"replicas"`
}

type BaseSuite struct {
	suite.Suite
	containers map[string]testcontainers.Container
	network    *testcontainers.DockerNetwork
	hostIP     string
}

func (s *BaseSuite) waitForTraefik(containerName string) {
	time.Sleep(1 * time.Second)

	// Wait for Traefik to turn ready.
	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8080/api/rawdata", nil)
	require.NoError(s.T(), err)

	err = try.Request(req, 2*time.Second, try.StatusCodeIs(http.StatusOK), try.BodyContains(containerName))
	require.NoError(s.T(), err)
}

func (s *BaseSuite) displayTraefikLogFile(path string) {
	if s.T().Failed() {
		if _, err := os.Stat(path); !os.IsNotExist(err) {
			content, errRead := os.ReadFile(path)
			// TODO TestName
			// fmt.Printf("%s: Traefik logs: \n", c.TestName())
			fmt.Print("Traefik logs: \n")
			if errRead == nil {
				fmt.Println(content)
			} else {
				fmt.Println(errRead)
			}
		} else {
			// fmt.Printf("%s: No Traefik logs.\n", c.TestName())
			fmt.Print("No Traefik logs.\n")
		}
		errRemove := os.Remove(path)
		if errRemove != nil {
			fmt.Println(errRemove)
		}
	}
}

func (s *BaseSuite) SetupSuite() {
	if isDockerDesktop(context.Background(), s.T()) {
		_, err := os.Stat(tailscaleSecretFilePath)
		require.NoError(s.T(), err, "Tailscale need to be configured when running integration tests with Docker Desktop: (https://doc.traefik.io/traefik/v2.11/contributing/building-testing/#testing)")
	}

	// configure default standard log.
	stdlog.SetFlags(stdlog.Lshortfile | stdlog.LstdFlags)
	// TODO
	// stdlog.SetOutput(log.Logger)

	ctx := context.Background()
	// Create docker network
	// docker network create traefik-test-network --driver bridge --subnet 172.31.42.0/24
	var opts []network.NetworkCustomizer
	opts = append(opts, network.WithDriver("bridge"))
	opts = append(opts, network.WithIPAM(&dockernetwork.IPAM{
		Driver: "default",
		Config: []dockernetwork.IPAMConfig{
			{
				Subnet: "172.31.42.0/24",
			},
		},
	}))
	dockerNetwork, err := network.New(ctx, opts...)
	require.NoError(s.T(), err)

	s.network = dockerNetwork
	s.hostIP = "172.31.42.1"
	if isDockerDesktop(ctx, s.T()) {
		s.hostIP = getDockerDesktopHostIP(ctx, s.T())
		s.setupVPN(tailscaleSecretFilePath)
	}
}

func getDockerDesktopHostIP(ctx context.Context, t *testing.T) string {
	t.Helper()

	req := testcontainers.ContainerRequest{
		Image: "alpine",
		HostConfigModifier: func(config *container.HostConfig) {
			config.AutoRemove = true
		},
		Cmd: []string{"getent", "hosts", "host.docker.internal"},
	}

	con, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
		ContainerRequest: req,
		Started:          true,
	})
	require.NoError(t, err)

	closer, err := con.Logs(ctx)
	require.NoError(t, err)

	all, err := io.ReadAll(closer)
	require.NoError(t, err)

	ipRegex := regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
	matches := ipRegex.FindAllString(string(all), -1)
	require.Len(t, matches, 1)

	return matches[0]
}

func isDockerDesktop(ctx context.Context, t *testing.T) bool {
	t.Helper()

	cli, err := testcontainers.NewDockerClientWithOpts(ctx)
	if err != nil {
		t.Fatalf("failed to create docker client: %s", err)
	}

	info, err := cli.Info(ctx)
	if err != nil {
		t.Fatalf("failed to get docker info: %s", err)
	}

	return info.OperatingSystem == "Docker Desktop"
}

func (s *BaseSuite) TearDownSuite() {
	s.composeDown()

	err := try.Do(5*time.Second, func() error {
		if s.network != nil {
			err := s.network.Remove(context.Background())
			if err != nil {
				return err
			}
		}

		return nil
	})
	require.NoError(s.T(), err)
}

// createComposeProject creates the docker compose project stored as a field in the BaseSuite.
// This method should be called before starting and/or stopping compose services.
func (s *BaseSuite) createComposeProject(name string) {
	composeFile := fmt.Sprintf("resources/compose/%s.yml", name)

	file, err := os.ReadFile(composeFile)
	require.NoError(s.T(), err)

	var composeConfigData composeConfig
	err = yaml.Unmarshal(file, &composeConfigData)
	require.NoError(s.T(), err)

	if s.containers == nil {
		s.containers = map[string]testcontainers.Container{}
	}

	ctx := context.Background()

	for id, containerConfig := range composeConfigData.Services {
		var mounts []mount.Mount
		for _, volume := range containerConfig.Volumes {
			split := strings.Split(volume, ":")
			if len(split) != 2 {
				continue
			}

			if strings.HasPrefix(split[0], "./") {
				path, err := os.Getwd()
				if err != nil {
					log.Err(err).Msg("can't determine current directory")
					continue
				}
				split[0] = strings.Replace(split[0], "./", path+"/", 1)
			}

			abs, err := filepath.Abs(split[0])
			require.NoError(s.T(), err)

			mounts = append(mounts, mount.Mount{Source: abs, Target: split[1], Type: mount.TypeBind})
		}

		if containerConfig.Deploy.Replicas > 0 {
			for i := range containerConfig.Deploy.Replicas {
				id = fmt.Sprintf("%s-%d", id, i+1)
				con, err := s.createContainer(ctx, containerConfig, id, mounts)
				require.NoError(s.T(), err)
				s.containers[id] = con
			}
			continue
		}

		con, err := s.createContainer(ctx, containerConfig, id, mounts)
		require.NoError(s.T(), err)
		s.containers[id] = con
	}
}

func (s *BaseSuite) createContainer(ctx context.Context, containerConfig composeService, id string, mounts []mount.Mount) (testcontainers.Container, error) {
	req := testcontainers.ContainerRequest{
		Image:      containerConfig.Image,
		Env:        containerConfig.Environment,
		Cmd:        containerConfig.Command,
		Labels:     containerConfig.Labels,
		Name:       id,
		Hostname:   containerConfig.Hostname,
		Privileged: containerConfig.Privileged,
		Networks:   []string{s.network.Name},
		HostConfigModifier: func(config *container.HostConfig) {
			if containerConfig.CapAdd != nil {
				config.CapAdd = containerConfig.CapAdd
			}
			if !isDockerDesktop(ctx, s.T()) {
				config.ExtraHosts = append(config.ExtraHosts, "host.docker.internal:"+s.hostIP)
			}
			config.Mounts = mounts
		},
	}
	con, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
		ContainerRequest: req,
		Started:          false,
	})

	return con, err
}

// composeUp starts the given services of the current docker compose project, if they are not already started.
// Already running services are not affected (i.e. not stopped).
func (s *BaseSuite) composeUp(services ...string) {
	for name, con := range s.containers {
		if len(services) == 0 || slices.Contains(services, name) {
			err := con.Start(context.Background())
			require.NoError(s.T(), err)
		}
	}
}

// composeStop stops the given services of the current docker compose project and removes the corresponding containers.
func (s *BaseSuite) composeStop(services ...string) {
	for name, con := range s.containers {
		if len(services) == 0 || slices.Contains(services, name) {
			timeout := 10 * time.Second
			err := con.Stop(context.Background(), &timeout)
			require.NoError(s.T(), err)
		}
	}
}

// composeDown stops all compose project services and removes the corresponding containers.
func (s *BaseSuite) composeDown() {
	for _, c := range s.containers {
		err := c.Terminate(context.Background())
		require.NoError(s.T(), err)
	}
	s.containers = map[string]testcontainers.Container{}
}

func (s *BaseSuite) cmdTraefik(args ...string) (*exec.Cmd, *bytes.Buffer) {
	binName := "traefik"
	if runtime.GOOS == "windows" {
		binName += ".exe"
	}

	traefikBinPath := filepath.Join("..", "dist", runtime.GOOS, runtime.GOARCH, binName)
	cmd := exec.Command(traefikBinPath, args...)

	s.T().Cleanup(func() {
		s.killCmd(cmd)
	})
	var out bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &out

	err := cmd.Start()
	require.NoError(s.T(), err)

	return cmd, &out
}

func (s *BaseSuite) killCmd(cmd *exec.Cmd) {
	if cmd.Process == nil {
		log.Error().Msg("No process to kill")
		return
	}
	err := cmd.Process.Kill()
	if err != nil {
		log.Error().Err(err).Msg("Kill")
	}

	time.Sleep(100 * time.Millisecond)
}

func (s *BaseSuite) traefikCmd(args ...string) *exec.Cmd {
	cmd, out := s.cmdTraefik(args...)

	s.T().Cleanup(func() {
		if s.T().Failed() || *showLog {
			s.displayLogK3S()
			s.displayLogCompose()
			s.displayTraefikLog(out)
		}
	})

	return cmd
}

func (s *BaseSuite) displayLogK3S() {
	filePath := "./fixtures/k8s/config.skip/k3s.log"
	if _, err := os.Stat(filePath); err == nil {
		content, errR := os.ReadFile(filePath)
		if errR != nil {
			log.Error().Err(errR).Send()
		}
		log.Print(string(content))
	}
	log.Print()
	log.Print("################################")
	log.Print()
}

func (s *BaseSuite) displayLogCompose() {
	for name, ctn := range s.containers {
		readCloser, err := ctn.Logs(context.Background())
		require.NoError(s.T(), err)
		for {
			b := make([]byte, 1024)
			_, err := readCloser.Read(b)
			if errors.Is(err, io.EOF) {
				break
			}
			require.NoError(s.T(), err)

			trimLogs := bytes.Trim(bytes.TrimSpace(b), string([]byte{0}))
			if len(trimLogs) > 0 {
				log.Info().Str("container", name).Msg(string(trimLogs))
			}
		}
	}
}

func (s *BaseSuite) displayTraefikLog(output *bytes.Buffer) {
	if output == nil || output.Len() == 0 {
		log.Info().Msg("No Traefik logs.")
	} else {
		for _, line := range strings.Split(output.String(), "\n") {
			log.Info().Msg(line)
		}
	}
}

func (s *BaseSuite) getDockerHost() string {
	dockerHost := os.Getenv("DOCKER_HOST")
	if dockerHost == "" {
		// Default docker socket
		dockerHost = "unix:///var/run/docker.sock"
	}

	return dockerHost
}

func (s *BaseSuite) adaptFile(path string, tempObjects interface{}) string {
	// Load file
	tmpl, err := template.ParseFiles(path)
	require.NoError(s.T(), err)

	folder, prefix := filepath.Split(path)
	tmpFile, err := os.CreateTemp(folder, strings.TrimSuffix(prefix, filepath.Ext(prefix))+"_*"+filepath.Ext(prefix))
	require.NoError(s.T(), err)
	defer tmpFile.Close()

	model := structs.Map(tempObjects)
	model["SelfFilename"] = tmpFile.Name()

	err = tmpl.ExecuteTemplate(tmpFile, prefix, model)
	require.NoError(s.T(), err)
	err = tmpFile.Sync()

	require.NoError(s.T(), err)

	s.T().Cleanup(func() {
		os.Remove(tmpFile.Name())
	})
	return tmpFile.Name()
}

func (s *BaseSuite) getComposeServiceIP(name string) string {
	container, ok := s.containers[name]
	if !ok {
		return ""
	}
	ip, err := container.ContainerIP(context.Background())
	if err != nil {
		return ""
	}
	return ip
}

func withConfigFile(file string) string {
	return "--configFile=" + file
}

// setupVPN starts Tailscale on the corresponding container, and makes it a subnet
// router, for all the other containers (whoamis, etc) subsequently started for the
// integration tests.
// It only does so if the file provided as argument exists, and contains a
// Tailscale auth key (an ephemeral, but reusable, one is recommended).
//
// Add this section to your tailscale ACLs to auto-approve the routes for the
// containers in the docker subnet:
//
//	"autoApprovers": {
//	  // Allow myself to automatically advertize routes for docker networks
//	  "routes": {
//	    "172.0.0.0/8": ["your_tailscale_identity"],
//	  },
//	},
func (s *BaseSuite) setupVPN(keyFile string) {
	data, err := os.ReadFile(keyFile)
	if err != nil {
		if !errors.Is(err, fs.ErrNotExist) {
			log.Error().Err(err).Send()
		}

		return
	}
	authKey := strings.TrimSpace(string(data))
	// // TODO: copy and create versions that don't need a check.C?
	s.createComposeProject("tailscale")
	s.composeUp()
	time.Sleep(5 * time.Second)
	// If we ever change the docker subnet in the Makefile,
	// we need to change this one below correspondingly.
	s.composeExec("tailscaled", "tailscale", "up", "--authkey="+authKey, "--advertise-routes=172.31.42.0/24")
}

// composeExec runs the command in the given args in the given compose service container.
// Already running services are not affected (i.e. not stopped).
func (s *BaseSuite) composeExec(service string, args ...string) string {
	require.Contains(s.T(), s.containers, service)

	_, reader, err := s.containers[service].Exec(context.Background(), args)
	require.NoError(s.T(), err)

	content, err := io.ReadAll(reader)
	require.NoError(s.T(), err)

	return string(content)
}