traefik/integration/integration_test.go

// This is the main file that sets up integration tests using go-check.
package integration

import (
	"bytes"
	"context"
	"errors"
	"flag"
	"fmt"
	"io/fs"
	stdlog "log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"text/template"
	"time"

	"github.com/compose-spec/compose-go/cli"
	"github.com/compose-spec/compose-go/types"
	"github.com/docker/cli/cli/config/configfile"
	"github.com/docker/compose/v2/cmd/formatter"
	composeapi "github.com/docker/compose/v2/pkg/api"
	"github.com/docker/compose/v2/pkg/compose"
	dockertypes "github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/client"
	"github.com/fatih/structs"
	"github.com/go-check/check"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/sirupsen/logrus"
	"github.com/traefik/traefik/v2/pkg/logs"
	checker "github.com/vdemeester/shakers"
)

var (
	integration = flag.Bool("integration", false, "run integration tests")
	showLog     = flag.Bool("tlog", false, "always show Traefik logs")
)

func Test(t *testing.T) {
	if !*integration {
		log.Info().Msg("Integration tests disabled.")
		return
	}

	log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}).
		With().Timestamp().Caller().Logger()
	zerolog.SetGlobalLevel(zerolog.InfoLevel)

	// Global logrus replacement
	logrus.StandardLogger().Out = logs.NoLevel(log.Logger, zerolog.DebugLevel)

	// configure default standard log.
	stdlog.SetFlags(stdlog.Lshortfile | stdlog.LstdFlags)
	stdlog.SetOutput(logs.NoLevel(log.Logger, zerolog.DebugLevel))

	// TODO(mpl): very niche optimization: do not start tailscale if none of the wanted tests actually need it (e.g. KeepAliveSuite does not).
	var (
		vpn    *tailscaleNotSuite
		useVPN bool
	)
	if os.Getenv("IN_DOCKER") != "true" {
		if vpn = setupVPN(nil, "tailscale.secret"); vpn != nil {
			defer vpn.TearDownSuite(nil)
			useVPN = true
		}
	}

	check.Suite(&AccessLogSuite{})
	if !useVPN {
		check.Suite(&AcmeSuite{})
	}
	check.Suite(&ConsulCatalogSuite{})
	check.Suite(&ConsulSuite{})
	check.Suite(&DockerComposeSuite{})
	check.Suite(&DockerSuite{})
	check.Suite(&ErrorPagesSuite{})
	check.Suite(&EtcdSuite{})
	check.Suite(&FileSuite{})
	check.Suite(&GRPCSuite{})
	check.Suite(&HeadersSuite{})
	check.Suite(&HealthCheckSuite{})
	check.Suite(&HostResolverSuite{})
	check.Suite(&HTTPSSuite{})
	check.Suite(&HTTPSuite{})
	if !useVPN {
		check.Suite(&K8sSuite{})
	}
	check.Suite(&KeepAliveSuite{})
	check.Suite(&LogRotationSuite{})
	if !useVPN {
		check.Suite(&ProxyProtocolSuite{})
	}
	check.Suite(&RateLimitSuite{})
	check.Suite(&RedisSuite{})
	check.Suite(&RestSuite{})
	check.Suite(&RetrySuite{})
	check.Suite(&SimpleSuite{})
	check.Suite(&TCPSuite{})
	check.Suite(&TimeoutSuite{})
	check.Suite(&ThrottlingSuite{})
	check.Suite(&TLSClientHeadersSuite{})
	check.Suite(&TracingSuite{})
	check.Suite(&UDPSuite{})
	check.Suite(&WebsocketSuite{})
	check.Suite(&ZookeeperSuite{})

	check.TestingT(t)
}

var traefikBinary = "../dist/traefik"

type BaseSuite struct {
	composeProject       *types.Project
	dockerComposeService composeapi.Service
	dockerClient         *client.Client
}

func (s *BaseSuite) TearDownSuite(c *check.C) {
	if s.composeProject != nil && s.dockerComposeService != nil {
		s.composeDown(c)
	}
}

// createComposeProject creates the docker compose project stored as a field in the BaseSuite.
// This method should be called before starting and/or stopping compose services.
func (s *BaseSuite) createComposeProject(c *check.C, name string) {
	projectName := fmt.Sprintf("traefik-integration-test-%s", name)
	composeFile := fmt.Sprintf("resources/compose/%s.yml", name)

	var err error
	s.dockerClient, err = client.NewClientWithOpts()
	c.Assert(err, checker.IsNil)

	s.dockerComposeService = compose.NewComposeService(s.dockerClient, &configfile.ConfigFile{})
	ops, err := cli.NewProjectOptions([]string{composeFile}, cli.WithName(projectName))
	c.Assert(err, checker.IsNil)

	s.composeProject, err = cli.ProjectFromOptions(ops)
	c.Assert(err, checker.IsNil)
}

// composeUp starts the given services of the current docker compose project, if they are not already started.
// Already running services are not affected (i.e. not stopped).
func (s *BaseSuite) composeUp(c *check.C, services ...string) {
	c.Assert(s.composeProject, check.NotNil)
	c.Assert(s.dockerComposeService, check.NotNil)

	// We use Create and Restart instead of Up, because the only option that actually works to control which containers
	// are started is within the RestartOptions.
	err := s.dockerComposeService.Create(context.Background(), s.composeProject, composeapi.CreateOptions{})
	c.Assert(err, checker.IsNil)

	err = s.dockerComposeService.Restart(context.Background(), s.composeProject, composeapi.RestartOptions{Services: services})
	c.Assert(err, checker.IsNil)
}

// composeExec runs the command in the given args in the given compose service container.
// Already running services are not affected (i.e. not stopped).
func (s *BaseSuite) composeExec(c *check.C, service string, args ...string) {
	c.Assert(s.composeProject, check.NotNil)
	c.Assert(s.dockerComposeService, check.NotNil)

	_, err := s.dockerComposeService.Exec(context.Background(), s.composeProject.Name, composeapi.RunOptions{
		Service: service,
		Stdin:   os.Stdin,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
		Command: args,
		Tty:     false,
		Index:   1,
	})
	c.Assert(err, checker.IsNil)
}

// composeStop stops the given services of the current docker compose project and removes the corresponding containers.
func (s *BaseSuite) composeStop(c *check.C, services ...string) {
	c.Assert(s.dockerComposeService, check.NotNil)
	c.Assert(s.composeProject, check.NotNil)

	err := s.dockerComposeService.Stop(context.Background(), s.composeProject, composeapi.StopOptions{Services: services})
	c.Assert(err, checker.IsNil)

	err = s.dockerComposeService.Remove(context.Background(), s.composeProject, composeapi.RemoveOptions{
		Services: services,
		Force:    true,
	})
	c.Assert(err, checker.IsNil)
}

// composeDown stops all compose project services and removes the corresponding containers.
func (s *BaseSuite) composeDown(c *check.C) {
	c.Assert(s.dockerComposeService, check.NotNil)
	c.Assert(s.composeProject, check.NotNil)

	err := s.dockerComposeService.Down(context.Background(), s.composeProject.Name, composeapi.DownOptions{})
	c.Assert(err, checker.IsNil)
}

func (s *BaseSuite) cmdTraefik(args ...string) (*exec.Cmd, *bytes.Buffer) {
	cmd := exec.Command(traefikBinary, args...)
	var out bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &out
	return cmd, &out
}

func (s *BaseSuite) killCmd(cmd *exec.Cmd) {
	err := cmd.Process.Kill()
	if err != nil {
		log.Error().Err(err).Msg("Kill")
	}

	time.Sleep(100 * time.Millisecond)
}

func (s *BaseSuite) traefikCmd(args ...string) (*exec.Cmd, func(*check.C)) {
	cmd, out := s.cmdTraefik(args...)
	return cmd, func(c *check.C) {
		if c.Failed() || *showLog {
			s.displayLogK3S()
			s.displayLogCompose(c)
			s.displayTraefikLog(c, out)
		}
	}
}

func (s *BaseSuite) displayLogK3S() {
	filePath := "./fixtures/k8s/config.skip/k3s.log"
	if _, err := os.Stat(filePath); err == nil {
		content, errR := os.ReadFile(filePath)
		if errR != nil {
			log.Error().Err(errR).Send()
		}
		log.Print(string(content))
	}
	log.Print()
	log.Print("################################")
	log.Print()
}

func (s *BaseSuite) displayLogCompose(c *check.C) {
	if s.dockerComposeService == nil || s.composeProject == nil {
		log.Info().Str("testName", c.TestName()).Msg("No docker compose logs.")
		return
	}

	log.Info().Str("testName", c.TestName()).Msg("docker compose logs")

	logConsumer := formatter.NewLogConsumer(context.Background(), logs.NoLevel(log.Logger, zerolog.InfoLevel), false, true)

	err := s.dockerComposeService.Logs(context.Background(), s.composeProject.Name, logConsumer, composeapi.LogOptions{})
	c.Assert(err, checker.IsNil)

	log.Print()
	log.Print("################################")
	log.Print()
}

func (s *BaseSuite) displayTraefikLog(c *check.C, output *bytes.Buffer) {
	if output == nil || output.Len() == 0 {
		log.Info().Str("testName", c.TestName()).Msg("No Traefik logs.")
	} else {
		log.Info().Str("testName", c.TestName()).
			Str("logs", output.String()).Msg("Traefik logs")
	}
}

func (s *BaseSuite) getDockerHost() string {
	dockerHost := os.Getenv("DOCKER_HOST")
	if dockerHost == "" {
		// Default docker socket
		dockerHost = "unix:///var/run/docker.sock"
	}

	return dockerHost
}

func (s *BaseSuite) adaptFile(c *check.C, path string, tempObjects interface{}) string {
	// Load file
	tmpl, err := template.ParseFiles(path)
	c.Assert(err, checker.IsNil)

	folder, prefix := filepath.Split(path)
	tmpFile, err := os.CreateTemp(folder, strings.TrimSuffix(prefix, filepath.Ext(prefix))+"_*"+filepath.Ext(prefix))
	c.Assert(err, checker.IsNil)
	defer tmpFile.Close()

	model := structs.Map(tempObjects)
	model["SelfFilename"] = tmpFile.Name()

	err = tmpl.ExecuteTemplate(tmpFile, prefix, model)
	c.Assert(err, checker.IsNil)
	err = tmpFile.Sync()

	c.Assert(err, checker.IsNil)

	return tmpFile.Name()
}

func (s *BaseSuite) getComposeServiceIP(c *check.C, name string) string {
	filter := filters.NewArgs(
		filters.Arg("label", fmt.Sprintf("%s=%s", composeapi.ProjectLabel, s.composeProject.Name)),
		filters.Arg("label", fmt.Sprintf("%s=%s", composeapi.ServiceLabel, name)),
	)

	containers, err := s.dockerClient.ContainerList(context.Background(), dockertypes.ContainerListOptions{Filters: filter})
	c.Assert(err, checker.IsNil)
	c.Assert(containers, checker.HasLen, 1)

	networkNames := s.composeProject.NetworkNames()
	c.Assert(networkNames, checker.HasLen, 1)

	network := s.composeProject.Networks[networkNames[0]]
	return containers[0].NetworkSettings.Networks[network.Name].IPAddress
}

func (s *BaseSuite) getContainerIP(c *check.C, name string) string {
	container, err := s.dockerClient.ContainerInspect(context.Background(), name)
	c.Assert(err, checker.IsNil)
	c.Assert(container.NetworkSettings.Networks, check.NotNil)

	for _, network := range container.NetworkSettings.Networks {
		return network.IPAddress
	}

	// Should never happen.
	c.Error("No network found")
	return ""
}

func withConfigFile(file string) string {
	return "--configFile=" + file
}

// tailscaleNotSuite includes a BaseSuite out of convenience, so we can benefit
// from composeUp et co., but it is not meant to function as a TestSuite per se.
type tailscaleNotSuite struct{ BaseSuite }

// setupVPN starts Tailscale on the corresponding container, and makes it a subnet
// router, for all the other containers (whoamis, etc) subsequently started for the
// integration tests.
// It only does so if the file provided as argument exists, and contains a
// Tailscale auth key (an ephemeral, but reusable, one is recommended).
//
// Add this section to your tailscale ACLs to auto-approve the routes for the
// containers in the docker subnet:
//
//	"autoApprovers": {
//	  // Allow myself to automatically advertize routes for docker networks
//	  "routes": {
//	    "172.0.0.0/8": ["your_tailscale_identity"],
//	  },
//	},
//
// TODO(mpl): we could maybe even move this setup to the Makefile, to start it
// and let it run (forever, or until voluntarily stopped).
func setupVPN(c *check.C, keyFile string) *tailscaleNotSuite {
	data, err := os.ReadFile(keyFile)
	if err != nil {
		if !errors.Is(err, fs.ErrNotExist) {
			log.Fatal().Err(err).Send()
		}
		return nil
	}
	authKey := strings.TrimSpace(string(data))
	// TODO: copy and create versions that don't need a check.C?
	vpn := &tailscaleNotSuite{}
	vpn.createComposeProject(c, "tailscale")
	vpn.composeUp(c)
	time.Sleep(5 * time.Second)
	// If we ever change the docker subnet in the Makefile,
	// we need to change this one below correspondingly.
	vpn.composeExec(c, "tailscaled", "tailscale", "up", "--authkey="+authKey, "--advertise-routes=172.31.42.0/24")
	return vpn
}