traefik/integration/k8s_conformance_test.go

package integration

import (
	"context"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/rs/zerolog/log"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
	"github.com/traefik/traefik/v3/integration/try"
	"github.com/traefik/traefik/v3/pkg/version"
	"gopkg.in/yaml.v3"
	ktypes "k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/sets"
	kclientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"sigs.k8s.io/controller-runtime/pkg/client"
	gatev1 "sigs.k8s.io/gateway-api/apis/v1"
	gatev1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
	gatev1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
	conformanceV1alpha1 "sigs.k8s.io/gateway-api/conformance/apis/v1alpha1"
	"sigs.k8s.io/gateway-api/conformance/tests"
	"sigs.k8s.io/gateway-api/conformance/utils/config"
	ksuite "sigs.k8s.io/gateway-api/conformance/utils/suite"
)

// K8sConformanceSuite tests suite.
type K8sConformanceSuite struct{ BaseSuite }

func TestK8sConformanceSuite(t *testing.T) {
	suite.Run(t, new(K8sConformanceSuite))
}

func (s *K8sConformanceSuite) SetupSuite() {
	s.BaseSuite.SetupSuite()

	s.createComposeProject("k8s")
	s.composeUp()

	abs, err := filepath.Abs("./fixtures/k8s/config.skip/kubeconfig.yaml")
	require.NoError(s.T(), err)

	err = try.Do(60*time.Second, func() error {
		_, err := os.Stat(abs)
		return err
	})
	require.NoError(s.T(), err)

	data, err := os.ReadFile(abs)
	require.NoError(s.T(), err)

	content := strings.ReplaceAll(string(data), "https://server:6443", fmt.Sprintf("https://%s", net.JoinHostPort(s.getComposeServiceIP("server"), "6443")))

	err = os.WriteFile(abs, []byte(content), 0o644)
	require.NoError(s.T(), err)

	err = os.Setenv("KUBECONFIG", abs)
	require.NoError(s.T(), err)
}

func (s *K8sConformanceSuite) TearDownSuite() {
	s.BaseSuite.TearDownSuite()

	generatedFiles := []string{
		"./fixtures/k8s/config.skip/kubeconfig.yaml",
		"./fixtures/k8s/config.skip/k3s.log",
		"./fixtures/k8s/rolebindings.yaml",
		"./fixtures/k8s/ccm.yaml",
	}

	for _, filename := range generatedFiles {
		if err := os.Remove(filename); err != nil {
			log.Warn().Err(err).Send()
		}
	}
}

func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
	if !*k8sConformance {
		s.T().Skip("Skip because it can take a long time to execute. To enable pass the `k8sConformance` flag.")
	}

	configFromFlags, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG"))
	if err != nil {
		s.T().Fatal(err)
	}

	kClient, err := client.New(configFromFlags, client.Options{})
	if err != nil {
		s.T().Fatalf("Error initializing Kubernetes client: %v", err)
	}

	kClientSet, err := kclientset.NewForConfig(configFromFlags)
	if err != nil {
		s.T().Fatal(err)
	}

	err = gatev1alpha2.AddToScheme(kClient.Scheme())
	require.NoError(s.T(), err)
	err = gatev1beta1.AddToScheme(kClient.Scheme())
	require.NoError(s.T(), err)
	err = gatev1.AddToScheme(kClient.Scheme())
	require.NoError(s.T(), err)

	s.traefikCmd(withConfigFile("fixtures/k8s_gateway_conformance.toml"))

	// Wait for traefik to start
	err = try.GetRequest("http://127.0.0.1:8080/api/entrypoints", 10*time.Second, try.BodyContains(`"name":"web"`))
	require.NoError(s.T(), err)

	err = try.Do(10*time.Second, func() error {
		gwc := &gatev1.GatewayClass{}
		err := kClient.Get(context.Background(), ktypes.NamespacedName{Name: "my-gateway-class"}, gwc)
		if err != nil {
			return fmt.Errorf("error fetching GatewayClass: %w", err)
		}

		return nil
	})
	require.NoError(s.T(), err)

	opts := ksuite.Options{
		Client:               kClient,
		RestConfig:           configFromFlags,
		Clientset:            kClientSet,
		GatewayClassName:     "my-gateway-class",
		Debug:                true,
		CleanupBaseResources: true,
		TimeoutConfig: config.TimeoutConfig{
			CreateTimeout:                     5 * time.Second,
			DeleteTimeout:                     5 * time.Second,
			GetTimeout:                        5 * time.Second,
			GatewayMustHaveAddress:            5 * time.Second,
			GatewayMustHaveCondition:          5 * time.Second,
			GatewayStatusMustHaveListeners:    10 * time.Second,
			GatewayListenersMustHaveCondition: 5 * time.Second,
			GWCMustBeAccepted:                 60 * time.Second, // Pod creation in k3s cluster can be long.
			HTTPRouteMustNotHaveParents:       5 * time.Second,
			HTTPRouteMustHaveCondition:        5 * time.Second,
			TLSRouteMustHaveCondition:         5 * time.Second,
			RouteMustHaveParents:              5 * time.Second,
			ManifestFetchTimeout:              5 * time.Second,
			MaxTimeToConsistency:              5 * time.Second,
			NamespacesMustBeReady:             60 * time.Second, // Pod creation in k3s cluster can be long.
			RequestTimeout:                    5 * time.Second,
			LatestObservedGenerationSet:       5 * time.Second,
			RequiredConsecutiveSuccesses:      0,
		},
		SupportedFeatures: sets.New[ksuite.SupportedFeature]().
			Insert(ksuite.GatewayCoreFeatures.UnsortedList()...),
		EnableAllSupportedFeatures: false,
		RunTest:                    *k8sConformanceRunTest,
		// Until the feature are all supported, following tests are skipped.
		SkipTests: []string{
			"HTTPExactPathMatching",
			"HTTPRouteHostnameIntersection",
			"GatewaySecretReferenceGrantAllInNamespace",
			"HTTPRouteListenerHostnameMatching",
			"HTTPRouteRequestHeaderModifier",
			"GatewaySecretInvalidReferenceGrant",
			"GatewayClassObservedGenerationBump",
			"HTTPRouteInvalidNonExistentBackendRef",
			"GatewayWithAttachedRoutes",
			"HTTPRouteCrossNamespace",
			"HTTPRouteDisallowedKind",
			"HTTPRouteInvalidReferenceGrant",
			"HTTPRouteObservedGenerationBump",
			"GatewayInvalidRouteKind",
			"TLSRouteSimpleSameNamespace",
			"TLSRouteInvalidReferenceGrant",
			"HTTPRouteInvalidCrossNamespaceParentRef",
			"HTTPRouteInvalidParentRefNotMatchingSectionName",
			"GatewaySecretReferenceGrantSpecific",
			"GatewayModifyListeners",
			"GatewaySecretMissingReferenceGrant",
			"GatewayInvalidTLSConfiguration",
			"HTTPRouteInvalidCrossNamespaceBackendRef",
			"HTTPRouteMatchingAcrossRoutes",
			"HTTPRoutePartiallyInvalidViaInvalidReferenceGrant",
			"HTTPRouteRedirectHostAndStatus",
			"HTTPRouteInvalidBackendRefUnknownKind",
			"HTTPRoutePathMatchOrder",
			"HTTPRouteSimpleSameNamespace",
			"HTTPRouteMatching",
			"HTTPRouteHeaderMatching",
			"HTTPRouteReferenceGrant",
		},
	}

	cSuite, err := ksuite.NewExperimentalConformanceTestSuite(ksuite.ExperimentalConformanceOptions{
		Options: opts,
		Implementation: conformanceV1alpha1.Implementation{
			Organization: "traefik",
			Project:      "traefik",
			URL:          "https://traefik.io/",
			Version:      version.Version,
			Contact:      []string{"@traefik/maintainers"},
		},
		ConformanceProfiles: sets.New[ksuite.ConformanceProfileName](
			ksuite.HTTPConformanceProfileName,
			ksuite.TLSConformanceProfileName,
		),
	})
	require.NoError(s.T(), err)

	cSuite.Setup(s.T())
	err = cSuite.Run(s.T(), tests.ConformanceTests)
	require.NoError(s.T(), err)

	report, err := cSuite.Report()
	require.NoError(s.T(), err, "failed generating conformance report")

	report.GatewayAPIVersion = "1.0.0"

	rawReport, err := yaml.Marshal(report)
	require.NoError(s.T(), err)
	s.T().Logf("Conformance report:\n%s", string(rawReport))

	require.NoError(s.T(), os.MkdirAll("./conformance-reports", 0o755))
	outFile := filepath.Join("conformance-reports", fmt.Sprintf("traefik-traefik-%d.yaml", time.Now().UnixNano()))
	require.NoError(s.T(), os.WriteFile(outFile, rawReport, 0o600))
	s.T().Logf("Report written to: %s", outFile)
}