Add versioning for Gateway API Conformance Test Report

.github/workflows/test-conformance.yaml

@@ -31,5 +31,7 @@ jobs:
       - name: Avoid generating webui
         run: touch webui/static/index.html
 
-      - name: K8s Gateway API conformance test
-        run: make test-gateway-api-conformance
+      - name: K8s Gateway API conformance test and report
+        run: |
+          make test-gateway-api-conformance
+          git diff --exit-code

.gitignore

@@ -19,4 +19,4 @@ plugins-storage/
 plugins-local/
 traefik_changelog.md
 integration/tailscale.secret
-integration/conformance-reports/
+integration/conformance-reports/**/experimental-dev-default-report.yaml

Makefile

@@ -103,7 +103,8 @@ test-integration: binary
 .PHONY: test-gateway-api-conformance
 #? test-gateway-api-conformance: Run the conformance tests
 test-gateway-api-conformance: build-image-dirty
-	GOOS=$(GOOS) GOARCH=$(GOARCH) go test ./integration -v -test.run K8sConformanceSuite -k8sConformance $(TESTFLAGS)
+	# In case of a new Minor/Major version, the k8sConformanceTraefikVersion needs to be updated.
+	GOOS=$(GOOS) GOARCH=$(GOARCH) go test ./integration -v -test.run K8sConformanceSuite -k8sConformance -k8sConformanceTraefikVersion="v3.1" $(TESTFLAGS)
 
 .PHONY: test-ui-unit
 #? test-ui-unit: Run the unit tests for the webui

docs/content/contributing/submitting-pull-requests.md

@@ -91,6 +91,8 @@ You must run these local verifications before you submit your pull request to pr
 Your PR will not be reviewed until these are green on the CI.
 
 * `make generate`
+* `make generate-crd`
+* `make test-gateway-api-conformance`
 * `make validate`
 * `make pull-images`
 * `make test`

integration/conformance-reports/v1.1.0/experimental-v3.1-default-report.yaml

@@ -0,0 +1,55 @@
+apiVersion: gateway.networking.k8s.io/v1alpha1
+date: '-'
+gatewayAPIChannel: experimental
+gatewayAPIVersion: v1.1.0
+implementation:
+  contact:
+  - '@traefik/maintainers'
+  organization: traefik
+  project: traefik
+  url: https://traefik.io/
+  version: v3.1
+kind: ConformanceReport
+mode: default
+profiles:
+- core:
+    result: success
+    statistics:
+      Failed: 0
+      Passed: 12
+      Skipped: 0
+  name: GATEWAY-GRPC
+  summary: Core tests succeeded.
+- core:
+    result: success
+    statistics:
+      Failed: 0
+      Passed: 33
+      Skipped: 0
+  extended:
+    result: success
+    statistics:
+      Failed: 0
+      Passed: 10
+      Skipped: 0
+    supportedFeatures:
+    - GatewayPort8080
+    - HTTPRouteHostRewrite
+    - HTTPRouteMethodMatching
+    - HTTPRoutePathRedirect
+    - HTTPRoutePathRewrite
+    - HTTPRoutePortRedirect
+    - HTTPRouteQueryParamMatching
+    - HTTPRouteResponseHeaderModification
+    - HTTPRouteSchemeRedirect
+    unsupportedFeatures:
+    - GatewayHTTPListenerIsolation
+    - GatewayStaticAddresses
+    - HTTPRouteBackendRequestHeaderModification
+    - HTTPRouteBackendTimeout
+    - HTTPRouteParentRefPort
+    - HTTPRouteRequestMirror
+    - HTTPRouteRequestMultipleMirrors
+    - HTTPRouteRequestTimeout
+  name: GATEWAY-HTTP
+  summary: Core tests succeeded. Extended tests succeeded.

integration/integration_test.go

@@ -36,9 +36,10 @@ import (
 )
 
 var (
-	showLog               = flag.Bool("tlog", false, "always show Traefik logs")
-	k8sConformance        = flag.Bool("k8sConformance", false, "run K8s Gateway API conformance test")
-	k8sConformanceRunTest = flag.String("k8sConformanceRunTest", "", "run a specific K8s Gateway API conformance test")
+	showLog                      = flag.Bool("tlog", false, "always show Traefik logs")
+	k8sConformance               = flag.Bool("k8sConformance", false, "run K8s Gateway API conformance test")
+	k8sConformanceRunTest        = flag.String("k8sConformanceRunTest", "", "run a specific K8s Gateway API conformance test")
+	k8sConformanceTraefikVersion = flag.String("k8sConformanceTraefikVersion", "dev", "specify the Traefik version for the K8s Gateway API conformance report")
 )
 
 const tailscaleSecretFilePath = "tailscale.secret"

integration/k8s_conformance_test.go

@@ -17,7 +17,6 @@ import (
 	"github.com/testcontainers/testcontainers-go/modules/k3s"
 	"github.com/testcontainers/testcontainers-go/network"
 	"github.com/traefik/traefik/v3/integration/try"
-	"github.com/traefik/traefik/v3/pkg/version"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	kclientset "k8s.io/client-go/kubernetes"
@@ -191,7 +190,7 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
 			Organization: "traefik",
 			Project:      "traefik",
 			URL:          "https://traefik.io/",
-			Version:      version.Version,
+			Version:      *k8sConformanceTraefikVersion,
 			Contact:      []string{"@traefik/maintainers"},
 		},
 		ConformanceProfiles: sets.New(ksuite.GatewayHTTPConformanceProfileName, ksuite.GatewayGRPCConformanceProfileName),
@@ -220,12 +219,17 @@ func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
 	report, err := cSuite.Report()
 	require.NoError(s.T(), err, "failed generating conformance report")
 
+	// Ignore report date to avoid diff with CI job.
+	// However, we can track the date of the report thanks to the commit.
+	// TODO: to publish this report automatically, we have to figure out how to handle the date diff.
+	report.Date = "-"
+
 	rawReport, err := yaml.Marshal(report)
 	require.NoError(s.T(), err)
 	s.T().Logf("Conformance report:\n%s", string(rawReport))
 
-	require.NoError(s.T(), os.MkdirAll("./conformance-reports", 0o755))
-	outFile := filepath.Join("conformance-reports", fmt.Sprintf("%s-%s-%s-report.yaml", report.GatewayAPIChannel, report.Version, report.Mode))
+	require.NoError(s.T(), os.MkdirAll("./conformance-reports/"+report.GatewayAPIVersion, 0o755))
+	outFile := filepath.Join("conformance-reports/"+report.GatewayAPIVersion, fmt.Sprintf("%s-%s-%s-report.yaml", report.GatewayAPIChannel, report.Version, report.Mode))
 	require.NoError(s.T(), os.WriteFile(outFile, rawReport, 0o600))
 	s.T().Logf("Report written to: %s", outFile)
 }