From 373095f1a8119dbc81a6e8f9b15e0427da836c57 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 2 Oct 2024 10:34:04 +0200 Subject: [PATCH] Support NativeLB option in GatewayAPI provider Co-authored-by: Kevin Pollet --- .../reference/static-configuration/cli-ref.md | 3 + .../reference/static-configuration/env-ref.md | 3 + .../reference/static-configuration/file.toml | 1 + .../reference/static-configuration/file.yaml | 1 + .../kubernetes/crd/kubernetes_test.go | 20 +- .../kubernetes/gateway/annotations.go | 54 +++ .../kubernetes/gateway/annotations_test.go | 89 +++++ .../fixtures/httproute/simple_nativelb.yml | 51 +++ .../kubernetes/gateway/fixtures/services.yml | 44 +++ .../fixtures/tcproute/simple_nativelb.yml | 46 +++ .../fixtures/tlsroute/simple_nativelb.yml | 60 +++ pkg/provider/kubernetes/gateway/grpcroute.go | 2 +- pkg/provider/kubernetes/gateway/httproute.go | 2 +- pkg/provider/kubernetes/gateway/kubernetes.go | 28 +- .../kubernetes/gateway/kubernetes_test.go | 345 +++++++++++++++++- pkg/provider/kubernetes/gateway/tcproute.go | 2 +- pkg/provider/kubernetes/gateway/tlsroute.go | 2 +- 17 files changed, 734 insertions(+), 19 deletions(-) create mode 100644 pkg/provider/kubernetes/gateway/annotations.go create mode 100644 pkg/provider/kubernetes/gateway/annotations_test.go create mode 100644 pkg/provider/kubernetes/gateway/fixtures/httproute/simple_nativelb.yml create mode 100644 pkg/provider/kubernetes/gateway/fixtures/tcproute/simple_nativelb.yml create mode 100644 pkg/provider/kubernetes/gateway/fixtures/tlsroute/simple_nativelb.yml diff --git a/docs/content/reference/static-configuration/cli-ref.md b/docs/content/reference/static-configuration/cli-ref.md index 8bb8850a7..a4264b017 100644 --- a/docs/content/reference/static-configuration/cli-ref.md +++ b/docs/content/reference/static-configuration/cli-ref.md @@ -801,6 +801,9 @@ Kubernetes label selector to select specific GatewayClasses. `--providers.kubernetesgateway.namespaces`: Kubernetes namespaces. +`--providers.kubernetesgateway.nativelbbydefault`: +Defines whether to use Native Kubernetes load-balancing by default. (Default: ```false```) + `--providers.kubernetesgateway.statusaddress.hostname`: Hostname used for Kubernetes Gateway status address. diff --git a/docs/content/reference/static-configuration/env-ref.md b/docs/content/reference/static-configuration/env-ref.md index 480125d06..e835cfa83 100644 --- a/docs/content/reference/static-configuration/env-ref.md +++ b/docs/content/reference/static-configuration/env-ref.md @@ -801,6 +801,9 @@ Kubernetes label selector to select specific GatewayClasses. `TRAEFIK_PROVIDERS_KUBERNETESGATEWAY_NAMESPACES`: Kubernetes namespaces. +`TRAEFIK_PROVIDERS_KUBERNETESGATEWAY_NATIVELBBYDEFAULT`: +Defines whether to use Native Kubernetes load-balancing by default. (Default: ```false```) + `TRAEFIK_PROVIDERS_KUBERNETESGATEWAY_STATUSADDRESS_HOSTNAME`: Hostname used for Kubernetes Gateway status address. diff --git a/docs/content/reference/static-configuration/file.toml b/docs/content/reference/static-configuration/file.toml index 92244954c..d5ad0411e 100644 --- a/docs/content/reference/static-configuration/file.toml +++ b/docs/content/reference/static-configuration/file.toml @@ -158,6 +158,7 @@ labelSelector = "foobar" throttleDuration = "42s" experimentalChannel = true + nativeLBByDefault = true [providers.kubernetesGateway.statusAddress] ip = "foobar" hostname = "foobar" diff --git a/docs/content/reference/static-configuration/file.yaml b/docs/content/reference/static-configuration/file.yaml index a547708fc..8823bb754 100644 --- a/docs/content/reference/static-configuration/file.yaml +++ b/docs/content/reference/static-configuration/file.yaml @@ -183,6 +183,7 @@ providers: service: name: foobar namespace: foobar + nativeLBByDefault: true rest: insecure: true consulCatalog: diff --git a/pkg/provider/kubernetes/crd/kubernetes_test.go b/pkg/provider/kubernetes/crd/kubernetes_test.go index 49b7bc686..b433662ae 100644 --- a/pkg/provider/kubernetes/crd/kubernetes_test.go +++ b/pkg/provider/kubernetes/crd/kubernetes_test.go @@ -1609,7 +1609,7 @@ func TestLoadIngressRouteTCPs(t *testing.T) { k8sObjects, crdObjects := readResources(t, test.paths) - kubeClient := kubefake.NewSimpleClientset(k8sObjects...) + kubeClient := kubefake.NewClientset(k8sObjects...) crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...) client := newClientImpl(kubeClient, crdClient) @@ -4891,7 +4891,7 @@ func TestLoadIngressRoutes(t *testing.T) { k8sObjects, crdObjects := readResources(t, test.paths) - kubeClient := kubefake.NewSimpleClientset(k8sObjects...) + kubeClient := kubefake.NewClientset(k8sObjects...) crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...) client := newClientImpl(kubeClient, crdClient) @@ -4972,7 +4972,7 @@ func TestLoadIngressRoutes_multipleEndpointAddresses(t *testing.T) { k8sObjects, crdObjects := readResources(t, []string{"services.yml", "with_multiple_endpointslices.yml"}) - kubeClient := kubefake.NewSimpleClientset(k8sObjects...) + kubeClient := kubefake.NewClientset(k8sObjects...) crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...) client := newClientImpl(kubeClient, crdClient) @@ -5481,7 +5481,7 @@ func TestLoadIngressRouteUDPs(t *testing.T) { k8sObjects, crdObjects := readResources(t, test.paths) - kubeClient := kubefake.NewSimpleClientset(k8sObjects...) + kubeClient := kubefake.NewClientset(k8sObjects...) crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...) client := newClientImpl(kubeClient, crdClient) @@ -6971,7 +6971,7 @@ func TestCrossNamespace(t *testing.T) { k8sObjects, crdObjects := readResources(t, test.paths) - kubeClient := kubefake.NewSimpleClientset(k8sObjects...) + kubeClient := kubefake.NewClientset(k8sObjects...) crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...) client := newClientImpl(kubeClient, crdClient) @@ -7240,7 +7240,7 @@ func TestExternalNameService(t *testing.T) { k8sObjects, crdObjects := readResources(t, test.paths) - kubeClient := kubefake.NewSimpleClientset(k8sObjects...) + kubeClient := kubefake.NewClientset(k8sObjects...) crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...) client := newClientImpl(kubeClient, crdClient) @@ -7421,7 +7421,7 @@ func TestNativeLB(t *testing.T) { k8sObjects, crdObjects := readResources(t, test.paths) - kubeClient := kubefake.NewSimpleClientset(k8sObjects...) + kubeClient := kubefake.NewClientset(k8sObjects...) crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...) client := newClientImpl(kubeClient, crdClient) @@ -7686,7 +7686,7 @@ func TestNodePortLB(t *testing.T) { k8sObjects, crdObjects := readResources(t, test.paths) - kubeClient := kubefake.NewSimpleClientset(k8sObjects...) + kubeClient := kubefake.NewClientset(k8sObjects...) crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...) client := newClientImpl(kubeClient, crdClient) @@ -7727,7 +7727,7 @@ func TestCreateBasicAuthCredentials(t *testing.T) { } } - kubeClient := kubefake.NewSimpleClientset(k8sObjects...) + kubeClient := kubefake.NewClientset(k8sObjects...) crdClient := traefikcrdfake.NewSimpleClientset() client := newClientImpl(kubeClient, crdClient) @@ -8198,7 +8198,7 @@ func TestGlobalNativeLB(t *testing.T) { } } - kubeClient := kubefake.NewSimpleClientset(k8sObjects...) + kubeClient := kubefake.NewClientset(k8sObjects...) crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...) client := newClientImpl(kubeClient, crdClient) diff --git a/pkg/provider/kubernetes/gateway/annotations.go b/pkg/provider/kubernetes/gateway/annotations.go new file mode 100644 index 000000000..8e82ae603 --- /dev/null +++ b/pkg/provider/kubernetes/gateway/annotations.go @@ -0,0 +1,54 @@ +package gateway + +import ( + "fmt" + "strings" + + "github.com/traefik/traefik/v3/pkg/config/label" +) + +const annotationsPrefix = "traefik.io/" + +// ServiceConfig is the service's root configuration from annotations. +type ServiceConfig struct { + Service Service `json:"service"` +} + +// Service is the service's configuration from annotations. +type Service struct { + NativeLB bool `json:"nativeLB"` +} + +func parseServiceAnnotations(annotations map[string]string) (ServiceConfig, error) { + var svcConf ServiceConfig + + labels := convertAnnotations(annotations) + if len(labels) == 0 { + return svcConf, nil + } + + if err := label.Decode(labels, &svcConf, "traefik.service."); err != nil { + return svcConf, fmt.Errorf("decoding labels: %w", err) + } + + return svcConf, nil +} + +func convertAnnotations(annotations map[string]string) map[string]string { + if len(annotations) == 0 { + return nil + } + + result := make(map[string]string) + + for key, value := range annotations { + if !strings.HasPrefix(key, annotationsPrefix) { + continue + } + + newKey := strings.ReplaceAll(key, "io/", "") + result[newKey] = value + } + + return result +} diff --git a/pkg/provider/kubernetes/gateway/annotations_test.go b/pkg/provider/kubernetes/gateway/annotations_test.go new file mode 100644 index 000000000..80537526a --- /dev/null +++ b/pkg/provider/kubernetes/gateway/annotations_test.go @@ -0,0 +1,89 @@ +package gateway + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_parseServiceConfig(t *testing.T) { + testCases := []struct { + desc string + annotations map[string]string + expected ServiceConfig + }{ + { + desc: "service annotations", + annotations: map[string]string{ + "ingress.kubernetes.io/foo": "bar", + "traefik.io/foo": "bar", + "traefik.io/service.nativelb": "true", + }, + expected: ServiceConfig{ + Service: Service{ + NativeLB: true, + }, + }, + }, + { + desc: "empty map", + annotations: map[string]string{}, + expected: ServiceConfig{}, + }, + { + desc: "nil map", + annotations: nil, + expected: ServiceConfig{}, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + cfg, err := parseServiceAnnotations(test.annotations) + require.NoError(t, err) + + assert.Equal(t, test.expected, cfg) + }) + } +} + +func Test_convertAnnotations(t *testing.T) { + testCases := []struct { + desc string + annotations map[string]string + expected map[string]string + }{ + { + desc: "service annotations", + annotations: map[string]string{ + "traefik.io/service.nativelb": "true", + }, + expected: map[string]string{ + "traefik.service.nativelb": "true", + }, + }, + { + desc: "empty map", + annotations: map[string]string{}, + expected: nil, + }, + { + desc: "nil map", + annotations: nil, + expected: nil, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + labels := convertAnnotations(test.annotations) + + assert.Equal(t, test.expected, labels) + }) + } +} diff --git a/pkg/provider/kubernetes/gateway/fixtures/httproute/simple_nativelb.yml b/pkg/provider/kubernetes/gateway/fixtures/httproute/simple_nativelb.yml new file mode 100644 index 000000000..0d2febefa --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/httproute/simple_nativelb.yml @@ -0,0 +1,51 @@ +--- +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway-class +spec: + controllerName: traefik.io/gateway-controller + +--- +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + kinds: + - kind: HTTPRoute + group: gateway.networking.k8s.io + namespaces: + from: Same + +--- +kind: HTTPRoute +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: http-app-1 + namespace: default +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + hostnames: + - "foo.com" + rules: + - matches: + - path: + type: Exact + value: /bar + backendRefs: + - name: whoami-native + port: 80 + weight: 1 + kind: Service + group: "" diff --git a/pkg/provider/kubernetes/gateway/fixtures/services.yml b/pkg/provider/kubernetes/gateway/fixtures/services.yml index fe7cf9d80..d2872a0d7 100644 --- a/pkg/provider/kubernetes/gateway/fixtures/services.yml +++ b/pkg/provider/kubernetes/gateway/fixtures/services.yml @@ -5,6 +5,7 @@ metadata: namespace: default spec: + clusterIP: 10.10.10.1 ports: - name: web2 protocol: TCP @@ -262,6 +263,7 @@ metadata: namespace: default spec: + clusterIP: 10.10.10.1 ports: - protocol: TCP port: 9000 @@ -424,3 +426,45 @@ spec: port: 80 name: wss appProtocol: kubernetes.io/wss + +--- +apiVersion: v1 +kind: Service +metadata: + name: whoami-native + namespace: default + annotations: + traefik.io/service.nativelb: "true" +spec: + clusterIP: 10.10.10.1 + ports: + - name: web2 + protocol: TCP + port: 8000 + targetPort: web2 + - name: web + protocol: TCP + port: 80 + targetPort: web + selector: + app: containous + task: whoami + +--- +apiVersion: v1 +kind: Service +metadata: + name: whoamitcp-native + namespace: default + annotations: + traefik.io/service.nativelb: "true" + +spec: + clusterIP: 10.10.10.1 + ports: + - protocol: TCP + port: 9000 + name: tcp-1 + - protocol: TCP + port: 10000 + name: tcp-2 diff --git a/pkg/provider/kubernetes/gateway/fixtures/tcproute/simple_nativelb.yml b/pkg/provider/kubernetes/gateway/fixtures/tcproute/simple_nativelb.yml new file mode 100644 index 000000000..a852ee913 --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/tcproute/simple_nativelb.yml @@ -0,0 +1,46 @@ +--- +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway-class + namespace: default +spec: + controllerName: traefik.io/gateway-controller + +--- +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-tcp-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: tcp + protocol: TCP + port: 9000 + allowedRoutes: + namespaces: + from: Same + kinds: + - kind: TCPRoute + group: gateway.networking.k8s.io + +--- +kind: TCPRoute +apiVersion: gateway.networking.k8s.io/v1alpha2 +metadata: + name: tcp-app-1 + namespace: default +spec: + parentRefs: + - name: my-tcp-gateway + kind: Gateway + group: gateway.networking.k8s.io + rules: + - backendRefs: + - name: whoamitcp-native + port: 9000 + weight: 1 + kind: Service + group: "" diff --git a/pkg/provider/kubernetes/gateway/fixtures/tlsroute/simple_nativelb.yml b/pkg/provider/kubernetes/gateway/fixtures/tlsroute/simple_nativelb.yml new file mode 100644 index 000000000..253e59b9b --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/tlsroute/simple_nativelb.yml @@ -0,0 +1,60 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: supersecret + namespace: default + +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0= + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0= + +--- +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway-class + namespace: default +spec: + controllerName: traefik.io/gateway-controller + +--- +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-tls-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: tls + protocol: TLS + hostname: foo.example.com + port: 9000 + tls: + mode: Passthrough + allowedRoutes: + kinds: + - kind: TLSRoute + group: gateway.networking.k8s.io + namespaces: + from: Same + +--- +kind: TLSRoute +apiVersion: gateway.networking.k8s.io/v1alpha2 +metadata: + name: tls-app-1 + namespace: default +spec: + parentRefs: + - name: my-tls-gateway + kind: Gateway + group: gateway.networking.k8s.io + rules: + - backendRefs: + - name: whoamitcp-native + port: 9000 + weight: 1 + kind: Service + group: "" diff --git a/pkg/provider/kubernetes/gateway/grpcroute.go b/pkg/provider/kubernetes/gateway/grpcroute.go index 8dec437a2..21fc5e5d6 100644 --- a/pkg/provider/kubernetes/gateway/grpcroute.go +++ b/pkg/provider/kubernetes/gateway/grpcroute.go @@ -350,7 +350,7 @@ func (p *Provider) loadGRPCServers(namespace string, route *gatev1.GRPCRoute, ba for _, ba := range backendAddresses { lb.Servers = append(lb.Servers, dynamic.Server{ - URL: fmt.Sprintf("h2c://%s", net.JoinHostPort(ba.Address, strconv.Itoa(int(ba.Port)))), + URL: fmt.Sprintf("h2c://%s", net.JoinHostPort(ba.IP, strconv.Itoa(int(ba.Port)))), }) } return lb, nil diff --git a/pkg/provider/kubernetes/gateway/httproute.go b/pkg/provider/kubernetes/gateway/httproute.go index 7d4502cec..5fa5ef9ff 100644 --- a/pkg/provider/kubernetes/gateway/httproute.go +++ b/pkg/provider/kubernetes/gateway/httproute.go @@ -482,7 +482,7 @@ func (p *Provider) loadHTTPServers(namespace string, route *gatev1.HTTPRoute, ba for _, ba := range backendAddresses { lb.Servers = append(lb.Servers, dynamic.Server{ - URL: fmt.Sprintf("%s://%s", protocol, net.JoinHostPort(ba.Address, strconv.Itoa(int(ba.Port)))), + URL: fmt.Sprintf("%s://%s", protocol, net.JoinHostPort(ba.IP, strconv.Itoa(int(ba.Port)))), }) } return lb, svcPort, nil diff --git a/pkg/provider/kubernetes/gateway/kubernetes.go b/pkg/provider/kubernetes/gateway/kubernetes.go index 6be790eb3..2588182b5 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes.go +++ b/pkg/provider/kubernetes/gateway/kubernetes.go @@ -65,6 +65,7 @@ type Provider struct { ThrottleDuration ptypes.Duration `description:"Kubernetes refresh throttle duration" json:"throttleDuration,omitempty" toml:"throttleDuration,omitempty" yaml:"throttleDuration,omitempty" export:"true"` ExperimentalChannel bool `description:"Toggles Experimental Channel resources support (TCPRoute, TLSRoute...)." json:"experimentalChannel,omitempty" toml:"experimentalChannel,omitempty" yaml:"experimentalChannel,omitempty" export:"true"` StatusAddress *StatusAddress `description:"Defines the Kubernetes Gateway status address." json:"statusAddress,omitempty" toml:"statusAddress,omitempty" yaml:"statusAddress,omitempty" export:"true"` + NativeLBByDefault bool `description:"Defines whether to use Native Kubernetes load-balancing by default." json:"nativeLBByDefault,omitempty" toml:"nativeLBByDefault,omitempty" yaml:"nativeLBByDefault,omitempty" export:"true"` EntryPoints map[string]Entrypoint `json:"-" toml:"-" yaml:"-" label:"-" file:"-"` @@ -873,8 +874,8 @@ func (p *Provider) allowedNamespaces(gatewayNamespace string, routeNamespaces *g } type backendAddress struct { - Address string - Port int32 + IP string + Port int32 } func (p *Provider) getBackendAddresses(namespace string, ref gatev1.BackendRef) ([]backendAddress, corev1.ServicePort, error) { @@ -889,6 +890,9 @@ func (p *Provider) getBackendAddresses(namespace string, ref gatev1.BackendRef) if !exists { return nil, corev1.ServicePort{}, errors.New("service not found") } + if service.Spec.Type == corev1.ServiceTypeExternalName { + return nil, corev1.ServicePort{}, errors.New("type ExternalName is not supported for Kubernetes Service reference") + } var svcPort *corev1.ServicePort for _, p := range service.Spec.Ports { @@ -901,6 +905,22 @@ func (p *Provider) getBackendAddresses(namespace string, ref gatev1.BackendRef) return nil, corev1.ServicePort{}, fmt.Errorf("service port %d not found", *ref.Port) } + annotationsConfig, err := parseServiceAnnotations(service.Annotations) + if err != nil { + return nil, corev1.ServicePort{}, fmt.Errorf("parsing service annotations config: %w", err) + } + + if p.NativeLBByDefault || annotationsConfig.Service.NativeLB { + if service.Spec.ClusterIP == "" || service.Spec.ClusterIP == "None" { + return nil, corev1.ServicePort{}, fmt.Errorf("no clusterIP found for service: %s/%s", service.Namespace, service.Name) + } + + return []backendAddress{{ + IP: service.Spec.ClusterIP, + Port: svcPort.Port, + }}, *svcPort, nil + } + endpointSlices, err := p.client.ListEndpointSlicesForService(namespace, string(ref.Name)) if err != nil { return nil, corev1.ServicePort{}, fmt.Errorf("getting endpointslices: %w", err) @@ -935,8 +955,8 @@ func (p *Provider) getBackendAddresses(namespace string, ref gatev1.BackendRef) uniqAddresses[address] = struct{}{} backendServers = append(backendServers, backendAddress{ - Address: address, - Port: port, + IP: address, + Port: port, }) } } diff --git a/pkg/provider/kubernetes/gateway/kubernetes_test.go b/pkg/provider/kubernetes/gateway/kubernetes_test.go index 211381d75..e8929b639 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes_test.go +++ b/pkg/provider/kubernetes/gateway/kubernetes_test.go @@ -57,6 +57,7 @@ func TestLoadHTTPRoutes(t *testing.T) { expected *dynamic.Configuration entryPoints map[string]Entrypoint experimentalChannel bool + nativeLB bool }{ { desc: "Empty", @@ -2334,6 +2335,123 @@ func TestLoadHTTPRoutes(t *testing.T) { TLS: &dynamic.TLSConfiguration{}, }, }, + { + desc: "Simple HTTPRoute with NativeLBByDefault enabled", + paths: []string{"services.yml", "httproute/simple.yml"}, + nativeLB: true, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06": { + EntryPoints: []string{"web"}, + Service: "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr", + Rule: "Host(`foo.com`) && Path(`/bar`)", + Priority: 100008, + RuleSyntax: "v3", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "default-whoami-80", + Weight: ptr.To(1), + }, + }, + }, + }, + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.10.1:80", + }, + }, + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: ptypes.Duration(100 * time.Millisecond), + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Simple HTTPRoute with NativeLB annotation", + paths: []string{"services.yml", "httproute/simple_nativelb.yml"}, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06": { + EntryPoints: []string{"web"}, + Service: "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr", + Rule: "Host(`foo.com`) && Path(`/bar`)", + Priority: 100008, + RuleSyntax: "v3", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "default-whoami-native-80", + Weight: ptr.To(1), + }, + }, + }, + }, + "default-whoami-native-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.10.1:80", + }, + }, + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: ptypes.Duration(100 * time.Millisecond), + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, } for _, test := range testCases { @@ -2363,6 +2481,7 @@ func TestLoadHTTPRoutes(t *testing.T) { p := Provider{ EntryPoints: test.entryPoints, ExperimentalChannel: test.experimentalChannel, + NativeLBByDefault: test.nativeLB, client: client, } @@ -3078,6 +3197,7 @@ func TestLoadTCPRoutes(t *testing.T) { paths []string expected *dynamic.Configuration entryPoints map[string]Entrypoint + nativeLB bool }{ { desc: "Empty", @@ -3826,6 +3946,113 @@ func TestLoadTCPRoutes(t *testing.T) { TLS: &dynamic.TLSConfiguration{}, }, }, + { + desc: "Simple TCPRoute with NativeLBByDefault", + paths: []string{"services.yml", "tcproute/simple.yml"}, + nativeLB: true, + entryPoints: map[string]Entrypoint{ + "tcp": {Address: ":9000"}, + }, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{ + "default-tcp-app-1-my-tcp-gateway-tcp-0-e3b0c44298fc1c149afb": { + EntryPoints: []string{"tcp"}, + Service: "default-tcp-app-1-my-tcp-gateway-tcp-0-e3b0c44298fc1c149afb-wrr", + Rule: "HostSNI(`*`)", + RuleSyntax: "v3", + }, + }, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{ + "default-tcp-app-1-my-tcp-gateway-tcp-0-e3b0c44298fc1c149afb-wrr": { + Weighted: &dynamic.TCPWeightedRoundRobin{ + Services: []dynamic.TCPWRRService{ + { + Name: "default-whoamitcp-9000", + Weight: ptr.To(1), + }, + }, + }, + }, + "default-whoamitcp-9000": { + LoadBalancer: &dynamic.TCPServersLoadBalancer{ + Servers: []dynamic.TCPServer{ + { + Address: "10.10.10.1:9000", + }, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Simple TCPRoute with NativeLB annotation", + paths: []string{"services.yml", "tcproute/simple_nativelb.yml"}, + entryPoints: map[string]Entrypoint{ + "tcp": {Address: ":9000"}, + }, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{ + "default-tcp-app-1-my-tcp-gateway-tcp-0-e3b0c44298fc1c149afb": { + EntryPoints: []string{"tcp"}, + Service: "default-tcp-app-1-my-tcp-gateway-tcp-0-e3b0c44298fc1c149afb-wrr", + Rule: "HostSNI(`*`)", + RuleSyntax: "v3", + }, + }, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{ + "default-tcp-app-1-my-tcp-gateway-tcp-0-e3b0c44298fc1c149afb-wrr": { + Weighted: &dynamic.TCPWeightedRoundRobin{ + Services: []dynamic.TCPWRRService{ + { + Name: "default-whoamitcp-native-9000", + Weight: ptr.To(1), + }, + }, + }, + }, + "default-whoamitcp-native-9000": { + LoadBalancer: &dynamic.TCPServersLoadBalancer{ + Servers: []dynamic.TCPServer{ + { + Address: "10.10.10.1:9000", + }, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, } for _, test := range testCases { @@ -3854,6 +4081,7 @@ func TestLoadTCPRoutes(t *testing.T) { p := Provider{ EntryPoints: test.entryPoints, + NativeLBByDefault: test.nativeLB, ExperimentalChannel: true, client: client, } @@ -3869,8 +4097,9 @@ func TestLoadTLSRoutes(t *testing.T) { desc string ingressClass string paths []string - expected *dynamic.Configuration entryPoints map[string]Entrypoint + nativeLB bool + expected *dynamic.Configuration }{ { desc: "Empty", @@ -4975,6 +5204,119 @@ func TestLoadTLSRoutes(t *testing.T) { TLS: &dynamic.TLSConfiguration{}, }, }, + { + desc: "Simple TLSRoute with NativeLBByDefault", + paths: []string{"services.yml", "tlsroute/simple_TLS_to_TLSRoute.yml"}, + nativeLB: true, + entryPoints: map[string]Entrypoint{ + "tcp": {Address: ":9000"}, + }, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{ + "default-tls-app-1-my-tls-gateway-tcp-0-e3b0c44298fc1c149afb": { + EntryPoints: []string{"tcp"}, + Service: "default-tls-app-1-my-tls-gateway-tcp-0-e3b0c44298fc1c149afb-wrr", + Rule: "HostSNI(`foo.example.com`)", + RuleSyntax: "v3", + TLS: &dynamic.RouterTCPTLSConfig{ + Passthrough: true, + }, + }, + }, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{ + "default-tls-app-1-my-tls-gateway-tcp-0-e3b0c44298fc1c149afb-wrr": { + Weighted: &dynamic.TCPWeightedRoundRobin{ + Services: []dynamic.TCPWRRService{ + { + Name: "default-whoamitcp-9000", + Weight: ptr.To(1), + }, + }, + }, + }, + "default-whoamitcp-9000": { + LoadBalancer: &dynamic.TCPServersLoadBalancer{ + Servers: []dynamic.TCPServer{ + { + Address: "10.10.10.1:9000", + }, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Simple TLSRoute with NativeLB annotation", + paths: []string{"services.yml", "tlsroute/simple_nativelb.yml"}, + entryPoints: map[string]Entrypoint{ + "tcp": {Address: ":9000"}, + }, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{ + "default-tls-app-1-my-tls-gateway-tcp-0-e3b0c44298fc1c149afb": { + EntryPoints: []string{"tcp"}, + Service: "default-tls-app-1-my-tls-gateway-tcp-0-e3b0c44298fc1c149afb-wrr", + Rule: "HostSNI(`foo.example.com`)", + RuleSyntax: "v3", + TLS: &dynamic.RouterTCPTLSConfig{ + Passthrough: true, + }, + }, + }, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{ + "default-tls-app-1-my-tls-gateway-tcp-0-e3b0c44298fc1c149afb-wrr": { + Weighted: &dynamic.TCPWeightedRoundRobin{ + Services: []dynamic.TCPWRRService{ + { + Name: "default-whoamitcp-native-9000", + Weight: ptr.To(1), + }, + }, + }, + }, + "default-whoamitcp-native-9000": { + LoadBalancer: &dynamic.TCPServersLoadBalancer{ + Servers: []dynamic.TCPServer{ + { + Address: "10.10.10.1:9000", + }, + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{}, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{}, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, } for _, test := range testCases { @@ -5003,6 +5345,7 @@ func TestLoadTLSRoutes(t *testing.T) { p := Provider{ EntryPoints: test.entryPoints, + NativeLBByDefault: test.nativeLB, ExperimentalChannel: true, client: client, } diff --git a/pkg/provider/kubernetes/gateway/tcproute.go b/pkg/provider/kubernetes/gateway/tcproute.go index 6d7cfb8a4..90e98b8ae 100644 --- a/pkg/provider/kubernetes/gateway/tcproute.go +++ b/pkg/provider/kubernetes/gateway/tcproute.go @@ -286,7 +286,7 @@ func (p *Provider) loadTCPServers(namespace string, route *gatev1alpha2.TCPRoute for _, ba := range backendAddresses { lb.Servers = append(lb.Servers, dynamic.TCPServer{ - Address: net.JoinHostPort(ba.Address, strconv.Itoa(int(ba.Port))), + Address: net.JoinHostPort(ba.IP, strconv.Itoa(int(ba.Port))), }) } return lb, nil diff --git a/pkg/provider/kubernetes/gateway/tlsroute.go b/pkg/provider/kubernetes/gateway/tlsroute.go index bf6f43758..ea3a54270 100644 --- a/pkg/provider/kubernetes/gateway/tlsroute.go +++ b/pkg/provider/kubernetes/gateway/tlsroute.go @@ -289,7 +289,7 @@ func (p *Provider) loadTLSServers(namespace string, route *gatev1alpha2.TLSRoute for _, ba := range backendAddresses { lb.Servers = append(lb.Servers, dynamic.TCPServer{ // TODO determine whether the servers needs TLS, from the port? - Address: net.JoinHostPort(ba.Address, strconv.Itoa(int(ba.Port))), + Address: net.JoinHostPort(ba.IP, strconv.Itoa(int(ba.Port))), }) } return lb, nil