test: add a test for inline machine config trusted roots

Run SideroLink API server via TLS with self-signed certificate, inject that certificate into Talos via `talos.config.inline=`. Fix a couple of place where our special TLS root CA provider supporting reloading on the fly was not used. Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com> (cherry picked from commit 8d6884a8e28e1bfa29f9a479e0f7179819cf70cd)
2024-09-12 18:16:56 +04:00 · 2024-09-12 18:16:56 +04:00 · bd91675121
commit bd91675121
parent 073ba25855
11 changed files with 171 additions and 20 deletions
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-09-09T12:40:13Z by kres 8be5fa7.
+# Generated on 2024-09-12T16:43:46Z by kres 8be5fa7.

 name: default
 concurrency:
@ -2385,6 +2385,14 @@ jobs:
          WITH_SIDEROLINK_AGENT: tunnel
        run: |
          sudo -E make e2e-qemu
+      - name: e2e-siderolink-tls
+        env:
+          IMAGE_REGISTRY: registry.dev.siderolabs.io
+          SHORT_INTEGRATION_TEST: "yes"
+          VIA_MAINTENANCE_MODE: "true"
+          WITH_SIDEROLINK_AGENT: wireguard+tls
+        run: |
+          sudo -E make e2e-qemu
      - name: e2e-apparmor
        env:
          IMAGE_REGISTRY: registry.dev.siderolabs.io
--- a/.github/workflows/integration-misc-4-cron.yaml
+++ b/.github/workflows/integration-misc-4-cron.yaml
@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-09-09T12:40:13Z by kres 8be5fa7.
+# Generated on 2024-09-12T16:43:46Z by kres 8be5fa7.

 name: integration-misc-4-cron
 concurrency:
@ -94,6 +94,14 @@ jobs:
          WITH_SIDEROLINK_AGENT: tunnel
        run: |
          sudo -E make e2e-qemu
+      - name: e2e-siderolink-tls
+        env:
+          IMAGE_REGISTRY: registry.dev.siderolabs.io
+          SHORT_INTEGRATION_TEST: "yes"
+          VIA_MAINTENANCE_MODE: "true"
+          WITH_SIDEROLINK_AGENT: wireguard+tls
+        run: |
+          sudo -E make e2e-qemu
      - name: e2e-apparmor
        env:
          IMAGE_REGISTRY: registry.dev.siderolabs.io
--- a/.kres.yaml
+++ b/.kres.yaml
@ -926,6 +926,14 @@ spec:
            WITH_SIDEROLINK_AGENT: tunnel
            VIA_MAINTENANCE_MODE: true
            IMAGE_REGISTRY: registry.dev.siderolabs.io
+        - name: e2e-siderolink-tls
+          command: e2e-qemu
+          withSudo: true
+          environment:
+            SHORT_INTEGRATION_TEST: yes
+            WITH_SIDEROLINK_AGENT: wireguard+tls
+            VIA_MAINTENANCE_MODE: true
+            IMAGE_REGISTRY: registry.dev.siderolabs.io
        - name: e2e-apparmor
          command: e2e-qemu
          withSudo: true
--- a/cmd/talosctl/cmd/mgmt/cluster/create.go
+++ b/cmd/talosctl/cmd/mgmt/cluster/create.go
@ -5,7 +5,9 @@
 package cluster

 import (
+	"bytes"
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"math/big"
@ -23,6 +25,8 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/google/uuid"
 	"github.com/hashicorp/go-getter/v2"
+	"github.com/klauspost/compress/zstd"
+	"github.com/siderolabs/crypto/x509"
 	"github.com/siderolabs/gen/maps"
 	"github.com/siderolabs/go-blockdevice/v2/encryption"
 	"github.com/siderolabs/go-kubeconfig"
@ -40,10 +44,12 @@ import (
 	clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config"
 	"github.com/siderolabs/talos/pkg/machinery/config"
 	"github.com/siderolabs/talos/pkg/machinery/config/bundle"
+	"github.com/siderolabs/talos/pkg/machinery/config/configloader"
 	"github.com/siderolabs/talos/pkg/machinery/config/configpatcher"
 	"github.com/siderolabs/talos/pkg/machinery/config/encoder"
 	"github.com/siderolabs/talos/pkg/machinery/config/generate"
 	"github.com/siderolabs/talos/pkg/machinery/config/machine"
+	"github.com/siderolabs/talos/pkg/machinery/config/types/security"
 	"github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1"
 	"github.com/siderolabs/talos/pkg/machinery/constants"
 	"github.com/siderolabs/talos/pkg/machinery/nethelpers"
@ -752,6 +758,24 @@ func create(ctx context.Context) error {
 		)
 	}

+	var slb *siderolinkBuilder
+
+	if withSiderolinkAgent.IsEnabled() {
+		slb, err = newSiderolinkBuilder(gatewayIPs[0].String(), withSiderolinkAgent.IsTLS())
+		if err != nil {
+			return err
+		}
+	}
+
+	if trustedRootsConfig := slb.TrustedRootsConfig(); trustedRootsConfig != nil {
+		trustedRootsPatch, err := configloader.NewFromBytes(trustedRootsConfig)
+		if err != nil {
+			return fmt.Errorf("error loading trusted roots config: %w", err)
+		}
+
+		configBundleOpts = append(configBundleOpts, bundle.WithPatch([]configpatcher.Patch{configpatcher.NewStrategicMergePatch(trustedRootsPatch)}))
+	}
+
 	configBundle, err := bundle.NewBundle(configBundleOpts...)
 	if err != nil {
 		return err
@ -795,15 +819,6 @@ func create(ctx context.Context) error {
 		extraKernelArgs = procfs.NewCmdline(extraBootKernelArgs)
 	}

-	var slb *siderolinkBuilder
-
-	if withSiderolinkAgent.IsEnabled() {
-		slb, err = newSiderolinkBuilder(gatewayIPs[0].String())
-		if err != nil {
-			return err
-		}
-	}
-
 	err = slb.SetKernelArgs(extraKernelArgs, withSiderolinkAgent.IsTunnel())
 	if err != nil {
 		return err
@ -1255,7 +1270,7 @@ func init() {
 	Cmd.AddCommand(createCmd)
 }

-func newSiderolinkBuilder(wgHost string) (*siderolinkBuilder, error) {
+func newSiderolinkBuilder(wgHost string, useTLS bool) (*siderolinkBuilder, error) {
 	prefix, err := networkPrefix("")
 	if err != nil {
 		return nil, err
@ -1268,6 +1283,16 @@ func newSiderolinkBuilder(wgHost string) (*siderolinkBuilder, error) {
 		nodeIPv6Addr: prefix.Addr().Next().String(),
 	}

+	if useTLS {
+		ca, err := x509.NewSelfSignedCertificateAuthority(x509.ECDSA(true), x509.IPAddresses([]net.IP{net.ParseIP(wgHost)}))
+		if err != nil {
+			return nil, err
+		}
+
+		result.apiCert = ca.CrtPEM
+		result.apiKey = ca.KeyPEM
+	}
+
 	var resultErr error

 	for range 10 {
@ -1312,6 +1337,9 @@ type siderolinkBuilder struct {
 	apiPort      int
 	sinkPort     int
 	logPort      int
+
+	apiCert []byte
+	apiKey  []byte
 }

 // DefineIPv6ForUUID defines an IPv6 address for a given UUID. It is safe to call this method on a nil pointer.
@ -1340,6 +1368,8 @@ func (slb *siderolinkBuilder) SiderolinkRequest() provision.SiderolinkRequest {
 	return provision.SiderolinkRequest{
 		WireguardEndpoint: net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.wgPort)),
 		APIEndpoint:       ":" + strconv.Itoa(slb.apiPort),
+		APICertificate:    slb.apiCert,
+		APIKey:            slb.apiKey,
 		SinkEndpoint:      ":" + strconv.Itoa(slb.sinkPort),
 		LogEndpoint:       ":" + strconv.Itoa(slb.logPort),
 		SiderolinkBind: maps.ToSlice(slb.binds, func(k uuid.UUID, v netip.Addr) provision.SiderolinkBind {
@ -1351,6 +1381,24 @@ func (slb *siderolinkBuilder) SiderolinkRequest() provision.SiderolinkRequest {
 	}
 }

+// TrustedRootsConfig returns the trusted roots config for the current builder.
+func (slb *siderolinkBuilder) TrustedRootsConfig() []byte {
+	if slb == nil || slb.apiCert == nil {
+		return nil
+	}
+
+	trustedRootsConfig := security.NewTrustedRootsConfigV1Alpha1()
+	trustedRootsConfig.MetaName = "siderolink-ca"
+	trustedRootsConfig.Certificates = string(slb.apiCert)
+
+	marshaled, err := encoder.NewEncoder(trustedRootsConfig, encoder.WithComments(encoder.CommentsDisabled)).Encode()
+	if err != nil {
+		panic(fmt.Sprintf("failed to marshal trusted roots config: %s", err))
+	}
+
+	return marshaled
+}
+
 // SetKernelArgs sets the kernel arguments for the current builder. It is safe to call this method on a nil pointer.
 func (slb *siderolinkBuilder) SetKernelArgs(extraKernelArgs *procfs.Cmdline, tunnel bool) error {
 	switch {
@ -1361,7 +1409,13 @@ func (slb *siderolinkBuilder) SetKernelArgs(extraKernelArgs *procfs.Cmdline, tun
 		extraKernelArgs.Get("talos.logging.kernel") != nil:
 		return errors.New("siderolink kernel arguments are already set, cannot run with --with-siderolink")
 	default:
-		apiLink := "grpc://" + net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.apiPort)) + "?jointoken=foo"
+		scheme := "grpc://"
+
+		if slb.apiCert != nil {
+			scheme = "https://"
+		}
+
+		apiLink := scheme + net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.apiPort)) + "?jointoken=foo"

 		if tunnel {
 			apiLink += "&grpc_tunnel=true"
@ -1371,6 +1425,26 @@ func (slb *siderolinkBuilder) SetKernelArgs(extraKernelArgs *procfs.Cmdline, tun
 		extraKernelArgs.Append("talos.events.sink", net.JoinHostPort(slb.nodeIPv6Addr, strconv.Itoa(slb.sinkPort)))
 		extraKernelArgs.Append("talos.logging.kernel", "tcp://"+net.JoinHostPort(slb.nodeIPv6Addr, strconv.Itoa(slb.logPort)))

+		if trustedRootsConfig := slb.TrustedRootsConfig(); trustedRootsConfig != nil {
+			var buf bytes.Buffer
+
+			zencoder, err := zstd.NewWriter(&buf)
+			if err != nil {
+				return fmt.Errorf("failed to create zstd encoder: %w", err)
+			}
+
+			_, err = zencoder.Write(trustedRootsConfig)
+			if err != nil {
+				return fmt.Errorf("failed to write zstd data: %w", err)
+			}
+
+			if err = zencoder.Close(); err != nil {
+				return fmt.Errorf("failed to close zstd encoder: %w", err)
+			}
+
+			extraKernelArgs.Append(constants.KernelParamConfigInline, base64.StdEncoding.EncodeToString(buf.Bytes()))
+		}
+
 		return nil
 	}
 }
@ -1444,6 +1518,10 @@ func (a *agentFlag) String() string {
 		return "wireguard"
 	case 2:
 		return "grpc-tunnel"
+	case 3:
+		return "wireguard+tls"
+	case 4:
+		return "grpc-tunnel+tls"
 	default:
 		return "none"
 	}
@ -1455,8 +1533,12 @@ func (a *agentFlag) Set(s string) error {
 		*a = 1
 	case "tunnel":
 		*a = 2
+	case "wireguard+tls":
+		*a = 3
+	case "grpc-tunnel+tls":
+		*a = 4
 	default:
-		return fmt.Errorf("unknown type: %s, possible values: 'true', 'wireguard' for the usual WG; 'tunnel' for WG over GRPC", s)
+		return fmt.Errorf("unknown type: %s, possible values: 'true', 'wireguard' for the usual WG; 'tunnel' for WG over GRPC, add '+tls' to enable TLS for API", s)
 	}

 	return nil
@ -1464,4 +1546,5 @@ func (a *agentFlag) Set(s string) error {

 func (a *agentFlag) Type() string    { return "agent" }
 func (a *agentFlag) IsEnabled() bool { return *a != 0 }
-func (a *agentFlag) IsTunnel() bool  { return *a == 2 }
+func (a *agentFlag) IsTunnel() bool  { return *a == 2 || *a == 4 }
+func (a *agentFlag) IsTLS() bool     { return *a == 3 || *a == 4 }
--- a/cmd/talosctl/cmd/mgmt/siderolink_launch_linux.go
+++ b/cmd/talosctl/cmd/mgmt/siderolink_launch_linux.go
@ -6,6 +6,7 @@ package mgmt

 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"os"
 	"os/signal"
@ -23,6 +24,8 @@ var siderolinkFlags struct {
 	wireguardEndpoint string
 	sinkEndpoint      string
 	apiEndpoint       string
+	apiCertPath       string
+	apiKeyPath        string
 	logEndpoint       string
 	predefinedPairs   []string
 }
@ -46,6 +49,8 @@ func init() {
 	siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.wireguardEndpoint, "sidero-link-wireguard-endpoint", "", "advertised Wireguard endpoint")
 	siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.sinkEndpoint, "event-sink-endpoint", "", "gRPC API endpoint for the Event Sink")
 	siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.apiEndpoint, "sidero-link-api-endpoint", "", "gRPC API endpoint for the SideroLink")
+	siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.apiCertPath, "sidero-link-api-cert", "", "path to the API server certificate (optional)")
+	siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.apiKeyPath, "sidero-link-api-key", "", "path to the API server key (optional)")
 	siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.logEndpoint, "log-receiver-endpoint", "", "TCP log receiver endpoint")
 	siderolinkCmd.PersistentFlags().StringArrayVar(&siderolinkFlags.predefinedPairs, "predefined-pair", nil, "predefined pairs of UUID=IPv6 addrs for the nodes")

@ -68,11 +73,25 @@ func run(ctx context.Context) error {
 	logger.Info("starting embedded siderolink agent")
 	defer logger.Info("stopping embedded siderolink agent")

+	var apiTLSConfig *tls.Config
+
+	if siderolinkFlags.apiCertPath != "" && siderolinkFlags.apiKeyPath != "" {
+		apiCert, err := tls.LoadX509KeyPair(siderolinkFlags.apiCertPath, siderolinkFlags.apiKeyPath)
+		if err != nil {
+			return fmt.Errorf("failed to load API server certificate: %w", err)
+		}
+
+		apiTLSConfig = &tls.Config{
+			Certificates: []tls.Certificate{apiCert},
+		}
+	}
+
 	err = agent.Run(
 		ctx,
 		agent.Config{
 			WireguardEndpoint: siderolinkFlags.wireguardEndpoint,
 			APIEndpoint:       siderolinkFlags.apiEndpoint,
+			APITLSConfig:      apiTLSConfig,
 			JoinToken:         siderolinkFlags.joinToken,
 			SinkEndpoint:      siderolinkFlags.sinkEndpoint,
 			LogEndpoint:       siderolinkFlags.logEndpoint,
--- a/go.mod
+++ b/go.mod
@ -152,7 +152,7 @@ require (
 	github.com/siderolabs/kms-client v0.1.0
 	github.com/siderolabs/net v0.4.0
 	github.com/siderolabs/protoenc v0.2.1
-	github.com/siderolabs/siderolink v0.3.9
+	github.com/siderolabs/siderolink v0.3.10
 	github.com/siderolabs/talos/pkg/machinery v1.8.0-beta.0
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
--- a/go.sum
+++ b/go.sum
@ -629,8 +629,8 @@ github.com/siderolabs/net v0.4.0 h1:1bOgVay/ijPkJz4qct98nHsiB/ysLQU0KLoBC4qLm7I=
 github.com/siderolabs/net v0.4.0/go.mod h1:/ibG+Hm9HU27agp5r9Q3eZicEfjquzNzQNux5uEk0kM=
 github.com/siderolabs/protoenc v0.2.1 h1:BqxEmeWQeMpNP3R6WrPqDatX8sM/r4t97OP8mFmg6GA=
 github.com/siderolabs/protoenc v0.2.1/go.mod h1:StTHxjet1g11GpNAWiATgc8K0HMKiFSEVVFOa/H0otc=
-github.com/siderolabs/siderolink v0.3.9 h1:lvHFCu+CdfUyMk90g1Zt5r7n1Dw3jhXMxyzXmQ0776o=
-github.com/siderolabs/siderolink v0.3.9/go.mod h1:QbGnXpHI5MDq6qMZkCFnxYOOw5eE+lkLx53L5ZgjLMQ=
+github.com/siderolabs/siderolink v0.3.10 h1:M8OrRyfzmyyGksHalOqvRSxvb1Fwi7S3AFQx6ERap44=
+github.com/siderolabs/siderolink v0.3.10/go.mod h1:QbGnXpHI5MDq6qMZkCFnxYOOw5eE+lkLx53L5ZgjLMQ=
 github.com/siderolabs/tcpproxy v0.1.0 h1:IbkS9vRhjMOscc1US3M5P1RnsGKFgB6U5IzUk+4WkKA=
 github.com/siderolabs/tcpproxy v0.1.0/go.mod h1:onn6CPPj/w1UNqQ0U97oRPF0CqbrgEApYCw4P9IiCW8=
 github.com/siderolabs/wgctrl-go v0.0.0-20240401105613-579af3342774 h1:wLhs5zMQVjA6LN9WpF2owOdtcoRp40zL8AaQSle+9EE=
--- a/internal/app/machined/pkg/controllers/siderolink/manager.go
+++ b/internal/app/machined/pkg/controllers/siderolink/manager.go
@ -30,6 +30,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"

 	networkutils "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/network/utils"
+	"github.com/siderolabs/talos/pkg/httpdefaults"
 	"github.com/siderolabs/talos/pkg/machinery/constants"
 	"github.com/siderolabs/talos/pkg/machinery/nethelpers"
 	"github.com/siderolabs/talos/pkg/machinery/resources/config"
@ -487,7 +488,9 @@ func withTransportCredentials(insec bool) grpc.DialOption {
 	if insec {
 		transportCredentials = insecure.NewCredentials()
 	} else {
-		transportCredentials = credentials.NewTLS(&tls.Config{})
+		transportCredentials = credentials.NewTLS(&tls.Config{
+			RootCAs: httpdefaults.RootCAs(),
+		})
 	}

 	return grpc.WithTransportCredentials(transportCredentials)
--- a/internal/pkg/encryption/keys/kms.go
+++ b/internal/pkg/encryption/keys/kms.go
@ -23,6 +23,7 @@ import (

 	"github.com/siderolabs/talos/internal/pkg/encryption/helpers"
 	"github.com/siderolabs/talos/internal/pkg/endpoint"
+	"github.com/siderolabs/talos/pkg/httpdefaults"
 )

 // KMSToken is the userdata stored in the partition token metadata.
@ -130,7 +131,9 @@ func (h *KMSKeyHandler) getConn() (*grpc.ClientConn, error) {
 	if endpoint.Insecure {
 		transportCredentials = insecure.NewCredentials()
 	} else {
-		transportCredentials = credentials.NewTLS(&tls.Config{})
+		transportCredentials = credentials.NewTLS(&tls.Config{
+			RootCAs: httpdefaults.RootCAs(),
+		})
 	}

 	return grpc.NewClient(
--- a/pkg/provision/providers/vm/siderolink-agent.go
+++ b/pkg/provision/providers/vm/siderolink-agent.go
@ -18,6 +18,8 @@ import (
 const (
 	siderolinkAgentPid = "siderolink-agent.pid"
 	siderolinkAgentLog = "siderolink-agent.log"
+	siderolinkCert     = "siderolink-agent-cert.pem"
+	siderolinkKey      = "siderolink-agent-key.pem"
 )

 // CreateSiderolinkAgent creates siderlink agent.
@ -40,6 +42,21 @@ func (p *Provisioner) CreateSiderolinkAgent(state *State, clusterReq provision.C
 		"--log-receiver-endpoint", clusterReq.SiderolinkRequest.LogEndpoint,
 	}

+	if clusterReq.SiderolinkRequest.APICertificate != nil && clusterReq.SiderolinkRequest.APIKey != nil {
+		apiCertPath := state.GetRelativePath(siderolinkCert)
+		apiKeyPath := state.GetRelativePath(siderolinkKey)
+
+		if err = os.WriteFile(apiCertPath, clusterReq.SiderolinkRequest.APICertificate, 0o600); err != nil {
+			return fmt.Errorf("error writing SideroLink API certificate: %w", err)
+		}
+
+		if err = os.WriteFile(apiKeyPath, clusterReq.SiderolinkRequest.APIKey, 0o600); err != nil {
+			return fmt.Errorf("error writing SideroLink API key: %w", err)
+		}
+
+		args = append(args, "--sidero-link-api-cert", apiCertPath, "--sidero-link-api-key", apiKeyPath)
+	}
+
 	for _, bind := range clusterReq.SiderolinkRequest.SiderolinkBind {
 		args = append(args, "--predefined-pair", bind.UUID.String()+"="+bind.Addr.String())
 	}
--- a/pkg/provision/request.go
+++ b/pkg/provision/request.go
@ -212,6 +212,8 @@ type NodeRequest struct {
 type SiderolinkRequest struct {
 	WireguardEndpoint string
 	APIEndpoint       string
+	APICertificate    []byte
+	APIKey            []byte
 	SinkEndpoint      string
 	LogEndpoint       string
 	SiderolinkBind    []SiderolinkBind