alt-orchestra/talos
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: update etcd certificates when node addresses changes

Browse Source 
Fixes #6110

I somehow missed the fact that etcd certs were not made fully reactive
to node address changes (I wrongly assume it was already the fact).

This PR refactors etcd certificate generation process to be
resource-based and introduces unit-tests for the controller.

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
This commit is contained in:
Andrey Smirnov 2022-08-24 23:28:29 +04:00
parent 11edb2c6f8
commit 053af1d59e
No known key found for this signature in database
GPG Key ID: 7B26396447AB6DFD
3 changed files with 270 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
internal/app/machined/pkg/controllers/secrets/etcd.go
View File

@ -10,11 +10,13 @@ import (
"github.com/cosi-project/runtime/pkg/controller"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/safe"
"github.com/cosi-project/runtime/pkg/state"
"github.com/siderolabs/go-pointer"
"go.uber.org/zap"
"github.com/talos-systems/talos/internal/pkg/etcd"
"github.com/talos-systems/talos/pkg/machinery/resources/k8s"
"github.com/talos-systems/talos/pkg/machinery/resources/network"
"github.com/talos-systems/talos/pkg/machinery/resources/secrets"
"github.com/talos-systems/talos/pkg/machinery/resources/time"
@ -50,6 +52,18 @@ func (ctrl *EtcdController) Inputs() []controller.Input {
ID: pointer.To(time.StatusID),
Kind: controller.InputWeak,
},
{
Namespace: network.NamespaceName,
Type: network.HostnameStatusType,
ID: pointer.To(network.HostnameID),
Kind: controller.InputWeak,
},
{
Namespace: network.NamespaceName,
Type: network.NodeAddressType,
ID: pointer.To(network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s)),
Kind: controller.InputWeak,
},
}
}
@ -74,7 +88,7 @@ func (ctrl *EtcdController) Run(ctx context.Context, r controller.Runtime, logge
case <-r.EventCh():
}
etcdRootRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdRootType, secrets.EtcdRootID, resource.VersionUndefined))
etcdRootRes, err := safe.ReaderGet[*secrets.EtcdRoot](ctx, r, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdRootType, secrets.EtcdRootID, resource.VersionUndefined))
if err != nil {
if state.IsNotFoundError(err) {
if err = ctrl.teardownAll(ctx, r); err != nil {
@ -87,10 +101,10 @@ func (ctrl *EtcdController) Run(ctx context.Context, r controller.Runtime, logge
return fmt.Errorf("error getting etcd root secrets: %w", err)
}
etcdRoot := etcdRootRes.(*secrets.EtcdRoot).TypedSpec()
etcdRoot := etcdRootRes.TypedSpec()
// wait for network to be ready as it might change IPs/hostname
networkResource, err := r.Get(ctx, resource.NewMetadata(network.NamespaceName, network.StatusType, network.StatusID, resource.VersionUndefined))
networkResource, err := safe.ReaderGet[*network.Status](ctx, r, resource.NewMetadata(network.NamespaceName, network.StatusType, network.StatusID, resource.VersionUndefined))
if err != nil {
if state.IsNotFoundError(err) {
continue
@ -99,7 +113,7 @@ func (ctrl *EtcdController) Run(ctx context.Context, r controller.Runtime, logge
return err
}
networkStatus := networkResource.(*network.Status).TypedSpec()
networkStatus := networkResource.TypedSpec()
if !(networkStatus.AddressReady && networkStatus.HostnameReady) {
continue
@ -119,33 +133,67 @@ func (ctrl *EtcdController) Run(ctx context.Context, r controller.Runtime, logge
continue
}
if err = r.Modify(ctx, secrets.NewEtcd(), func(r resource.Resource) error {
return ctrl.updateSecrets(etcdRoot, r.(*secrets.Etcd).TypedSpec())
hostnameStatus, err := safe.ReaderGet[*network.HostnameStatus](ctx, r, resource.NewMetadata(network.NamespaceName, network.HostnameStatusType, network.HostnameID, resource.VersionUndefined))
if err != nil {
if state.IsNotFoundError(err) {
continue
}
return fmt.Errorf("error getting hostname status: %w", err)
}
nodeAddrs, err := safe.ReaderGet[*network.NodeAddress](
ctx,
r,
resource.NewMetadata(
network.NamespaceName,
network.NodeAddressType,
network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s),
resource.VersionUndefined,
),
)
if err != nil {
if state.IsNotFoundError(err) {
continue
}
return fmt.Errorf("error getting addresses: %w", err)
}
if err = safe.WriterModify(ctx, r, secrets.NewEtcd(), func(r *secrets.Etcd) error {
return ctrl.updateSecrets(etcdRoot, nodeAddrs, hostnameStatus, r.TypedSpec())
}); err != nil {
return err
}
}
}
func (ctrl *EtcdController) updateSecrets(etcdRoot *secrets.EtcdRootSpec, etcdCerts *secrets.EtcdCertsSpec) error {
func (ctrl *EtcdController) updateSecrets(etcdRoot *secrets.EtcdRootSpec, nodeAddress *network.NodeAddress, hostnameStatus *network.HostnameStatus, etcdCerts *secrets.EtcdCertsSpec) error {
generator := etcd.CertificateGenerator{
CA: etcdRoot.EtcdCA,
NodeAddresses: nodeAddress,
HostnameStatus: hostnameStatus,
}
var err error
etcdCerts.Etcd, err = etcd.GenerateServerCert(etcdRoot.EtcdCA)
etcdCerts.Etcd, err = generator.GenerateServerCert()
if err != nil {
return fmt.Errorf("error generating etcd client certs: %w", err)
}
etcdCerts.EtcdPeer, err = etcd.GeneratePeerCert(etcdRoot.EtcdCA)
etcdCerts.EtcdPeer, err = generator.GeneratePeerCert()
if err != nil {
return fmt.Errorf("error generating etcd peer certs: %w", err)
}
etcdCerts.EtcdAdmin, err = etcd.GenerateClientCert(etcdRoot.EtcdCA, "talos")
etcdCerts.EtcdAdmin, err = generator.GenerateClientCert("talos")
if err != nil {
return fmt.Errorf("error generating admin client certs: %w", err)
}
etcdCerts.EtcdAPIServer, err = etcd.GenerateClientCert(etcdRoot.EtcdCA, "kube-apiserver")
etcdCerts.EtcdAPIServer, err = generator.GenerateClientCert("kube-apiserver")
if err != nil {
return fmt.Errorf("error generating kube-apiserver etcd client certs: %w", err)
}

178
internal/app/machined/pkg/controllers/secrets/etcd_test.go Normal file
View File

@ -0,0 +1,178 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//nolint:dupl
package secrets_test
import (
"fmt"
"testing"
"time"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/state"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/talos-systems/crypto/x509"
"inet.af/netaddr"
"github.com/talos-systems/talos/internal/app/machined/pkg/controllers/ctest"
secretsctrl "github.com/talos-systems/talos/internal/app/machined/pkg/controllers/secrets"
"github.com/talos-systems/talos/pkg/machinery/resources/k8s"
"github.com/talos-systems/talos/pkg/machinery/resources/network"
"github.com/talos-systems/talos/pkg/machinery/resources/secrets"
timeres "github.com/talos-systems/talos/pkg/machinery/resources/time"
)
func TestEtcdSuite(t *testing.T) {
suite.Run(t, &EtcdSuite{
DefaultSuite: ctest.DefaultSuite{
AfterSetup: func(suite *ctest.DefaultSuite) {
suite.Require().NoError(suite.Runtime().RegisterController(&secretsctrl.EtcdController{}))
},
},
})
}
type EtcdSuite struct {
ctest.DefaultSuite
}
func (suite *EtcdSuite) TestReconcile() {
rootSecrets := secrets.NewEtcdRoot(secrets.EtcdRootID)
etcdCA, err := x509.NewSelfSignedCertificateAuthority(
x509.Organization("talos"),
x509.ECDSA(true),
)
suite.Require().NoError(err)
rootSecrets.TypedSpec().EtcdCA = &x509.PEMEncodedCertificateAndKey{
Crt: etcdCA.CrtPEM,
Key: etcdCA.KeyPEM,
}
suite.Require().NoError(suite.State().Create(suite.Ctx(), rootSecrets))
networkStatus := network.NewStatus(network.NamespaceName, network.StatusID)
networkStatus.TypedSpec().AddressReady = true
networkStatus.TypedSpec().HostnameReady = true
suite.Require().NoError(suite.State().Create(suite.Ctx(), networkStatus))
hostnameStatus := network.NewHostnameStatus(network.NamespaceName, network.HostnameID)
hostnameStatus.TypedSpec().Hostname = "host"
hostnameStatus.TypedSpec().Domainname = "domain"
suite.Require().NoError(suite.State().Create(suite.Ctx(), hostnameStatus))
nodeAddresses := network.NewNodeAddress(network.NamespaceName, network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s))
nodeAddresses.TypedSpec().Addresses = []netaddr.IPPrefix{
netaddr.MustParseIPPrefix("10.3.4.5/24"),
netaddr.MustParseIPPrefix("2001:db8::1eaf/64"),
}
suite.Require().NoError(suite.State().Create(suite.Ctx(), nodeAddresses))
timeSync := timeres.NewStatus()
timeSync.TypedSpec().Synced = true
suite.Require().NoError(suite.State().Create(suite.Ctx(), timeSync))
suite.AssertWithin(3*time.Second, 100*time.Millisecond,
ctest.WrapRetry(func(assert *assert.Assertions, require *require.Assertions) {
certs, err := ctest.Get[*secrets.Etcd](
suite,
resource.NewMetadata(
secrets.NamespaceName,
secrets.EtcdType,
secrets.EtcdID,
resource.VersionUndefined,
),
)
if err != nil {
if state.IsNotFoundError(err) {
assert.NoError(err)
} else {
require.NoError(err)
}
return
}
etcdCerts := certs.TypedSpec()
serverCert, err := etcdCerts.Etcd.GetCert()
require.NoError(err)
assert.Equal([]string{"host", "host.domain", "localhost"}, serverCert.DNSNames)
assert.Equal("[10.3.4.5 2001:db8::1eaf 127.0.0.1 ::1]", fmt.Sprintf("%v", serverCert.IPAddresses))
assert.Equal("host", serverCert.Subject.CommonName)
peerCert, err := etcdCerts.EtcdPeer.GetCert()
require.NoError(err)
assert.Equal([]string{"host", "host.domain"}, peerCert.DNSNames)
assert.Equal("[10.3.4.5 2001:db8::1eaf]", fmt.Sprintf("%v", peerCert.IPAddresses))
assert.Equal("host", peerCert.Subject.CommonName)
adminCert, err := etcdCerts.EtcdAdmin.GetCert()
require.NoError(err)
assert.Empty(adminCert.DNSNames)
assert.Empty(adminCert.IPAddresses)
assert.Equal("talos", adminCert.Subject.CommonName)
kubeAPICert, err := etcdCerts.EtcdAPIServer.GetCert()
require.NoError(err)
assert.Empty(kubeAPICert.DNSNames)
assert.Empty(kubeAPICert.IPAddresses)
assert.Equal("kube-apiserver", kubeAPICert.Subject.CommonName)
}))
// update node addresses, certs should be updated
oldVersion := nodeAddresses.Metadata().Version()
nodeAddresses.TypedSpec().Addresses = []netaddr.IPPrefix{
netaddr.MustParseIPPrefix("10.3.4.5/24"),
}
nodeAddresses.Metadata().BumpVersion()
suite.Require().NoError(suite.State().Update(suite.Ctx(), oldVersion, nodeAddresses))
suite.AssertWithin(3*time.Second, 100*time.Millisecond,
ctest.WrapRetry(func(assert *assert.Assertions, require *require.Assertions) {
certs, err := ctest.Get[*secrets.Etcd](
suite,
resource.NewMetadata(
secrets.NamespaceName,
secrets.EtcdType,
secrets.EtcdID,
resource.VersionUndefined,
),
)
if err != nil {
require.NoError(err)
return
}
etcdCerts := certs.TypedSpec()
serverCert, err := etcdCerts.Etcd.GetCert()
require.NoError(err)
assert.Equal([]string{"host", "host.domain", "localhost"}, serverCert.DNSNames)
assert.Equal("[10.3.4.5 127.0.0.1]", fmt.Sprintf("%v", serverCert.IPAddresses))
assert.Equal("host", serverCert.Subject.CommonName)
peerCert, err := etcdCerts.EtcdPeer.GetCert()
require.NoError(err)
assert.Equal([]string{"host", "host.domain"}, peerCert.DNSNames)
assert.Equal("[10.3.4.5]", fmt.Sprintf("%v", peerCert.IPAddresses))
assert.Equal("host", peerCert.Subject.CommonName)
}))
}

75
internal/pkg/etcd/certs.go
View File

@ -7,41 +7,41 @@ package etcd
import (
stdlibx509 "crypto/x509"
"fmt"
stdlibnet "net"
"os"
"time"
"github.com/talos-systems/crypto/x509"
"github.com/talos-systems/net"
"inet.af/netaddr"
"github.com/talos-systems/talos/pkg/machinery/nethelpers"
"github.com/talos-systems/talos/pkg/machinery/resources/network"
)
// buildOptions set common certificate options.
func buildOptions(autoSANs, includeLocalhost bool) ([]x509.Option, error) {
ips, err := net.IPAddrs()
if err != nil {
return nil, fmt.Errorf("failed to discover IP addresses: %w", err)
}
// CertificateGenerator contains etcd certificate options.
type CertificateGenerator struct {
CA *x509.PEMEncodedCertificateAndKey
ips = net.IPFilter(ips, network.NotSideroLinkStdIP)
NodeAddresses *network.NodeAddress
HostnameStatus *network.HostnameStatus
}
// buildOptions set common certificate options.
func (gen *CertificateGenerator) buildOptions(autoSANs, includeLocalhost bool) []x509.Option {
addresses := gen.NodeAddresses.TypedSpec().IPs()
if includeLocalhost {
ips = append(ips, stdlibnet.ParseIP("127.0.0.1"))
if net.IsIPv6(ips...) {
ips = append(ips, stdlibnet.ParseIP("::1"))
addresses = append(addresses, netaddr.MustParseIP("127.0.0.1"))
for _, addr := range addresses {
if addr.Is6() {
addresses = append(addresses, netaddr.MustParseIP("::1"))
break
}
}
}
hostname, err := os.Hostname()
if err != nil {
return nil, fmt.Errorf("failed to get hostname: %w", err)
}
dnsNames, err := net.DNSNames()
if err != nil {
return nil, fmt.Errorf("failed to get host DNS names: %w", err)
}
hostname := gen.HostnameStatus.TypedSpec().Hostname
dnsNames := gen.HostnameStatus.TypedSpec().DNSNames()
if includeLocalhost {
dnsNames = append(dnsNames, "localhost")
@ -56,21 +56,18 @@ func buildOptions(autoSANs, includeLocalhost bool) ([]x509.Option, error) {
result = append(result,
x509.CommonName(hostname),
x509.DNSNames(dnsNames),
x509.IPAddresses(ips),
x509.IPAddresses(nethelpers.MapNetAddrToStd(addresses)),
)
}
return result, nil
return result
}
// GeneratePeerCert generates etcd peer certificate and key from etcd CA.
//
//nolint:dupl
func GeneratePeerCert(etcdCA *x509.PEMEncodedCertificateAndKey) (*x509.PEMEncodedCertificateAndKey, error) {
opts, err := buildOptions(true, false)
if err != nil {
return nil, err
}
func (gen *CertificateGenerator) GeneratePeerCert() (*x509.PEMEncodedCertificateAndKey, error) {
opts := gen.buildOptions(true, false)
opts = append(opts,
x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{
@ -79,7 +76,7 @@ func GeneratePeerCert(etcdCA *x509.PEMEncodedCertificateAndKey) (*x509.PEMEncode
}),
)
ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(etcdCA)
ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(gen.CA)
if err != nil {
return nil, fmt.Errorf("failed loading CA from config: %w", err)
}
@ -95,11 +92,8 @@ func GeneratePeerCert(etcdCA *x509.PEMEncodedCertificateAndKey) (*x509.PEMEncode
// GenerateServerCert generates server etcd certificate and key from etcd CA.
//
//nolint:dupl
func GenerateServerCert(etcdCA *x509.PEMEncodedCertificateAndKey) (*x509.PEMEncodedCertificateAndKey, error) {
opts, err := buildOptions(true, true)
if err != nil {
return nil, err
}
func (gen *CertificateGenerator) GenerateServerCert() (*x509.PEMEncodedCertificateAndKey, error) {
opts := gen.buildOptions(true, true)
opts = append(opts,
x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{
@ -108,7 +102,7 @@ func GenerateServerCert(etcdCA *x509.PEMEncodedCertificateAndKey) (*x509.PEMEnco
}),
)
ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(etcdCA)
ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(gen.CA)
if err != nil {
return nil, fmt.Errorf("failed loading CA from config: %w", err)
}
@ -122,11 +116,8 @@ func GenerateServerCert(etcdCA *x509.PEMEncodedCertificateAndKey) (*x509.PEMEnco
}
// GenerateClientCert generates client certificate and key from etcd CA.
func GenerateClientCert(etcdCA *x509.PEMEncodedCertificateAndKey, commonName string) (*x509.PEMEncodedCertificateAndKey, error) {
opts, err := buildOptions(false, false)
if err != nil {
return nil, err
}
func (gen *CertificateGenerator) GenerateClientCert(commonName string) (*x509.PEMEncodedCertificateAndKey, error) {
opts := gen.buildOptions(false, false)
opts = append(opts, x509.CommonName(commonName))
opts = append(opts,
@ -135,7 +126,7 @@ func GenerateClientCert(etcdCA *x509.PEMEncodedCertificateAndKey, commonName str
}),
)
ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(etcdCA)
ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(gen.CA)
if err != nil {
return nil, fmt.Errorf("failed loading CA from config: %w", err)
}