alt-orchestra/talos
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

fix: refresh kubelet self-issued serving certificates

Browse Source 
Kubelet doesn't refresh self-issued serving certificates, so force it by
removing the cert on each restart.

Fix the code which was forcing rejoin when the nodename changes, it was
broken, as it was checking serving certificate instead of client
certificate. It worked by accident when not using controlplane-issued
serving certificates.

Fixes #7235

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
This commit is contained in:
Andrey Smirnov 2023-05-18 22:19:34 +04:00
parent bb02dd263c
commit dd8336c9ee
No known key found for this signature in database
GPG Key ID: 7B26396447AB6DFD
1 changed files with 39 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
internal/app/machined/pkg/controllers/k8s/kubelet_service.go
View File

@ -20,7 +20,6 @@ import (
"github.com/cosi-project/runtime/pkg/controller"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/state"
"github.com/siderolabs/gen/slices"
"github.com/siderolabs/go-pointer"
"go.uber.org/zap"
"k8s.io/apimachinery/pkg/runtime"
@ -191,14 +190,17 @@ func (ctrl *KubeletServiceController) Run(ctx context.Context, r controller.Runt
// refresh certs only if we are managing the node name (not overridden by the user)
if cfgSpec.ExpectedNodename != "" {
err = ctrl.refreshKubeletCerts(cfgSpec.ExpectedNodename)
err = ctrl.refreshKubeletCerts(logger, cfgSpec.ExpectedNodename)
if err != nil {
return err
}
}
err = updateKubeconfig(logger, secretSpec.Endpoint)
if err != nil {
if err = ctrl.refreshSelfServingCert(); err != nil {
return err
}
if err = ctrl.updateKubeconfig(logger, secretSpec.Endpoint); err != nil {
return err
}
@ -287,7 +289,7 @@ func (ctrl *KubeletServiceController) writeConfig(cfgSpec *k8s.KubeletSpecSpec)
}
// updateKubeconfig updates the kubeconfig of kubelet with the given endpoint if it exists.
func updateKubeconfig(logger *zap.Logger, newEndpoint *url.URL) error {
func (ctrl *KubeletServiceController) updateKubeconfig(logger *zap.Logger, newEndpoint *url.URL) error {
config, err := clientcmd.LoadFromFile(constants.KubeletKubeconfig)
if errors.Is(err, os.ErrNotExist) {
return nil
@ -326,8 +328,8 @@ func updateKubeconfig(logger *zap.Logger, newEndpoint *url.URL) error {
// refreshKubeletCerts checks if the existing kubelet certificates match the node hostname.
// If they don't match, it clears the certificate directory and the removes kubelet's kubeconfig so that
// they can be regenerated next time kubelet is started.
func (ctrl *KubeletServiceController) refreshKubeletCerts(hostname string) error {
cert, err := ctrl.readKubeletCertificate()
func (ctrl *KubeletServiceController) refreshKubeletCerts(logger *zap.Logger, nodename string) error {
cert, err := ctrl.readKubeletClientCertificate()
if err != nil {
return err
}
@ -336,15 +338,20 @@ func (ctrl *KubeletServiceController) refreshKubeletCerts(hostname string) error
return nil
}
valid := slices.Contains(cert.DNSNames, func(name string) bool {
return name == hostname
})
expectedCommonName := fmt.Sprintf("system:node:%s", nodename)
valid := expectedCommonName == cert.Subject.CommonName
if valid {
// certificate looks good, no need to refresh
return nil
}
logger.Info("kubelet client certificate does not match expected nodename, removing",
zap.String("expected", expectedCommonName),
zap.String("actual", cert.Subject.CommonName),
)
// remove the pki directory
err = os.RemoveAll(constants.KubeletPKIDir)
if err != nil {
@ -360,8 +367,28 @@ func (ctrl *KubeletServiceController) refreshKubeletCerts(hostname string) error
return err
}
func (ctrl *KubeletServiceController) readKubeletCertificate() (*x509.Certificate, error) {
raw, err := os.ReadFile(filepath.Join(constants.KubeletPKIDir, "kubelet.crt"))
// refreshSelfServingCert removes the self-signed serving certificate (if exists) to force the kubelet to renew it.
func (ctrl *KubeletServiceController) refreshSelfServingCert() error {
for _, filename := range []string{
"kubelet.crt",
"kubelet.key",
} {
path := filepath.Join(constants.KubeletPKIDir, filename)
_, err := os.Stat(path)
if err == nil {
err = os.Remove(path)
if err != nil {
return fmt.Errorf("error removing self-signed certificate: %w", err)
}
}
}
return nil
}
func (ctrl *KubeletServiceController) readKubeletClientCertificate() (*x509.Certificate, error) {
raw, err := os.ReadFile(filepath.Join(constants.KubeletPKIDir, "kubelet-client-current.pem"))
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}