feat: update controller-runtime logs to console level on config.debug

This PR enables debug logging of controller-runtime to the server
console if the machine configuration .debug field is set to true (log
verbosity can be also changed on the fly).

For control plane nodes, don't send kubelet logs to the console (as they
tend to flood the console with messages), this leaves reasonable amount
of logging for early boot troubleshooting.

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go

@@ -101,9 +101,11 @@ func (r *Runtime) CanApplyImmediate(b []byte) error {
 
 	// the config changes allowed to be applied immediately are:
 	// * cluster config
+	// * .machine.debug
 	// * .machine.time
 	// * .machine.network
 	newConfig.ClusterConfig = currentConfig.ClusterConfig
+	newConfig.ConfigDebug = currentConfig.ConfigDebug
 
 	if newConfig.MachineConfig != nil && currentConfig.MachineConfig != nil {
 		newConfig.MachineConfig.MachineTime = currentConfig.MachineConfig.MachineTime

internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go

@@ -9,7 +9,10 @@ import (
 
 	"github.com/cosi-project/runtime/pkg/controller"
 	osruntime "github.com/cosi-project/runtime/pkg/controller/runtime"
+	"github.com/cosi-project/runtime/pkg/resource"
+	"github.com/cosi-project/runtime/pkg/state"
 	"github.com/talos-systems/go-procfs/procfs"
+	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
 	"github.com/talos-systems/talos/internal/app/machined/pkg/controllers/config"
@@ -23,11 +26,14 @@ import (
 	"github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
 	"github.com/talos-systems/talos/pkg/logging"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
+	configresource "github.com/talos-systems/talos/pkg/resources/config"
 )
 
 // Controller implements runtime.V1alpha2Controller.
 type Controller struct {
 	controllerRuntime *osruntime.Runtime
+	consoleLogLevel   zap.AtomicLevel
+	logger            *zap.Logger
 
 	v1alpha1Runtime runtime.Runtime
 }
@@ -36,6 +42,7 @@ type Controller struct {
 func NewController(v1alpha1Runtime runtime.Runtime, loggingManager runtime.LoggingManager) (*Controller, error) {
 	ctrl := &Controller{
 		v1alpha1Runtime: v1alpha1Runtime,
+		consoleLogLevel: zap.NewAtomicLevel(),
 	}
 
 	logWriter, err := loggingManager.ServiceLog("controller-runtime").Writer()
@@ -43,18 +50,21 @@ func NewController(v1alpha1Runtime runtime.Runtime, loggingManager runtime.Loggi
 		return nil, err
 	}
 
-	logger := logging.ZapLogger(
+	ctrl.logger = logging.ZapLogger(
 		logging.NewLogDestination(logWriter, zapcore.DebugLevel, logging.WithColoredLevels()),
-		logging.NewLogDestination(logging.StdWriter, zapcore.InfoLevel, logging.WithoutTimestamp(), logging.WithoutLogLevels()),
+		logging.NewLogDestination(logging.StdWriter, ctrl.consoleLogLevel, logging.WithoutTimestamp(), logging.WithoutLogLevels()),
 	).With(logging.Component("controller-runtime"))
 
-	ctrl.controllerRuntime, err = osruntime.NewRuntime(v1alpha1Runtime.State().V1Alpha2().Resources(), logger)
+	ctrl.controllerRuntime, err = osruntime.NewRuntime(v1alpha1Runtime.State().V1Alpha2().Resources(), ctrl.logger)
 
 	return ctrl, err
 }
 
 // Run the controller runtime.
 func (ctrl *Controller) Run(ctx context.Context) error {
+	// adjust the log level based on machine configuration
+	go ctrl.watchMachineConfig(ctx, ctrl.logger)
+
 	for _, c := range []controller.Controller{
 		&v1alpha1.BootstrapStatusController{},
 		&v1alpha1.ServiceController{
@@ -142,3 +152,38 @@ func (ctrl *Controller) Run(ctx context.Context) error {
 func (ctrl *Controller) DependencyGraph() (*controller.DependencyGraph, error) {
 	return ctrl.controllerRuntime.GetDependencyGraph()
 }
+
+func (ctrl *Controller) watchMachineConfig(ctx context.Context, logger *zap.Logger) {
+	watchCh := make(chan state.Event)
+
+	if err := ctrl.v1alpha1Runtime.State().V1Alpha2().Resources().Watch(
+		ctx,
+		resource.NewMetadata(configresource.NamespaceName, configresource.MachineConfigType, configresource.V1Alpha1ID, resource.VersionUndefined),
+		watchCh,
+	); err != nil {
+		logger.Warn("error watching machine configuration", zap.Error(err))
+
+		return
+	}
+
+	for {
+		logLevel := zapcore.InfoLevel
+
+		select {
+		case event := <-watchCh:
+			if event.Type != state.Destroyed {
+				if event.Resource.(*configresource.MachineConfig).Config().Debug() {
+					logLevel = zapcore.DebugLevel
+				}
+			}
+		case <-ctx.Done():
+			return
+		}
+
+		if ctrl.consoleLogLevel.Level() != logLevel {
+			ctrl.consoleLogLevel.SetLevel(logLevel)
+
+			ctrl.logger.Info("setting console log level", zap.Stringer("level", logLevel))
+		}
+	}
+}

internal/app/machined/pkg/system/services/kubelet.go

@@ -34,6 +34,7 @@ import (
 	"github.com/talos-systems/talos/internal/pkg/containers/image"
 	"github.com/talos-systems/talos/pkg/argsbuilder"
 	"github.com/talos-systems/talos/pkg/conditions"
+	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
 	"github.com/talos-systems/talos/pkg/resources/network"
 	timeresource "github.com/talos-systems/talos/pkg/resources/time"
@@ -185,7 +186,7 @@ func (k *Kubelet) Runner(r runtime.Runtime) (runner.Runner, error) {
 	}
 
 	return restart.New(containerd.NewRunner(
-		r.Config().Debug(),
+		r.Config().Debug() && r.Config().Machine().Type() == machine.TypeJoin, // enable debug logs only for the worker nodes
 		&args,
 		runner.WithLoggingManager(r.Logging()),
 		runner.WithNamespace(constants.SystemContainerdNamespace),

pkg/logging/zap.go

@@ -61,7 +61,7 @@ var StdWriter = &LogWrapper{nil}
 // LogDestination defines logging destination Config.
 type LogDestination struct {
 	// Level log level.
-	Level  zap.AtomicLevel
+	Level  zapcore.LevelEnabler
 	writer io.Writer
 	config zapcore.EncoderConfig
 }
@@ -91,7 +91,7 @@ func WithColoredLevels() EncoderOption {
 }
 
 // NewLogDestination creates new log destination.
-func NewLogDestination(writer io.Writer, logLevel zapcore.Level, options ...EncoderOption) *LogDestination {
+func NewLogDestination(writer io.Writer, logLevel zapcore.LevelEnabler, options ...EncoderOption) *LogDestination {
 	config := zap.NewDevelopmentEncoderConfig()
 	config.ConsoleSeparator = " "
 	config.StacktraceKey = "error"
@@ -101,7 +101,7 @@ func NewLogDestination(writer io.Writer, logLevel zapcore.Level, options ...Enco
 	}
 
 	return &LogDestination{
-		Level:  zap.NewAtomicLevelAt(logLevel),
+		Level:  logLevel,
 		config: config,
 		writer: writer,
 	}