feat: implement sysctl controllers

Fixed: https://github.com/talos-systems/talos/issues/3686

Replaced sequencer tasks for KSPP and Kubernetes required sysctl props
by the ones set by controllers.

KernelParam flow includes of 3 controllers and 2 resources:
- `KernelParamConfigController` - handles user sysctls coming from v1alpha1
config.
- `KernelParamDefaultsController` - handles our built-in KSPP and K8s
required sysctls.
- `KernelParamSpecController` - consumes `KernelParamSpec`s created by the
previous two controllers, applies them and updates the corresponding
`KernelParamStatus`.

Signed-off-by: Artem Chernyshev <artem.0xD2@gmail.com>
This commit is contained in:
Artem Chernyshev 2021-08-02 20:06:59 +03:00 committed by talos-bot
parent fdf6b2433c
commit e08b4f8f9e
20 changed files with 1212 additions and 99 deletions

View File

@ -36,6 +36,14 @@ before upgrading to Talos 0.12.
Current control plane status can be checked with `talosctl get bootstrapstatus` before performing upgrade to Talos 0.12.
"""
[notes.cosi]
title = "Sysctl Configuration"
description = """\
Sysctl Kernel Params configuration was completely rewritten to be based on controllers and resources,
which makes it possible to apply `.machine.sysctls` in immediate mode (without a reboot).
`talosctl get kernelparams` returns merged list of KSPP, Kubernetes and user defined params along with
the default values overwritten by Talos.
"""
[make_deps]

View File

@ -0,0 +1,102 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package runtime_test
import (
"context"
"fmt"
"log"
"sync"
"time"
"github.com/cosi-project/runtime/pkg/controller/runtime"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/state"
"github.com/cosi-project/runtime/pkg/state/impl/inmem"
"github.com/cosi-project/runtime/pkg/state/impl/namespaced"
"github.com/stretchr/testify/suite"
"github.com/talos-systems/go-retry/retry"
"github.com/talos-systems/talos/pkg/logging"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
"github.com/talos-systems/talos/pkg/resources/config"
)
const (
fsFileMax = "fs.file-max"
)
type KernelParamSuite struct {
suite.Suite
state state.State
runtime *runtime.Runtime
wg sync.WaitGroup
ctx context.Context
ctxCancel context.CancelFunc
}
func (suite *KernelParamSuite) SetupTest() {
suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute)
suite.state = state.WrapCore(namespaced.NewState(inmem.Build))
var err error
logger := logging.Wrap(log.Writer())
suite.runtime, err = runtime.NewRuntime(suite.state, logger)
suite.Require().NoError(err)
}
func (suite *KernelParamSuite) startRuntime() {
suite.wg.Add(1)
go func() {
defer suite.wg.Done()
suite.Assert().NoError(suite.runtime.Run(suite.ctx))
}()
}
func (suite *KernelParamSuite) assertResource(md resource.Metadata, compare func(res resource.Resource) bool) func() error {
return func() error {
r, err := suite.state.Get(suite.ctx, md)
if err != nil {
if state.IsNotFoundError(err) {
return retry.ExpectedError(err)
}
return err
}
if !compare(r) {
return fmt.Errorf("resource is not equal to the expected one")
}
return nil
}
}
func (suite *KernelParamSuite) TearDownTest() {
suite.T().Log("tear down")
suite.ctxCancel()
suite.wg.Wait()
// trigger updates in resources to stop watch loops
err := suite.state.Create(context.Background(), config.NewMachineConfig(&v1alpha1.Config{
ConfigVersion: "v1alpha1",
MachineConfig: &v1alpha1.MachineConfig{},
}))
if state.IsConflictError(err) {
err = suite.state.Destroy(context.Background(), config.NewMachineConfig(nil).Metadata())
}
suite.Assert().NoError(err)
}

View File

@ -0,0 +1,105 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package runtime
import (
"context"
"fmt"
"github.com/AlekSi/pointer"
"github.com/cosi-project/runtime/pkg/controller"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/state"
"go.uber.org/zap"
"github.com/talos-systems/talos/pkg/resources/config"
"github.com/talos-systems/talos/pkg/resources/runtime"
)
// KernelParamConfigController watches v1alpha1.Config, creates/updates/deletes kernel param specs.
type KernelParamConfigController struct{}
// Name implements controller.Controller interface.
func (ctrl *KernelParamConfigController) Name() string {
return "runtime.KernelParamConfigController"
}
// Inputs implements controller.Controller interface.
func (ctrl *KernelParamConfigController) Inputs() []controller.Input {
return []controller.Input{
{
Namespace: config.NamespaceName,
Type: config.MachineConfigType,
ID: pointer.ToString(config.V1Alpha1ID),
},
}
}
// Outputs implements controller.Controller interface.
func (ctrl *KernelParamConfigController) Outputs() []controller.Output {
return []controller.Output{
{
Type: runtime.KernelParamSpecType,
Kind: controller.OutputShared,
},
}
}
// Run implements controller.Controller interface.
//
//nolint:gocyclo
func (ctrl *KernelParamConfigController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
for {
select {
case <-ctx.Done():
return nil
case <-r.EventCh():
cfg, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.MachineConfigType, config.V1Alpha1ID, resource.VersionUndefined))
if err != nil {
if !state.IsNotFoundError(err) {
return fmt.Errorf("error getting config: %w", err)
}
}
touchedIDs := make(map[resource.ID]struct{})
if cfg != nil {
c, _ := cfg.(*config.MachineConfig) //nolint:errcheck
for key, value := range c.Config().Machine().Sysctls() {
touchedIDs[key] = struct{}{}
value := value
item := runtime.NewKernelParamSpec(runtime.NamespaceName, key)
if err = r.Modify(ctx, item, func(res resource.Resource) error {
res.(*runtime.KernelParamSpec).TypedSpec().Value = value
return nil
}); err != nil {
return err
}
}
}
// list keys for cleanup
list, err := r.List(ctx, resource.NewMetadata(runtime.NamespaceName, runtime.KernelParamSpecType, "", resource.VersionUndefined))
if err != nil {
return fmt.Errorf("error listing resources: %w", err)
}
for _, res := range list.Items {
if res.Metadata().Owner() != ctrl.Name() {
continue
}
if _, ok := touchedIDs[res.Metadata().ID()]; !ok {
if err = r.Destroy(ctx, res.Metadata()); err != nil {
return fmt.Errorf("error cleaning up specs: %w", err)
}
}
}
}
}
}

View File

@ -0,0 +1,93 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package runtime_test
import (
"fmt"
"testing"
"time"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/state"
"github.com/stretchr/testify/suite"
"github.com/talos-systems/go-retry/retry"
runtimecontrollers "github.com/talos-systems/talos/internal/app/machined/pkg/controllers/runtime"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
"github.com/talos-systems/talos/pkg/resources/config"
runtimeresource "github.com/talos-systems/talos/pkg/resources/runtime"
)
type KernelParamConfigSuite struct {
KernelParamSuite
}
func (suite *KernelParamConfigSuite) TestReconcileConfig() {
suite.Require().NoError(suite.runtime.RegisterController(&runtimecontrollers.KernelParamConfigController{}))
suite.startRuntime()
value := "500000"
cfg := config.NewMachineConfig(&v1alpha1.Config{
ConfigVersion: "v1alpha1",
MachineConfig: &v1alpha1.MachineConfig{
MachineSysctls: map[string]string{
fsFileMax: value,
},
},
ClusterConfig: &v1alpha1.ClusterConfig{},
})
suite.Require().NoError(suite.state.Create(suite.ctx, cfg))
specMD := resource.NewMetadata(runtimeresource.NamespaceName, runtimeresource.KernelParamSpecType, fsFileMax, resource.VersionUndefined)
suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
suite.assertResource(
specMD,
func(res resource.Resource) bool {
return res.(*runtimeresource.KernelParamSpec).TypedSpec().Value == value
},
),
))
cfg = config.NewMachineConfig(&v1alpha1.Config{
ConfigVersion: "v1alpha1",
MachineConfig: &v1alpha1.MachineConfig{
MachineSysctls: map[string]string{},
},
ClusterConfig: &v1alpha1.ClusterConfig{},
})
old := cfg.Metadata().Version()
cfg.Metadata().BumpVersion()
suite.Require().NoError(suite.state.Update(suite.ctx, old, cfg))
var err error
// wait for the resource to be removed
suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
func() error {
for _, md := range []resource.Metadata{specMD} {
_, err = suite.state.Get(suite.ctx, md)
if err != nil {
if state.IsNotFoundError(err) {
return nil
}
return err
}
}
return retry.ExpectedError(fmt.Errorf("resource still exists"))
},
))
}
func TestKernelParamConfigSuite(t *testing.T) {
suite.Run(t, new(KernelParamConfigSuite))
}

View File

@ -0,0 +1,110 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package runtime
import (
"context"
"github.com/cosi-project/runtime/pkg/controller"
"github.com/cosi-project/runtime/pkg/resource"
"go.uber.org/zap"
v1alpha1runtime "github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
"github.com/talos-systems/talos/pkg/kernel"
"github.com/talos-systems/talos/pkg/kernel/kspp"
"github.com/talos-systems/talos/pkg/resources/runtime"
)
// KernelParamDefaultsController creates default kernel params.
type KernelParamDefaultsController struct {
V1Alpha1Mode v1alpha1runtime.Mode
}
// Name implements controller.Controller interface.
func (ctrl *KernelParamDefaultsController) Name() string {
return "runtime.KernelParamDefaultsController"
}
// Inputs implements controller.Controller interface.
func (ctrl *KernelParamDefaultsController) Inputs() []controller.Input {
return nil
}
// Outputs implements controller.Controller interface.
func (ctrl *KernelParamDefaultsController) Outputs() []controller.Output {
return []controller.Output{
{
Type: runtime.KernelParamSpecType,
Kind: controller.OutputShared,
},
}
}
// Run implements controller.Controller interface.
func (ctrl *KernelParamDefaultsController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
select {
case <-ctx.Done():
return nil
case <-r.EventCh():
kernels := append(
kspp.GetKernelParams(),
ctrl.getKernelParams()...,
)
for _, prop := range kernels {
value := prop.Value
item := runtime.NewKernelParamSpec(runtime.NamespaceName, prop.Key)
if err := r.Modify(ctx, item, func(res resource.Resource) error {
res.(*runtime.KernelParamSpec).TypedSpec().Value = value
if item.Metadata().ID() == "net.ipv6.conf.default.forwarding" {
res.(*runtime.KernelParamSpec).TypedSpec().IgnoreErrors = true
}
return nil
}); err != nil {
return err
}
}
}
return nil
}
func (ctrl *KernelParamDefaultsController) getKernelParams() []*kernel.Param {
res := []*kernel.Param{
{
Key: "net.ipv4.ip_forward",
Value: "1",
},
}
if ctrl.V1Alpha1Mode != v1alpha1runtime.ModeContainer {
res = append(res, []*kernel.Param{
{
Key: "net.bridge.bridge-nf-call-iptables",
Value: "1",
},
{
Key: "net.bridge.bridge-nf-call-ip6tables",
Value: "1",
},
}...)
}
res = append(res, []*kernel.Param{
{
Key: "net.ipv6.conf.default.forwarding",
Value: "1",
},
{
Key: "kernel.pid_max",
Value: "262144",
},
}...)
return res
}

View File

@ -0,0 +1,107 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package runtime_test
import (
"testing"
"time"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/stretchr/testify/suite"
"github.com/talos-systems/go-retry/retry"
runtimecontrollers "github.com/talos-systems/talos/internal/app/machined/pkg/controllers/runtime"
"github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
"github.com/talos-systems/talos/pkg/kernel"
runtimeresource "github.com/talos-systems/talos/pkg/resources/runtime"
)
type KernelParamDefaultsSuite struct {
KernelParamSuite
}
func getParams(mode runtime.Mode) []*kernel.Param {
res := []*kernel.Param{
{
Key: "net.ipv4.ip_forward",
Value: "1",
},
{
Key: "net.ipv6.conf.default.forwarding",
Value: "1",
},
{
Key: "kernel.pid_max",
Value: "262144",
},
}
if mode != runtime.ModeContainer {
res = append(res, []*kernel.Param{
{
Key: "net.bridge.bridge-nf-call-iptables",
Value: "1",
},
{
Key: "net.bridge.bridge-nf-call-ip6tables",
Value: "1",
},
}...)
}
return res
}
//nolint:dupl
func (suite *KernelParamDefaultsSuite) TestContainerMode() {
controller := &runtimecontrollers.KernelParamDefaultsController{
runtime.ModeContainer,
}
suite.Require().NoError(suite.runtime.RegisterController(controller))
suite.startRuntime()
for _, prop := range getParams(runtime.ModeContainer) {
prop := prop
suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
suite.assertResource(
resource.NewMetadata(runtimeresource.NamespaceName, runtimeresource.KernelParamSpecType, prop.Key, resource.VersionUndefined),
func(res resource.Resource) bool {
return res.(*runtimeresource.KernelParamSpec).TypedSpec().Value == prop.Value
},
),
))
}
}
//nolint:dupl
func (suite *KernelParamDefaultsSuite) TestMetalMode() {
controller := &runtimecontrollers.KernelParamDefaultsController{
runtime.ModeMetal,
}
suite.Require().NoError(suite.runtime.RegisterController(controller))
suite.startRuntime()
for _, prop := range getParams(runtime.ModeMetal) {
prop := prop
suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
suite.assertResource(
resource.NewMetadata(runtimeresource.NamespaceName, runtimeresource.KernelParamSpecType, prop.Key, resource.VersionUndefined),
func(res resource.Resource) bool {
return res.(*runtimeresource.KernelParamSpec).TypedSpec().Value == prop.Value
},
),
))
}
}
func TestKernelParamDefaultsSuite(t *testing.T) {
suite.Run(t, new(KernelParamDefaultsSuite))
}

View File

@ -0,0 +1,168 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package runtime
import (
"context"
"errors"
"os"
"strings"
"github.com/cosi-project/runtime/pkg/controller"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/hashicorp/go-multierror"
"go.uber.org/zap"
"github.com/talos-systems/talos/pkg/kernel"
"github.com/talos-systems/talos/pkg/resources/runtime"
)
// KernelParamSpecController watches KernelParamSpecs, sets/resets kernel params.
type KernelParamSpecController struct {
defaults map[string]string
state map[string]string
}
// Name implements controller.Controller interface.
func (ctrl *KernelParamSpecController) Name() string {
return "runtime.KernelParamSpecController"
}
// Inputs implements controller.Controller interface.
func (ctrl *KernelParamSpecController) Inputs() []controller.Input {
return []controller.Input{
{
Namespace: runtime.NamespaceName,
Type: runtime.KernelParamSpecType,
Kind: controller.InputStrong,
},
}
}
// Outputs implements controller.Controller interface.
func (ctrl *KernelParamSpecController) Outputs() []controller.Output {
return []controller.Output{
{
Type: runtime.KernelParamStatusType,
Kind: controller.OutputExclusive,
},
}
}
// Run implements controller.Controller interface.
//nolint:gocyclo
func (ctrl *KernelParamSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
if ctrl.state == nil {
ctrl.state = map[string]string{}
}
if ctrl.defaults == nil {
ctrl.defaults = map[string]string{}
}
for {
select {
case <-ctx.Done():
return nil
case <-r.EventCh():
list, err := r.List(ctx, resource.NewMetadata(runtime.NamespaceName, runtime.KernelParamSpecType, "", resource.VersionUndefined))
if err != nil {
return err
}
touchedIDs := map[string]struct{}{}
var errs *multierror.Error
for _, item := range list.Items {
spec := item.(*runtime.KernelParamSpec).TypedSpec()
key := item.Metadata().ID()
if err = ctrl.updateKernelParam(ctx, r, key, spec.Value); err != nil {
if errors.Is(err, os.ErrNotExist) && spec.IgnoreErrors {
status := runtime.NewKernelParamStatus(runtime.NamespaceName, key)
if e := r.Modify(ctx, status, func(res resource.Resource) error {
res.(*runtime.KernelParamStatus).TypedSpec().Unsupported = true
return nil
}); e != nil {
errs = multierror.Append(errs, err)
}
} else {
errs = multierror.Append(errs, err)
}
continue
}
touchedIDs[item.Metadata().ID()] = struct{}{}
}
for key := range ctrl.state {
if _, ok := touchedIDs[key]; ok {
continue
}
if err = ctrl.resetKernelParam(ctx, r, key); err != nil {
errs = multierror.Append(errs, err)
}
}
if errs != nil {
return errs
}
}
}
}
func (ctrl *KernelParamSpecController) updateKernelParam(ctx context.Context, r controller.Runtime, key, value string) error {
prop := &kernel.Param{Key: key, Value: value}
if _, ok := ctrl.defaults[key]; !ok {
if data, err := kernel.ReadParam(prop); err == nil {
ctrl.defaults[key] = string(data)
} else if !errors.Is(err, os.ErrNotExist) {
return err
}
}
if err := kernel.WriteParam(prop); err != nil {
return err
}
ctrl.state[key] = value
status := runtime.NewKernelParamStatus(runtime.NamespaceName, key)
return r.Modify(ctx, status, func(res resource.Resource) error {
res.(*runtime.KernelParamStatus).TypedSpec().Current = value
res.(*runtime.KernelParamStatus).TypedSpec().Default = strings.TrimSpace(ctrl.defaults[key])
return nil
})
}
func (ctrl *KernelParamSpecController) resetKernelParam(ctx context.Context, r controller.Runtime, key string) error {
var err error
if def, ok := ctrl.defaults[key]; ok {
err = kernel.WriteParam(&kernel.Param{
Key: key,
Value: def,
})
} else {
err = kernel.DeleteParam(&kernel.Param{Key: key})
}
if err != nil {
return err
}
delete(ctrl.defaults, key)
delete(ctrl.state, key)
return r.Destroy(ctx, resource.NewMetadata(runtime.NamespaceName, runtime.KernelParamStatusType, key, resource.VersionUndefined))
}

View File

@ -0,0 +1,109 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package runtime_test
import (
"fmt"
"strings"
"testing"
"time"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/state"
"github.com/stretchr/testify/suite"
"github.com/talos-systems/go-retry/retry"
runtimecontrollers "github.com/talos-systems/talos/internal/app/machined/pkg/controllers/runtime"
"github.com/talos-systems/talos/pkg/kernel"
runtimeresource "github.com/talos-systems/talos/pkg/resources/runtime"
)
type KernelParamSpecSuite struct {
KernelParamSuite
}
func (suite *KernelParamSpecSuite) TestParamsSynced() {
suite.Require().NoError(suite.runtime.RegisterController(&runtimecontrollers.KernelParamSpecController{}))
suite.startRuntime()
value := "500000"
def := ""
spec := runtimeresource.NewKernelParamSpec(runtimeresource.NamespaceName, fsFileMax)
spec.TypedSpec().Value = value
suite.Require().NoError(suite.state.Create(suite.ctx, spec))
statusMD := resource.NewMetadata(runtimeresource.NamespaceName, runtimeresource.KernelParamStatusType, fsFileMax, resource.VersionUndefined)
suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
suite.assertResource(
statusMD,
func(res resource.Resource) bool {
def = res.(*runtimeresource.KernelParamStatus).TypedSpec().Default
return res.(*runtimeresource.KernelParamStatus).TypedSpec().Current == value
},
),
))
prop, err := kernel.ReadParam(&kernel.Param{Key: fsFileMax})
suite.Assert().NoError(err)
suite.Require().Equal(value, strings.TrimSpace(string(prop)))
suite.Require().NoError(suite.state.Destroy(suite.ctx, spec.Metadata()))
// wait for the resource to be removed
suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
func() error {
for _, md := range []resource.Metadata{statusMD} {
_, err = suite.state.Get(suite.ctx, md)
if err != nil {
if state.IsNotFoundError(err) {
return nil
}
return err
}
}
return retry.ExpectedError(fmt.Errorf("resource still exists"))
},
))
prop, err = kernel.ReadParam(&kernel.Param{Key: fsFileMax})
suite.Assert().NoError(err)
suite.Require().Equal(def, strings.TrimSpace(string(prop)))
}
func (suite *KernelParamSpecSuite) TestParamsUnsupported() {
suite.Require().NoError(suite.runtime.RegisterController(&runtimecontrollers.KernelParamSpecController{}))
suite.startRuntime()
id := "some.really.not.existing.sysctl"
spec := runtimeresource.NewKernelParamSpec(runtimeresource.NamespaceName, id)
spec.TypedSpec().Value = "value"
spec.TypedSpec().IgnoreErrors = true
suite.Require().NoError(suite.state.Create(suite.ctx, spec))
statusMD := resource.NewMetadata(runtimeresource.NamespaceName, runtimeresource.KernelParamStatusType, id, resource.VersionUndefined)
suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
suite.assertResource(
statusMD,
func(res resource.Resource) bool {
return res.(*runtimeresource.KernelParamStatus).TypedSpec().Unsupported == true
},
),
))
}
func TestKernelParamSpecSuite(t *testing.T) {
suite.Run(t, new(KernelParamSpecSuite))
}

View File

@ -103,6 +103,7 @@ func (r *Runtime) CanApplyImmediate(b []byte) error {
// * .machine.time
// * .machine.network
// * .machine.certCANs
// * .machine.sysctls
newConfig.ClusterConfig = currentConfig.ClusterConfig
newConfig.ConfigDebug = currentConfig.ConfigDebug
@ -110,6 +111,7 @@ func (r *Runtime) CanApplyImmediate(b []byte) error {
newConfig.MachineConfig.MachineTime = currentConfig.MachineConfig.MachineTime
newConfig.MachineConfig.MachineCertSANs = currentConfig.MachineConfig.MachineCertSANs
newConfig.MachineConfig.MachineNetwork = currentConfig.MachineConfig.MachineNetwork
newConfig.MachineConfig.MachineSysctls = currentConfig.MachineConfig.MachineSysctls
}
if !reflect.DeepEqual(currentConfig, newConfig) {

View File

@ -86,7 +86,6 @@ func (*Sequencer) Initialize(r runtime.Runtime) []runtime.Phase {
SetupLogger,
).Append(
"systemRequirements",
WriteRequiredSysctlsForContainer,
SetupSystemDirectory,
).Append(
"etc",
@ -103,7 +102,6 @@ func (*Sequencer) Initialize(r runtime.Runtime) []runtime.Phase {
).Append(
"systemRequirements",
EnforceKSPPRequirements,
WriteRequiredSysctls,
SetupSystemDirectory,
MountBPFFS,
MountCgroups,
@ -235,7 +233,6 @@ func (*Sequencer) Boot(r runtime.Runtime) []runtime.Phase {
).Append(
"userSetup",
WriteUserFiles,
WriteUserSysctls,
).AppendWhen(
r.State().Platform().Mode() != runtime.ModeContainer,
"lvm",

View File

@ -52,11 +52,12 @@ import (
"github.com/talos-systems/talos/internal/pkg/containers/cri/containerd"
"github.com/talos-systems/talos/internal/pkg/cri"
"github.com/talos-systems/talos/internal/pkg/etcd"
"github.com/talos-systems/talos/internal/pkg/kernel/kspp"
"github.com/talos-systems/talos/internal/pkg/mount"
"github.com/talos-systems/talos/internal/pkg/partition"
"github.com/talos-systems/talos/pkg/conditions"
"github.com/talos-systems/talos/pkg/images"
"github.com/talos-systems/talos/pkg/kernel"
"github.com/talos-systems/talos/pkg/kernel/kspp"
"github.com/talos-systems/talos/pkg/kubernetes"
machineapi "github.com/talos-systems/talos/pkg/machinery/api/machine"
"github.com/talos-systems/talos/pkg/machinery/config"
@ -64,7 +65,7 @@ import (
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
"github.com/talos-systems/talos/pkg/machinery/constants"
"github.com/talos-systems/talos/pkg/sysctl"
resourceruntime "github.com/talos-systems/talos/pkg/resources/runtime"
"github.com/talos-systems/talos/pkg/version"
)
@ -86,7 +87,7 @@ func SetupLogger(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionF
// disable ratelimiting for kmsg, otherwise logs might be not visible.
// this should be set via kernel arg, but in case it's not set, try to force it.
if err = sysctl.WriteSystemProperty(&sysctl.SystemProperty{
if err = kernel.WriteParam(&kernel.Param{
Key: "kernel.printk_devkmsg",
Value: "on\n",
}); err != nil {
@ -112,7 +113,7 @@ func EnforceKSPPRequirements(seq runtime.Sequence, data interface{}) (runtime.Ta
return err
}
return kspp.EnforceKSPPSysctls()
return resourceruntime.NewKernelParamsSetCondition(r.State().V1Alpha2().Resources(), kspp.GetKernelParams()...).Wait(ctx)
}, "enforceKSPPRequirements"
}
@ -222,60 +223,6 @@ func MountPseudoFilesystems(seq runtime.Sequence, data interface{}) (runtime.Tas
}, "mountPseudoFilesystems"
}
// WriteRequiredSysctlsForContainer represents the WriteRequiredSysctlsForContainer task.
func WriteRequiredSysctlsForContainer(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) {
return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) {
var multiErr *multierror.Error
if err := sysctl.WriteSystemProperty(&sysctl.SystemProperty{Key: "net.ipv4.ip_forward", Value: "1"}); err != nil {
multiErr = multierror.Append(multiErr, fmt.Errorf("failed to set net.ipv4.ip_forward: %w", err))
}
if err := sysctl.WriteSystemProperty(&sysctl.SystemProperty{Key: "net.ipv6.conf.default.forwarding", Value: "1"}); err != nil {
if !errors.Is(err, os.ErrNotExist) { // ignore error if ipv6 is disabled
multiErr = multierror.Append(multiErr, fmt.Errorf("failed to set net.ipv6.conf.default.forwarding: %w", err))
}
}
if err := sysctl.WriteSystemProperty(&sysctl.SystemProperty{Key: "kernel.pid_max", Value: "262144"}); err != nil {
multiErr = multierror.Append(multiErr, fmt.Errorf("failed to set kernel.pid_max: %w", err))
}
return multiErr.ErrorOrNil()
}, "writeRequiredSysctlsForContainer"
}
// WriteRequiredSysctls represents the WriteRequiredSysctls task.
func WriteRequiredSysctls(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) {
return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) {
var multiErr *multierror.Error
if err := sysctl.WriteSystemProperty(&sysctl.SystemProperty{Key: "net.ipv4.ip_forward", Value: "1"}); err != nil {
multiErr = multierror.Append(multiErr, fmt.Errorf("failed to set net.ipv4.ip_forward: %w", err))
}
if err := sysctl.WriteSystemProperty(&sysctl.SystemProperty{Key: "net.bridge.bridge-nf-call-iptables", Value: "1"}); err != nil {
multiErr = multierror.Append(multiErr, fmt.Errorf("failed to set net.bridge.bridge-nf-call-iptables: %w", err))
}
if err := sysctl.WriteSystemProperty(&sysctl.SystemProperty{Key: "net.bridge.bridge-nf-call-ip6tables", Value: "1"}); err != nil {
multiErr = multierror.Append(multiErr, fmt.Errorf("failed to set net.bridge.bridge-nf-call-ip6tables: %w", err))
}
if err := sysctl.WriteSystemProperty(&sysctl.SystemProperty{Key: "net.ipv6.conf.default.forwarding", Value: "1"}); err != nil {
if !errors.Is(err, os.ErrNotExist) { // ignore error if ipv6 is disabled
multiErr = multierror.Append(multiErr, fmt.Errorf("failed to set net.ipv6.conf.default.forwarding: %w", err))
}
}
if err := sysctl.WriteSystemProperty(&sysctl.SystemProperty{Key: "kernel.pid_max", Value: "262144"}); err != nil {
multiErr = multierror.Append(multiErr, fmt.Errorf("failed to set kernel.pid_max: %w", err))
}
return multiErr.ErrorOrNil()
}, "writeRequiredSysctls"
}
// SetRLimit represents the SetRLimit task.
func SetRLimit(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) {
return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) {
@ -1021,21 +968,6 @@ func existsAndIsFile(p string) (err error) {
return nil
}
// WriteUserSysctls represents the WriteUserSysctls task.
func WriteUserSysctls(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) {
return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) {
var result *multierror.Error
for k, v := range r.Config().Machine().Sysctls() {
if err = sysctl.WriteSystemProperty(&sysctl.SystemProperty{Key: k, Value: v}); err != nil {
return err
}
}
return result.ErrorOrNil()
}, "writeUserSysctls"
}
// UnmountOverlayFilesystems represents the UnmountOverlayFilesystems task.
func UnmountOverlayFilesystems(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) {
return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) {

View File

@ -20,6 +20,7 @@ import (
"github.com/talos-systems/talos/internal/app/machined/pkg/controllers/k8s"
"github.com/talos-systems/talos/internal/app/machined/pkg/controllers/network"
"github.com/talos-systems/talos/internal/app/machined/pkg/controllers/perf"
runtimecontrollers "github.com/talos-systems/talos/internal/app/machined/pkg/controllers/runtime"
"github.com/talos-systems/talos/internal/app/machined/pkg/controllers/secrets"
"github.com/talos-systems/talos/internal/app/machined/pkg/controllers/time"
"github.com/talos-systems/talos/internal/app/machined/pkg/controllers/v1alpha1"
@ -135,8 +136,13 @@ func (ctrl *Controller) Run(ctx context.Context) error {
Cmdline: procfs.ProcCmdline(),
},
&network.TimeServerMergeController{},
&perf.StatsController{},
&network.TimeServerSpecController{},
&perf.StatsController{},
&runtimecontrollers.KernelParamConfigController{},
&runtimecontrollers.KernelParamDefaultsController{
V1Alpha1Mode: ctrl.v1alpha1Runtime.State().Platform().Mode(),
},
&runtimecontrollers.KernelParamSpecController{},
&secrets.APIController{},
&secrets.EtcdController{},
&secrets.KubernetesController{},

View File

@ -19,6 +19,7 @@ import (
"github.com/talos-systems/talos/pkg/resources/k8s"
"github.com/talos-systems/talos/pkg/resources/network"
"github.com/talos-systems/talos/pkg/resources/perf"
"github.com/talos-systems/talos/pkg/resources/runtime"
"github.com/talos-systems/talos/pkg/resources/secrets"
"github.com/talos-systems/talos/pkg/resources/time"
"github.com/talos-systems/talos/pkg/resources/v1alpha1"
@ -102,6 +103,8 @@ func NewState() (*State, error) {
&network.TimeServerSpec{},
&perf.CPU{},
&perf.Memory{},
&runtime.KernelParamSpec{},
&runtime.KernelParamStatus{},
&secrets.API{},
&secrets.Etcd{},
&secrets.Kubernetes{},

View File

@ -2,31 +2,37 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package sysctl
package kernel
import (
"io/ioutil"
"os"
"path"
"strings"
)
// SystemProperty represents a kernel system property.
type SystemProperty struct {
// Param represents a kernel system property.
type Param struct {
Key string
Value string
}
// WriteSystemProperty writes a value to a key under /proc/sys.
func WriteSystemProperty(prop *SystemProperty) error {
// WriteParam writes a value to a key under /proc/sys.
func WriteParam(prop *Param) error {
return ioutil.WriteFile(prop.Path(), []byte(prop.Value), 0o644)
}
// ReadSystemProperty reads a value from a key under /proc/sys.
func ReadSystemProperty(prop *SystemProperty) ([]byte, error) {
// ReadParam reads a value from a key under /proc/sys.
func ReadParam(prop *Param) ([]byte, error) {
return ioutil.ReadFile(prop.Path())
}
// DeleteParam deletes a value from a key under /proc/sys.
func DeleteParam(prop *Param) error {
return os.Remove(prop.Path())
}
// Path returns the path to the systctl file under /proc/sys.
func (prop *SystemProperty) Path() string {
func (prop *Param) Path() string {
return path.Join("/proc/sys", strings.ReplaceAll(prop.Key, ".", "/"))
}

View File

@ -10,7 +10,7 @@ import (
"github.com/hashicorp/go-multierror"
"github.com/talos-systems/go-procfs/procfs"
"github.com/talos-systems/talos/pkg/sysctl"
"github.com/talos-systems/talos/pkg/kernel"
)
// RequiredKSPPKernelParameters is the set of kernel parameters required to
@ -46,10 +46,9 @@ func EnforceKSPPKernelParameters() error {
return result.ErrorOrNil()
}
// EnforceKSPPSysctls verifies that all required KSPP kernel sysctls are set
// with the right value.
func EnforceKSPPSysctls() (err error) {
props := []*sysctl.SystemProperty{
// GetKernelParams returns the list of KSPP kernels.
func GetKernelParams() []*kernel.Param {
return []*kernel.Param{
{
Key: "kernel.kptr_restrict",
Value: "1",
@ -62,7 +61,7 @@ func EnforceKSPPSysctls() (err error) {
Key: "kernel.perf_event_paranoid",
Value: "3",
},
// We can skip this sysctl because CONFIG_KEXEC is not set.
// We can skip this kernel because CONFIG_KEXEC is not set.
// {
// Key: "kernel.kexec_load_disabled",
// Value: "1",
@ -84,12 +83,4 @@ func EnforceKSPPSysctls() (err error) {
Value: "2",
},
}
for _, prop := range props {
if err = sysctl.WriteSystemProperty(prop); err != nil {
return
}
}
return nil
}

View File

@ -0,0 +1,59 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package runtime
import (
"context"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/state"
"github.com/talos-systems/talos/pkg/kernel"
)
// KernelParamsSetCondition implements condition which waits for the kernels to be in sync.
type KernelParamsSetCondition struct {
state state.State
props []*kernel.Param
}
// NewKernelParamsSetCondition builds a coondition which waits for the kernel to be in sync.
func NewKernelParamsSetCondition(state state.State, props ...*kernel.Param) *KernelParamsSetCondition {
return &KernelParamsSetCondition{
state: state,
props: props,
}
}
func (condition *KernelParamsSetCondition) String() string {
return "kernelParams"
}
// Wait implements condition interface.
func (condition *KernelParamsSetCondition) Wait(ctx context.Context) error {
for _, prop := range condition.props {
prop := prop
if _, err := condition.state.WatchFor(
ctx,
resource.NewMetadata(NamespaceName, KernelParamStatusType, prop.Key, resource.VersionUndefined),
state.WithCondition(func(r resource.Resource) (bool, error) {
if resource.IsTombstone(r) {
return false, nil
}
status := r.(*KernelParamStatus).TypedSpec()
if status.Current != prop.Value {
return false, nil
}
return true, nil
}),
); err != nil {
return err
}
}
return nil
}

View File

@ -0,0 +1,111 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package runtime_test
import (
"context"
"errors"
"testing"
"time"
"github.com/cosi-project/runtime/pkg/state"
"github.com/cosi-project/runtime/pkg/state/impl/inmem"
"github.com/cosi-project/runtime/pkg/state/impl/namespaced"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/talos-systems/talos/pkg/kernel"
"github.com/talos-systems/talos/pkg/kernel/kspp"
"github.com/talos-systems/talos/pkg/resources/runtime"
)
func TestCondition(t *testing.T) {
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second)
t.Cleanup(ctxCancel)
t.Parallel()
for _, tt := range []struct {
Name string
ActualKernelParams []*kernel.Param
AwaitKernelParams []*kernel.Param
Succeeds bool
}{
{
Name: "okay",
ActualKernelParams: []*kernel.Param{
{
Key: "kernel.kptr_restrict",
Value: "1",
},
},
AwaitKernelParams: []*kernel.Param{
{
Key: "kernel.kptr_restrict",
Value: "1",
},
},
Succeeds: true,
},
{
Name: "timeout",
ActualKernelParams: []*kernel.Param{},
AwaitKernelParams: []*kernel.Param{
{
Key: "kernel.kptr_restrict",
Value: "1",
},
},
Succeeds: false,
},
{
Name: "value differs",
ActualKernelParams: []*kernel.Param{
{
Key: "kernel.kptr_restrict",
Value: "0",
},
},
AwaitKernelParams: []*kernel.Param{
{
Key: "kernel.kptr_restrict",
Value: "1",
},
},
Succeeds: false,
},
{
Name: "multiple values",
ActualKernelParams: kspp.GetKernelParams(),
AwaitKernelParams: kspp.GetKernelParams(),
Succeeds: true,
},
} {
tt := tt
t.Run(tt.Name, func(t *testing.T) {
t.Parallel()
state := state.WrapCore(namespaced.NewState(inmem.Build))
for _, prop := range tt.ActualKernelParams {
status := runtime.NewKernelParamStatus(runtime.NamespaceName, prop.Key)
*status.TypedSpec() = runtime.KernelParamStatusSpec{
Current: prop.Value,
}
require.NoError(t, state.Create(ctx, status))
}
err := runtime.NewKernelParamsSetCondition(state, tt.AwaitKernelParams...).Wait(ctx)
if tt.Succeeds {
assert.NoError(t, err)
} else {
assert.True(t, errors.Is(err, context.DeadlineExceeded), "error is %v", err)
}
})
}
}

View File

@ -0,0 +1,81 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package runtime
import (
"fmt"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/resource/meta"
"github.com/talos-systems/talos/pkg/resources/v1alpha1"
)
// NamespaceName contains configuration resources.
const NamespaceName resource.Namespace = v1alpha1.NamespaceName
// KernelParamSpecType is type of KernelParam resource.
const KernelParamSpecType = resource.Type("KernelParamSpecs.runtime.talos.dev")
// KernelParamSpec resource holds sysctl flags to define.
type KernelParamSpec struct {
md resource.Metadata
spec KernelParamSpecSpec
}
// KernelParamSpecSpec describes status of the defined sysctls.
type KernelParamSpecSpec struct {
Value string `yaml:"value"`
IgnoreErrors bool `yaml:"ignoreErrors"`
}
// NewKernelParamSpec initializes a KernelParamSpec resource.
func NewKernelParamSpec(namespace resource.Namespace, id resource.ID) *KernelParamSpec {
r := &KernelParamSpec{
md: resource.NewMetadata(namespace, KernelParamSpecType, id, resource.VersionUndefined),
spec: KernelParamSpecSpec{},
}
r.md.BumpVersion()
return r
}
// Metadata implements resource.Resource.
func (r *KernelParamSpec) Metadata() *resource.Metadata {
return &r.md
}
// Spec implements resource.Resource.
func (r *KernelParamSpec) Spec() interface{} {
return r.spec
}
func (r *KernelParamSpec) String() string {
return fmt.Sprintf("runtime.KernelParamSpec.(%q)", r.md.ID())
}
// DeepCopy implements resource.Resource.
func (r *KernelParamSpec) DeepCopy() resource.Resource {
return &KernelParamSpec{
md: r.md,
spec: r.spec,
}
}
// ResourceDefinition implements meta.ResourceDefinitionProvider interface.
func (r *KernelParamSpec) ResourceDefinition() meta.ResourceDefinitionSpec {
return meta.ResourceDefinitionSpec{
Type: KernelParamSpecType,
Aliases: []resource.Type{},
DefaultNamespace: NamespaceName,
PrintColumns: []meta.PrintColumn{},
}
}
// TypedSpec allows to access the KernelParamSpecSpec with the proper type.
func (r *KernelParamSpec) TypedSpec() *KernelParamSpecSpec {
return &r.spec
}

View File

@ -0,0 +1,90 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package runtime
import (
"fmt"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/resource/meta"
)
// KernelParamStatusType is type of KernelParam resource.
const KernelParamStatusType = resource.Type("KernelParamStatuses.runtime.talos.dev")
// KernelParamStatus resource holds defined sysctl flags status.
type KernelParamStatus struct {
md resource.Metadata
spec KernelParamStatusSpec
}
// KernelParamStatusSpec describes status of the defined sysctls.
type KernelParamStatusSpec struct {
Current string `yaml:"current"`
Default string `yaml:"default"`
Unsupported bool `yaml:"unsupported"`
}
// NewKernelParamStatus initializes a KernelParamStatus resource.
func NewKernelParamStatus(namespace resource.Namespace, id resource.ID) *KernelParamStatus {
r := &KernelParamStatus{
md: resource.NewMetadata(namespace, KernelParamStatusType, id, resource.VersionUndefined),
spec: KernelParamStatusSpec{},
}
r.md.BumpVersion()
return r
}
// Metadata implements resource.Resource.
func (r *KernelParamStatus) Metadata() *resource.Metadata {
return &r.md
}
// Spec implements resource.Resource.
func (r *KernelParamStatus) Spec() interface{} {
return r.spec
}
func (r *KernelParamStatus) String() string {
return fmt.Sprintf("runtime.KernelParamStatus.(%q)", r.md.ID())
}
// DeepCopy implements resource.Resource.
func (r *KernelParamStatus) DeepCopy() resource.Resource {
return &KernelParamStatus{
md: r.md,
spec: r.spec,
}
}
// ResourceDefinition implements meta.ResourceDefinitionProvider interface.
func (r *KernelParamStatus) ResourceDefinition() meta.ResourceDefinitionSpec {
return meta.ResourceDefinitionSpec{
Type: KernelParamStatusType,
Aliases: []resource.Type{"Sysctls", "KernelParameters", "KernelParams"},
DefaultNamespace: NamespaceName,
PrintColumns: []meta.PrintColumn{
{
Name: "Current",
JSONPath: `{.current}`,
},
{
Name: "Default",
JSONPath: `{.default}`,
},
{
Name: "Unsupported",
JSONPath: `{.unsupported}`,
},
},
}
}
// TypedSpec allows to access the KernelParamStatusSpec with the proper type.
func (r *KernelParamStatus) TypedSpec() *KernelParamStatusSpec {
return &r.spec
}

View File

@ -0,0 +1,33 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package runtime_test
import (
"context"
"testing"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/state"
"github.com/cosi-project/runtime/pkg/state/impl/inmem"
"github.com/cosi-project/runtime/pkg/state/impl/namespaced"
"github.com/cosi-project/runtime/pkg/state/registry"
"github.com/stretchr/testify/assert"
"github.com/talos-systems/talos/pkg/resources/runtime"
)
func TestRegisterResource(t *testing.T) {
ctx := context.TODO()
resources := state.WrapCore(namespaced.NewState(inmem.Build))
resourceRegistry := registry.NewResourceRegistry(resources)
for _, resource := range []resource.Resource{
&runtime.KernelParamSpec{},
&runtime.KernelParamStatus{},
} {
assert.NoError(t, resourceRegistry.Register(ctx, resource))
}
}