alt-orchestra/talos
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

feat: enable xfs project quota support, kubelet feature

Browse Source 
This is controlled with a feature flag which gets enabled automatically
for Talos 1.5+.

Fixes #7181

If enabled, configures kubelet to use project quotas to track xfs volume
usage, which is much more efficient than doing `du` periodically.

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
This commit is contained in:
Andrey Smirnov 2023-05-18 16:39:28 +04:00
parent eba8185642
commit 10155c390e
No known key found for this signature in database
GPG Key ID: 7B26396447AB6DFD
25 changed files with 282 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/resource/definitions/k8s/k8s.proto
View File

@ -116,6 +116,7 @@ message KubeletConfigSpec {
bool skip_node_registration = 9;
string static_pod_list_url = 10;
bool disable_manifests_directory = 11;
bool enable_fs_quota_monitoring = 12;
}
// KubeletSpecSpec holds the source of kubelet configuration.

14
hack/release.toml
View File

@ -25,6 +25,20 @@ preface = """\
* Kubernetes: 1.27.2
Talos is built with Go 1.20.4.
"""
[notes.quota]
title = "XFS Quota"
description="""\
Talos 1.5+ enables XFS project quota support by default, also enabling by default
kubelet feature gate `LocalStorageCapacityIsolationFSQuotaMonitoring` to use xfs quotas
to monitor volume usage instead of `du`.
This feature is controlled by the `.machine.features.diskQuotaSupport` field in the machine config,
it is set to true for new clusters.
When upgrading from a previous version, the feature can be enabled by setting the field to true.
On the first mount of a volume, the quota information will be recalculated, which may take some time.
"""
[make_deps]

1
internal/app/machined/pkg/controllers/k8s/kubelet_config.go
View File

@ -144,6 +144,7 @@ func modifyKubeletConfig(cfgProvider talosconfig.Provider, staticPodListURL stri
kubeletConfig.SkipNodeRegistration = cfgProvider.Machine().Kubelet().SkipNodeRegistration()
kubeletConfig.StaticPodListURL = staticPodListURL
kubeletConfig.DisableManifestsDirectory = cfgProvider.Machine().Kubelet().DisableManifestsDirectory()
kubeletConfig.EnableFSQuotaMonitoring = cfgProvider.Machine().Features().DiskQuotaSupportEnabled()
return nil
}

10
internal/app/machined/pkg/controllers/k8s/kubelet_spec.go
View File

@ -285,6 +285,16 @@ func NewKubeletConfiguration(cfgSpec *k8s.KubeletConfigSpec) (*kubeletconfig.Kub
}
}
if cfgSpec.EnableFSQuotaMonitoring {
if _, overridden := config.FeatureGates["LocalStorageCapacityIsolationFSQuotaMonitoring"]; !overridden {
if config.FeatureGates == nil {
config.FeatureGates = map[string]bool{}
}
config.FeatureGates["LocalStorageCapacityIsolationFSQuotaMonitoring"] = true
}
}
if cfgSpec.SkipNodeRegistration {
config.Authentication.Webhook.Enabled = pointer.To(false)
config.Authorization.Mode = kubeletconfig.KubeletAuthorizationModeAlwaysAllow

13
internal/app/machined/pkg/controllers/k8s/kubelet_spec_test.go
View File

@ -556,6 +556,19 @@ func TestNewKubeletConfigurationMerge(t *testing.T) {
kc.StaticPodPath = ""
},
},
{
name: "enable local FS quota monitoring",
cfgSpec: &k8s.KubeletConfigSpec{
ClusterDNS: []string{"10.0.0.5"},
ClusterDomain: "cluster.local",
EnableFSQuotaMonitoring: true,
},
expectedOverrides: func(kc *kubeletconfig.KubeletConfiguration) {
kc.FeatureGates = map[string]bool{
"LocalStorageCapacityIsolationFSQuotaMonitoring": true,
}
},
},
} {
tt := tt

10
internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go
View File

@ -1114,7 +1114,11 @@ func mountDisks(r runtime.Runtime) (err error) {
}
}
mountpoints.Set(partname, mount.NewMountPoint(partname, part.MountPoint(), "xfs", unix.MS_NOATIME, ""))
mountpoints.Set(partname,
mount.NewMountPoint(partname, part.MountPoint(), "xfs", unix.MS_NOATIME, "",
mount.WithProjectQuota(r.Config().Machine().Features().DiskQuotaSupportEnabled()),
),
)
}
}
@ -2147,7 +2151,9 @@ func UnmountStatePartition(runtime.Sequence, any) (runtime.TaskExecutionFunc, st
// MountEphemeralPartition mounts the ephemeral partition.
func MountEphemeralPartition(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) {
return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error {
return mount.SystemPartitionMount(r, logger, constants.EphemeralPartitionLabel, mount.WithFlags(mount.Resize))
return mount.SystemPartitionMount(r, logger, constants.EphemeralPartitionLabel,
mount.WithFlags(mount.Resize),
mount.WithProjectQuota(r.Config().Machine().Features().DiskQuotaSupportEnabled()))
}, "mountEphemeralPartition"
}

8
internal/pkg/mount/mount.go
View File

@ -205,6 +205,14 @@ func NewMountPoint(source, target, fstype string, flags uintptr, data string, se
p.target = filepath.Join(p.Prefix, p.target)
}
if p.Options.ProjectQuota {
if len(p.data) > 0 {
p.data += ","
}
p.data += "prjquota"
}
return p
}

8
internal/pkg/mount/options.go
View File

@ -46,6 +46,7 @@ type Options struct {
PostUnmountHooks []Hook
Encryption config.Encryption
Logger *log.Logger
ProjectQuota bool
}
// Option is the functional option func.
@ -103,6 +104,13 @@ func WithLogger(logger *log.Logger) Option {
}
}
// WithProjectQuota enables project quota mount option.
func WithProjectQuota(enable bool) Option {
return func(args *Options) {
args.ProjectQuota = enable
}
}
// Hook represents pre/post mount hook.
type Hook func(p *Point) error

11
internal/pkg/mount/system.go
View File

@ -21,7 +21,6 @@ import (
"github.com/siderolabs/talos/internal/app/machined/pkg/runtime/disk"
"github.com/siderolabs/talos/internal/pkg/encryption"
"github.com/siderolabs/talos/internal/pkg/partition"
"github.com/siderolabs/talos/pkg/machinery/config"
"github.com/siderolabs/talos/pkg/machinery/constants"
runtimeres "github.com/siderolabs/talos/pkg/machinery/resources/runtime"
"github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1"
@ -189,14 +188,12 @@ func SystemPartitionMount(r runtime.Runtime, logger *log.Logger, label string, o
return fmt.Errorf("failed to find device with partition labeled %s", label)
}
var encryptionConfig config.Encryption
if r.Config() != nil && r.Config().Machine() != nil {
encryptionConfig = r.Config().Machine().SystemDiskEncryption().Get(label)
}
encryptionConfig := r.Config().Machine().SystemDiskEncryption().Get(label)
if encryptionConfig != nil {
opts = append(opts, WithEncryptionConfig(encryptionConfig))
if encryptionConfig != nil {
opts = append(opts, WithEncryptionConfig(encryptionConfig))
}
}
opts = append(opts, WithLogger(logger))

220
pkg/machinery/api/resource/definitions/k8s/k8s.pb.go
View File

@ -890,6 +890,7 @@ type KubeletConfigSpec struct {
SkipNodeRegistration bool `protobuf:"varint,9,opt,name=skip_node_registration,json=skipNodeRegistration,proto3" json:"skip_node_registration,omitempty"`
StaticPodListUrl string `protobuf:"bytes,10,opt,name=static_pod_list_url,json=staticPodListUrl,proto3" json:"static_pod_list_url,omitempty"`
DisableManifestsDirectory bool `protobuf:"varint,11,opt,name=disable_manifests_directory,json=disableManifestsDirectory,proto3" json:"disable_manifests_directory,omitempty"`
EnableFsQuotaMonitoring bool `protobuf:"varint,12,opt,name=enable_fs_quota_monitoring,json=enableFsQuotaMonitoring,proto3" json:"enable_fs_quota_monitoring,omitempty"`
}
func (x *KubeletConfigSpec) Reset() {
@ -1001,6 +1002,13 @@ func (x *KubeletConfigSpec) GetDisableManifestsDirectory() bool {
return false
}
func (x *KubeletConfigSpec) GetEnableFsQuotaMonitoring() bool {
if x != nil {
return x.EnableFsQuotaMonitoring
}
return false
}
// KubeletSpecSpec holds the source of kubelet configuration.
type KubeletSpecSpec struct {
state protoimpl.MessageState
@ -1923,7 +1931,7 @@ var file_resource_definitions_k8s_k8s_proto_rawDesc = []byte{
0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28,
0x09, 0x52, 0x09, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1b, 0x0a, 0x09,
0x72, 0x65, 0x61, 0x64, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52,
0x08, 0x72, 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x22, 0xbc, 0x05, 0x0a, 0x11, 0x4b, 0x75,
0x08, 0x72, 0x65, 0x61, 0x64, 0x4f, 0x6e, 0x6c, 0x79, 0x22, 0xf9, 0x05, 0x0a, 0x11, 0x4b, 0x75,
0x62, 0x65, 0x6c, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x12,
0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72,
@ -1963,112 +1971,116 @@ var file_resource_definitions_k8s_k8s_proto_rawDesc = []byte{
0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x73, 0x5f,
0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52,
0x19, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74,
0x73, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x1a, 0x3c, 0x0a, 0x0e, 0x45, 0x78,
0x74, 0x72, 0x61, 0x41, 0x72, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03,
0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14,
0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76,
0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe5, 0x01, 0x0a, 0x0f, 0x4b, 0x75, 0x62,
0x65, 0x6c, 0x65, 0x74, 0x53, 0x70, 0x65, 0x63, 0x53, 0x70, 0x65, 0x63, 0x12, 0x14, 0x0a, 0x05,
0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61,
0x67, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x72, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09,
0x52, 0x04, 0x61, 0x72, 0x67, 0x73, 0x12, 0x4a, 0x0a, 0x0c, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f,
0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x74,
0x73, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x3b, 0x0a, 0x1a, 0x65, 0x6e,
0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x73, 0x5f, 0x71, 0x75, 0x6f, 0x74, 0x61, 0x5f, 0x6d, 0x6f,
0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17,
0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x73, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x4d, 0x6f, 0x6e,
0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x1a, 0x3c, 0x0a, 0x0e, 0x45, 0x78, 0x74, 0x72, 0x61,
0x41, 0x72, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76,
0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75,
0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe5, 0x01, 0x0a, 0x0f, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65,
0x74, 0x53, 0x70, 0x65, 0x63, 0x53, 0x70, 0x65, 0x63, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61,
0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12,
0x12, 0x0a, 0x04, 0x61, 0x72, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x61,
0x72, 0x67, 0x73, 0x12, 0x4a, 0x0a, 0x0c, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x6d, 0x6f, 0x75,
0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x74, 0x61, 0x6c, 0x6f,
0x73, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e,
0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4d, 0x6f, 0x75,
0x6e, 0x74, 0x52, 0x0b, 0x65, 0x78, 0x74, 0x72, 0x61, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12,
0x2b, 0x0a, 0x11, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x6e, 0x6f, 0x64, 0x65,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x65, 0x78, 0x70, 0x65,
0x63, 0x74, 0x65, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2f, 0x0a, 0x06,
0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67,
0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53,
0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x54, 0x0a,
0x0c, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x44, 0x0a,
0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x74,
0x61, 0x6c, 0x6f, 0x73, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x64, 0x65,
0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e,
0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x0b, 0x65, 0x78, 0x74, 0x72, 0x61, 0x4d, 0x6f, 0x75, 0x6e,
0x74, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x6e,
0x6f, 0x64, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x65,
0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12,
0x2f, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67,
0x22, 0x54, 0x0a, 0x0c, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x53, 0x70, 0x65, 0x63,
0x12, 0x44, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32,
0x2e, 0x2e, 0x74, 0x61, 0x6c, 0x6f, 0x73, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x6b, 0x38, 0x73,
0x2e, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52,
0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x41, 0x0a, 0x12, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65,
0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x2b, 0x0a, 0x11,
0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x73, 0x5f, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x65,
0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73,
0x74, 0x73, 0x41, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x64, 0x22, 0x60, 0x0a, 0x10, 0x4e, 0x6f, 0x64,
0x65, 0x49, 0x50, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x12, 0x23, 0x0a,
0x0d, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x01,
0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x53, 0x75, 0x62, 0x6e, 0x65,
0x74, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x73, 0x75,
0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x63,
0x6c, 0x75, 0x64, 0x65, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x22, 0x39, 0x0a, 0x0a, 0x4e,
0x6f, 0x64, 0x65, 0x49, 0x50, 0x53, 0x70, 0x65, 0x63, 0x12, 0x2b, 0x0a, 0x09, 0x61, 0x64, 0x64,
0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x63,
0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x65, 0x74, 0x49, 0x50, 0x52, 0x09, 0x61, 0x64, 0x64,
0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0x3b, 0x0a, 0x11, 0x4e, 0x6f, 0x64, 0x65, 0x4c, 0x61,
0x62, 0x65, 0x6c, 0x53, 0x70, 0x65, 0x63, 0x53, 0x70, 0x65, 0x63, 0x12, 0x10, 0x0a, 0x03, 0x6b,
0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x53, 0x69,
0x6e, 0x67, 0x6c, 0x65, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x05, 0x69, 0x74,
0x65, 0x6d, 0x73, 0x22, 0x41, 0x0a, 0x12, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x53,
0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x2b, 0x0a, 0x11, 0x6d, 0x61, 0x6e,
0x69, 0x66, 0x65, 0x73, 0x74, 0x73, 0x5f, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x65, 0x64, 0x18, 0x01,
0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x73, 0x41,
0x70, 0x70, 0x6c, 0x69, 0x65, 0x64, 0x22, 0x60, 0x0a, 0x10, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x50,
0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x12, 0x23, 0x0a, 0x0d, 0x76, 0x61,
0x6c, 0x69, 0x64, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,
0x09, 0x52, 0x0c, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x12,
0x27, 0x0a, 0x0f, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x73, 0x75, 0x62, 0x6e, 0x65,
0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64,
0x65, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x73, 0x22, 0x39, 0x0a, 0x0a, 0x4e, 0x6f, 0x64, 0x65,
0x49, 0x50, 0x53, 0x70, 0x65, 0x63, 0x12, 0x2b, 0x0a, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73,
0x73, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x63, 0x6f, 0x6d, 0x6d,
0x6f, 0x6e, 0x2e, 0x4e, 0x65, 0x74, 0x49, 0x50, 0x52, 0x09, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73,
0x73, 0x65, 0x73, 0x22, 0x3b, 0x0a, 0x11, 0x4e, 0x6f, 0x64, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c,
0x53, 0x70, 0x65, 0x63, 0x53, 0x70, 0x65, 0x63, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
0x22, 0x55, 0x0a, 0x0c, 0x4e, 0x6f, 0x64, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x53, 0x70, 0x65, 0x63,
0x12, 0x1a, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x29, 0x0a, 0x10,
0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65,
0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x86, 0x04, 0x0a, 0x13, 0x53, 0x63, 0x68, 0x65,
0x64, 0x75, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x12,
0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08,
0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61,
0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12,
0x61, 0x0a, 0x0a, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x18, 0x03, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x74, 0x61, 0x6c, 0x6f, 0x73, 0x2e, 0x72, 0x65, 0x73, 0x6f,
0x75, 0x72, 0x63, 0x65, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73,
0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x43, 0x6f,
0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x45, 0x78, 0x74, 0x72, 0x61, 0x41, 0x72,
0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x65, 0x78, 0x74, 0x72, 0x61, 0x41, 0x72,
0x67, 0x73, 0x12, 0x50, 0x0a, 0x0d, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x76, 0x6f, 0x6c, 0x75,
0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x74, 0x61, 0x6c, 0x6f,
0x73, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e,
0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x45, 0x78, 0x74, 0x72, 0x61,
0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x0c, 0x65, 0x78, 0x74, 0x72, 0x61, 0x56, 0x6f, 0x6c,
0x75, 0x6d, 0x65, 0x73, 0x12, 0x82, 0x01, 0x0a, 0x15, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e,
0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x05,
0x20, 0x03, 0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x74, 0x61, 0x6c, 0x6f, 0x73, 0x2e, 0x72, 0x65, 0x73,
0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e,
0x73, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x43,
0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f,
0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x45, 0x6e,
0x74, 0x72, 0x79, 0x52, 0x14, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74,
0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x1a, 0x3c, 0x0a, 0x0e, 0x45, 0x78, 0x74,
0x72, 0x61, 0x41, 0x72, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a,
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61,
0x6c, 0x75, 0x65, 0x22, 0x55, 0x0a, 0x0c, 0x4e, 0x6f, 0x64, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x53,
0x70, 0x65, 0x63, 0x12, 0x1a, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12,
0x29, 0x0a, 0x10, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73,
0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x68, 0x6f, 0x73, 0x74, 0x6e,
0x61, 0x6d, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x86, 0x04, 0x0a, 0x13, 0x53,
0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70,
0x65, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20,
0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05,
0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61,
0x67, 0x65, 0x12, 0x61, 0x0a, 0x0a, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x61, 0x72, 0x67, 0x73,
0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x74, 0x61, 0x6c, 0x6f, 0x73, 0x2e, 0x72,
0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69,
0x6f, 0x6e, 0x73, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65,
0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x45, 0x78, 0x74, 0x72,
0x61, 0x41, 0x72, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x65, 0x78, 0x74, 0x72,
0x61, 0x41, 0x72, 0x67, 0x73, 0x12, 0x50, 0x0a, 0x0d, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x76,
0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x74,
0x61, 0x6c, 0x6f, 0x73, 0x2e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x64, 0x65,
0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x45, 0x78,
0x74, 0x72, 0x61, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x0c, 0x65, 0x78, 0x74, 0x72, 0x61,
0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x12, 0x82, 0x01, 0x0a, 0x15, 0x65, 0x6e, 0x76, 0x69,
0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65,
0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x74, 0x61, 0x6c, 0x6f, 0x73, 0x2e,
0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74,
0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c,
0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x45, 0x6e, 0x76,
0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65,
0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x14, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d,
0x65, 0x6e, 0x74, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x1a, 0x3c, 0x0a, 0x0e,
0x45, 0x78, 0x74, 0x72, 0x61, 0x41, 0x72, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10,
0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79,
0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x47, 0x0a, 0x19, 0x45, 0x6e,
0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c,
0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c,
0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a,
0x02, 0x38, 0x01, 0x22, 0x43, 0x0a, 0x11, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x53, 0x74,
0x61, 0x74, 0x75, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64,
0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x12, 0x18,
0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x41, 0x0a, 0x0e, 0x53, 0x69, 0x6e, 0x67,
0x6c, 0x65, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x06, 0x6f, 0x62,
0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72,
0x75, 0x63, 0x74, 0x52, 0x06, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x22, 0x2d, 0x0a, 0x19, 0x53,
0x74, 0x61, 0x74, 0x69, 0x63, 0x50, 0x6f, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74,
0x61, 0x74, 0x75, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x3a, 0x0a, 0x0d, 0x53, 0x74,
0x61, 0x74, 0x69, 0x63, 0x50, 0x6f, 0x64, 0x53, 0x70, 0x65, 0x63, 0x12, 0x29, 0x0a, 0x03, 0x70,
0x6f, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63,
0x74, 0x52, 0x03, 0x70, 0x6f, 0x64, 0x22, 0x4d, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x74, 0x69, 0x63,
0x50, 0x6f, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x36, 0x0a,
0x0a, 0x70, 0x6f, 0x64, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x09, 0x70, 0x6f, 0x64, 0x53,
0x74, 0x61, 0x74, 0x75, 0x73, 0x42, 0x48, 0x5a, 0x46, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x69, 0x64, 0x65, 0x72, 0x6f, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x74,
0x61, 0x6c, 0x6f, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65,
0x72, 0x79, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2f,
0x64, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x6b, 0x38, 0x73, 0x62,
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x47, 0x0a, 0x19, 0x45, 0x6e, 0x76, 0x69, 0x72,
0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x45,
0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01,
0x22, 0x43, 0x0a, 0x11, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75,
0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01,
0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x76,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65,
0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x41, 0x0a, 0x0e, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x4d,
0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x06, 0x6f, 0x62, 0x6a, 0x65, 0x63,
0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74,
0x52, 0x06, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x22, 0x2d, 0x0a, 0x19, 0x53, 0x74, 0x61, 0x74,
0x69, 0x63, 0x50, 0x6f, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75,
0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x3a, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x69,
0x63, 0x50, 0x6f, 0x64, 0x53, 0x70, 0x65, 0x63, 0x12, 0x29, 0x0a, 0x03, 0x70, 0x6f, 0x64, 0x18,
0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x03,
0x70, 0x6f, 0x64, 0x22, 0x4d, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x74, 0x69, 0x63, 0x50, 0x6f, 0x64,
0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x6f,
0x64, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17,
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x09, 0x70, 0x6f, 0x64, 0x53, 0x74, 0x61, 0x74,
0x75, 0x73, 0x42, 0x48, 0x5a, 0x46, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
0x2f, 0x73, 0x69, 0x64, 0x65, 0x72, 0x6f, 0x6c, 0x61, 0x62, 0x73, 0x2f, 0x74, 0x61, 0x6c, 0x6f,
0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2f,
0x61, 0x70, 0x69, 0x2f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2f, 0x64, 0x65, 0x66,
0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2f, 0x6b, 0x38, 0x73, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
}
var (

33
pkg/machinery/api/resource/definitions/k8s/k8s_vtproto.pb.go
View File

@ -938,6 +938,16 @@ func (m *KubeletConfigSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) {
i -= len(m.unknownFields)
copy(dAtA[i:], m.unknownFields)
}
if m.EnableFsQuotaMonitoring {
i--
if m.EnableFsQuotaMonitoring {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i--
dAtA[i] = 0x60
}
if m.DisableManifestsDirectory {
i--
if m.DisableManifestsDirectory {
@ -2263,6 +2273,9 @@ func (m *KubeletConfigSpec) SizeVT() (n int) {
if m.DisableManifestsDirectory {
n += 2
}
if m.EnableFsQuotaMonitoring {
n += 2
}
n += len(m.unknownFields)
return n
}
@ -5600,6 +5613,26 @@ func (m *KubeletConfigSpec) UnmarshalVT(dAtA []byte) error {
}
}
m.DisableManifestsDirectory = bool(v != 0)
case 12:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field EnableFsQuotaMonitoring", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflow
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
m.EnableFsQuotaMonitoring = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skip(dAtA[iNdEx:])

5
pkg/machinery/config/contract.go
View File

@ -159,3 +159,8 @@ func (contract *VersionContract) KubeletManifestsDirectoryDisabled() bool {
func (contract *VersionContract) SecretboxEncryptionSupported() bool {
return contract.Greater(TalosVersion1_2)
}
// DiskQuotaSupportEnabled returns true if XFS filesystems should enable project quota.
func (contract *VersionContract) DiskQuotaSupportEnabled() bool {
return contract.Greater(TalosVersion1_4)
}

14
pkg/machinery/config/contract_test.go
View File

@ -65,6 +65,7 @@ func TestContractCurrent(t *testing.T) {
assert.True(t, contract.APIServerAuditPolicySupported())
assert.True(t, contract.KubeletManifestsDirectoryDisabled())
assert.True(t, contract.SecretboxEncryptionSupported())
assert.True(t, contract.DiskQuotaSupportEnabled())
}
func TestContract1_5(t *testing.T) {
@ -88,6 +89,7 @@ func TestContract1_5(t *testing.T) {
assert.True(t, contract.APIServerAuditPolicySupported())
assert.True(t, contract.KubeletManifestsDirectoryDisabled())
assert.True(t, contract.SecretboxEncryptionSupported())
assert.True(t, contract.DiskQuotaSupportEnabled())
}
func TestContract1_4(t *testing.T) {
@ -111,6 +113,7 @@ func TestContract1_4(t *testing.T) {
assert.True(t, contract.APIServerAuditPolicySupported())
assert.True(t, contract.KubeletManifestsDirectoryDisabled())
assert.True(t, contract.SecretboxEncryptionSupported())
assert.False(t, contract.DiskQuotaSupportEnabled())
}
func TestContract1_3(t *testing.T) {
@ -134,6 +137,7 @@ func TestContract1_3(t *testing.T) {
assert.True(t, contract.APIServerAuditPolicySupported())
assert.True(t, contract.KubeletManifestsDirectoryDisabled())
assert.True(t, contract.SecretboxEncryptionSupported())
assert.False(t, contract.DiskQuotaSupportEnabled())
}
func TestContract1_2(t *testing.T) {
@ -157,6 +161,7 @@ func TestContract1_2(t *testing.T) {
assert.False(t, contract.APIServerAuditPolicySupported())
assert.False(t, contract.KubeletManifestsDirectoryDisabled())
assert.False(t, contract.SecretboxEncryptionSupported())
assert.False(t, contract.DiskQuotaSupportEnabled())
}
func TestContract1_1(t *testing.T) {
@ -180,6 +185,7 @@ func TestContract1_1(t *testing.T) {
assert.False(t, contract.APIServerAuditPolicySupported())
assert.False(t, contract.KubeletManifestsDirectoryDisabled())
assert.False(t, contract.SecretboxEncryptionSupported())
assert.False(t, contract.DiskQuotaSupportEnabled())
}
func TestContract1_0(t *testing.T) {
@ -203,6 +209,7 @@ func TestContract1_0(t *testing.T) {
assert.False(t, contract.APIServerAuditPolicySupported())
assert.False(t, contract.KubeletManifestsDirectoryDisabled())
assert.False(t, contract.SecretboxEncryptionSupported())
assert.False(t, contract.DiskQuotaSupportEnabled())
}
func TestContract0_14(t *testing.T) {
@ -226,6 +233,7 @@ func TestContract0_14(t *testing.T) {
assert.False(t, contract.APIServerAuditPolicySupported())
assert.False(t, contract.KubeletManifestsDirectoryDisabled())
assert.False(t, contract.SecretboxEncryptionSupported())
assert.False(t, contract.DiskQuotaSupportEnabled())
}
func TestContract0_13(t *testing.T) {
@ -249,6 +257,7 @@ func TestContract0_13(t *testing.T) {
assert.False(t, contract.APIServerAuditPolicySupported())
assert.False(t, contract.KubeletManifestsDirectoryDisabled())
assert.False(t, contract.SecretboxEncryptionSupported())
assert.False(t, contract.DiskQuotaSupportEnabled())
}
func TestContract0_12(t *testing.T) {
@ -272,6 +281,7 @@ func TestContract0_12(t *testing.T) {
assert.False(t, contract.APIServerAuditPolicySupported())
assert.False(t, contract.KubeletManifestsDirectoryDisabled())
assert.False(t, contract.SecretboxEncryptionSupported())
assert.False(t, contract.DiskQuotaSupportEnabled())
}
func TestContract0_11(t *testing.T) {
@ -295,6 +305,7 @@ func TestContract0_11(t *testing.T) {
assert.False(t, contract.APIServerAuditPolicySupported())
assert.False(t, contract.KubeletManifestsDirectoryDisabled())
assert.False(t, contract.SecretboxEncryptionSupported())
assert.False(t, contract.DiskQuotaSupportEnabled())
}
func TestContract0_10(t *testing.T) {
@ -318,6 +329,7 @@ func TestContract0_10(t *testing.T) {
assert.False(t, contract.APIServerAuditPolicySupported())
assert.False(t, contract.KubeletManifestsDirectoryDisabled())
assert.False(t, contract.SecretboxEncryptionSupported())
assert.False(t, contract.DiskQuotaSupportEnabled())
}
func TestContract0_9(t *testing.T) {
@ -341,6 +353,7 @@ func TestContract0_9(t *testing.T) {
assert.False(t, contract.APIServerAuditPolicySupported())
assert.False(t, contract.KubeletManifestsDirectoryDisabled())
assert.False(t, contract.SecretboxEncryptionSupported())
assert.False(t, contract.DiskQuotaSupportEnabled())
}
func TestContract0_8(t *testing.T) {
@ -364,4 +377,5 @@ func TestContract0_8(t *testing.T) {
assert.False(t, contract.APIServerAuditPolicySupported())
assert.False(t, contract.KubeletManifestsDirectoryDisabled())
assert.False(t, contract.SecretboxEncryptionSupported())
assert.False(t, contract.DiskQuotaSupportEnabled())
}

1
pkg/machinery/config/provider.go
View File

@ -562,6 +562,7 @@ type Features interface {
StableHostnameEnabled() bool
KubernetesTalosAPIAccess() KubernetesTalosAPIAccess
ApidCheckExtKeyUsageEnabled() bool
DiskQuotaSupportEnabled() bool
}
// KubernetesTalosAPIAccess describes the Kubernetes Talos API access features.

6
pkg/machinery/config/types/v1alpha1/generate/init.go
View File

@ -15,7 +15,7 @@ import (
"github.com/siderolabs/talos/pkg/machinery/constants"
)
//nolint:gocyclo
//nolint:gocyclo,cyclop
func initUd(in *Input) (*v1alpha1.Config, error) {
config := &v1alpha1.Config{
ConfigVersion: "v1alpha1",
@ -69,6 +69,10 @@ func initUd(in *Input) (*v1alpha1.Config, error) {
machine.MachineFeatures.ApidCheckExtKeyUsage = pointer.To(true)
}
if in.VersionContract.DiskQuotaSupportEnabled() {
machine.MachineFeatures.DiskQuotaSupport = pointer.To(true)
}
if in.VersionContract.KubeletDefaultRuntimeSeccompProfileEnabled() {
machine.MachineKubelet.KubeletDefaultRuntimeSeccompProfileEnabled = pointer.To(true)
}

4
pkg/machinery/config/types/v1alpha1/generate/worker.go
View File

@ -70,6 +70,10 @@ func workerUd(in *Input) (*v1alpha1.Config, error) {
machine.MachineFeatures.ApidCheckExtKeyUsage = pointer.To(true)
}
if in.VersionContract.DiskQuotaSupportEnabled() {
machine.MachineFeatures.DiskQuotaSupport = pointer.To(true)
}
if in.VersionContract.KubeletDefaultRuntimeSeccompProfileEnabled() {
machine.MachineKubelet.KubeletDefaultRuntimeSeccompProfileEnabled = pointer.To(true)
}

7
pkg/machinery/config/types/v1alpha1/schemas/v1alpha1_config.schema.json
View File

@ -1340,6 +1340,13 @@
"description": "Enable checks for extended key usage of client certificates in apid.\n",
"markdownDescription": "Enable checks for extended key usage of client certificates in apid.",
"x-intellij-html-description": "\u003cp\u003eEnable checks for extended key usage of client certificates in apid.\u003c/p\u003e\n"
},
"diskQuotaSupport": {
"type": "boolean",
"title": "diskQuotaSupport",
"description": "Enable XFS project quota support for EPHEMERAL partition and user disks.\nAlso enables kubelet tracking of ephemeral disk usage in the kubelet via quota.\n",
"markdownDescription": "Enable XFS project quota support for EPHEMERAL partition and user disks.\nAlso enables kubelet tracking of ephemeral disk usage in the kubelet via quota.",
"x-intellij-html-description": "\u003cp\u003eEnable XFS project quota support for EPHEMERAL partition and user disks.\nAlso enables kubelet tracking of ephemeral disk usage in the kubelet via quota.\u003c/p\u003e\n"
}
},
"additionalProperties": false,

5
pkg/machinery/config/types/v1alpha1/v1alpha1_features.go
View File

@ -33,3 +33,8 @@ func (f *FeaturesConfig) KubernetesTalosAPIAccess() config.KubernetesTalosAPIAcc
func (f *FeaturesConfig) ApidCheckExtKeyUsageEnabled() bool {
return pointer.SafeDeref(f.ApidCheckExtKeyUsage)
}
// DiskQuotaSupportEnabled implements config.Features interface.
func (f *FeaturesConfig) DiskQuotaSupportEnabled() bool {
return pointer.SafeDeref(f.DiskQuotaSupport)
}

4
pkg/machinery/config/types/v1alpha1/v1alpha1_types.go
View File

@ -2603,6 +2603,10 @@ type FeaturesConfig struct {
// description: |
// Enable checks for extended key usage of client certificates in apid.
ApidCheckExtKeyUsage *bool `yaml:"apidCheckExtKeyUsage,omitempty"`
// description: |
// Enable XFS project quota support for EPHEMERAL partition and user disks.
// Also enables kubelet tracking of ephemeral disk usage in the kubelet via quota.
DiskQuotaSupport *bool `yaml:"diskQuotaSupport,omitempty"`
}
// KubernetesTalosAPIAccessConfig describes the configuration for the Talos API access from Kubernetes pods.

7
pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go
View File

@ -2368,7 +2368,7 @@ func init() {
FieldName: "features",
},
}
FeaturesConfigDoc.Fields = make([]encoder.Doc, 4)
FeaturesConfigDoc.Fields = make([]encoder.Doc, 5)
FeaturesConfigDoc.Fields[0].Name = "rbac"
FeaturesConfigDoc.Fields[0].Type = "bool"
FeaturesConfigDoc.Fields[0].Note = ""
@ -2391,6 +2391,11 @@ func init() {
FeaturesConfigDoc.Fields[3].Note = ""
FeaturesConfigDoc.Fields[3].Description = "Enable checks for extended key usage of client certificates in apid."
FeaturesConfigDoc.Fields[3].Comments[encoder.LineComment] = "Enable checks for extended key usage of client certificates in apid."
FeaturesConfigDoc.Fields[4].Name = "diskQuotaSupport"
FeaturesConfigDoc.Fields[4].Type = "bool"
FeaturesConfigDoc.Fields[4].Note = ""
FeaturesConfigDoc.Fields[4].Description = "Enable XFS project quota support for EPHEMERAL partition and user disks.\nAlso enables kubelet tracking of ephemeral disk usage in the kubelet via quota."
FeaturesConfigDoc.Fields[4].Comments[encoder.LineComment] = "Enable XFS project quota support for EPHEMERAL partition and user disks."
KubernetesTalosAPIAccessConfigDoc.Type = "KubernetesTalosAPIAccessConfig"
KubernetesTalosAPIAccessConfigDoc.Comments[encoder.LineComment] = "KubernetesTalosAPIAccessConfig describes the configuration for the Talos API access from Kubernetes pods."

5
pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go
View File

@ -958,6 +958,11 @@ func (in *FeaturesConfig) DeepCopyInto(out *FeaturesConfig) {
*out = new(bool)
**out = **in
}
if in.DiskQuotaSupport != nil {
in, out := &in.DiskQuotaSupport, &out.DiskQuotaSupport
*out = new(bool)
**out = **in
}
return
}

1
pkg/machinery/resources/k8s/kubelet_config.go
View File

@ -38,6 +38,7 @@ type KubeletConfigSpec struct {
SkipNodeRegistration bool `yaml:"skipNodeRegistration" protobuf:"9"`
StaticPodListURL string `yaml:"staticPodListURL" protobuf:"10"`
DisableManifestsDirectory bool `yaml:"disableManifestsDirectory" protobuf:"11"`
EnableFSQuotaMonitoring bool `yaml:"enableFSQuotaMonitoring" protobuf:"12"`
}
// NewKubeletConfig initializes an empty KubeletConfig resource.

1
website/content/v1.5/reference/api.md
View File

@ -1952,6 +1952,7 @@ KubeletConfigSpec holds the source of kubelet configuration.
| skip_node_registration | [bool](#bool) | | |
| static_pod_list_url | [string](#string) | | |
| disable_manifests_directory | [bool](#bool) | | |
| enable_fs_quota_monitoring | [bool](#bool) | | |

1
website/content/v1.5/reference/configuration.md
View File

@ -2688,6 +2688,7 @@ kubernetesTalosAPIAccess:
- kube-system
{{< /highlight >}}</details> | |
|`apidCheckExtKeyUsage` |bool |Enable checks for extended key usage of client certificates in apid. | |
|`diskQuotaSupport` |bool |<details><summary>Enable XFS project quota support for EPHEMERAL partition and user disks.</summary>Also enables kubelet tracking of ephemeral disk usage in the kubelet via quota.</details> | |

7
website/content/v1.5/schemas/v1alpha1_config.schema.json
View File

@ -1340,6 +1340,13 @@
"description": "Enable checks for extended key usage of client certificates in apid.\n",
"markdownDescription": "Enable checks for extended key usage of client certificates in apid.",
"x-intellij-html-description": "\u003cp\u003eEnable checks for extended key usage of client certificates in apid.\u003c/p\u003e\n"
},
"diskQuotaSupport": {
"type": "boolean",
"title": "diskQuotaSupport",
"description": "Enable XFS project quota support for EPHEMERAL partition and user disks.\nAlso enables kubelet tracking of ephemeral disk usage in the kubelet via quota.\n",
"markdownDescription": "Enable XFS project quota support for EPHEMERAL partition and user disks.\nAlso enables kubelet tracking of ephemeral disk usage in the kubelet via quota.",
"x-intellij-html-description": "\u003cp\u003eEnable XFS project quota support for EPHEMERAL partition and user disks.\nAlso enables kubelet tracking of ephemeral disk usage in the kubelet via quota.\u003c/p\u003e\n"
}
},
"additionalProperties": false,