alt-orchestra/talos
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

feat: add dynamic config decoder

Browse Source 
This adds the ability to dynamically decode mult-doc YAML files.

Signed-off-by: Andrew Rynhard <andrew@rynhard.io>
This commit is contained in:
Andrew Rynhard 2020-07-29 14:29:15 -07:00 committed by talos-bot
parent 26317071b6
commit 849959fefc
16 changed files with 430 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/talosctl/cmd/mgmt/config.go
View File

@ -13,7 +13,7 @@ import (
"strings"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"gopkg.in/yaml.v3"
"github.com/talos-systems/talos/cmd/talosctl/pkg/mgmt/helpers"
"github.com/talos-systems/talos/pkg/config/types/v1alpha1/bundle"

2
cmd/talosctl/pkg/talos/helpers/helpers_test.go
View File

@ -8,7 +8,7 @@ import (
"os"
"testing"
yaml "gopkg.in/yaml.v2"
yaml "gopkg.in/yaml.v3"
"github.com/stretchr/testify/assert"

2
go.mod
View File

@ -72,7 +72,7 @@ require (
google.golang.org/protobuf v1.25.0
gopkg.in/freddierice/go-losetup.v1 v1.0.0-20170407175016-fc9adea44124
gopkg.in/fsnotify.v1 v1.4.7
gopkg.in/yaml.v2 v2.3.0
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
gotest.tools v2.2.0+incompatible
inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252
k8s.io/api v0.19.0-rc.3

4
go.sum
View File

@ -1047,10 +1047,10 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=

2
internal/pkg/provision/providers/vm/reflect.go
View File

@ -10,7 +10,7 @@ import (
"os"
"path/filepath"
yaml "gopkg.in/yaml.v2"
yaml "gopkg.in/yaml.v3"
"github.com/talos-systems/talos/internal/pkg/provision"
)

2
internal/pkg/provision/providers/vm/state.go
View File

@ -9,7 +9,7 @@ import (
"os"
"path/filepath"
yaml "gopkg.in/yaml.v2"
yaml "gopkg.in/yaml.v3"
"github.com/containernetworking/cni/libcni"

2
pkg/client/config/config.go
View File

@ -9,7 +9,7 @@ import (
"os"
"path/filepath"
yaml "gopkg.in/yaml.v2"
yaml "gopkg.in/yaml.v3"
)
// Config represents the configuration file.

77
pkg/config/configloader/configloader.go
View File

@ -9,76 +9,47 @@ import (
"fmt"
"io/ioutil"
"gopkg.in/yaml.v2"
"github.com/talos-systems/talos/pkg/config"
"github.com/talos-systems/talos/pkg/config/decoder"
"github.com/talos-systems/talos/pkg/config/types/v1alpha1"
)
// content represents the raw config data.
//
//docgen: nodoc
type content struct {
Version string `yaml:"version"`
data []byte
}
// newConfig initializes and returns a Configurator.
func newConfig(c content) (config config.Provider, err error) {
switch c.Version {
case v1alpha1.Version:
return v1alpha1.Load(c.data)
default:
return nil, fmt.Errorf("unknown version: %q", c.Version)
func newConfig(source []byte) (config config.Provider, err error) {
dec := decoder.NewDecoder(source)
manifests, err := dec.Decode()
if err != nil {
return nil, err
}
// Look for the older flat v1alpha1 file first, since we have to handle it in
// a special way.
for _, manifest := range manifests {
if talosconfig, ok := manifest.(*v1alpha1.Config); ok {
return talosconfig, nil
}
}
return nil, fmt.Errorf("config not found")
}
// NewFromFile will take a filepath and attempt to parse a config file from it.
func NewFromFile(filepath string) (config.Provider, error) {
c, err := fromFile(filepath)
source, err := fromFile(filepath)
if err != nil {
return nil, err
}
return newConfig(c)
return newConfig(source)
}
// NewFromBytes will take a byteslice and attempt to parse a config file from it.
func NewFromBytes(in []byte) (config.Provider, error) {
c, err := fromBytes(in)
if err != nil {
return nil, err
}
return newConfig(c)
func NewFromBytes(source []byte) (config.Provider, error) {
return newConfig(source)
}
// fromFile is a convenience function that reads the config from disk, and
// unmarshals it.
func fromFile(p string) (c content, err error) {
b, err := ioutil.ReadFile(p)
if err != nil {
return c, fmt.Errorf("read config: %w", err)
}
return unmarshal(b)
}
// fromBytes is a convenience function that reads the config from a string, and
// unmarshals it.
func fromBytes(b []byte) (c content, err error) {
return unmarshal(b)
}
func unmarshal(b []byte) (c content, err error) {
c = content{
data: b,
}
if err = yaml.Unmarshal(b, &c); err != nil {
return c, fmt.Errorf("failed to parse config: %s", err.Error())
}
return c, nil
// fromFile is a convenience function that reads the config from disk.
func fromFile(p string) ([]byte, error) {
return ioutil.ReadFile(p)
}

11
pkg/config/configloader/configloader_test.go
View File

@ -9,8 +9,6 @@ import (
"testing"
"github.com/stretchr/testify/suite"
"github.com/talos-systems/talos/pkg/config/types/v1alpha1"
)
//docgen: nodoc
@ -26,13 +24,10 @@ func (suite *Suite) SetupSuite() {}
func (suite *Suite) TestNew() {
for _, t := range []struct {
content content
source []byte
errExpected bool
}{
{content{Version: v1alpha1.Version}, false},
{content{Version: ""}, true},
} {
_, err := newConfig(t.content)
}{} {
_, err := newConfig(t.source)
if t.errExpected {
suite.Require().Error(err)

168
pkg/config/decoder/decoder.go Normal file
View File

@ -0,0 +1,168 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package decoder
import (
"bytes"
"errors"
"fmt"
"io"
"gopkg.in/yaml.v3"
"github.com/talos-systems/talos/pkg/config"
)
var (
// ErrMissingVersion indicates that the manifest is missing a version.
ErrMissingVersion = errors.New("missing version")
// ErrMissingKind indicates that the manifest is missing a kind.
ErrMissingKind = errors.New("missing kind")
// ErrMissingSpec indicates that the manifest is missing a spec.
ErrMissingSpec = errors.New("missing spec")
// ErrMissingSpecConent indicates that the manifest spec is empty
ErrMissingSpecConent = errors.New("missing spec content")
)
const (
// ManifestVersionKey is the string indicating a manifest's version.
ManifestVersionKey = "version"
// ManifestKindKey is the string indicating a manifest's kind.
ManifestKindKey = "kind"
// ManifestSpecKey is represents a manifest's spec.
ManifestSpecKey = "spec"
// ManifestDeprecatedKey is represents the deprected v1alpha1 manifest.
ManifestDeprecatedKey = "machine"
)
// Decoder represents a multi-doc YAML decoder.
type Decoder struct {
source []byte
}
// Decode decodes all known manifests.
func (d *Decoder) Decode() ([]interface{}, error) {
return d.decode()
}
// NewDecoder initializes and returns a `Decoder`.
func NewDecoder(source []byte) *Decoder {
return &Decoder{
source: source,
}
}
func (d *Decoder) decode() ([]interface{}, error) {
return parse(d.source)
}
func parse(source []byte) (decoded []interface{}, err error) {
decoded = []interface{}{}
r := bytes.NewReader(source)
dec := yaml.NewDecoder(r)
dec.KnownFields(true)
// Iterate through all defined documents.
for {
var manifests yaml.Node
if err = dec.Decode(&manifests); err != nil {
if errors.Is(err, io.EOF) {
return decoded, nil
}
return nil, fmt.Errorf("decode error: %w", err)
}
if manifests.Kind != yaml.DocumentNode {
return nil, fmt.Errorf("expected a document")
}
for _, manifest := range manifests.Content {
var target interface{}
if target, err = decode(manifest); err != nil {
return nil, err
}
decoded = append(decoded, target)
}
}
}
//nolint: gocyclo
func decode(manifest *yaml.Node) (target interface{}, err error) {
var (
version string
kind string
spec *yaml.Node
)
for i, node := range manifest.Content {
switch node.Value {
case ManifestKindKey:
if len(manifest.Content) < i+1 {
return nil, fmt.Errorf("missing manifest content")
}
if err = manifest.Content[i+1].Decode(&kind); err != nil {
return nil, fmt.Errorf("kind decode: %w", err)
}
case ManifestVersionKey:
if len(manifest.Content) < i+1 {
return nil, fmt.Errorf("missing manifest content")
}
if err = manifest.Content[i+1].Decode(&version); err != nil {
return nil, fmt.Errorf("version decode: %w", err)
}
case ManifestSpecKey:
if len(manifest.Content) < i+1 {
return nil, fmt.Errorf("missing manifest content")
}
spec = manifest.Content[i+1]
case ManifestDeprecatedKey:
if target, err = config.New("v1alpha1", ""); err != nil {
return nil, fmt.Errorf("new deprecated config: %w", err)
}
if err = manifest.Decode(target); err != nil {
return nil, fmt.Errorf("deprecated decode: %w", err)
}
return target, nil
}
}
if kind == "" {
return nil, ErrMissingKind
}
if version == "" {
return nil, ErrMissingVersion
}
if spec == nil {
return nil, ErrMissingSpec
}
if spec.Content == nil {
return nil, ErrMissingSpecConent
}
if target, err = config.New(kind, version); err != nil {
return nil, fmt.Errorf("new config: %w", err)
}
if err = spec.Decode(target); err != nil {
return nil, fmt.Errorf("spec decode: %w", err)
}
return target, nil
}

155
pkg/config/decoder/decoder_test.go Normal file
View File

@ -0,0 +1,155 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//nolint: scopelint
package decoder_test
import (
"reflect"
"testing"
"github.com/talos-systems/talos/pkg/config"
"github.com/talos-systems/talos/pkg/config/decoder"
)
type Mock struct {
Test bool `yaml:"test"`
}
func init() {
config.Register("mock", func(verion string) interface{} {
return &Mock{}
})
}
func TestDecoder_Decode(t *testing.T) {
type fields struct {
source []byte
}
tests := []struct {
name string
fields fields
want []interface{}
wantErr bool
}{
{
name: "valid",
fields: fields{
source: []byte(`---
kind: mock
version: v1alpha1
spec:
test: true
`),
},
want: []interface{}{
&Mock{
Test: true,
},
},
wantErr: false,
},
{
name: "missing kind",
fields: fields{
source: []byte(`---
version: v1alpha1
spec:
test: true
`),
},
want: nil,
wantErr: true,
},
{
name: "empty kind",
fields: fields{
source: []byte(`---
kind:
version: v1alpha1
spec:
test: true
`),
},
want: nil,
wantErr: true,
},
{
name: "missing version",
fields: fields{
source: []byte(`---
kind: mock
spec:
test: true
`),
},
want: nil,
wantErr: true,
},
{
name: "empty version",
fields: fields{
source: []byte(`---
kind: mock
version:
spec:
test: true
`),
},
want: nil,
wantErr: true,
},
{
name: "missing spec",
fields: fields{
source: []byte(`---
kind: mock
version: v1alpha1
`),
},
want: nil,
wantErr: true,
},
{
name: "empty spec",
fields: fields{
source: []byte(`---
kind: mock
version: v1alpha1
spec:
`),
},
want: nil,
wantErr: true,
},
{
name: "tab instead of spaces",
fields: fields{
source: []byte(`---
kind: mock
version: v1alpha1
spec:
test: true
`),
},
want: nil,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
d := decoder.NewDecoder(tt.fields.source)
got, err := d.Decode()
if (err != nil) != tt.wantErr {
t.Errorf("Decoder.Decode() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("Decoder.Decode() = %v, want %v", got, tt.want)
}
})
}
}

62
pkg/config/registry.go Normal file
View File

@ -0,0 +1,62 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package config
import (
"errors"
"fmt"
"sync"
)
var (
// ErrNotRegistered indicates that the manifest kind is not registered.
ErrNotRegistered = errors.New("not registered")
// ErrExists indicates that the manifest is already registered.
ErrExists = errors.New("exists")
)
var registry = &Registry{
registered: map[string]func(string) interface{}{},
}
// Registry represents the provider registry.
type Registry struct {
registered map[string]func(string) interface{}
sync.Mutex
}
// Register registers a manifests with the registry.
func Register(kind string, f func(version string) interface{}) {
registry.register(kind, f)
}
// New creates a new instance of the requested manifest.
func New(kind, version string) (interface{}, error) {
return registry.new(kind, version)
}
func (r *Registry) register(kind string, f func(version string) interface{}) {
r.Lock()
defer r.Unlock()
if _, ok := r.registered[kind]; ok {
panic(ErrExists)
}
r.registered[kind] = f
}
func (r *Registry) new(kind, version string) (interface{}, error) {
r.Lock()
defer r.Unlock()
f, ok := r.registered[kind]
if ok {
return f(version), nil
}
return nil, fmt.Errorf("%q %q: %w", kind, version, ErrNotRegistered)
}

2
pkg/config/types/v1alpha1/bundle/bundle.go
View File

@ -10,7 +10,7 @@ import (
"path/filepath"
"strings"
yaml "gopkg.in/yaml.v2"
yaml "gopkg.in/yaml.v3"
clientconfig "github.com/talos-systems/talos/pkg/client/config"
"github.com/talos-systems/talos/pkg/config/types/v1alpha1"

21
pkg/config/types/v1alpha1/v1alpha1_loader.go
View File

@ -1,21 +0,0 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
package v1alpha1
import (
"fmt"
"gopkg.in/yaml.v2"
)
// Load config version v1alpha1.
func Load(data []byte) (config *Config, err error) {
config = &Config{}
if err = yaml.Unmarshal(data, config); err != nil {
return config, fmt.Errorf("failed to parse v1alpha1 config: %w", err)
}
return config, nil
}

2
pkg/config/types/v1alpha1/v1alpha1_provider.go
View File

@ -11,7 +11,7 @@ import (
"strings"
"time"
yaml "gopkg.in/yaml.v2"
yaml "gopkg.in/yaml.v3"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/talos-systems/bootkube-plugin/pkg/asset"

8
pkg/config/types/v1alpha1/v1alpha1_types.go
View File

@ -16,6 +16,14 @@ import (
"github.com/talos-systems/talos/pkg/crypto/x509"
)
func init() {
config.Register("v1alpha1", func(version string) (target interface{}) {
target = &Config{}
return target
})
}
// Config defines the v1alpha1 configuration file.
type Config struct {
// description: |