feat: allow additional manifests to be provided to bootkube

This PR will add an `additionalManifests` field to the config data that
allows users to specify a list of URLs that they'd like to fetch
manifests from. These manifests will then be added to the bootkube asset
directory and applied during the bootkube service.

Signed-off-by: Spencer Smith <robertspencersmith@gmail.com>

docs/website/content/v0.3/en/configuration/v1alpha1.md

@@ -484,6 +484,22 @@ etcd:
 
 ```
 
+#### extraManifests
+
+A list of urls that point to additional manifests.
+These will get automatically deployed by bootkube.
+
+Type: `array`
+
+Examples:
+
+```yaml
+extraManifests:
+  - "https://www.mysweethttpserver.com/manifest1.yaml"
+  - "https://www.mysweethttpserver.com/manifest2.yaml"
+
+```
+
 ---
 
 ### KubeletConfig

internal/app/machined/pkg/system/services/bootkube.go

@@ -341,15 +341,21 @@ func generateAssets(config runtime.Configurator) (err error) {
 		return err
 	}
 
+	if err = ioutil.WriteFile(filepath.Join(constants.AssetsDirectory, "manifests", "psp.yaml"), DefaultPodSecurityPolicy, 0600); err != nil {
+		return err
+	}
+
 	// If "custom" is the CNI, we expect the user to supply one or more urls that point to CNI yamls
 	if config.Cluster().Network().CNI().Name() == constants.CustomCNI {
-		if err = fetchCNIManifests(config.Cluster().Network().CNI().URLs()); err != nil {
+		if err = fetchManifests(config.Cluster().Network().CNI().URLs()); err != nil {
 			return err
 		}
 	}
 
-	if err = ioutil.WriteFile(filepath.Join(constants.AssetsDirectory, "manifests", "psp.yaml"), DefaultPodSecurityPolicy, 0600); err != nil {
-		return err
+	if len(config.Cluster().ExtraManifestURLs()) > 0 {
+		if err = fetchManifests(config.Cluster().ExtraManifestURLs()); err != nil {
+			return err
+		}
 	}
 
 	input, err := ioutil.ReadFile(constants.GeneratedKubeconfigAsset)
@@ -376,8 +382,8 @@ func altNamesFromURLs(urls []string) *tlsutil.AltNames {
 	return &an
 }
 
-// fetchCNIManifests will lay down provided CNI files to the bootkube assets directory
-func fetchCNIManifests(urls []string) error {
+// fetchManifests will lay down manifests in the provided urls to the bootkube assets directory
+func fetchManifests(urls []string) error {
 	ctx := context.Background()
 
 	var result *multierror.Error
@@ -400,10 +406,13 @@ func fetchCNIManifests(urls []string) error {
 		getter.Getters["http"] = httpGetter
 		getter.Getters["https"] = httpGetter
 
+		// We will squirrel all user-supplied manifests into a `zzz-talos` directory.
+		// Bootkube applies manifests alphabetically, so pushing these into a subdir with this name
+		// allows us to ensure they're the last things that get applied and things like PSPs and whatnot are present
 		client := &getter.Client{
 			Ctx:     ctx,
 			Src:     url,
-			Dst:     filepath.Join(constants.AssetsDirectory, "manifests", fileName),
+			Dst:     filepath.Join(constants.AssetsDirectory, "manifests", "zzz-talos", fileName),
 			Pwd:     pwd,
 			Mode:    getter.ClientModeFile,
 			Options: []getter.ClientOption{},

pkg/config/cluster/cluster.go

@@ -24,6 +24,7 @@ type Cluster interface {
 	Etcd() Etcd
 	Network() Network
 	LocalAPIServerPort() int
+	ExtraManifestURLs() []string
 }
 
 // Network defines the requirements for a config that pertains to cluster

pkg/config/types/v1alpha1/v1alpha1_configurator.go

@@ -323,6 +323,11 @@ func (c *ClusterConfig) ServiceCIDR() string {
 	return c.ClusterNetwork.ServiceSubnet[0]
 }
 
+// ExtraManifestURLs implements the Configurator interface.
+func (c *ClusterConfig) ExtraManifestURLs() []string {
+	return c.ExtraManifests
+}
+
 // Name implements the Configurator interface.
 func (c *CNIConfig) Name() string {
 	return c.CNIName

pkg/config/types/v1alpha1/v1alpha1_types.go

@@ -264,6 +264,15 @@ type ClusterConfig struct {
 	//           key: LS0tLS1CRUdJTiBSU0...
 	//         image: ...
 	EtcdConfig *EtcdConfig `yaml:"etcd,omitempty"`
+	//   description: |
+	//     A list of urls that point to additional manifests.
+	//     These will get automatically deployed by bootkube.
+	//   examples:
+	//     - |
+	//       extraManifests:
+	//         - "https://www.mysweethttpserver.com/manifest1.yaml"
+	//         - "https://www.mysweethttpserver.com/manifest2.yaml"
+	ExtraManifests []string `yaml:"extraManifests,omitempty"`
 }
 
 // KubeletConfig reperesents the kubelet config values