feat(init): Add upgrade endpoint (#623)

Signed-off-by: Brad Beam <brad.beam@talos-systems.com>

cmd/osctl/cmd/root.go

@@ -17,14 +17,14 @@ import (
 var (
 	ca           string
 	crt          string
+	csr          string
+	hours        int
+	ip           string
 	key          string
+	kubernetes   bool
+	name         string
 	organization string
 	rsa          bool
-	name         string
-	csr          string
-	ip           string
-	hours        int
-	kubernetes   bool
 	talosconfig  string
 	target       string
 	userdataFile string

cmd/osctl/cmd/upgrade.go

@@ -0,0 +1,63 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// nolint: dupl,golint
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/talos-systems/talos/cmd/osctl/pkg/client"
+	"github.com/talos-systems/talos/cmd/osctl/pkg/helpers"
+	"github.com/talos-systems/talos/internal/pkg/constants"
+)
+
+var (
+	assetURL string
+	local    bool
+)
+
+// upgradeCmd represents the processes command
+var upgradeCmd = &cobra.Command{
+	Use:   "upgrade",
+	Short: "Upgrade Talos on the target node",
+	Long:  ``,
+	Run: func(cmd *cobra.Command, args []string) {
+		var err error
+		if local {
+			if err = localUpgrade(); err != nil {
+				helpers.Fatalf("error upgrading host: %s", err)
+			}
+		} else {
+			if err = remoteUpgrade(); err != nil {
+				helpers.Fatalf("error upgrading host: %s", err)
+			}
+		}
+
+	},
+}
+
+func init() {
+	upgradeCmd.Flags().BoolVarP(&local, "local", "l", false, "operate in local mode")
+	upgradeCmd.Flags().StringVarP(&assetURL, "url", "u", "", "url hosting upgrade assets (excluding filename)")
+	upgradeCmd.Flags().StringVarP(&target, "target", "t", "", "target the specificed node")
+	rootCmd.AddCommand(upgradeCmd)
+}
+
+func remoteUpgrade() error {
+	creds, err := client.NewDefaultClientCredentials(talosconfig)
+	if err != nil {
+		return err
+	}
+	if target != "" {
+		creds.Target = target
+	}
+	c, err := client.NewClient(constants.OsdPort, creds)
+	if err != nil {
+		return err
+	}
+
+	// TODO: See if we can validate version and prevent
+	// starting upgrades to an unknown version
+	return c.Upgrade(assetURL)
+}

cmd/osctl/cmd/upgrade_darwin.go

@@ -0,0 +1,13 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// nolint: dupl,golint
+package cmd
+
+import "log"
+
+func localUpgrade() error {
+	log.Println("local upgrades are not supported on this platform")
+	return nil
+}

cmd/osctl/cmd/upgrade_linux.go

@@ -0,0 +1,14 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// nolint: dupl,golint
+package cmd
+
+import (
+	"github.com/talos-systems/talos/internal/pkg/upgrade"
+)
+
+func localUpgrade() error {
+	return upgrade.NewUpgrade(assetURL)
+}

cmd/osctl/pkg/client/client.go

@@ -312,3 +312,15 @@ func (c *Client) DF() (err error) {
 
 	return nil
 }
+
+// Upgrade initiates a Talos upgrade ... and implements the proto.OSDClient
+// interface
+func (c *Client) Upgrade(asseturl string) (err error) {
+	ctx := context.Background()
+	reply, err := c.initClient.Upgrade(ctx, &initproto.UpgradeRequest{Url: asseturl})
+	if err != nil {
+		return
+	}
+	fmt.Println(reply.Ack)
+	return
+}

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/containerd/cri v1.11.1
 	github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260 // indirect
 	github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd
+	github.com/coreos/go-semver v0.3.0
 	github.com/coreos/go-systemd v0.0.0-20180828140353-eee3db372b31 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible

go.sum

@@ -25,6 +25,8 @@ github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260 h1:XGyg7oTtD0DoRFh
 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
 github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd h1:JNn81o/xG+8NEo3bC/vx9pbi/g2WI8mtP2/nXzu297Y=
 github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20180828140353-eee3db372b31 h1:wRzCUSYhBIk1KvRIlx+nvScCRIxX0iIhSU5h9xj7MUU=
 github.com/coreos/go-systemd v0.0.0-20180828140353-eee3db372b31/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

internal/app/init/internal/reg/reg.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/golang/protobuf/ptypes/empty"
 	"github.com/talos-systems/talos/internal/app/init/proto"
+	"github.com/talos-systems/talos/internal/pkg/upgrade"
 	"github.com/talos-systems/talos/pkg/userdata"
 	"google.golang.org/grpc"
 )
@@ -62,3 +63,22 @@ func (r *Registrator) Shutdown(ctx context.Context, in *empty.Empty) (reply *pro
 
 	return
 }
+
+// Upgrade initiates a Talos upgrade
+func (r *Registrator) Upgrade(ctx context.Context, in *proto.UpgradeRequest) (data *proto.UpgradeReply, err error) {
+
+	if err = upgrade.NewUpgrade(in.Url); err != nil {
+		return data, err
+	}
+
+	// Trigger reboot
+	defer func() {
+		if _, err = r.Reboot(ctx, &empty.Empty{}); err != nil {
+			return
+		}
+	}()
+
+	// profit
+	data = &proto.UpgradeReply{Ack: "Upgrade completed, rebooting node"}
+	return data, err
+}

internal/app/init/proto/api.proto

@@ -9,6 +9,7 @@ import "google/protobuf/empty.proto";
 service Init {
   rpc Reboot(google.protobuf.Empty) returns (RebootReply) {}
   rpc Shutdown(google.protobuf.Empty) returns (ShutdownReply) {}
+  rpc Upgrade(UpgradeRequest) returns (UpgradeReply) {}
 }
 
 
@@ -18,3 +19,8 @@ message RebootReply {}
 // The response message containing the shutdown status.
 message ShutdownReply {}
 
+message UpgradeRequest {
+	string url = 1;
+}
+
+message UpgradeReply { string ack = 1; }

internal/app/osd/internal/reg/init_client.go

@@ -38,7 +38,12 @@ func (c *InitServiceClient) Reboot(ctx context.Context, empty *empty.Empty) (*pr
 	return c.InitClient.Reboot(ctx, empty)
 }
 
-// Shutdown executes init Shutdown() API
+// Shutdown executes init Shutdown() API.
 func (c *InitServiceClient) Shutdown(ctx context.Context, empty *empty.Empty) (*proto.ShutdownReply, error) {
 	return c.InitClient.Shutdown(ctx, empty)
 }
+
+// Upgrade executes the init Upgrade() API.
+func (c *InitServiceClient) Upgrade(ctx context.Context, in *proto.UpgradeRequest) (data *proto.UpgradeReply, err error) {
+	return c.InitClient.Upgrade(ctx, in)
+}

internal/app/osd/proto/api.proto

@@ -9,6 +9,7 @@ import "google/protobuf/empty.proto";
 //
 // OSD Service also implements all the API of Init Service
 service OSD {
+  rpc DF(google.protobuf.Empty) returns (DFReply) {}
   rpc Dmesg(google.protobuf.Empty) returns (Data) {}
   rpc Kubeconfig(google.protobuf.Empty) returns (Data) {}
   rpc Logs(LogsRequest) returns (stream Data) {}
@@ -18,7 +19,6 @@ service OSD {
   rpc Routes(google.protobuf.Empty) returns (RoutesReply) {}
   rpc Stats(StatsRequest) returns (StatsReply) {}
   rpc Top(google.protobuf.Empty) returns (TopReply) {}
-  rpc DF(google.protobuf.Empty) returns (DFReply) {}
   rpc Version(google.protobuf.Empty) returns (Data) {}
 }
 

internal/pkg/constants/constants.go

@@ -141,6 +141,15 @@ const (
 
 	// InitSocketPath is the path to file socket of init API
 	InitSocketPath = "/var/lib/init/init.sock"
+
+	// KernelAsset defines a well known name for our kernel filename
+	KernelAsset = "vmlinuz"
+
+	// InitramfsAsset defines a well known name for our initramfs filename
+	InitramfsAsset = "initramfs.xz"
+
+	// RootfsAsset defines a well known name for our rootfs filename
+	RootfsAsset = "rootfs.tar.gz"
 )
 
 // See https://linux.die.net/man/3/klogctl
@@ -168,9 +177,9 @@ func CurrentRootPartitionLabel() string {
 	}
 
 	switch *param {
-	case "B":
+	case RootBPartitionLabel:
 		return RootBPartitionLabel
-	case "A":
+	case RootAPartitionLabel:
 		fallthrough
 	default:
 		return RootAPartitionLabel

internal/pkg/upgrade/upgrade.go

@@ -0,0 +1,158 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package upgrade
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"github.com/talos-systems/talos/internal/pkg/blockdevice/bootloader/syslinux"
+	"github.com/talos-systems/talos/internal/pkg/blockdevice/probe"
+	"github.com/talos-systems/talos/internal/pkg/constants"
+	"github.com/talos-systems/talos/internal/pkg/install"
+	"github.com/talos-systems/talos/internal/pkg/kernel"
+	"github.com/talos-systems/talos/internal/pkg/kubernetes"
+	"github.com/talos-systems/talos/internal/pkg/mount"
+	"golang.org/x/sys/unix"
+)
+
+// NewUpgrade initiates a Talos upgrade
+func NewUpgrade(url string) (err error) {
+	if err = upgradeRoot(url); err != nil {
+		return
+	}
+	if err = upgradeBoot(url); err != nil {
+		return
+	}
+
+	// cordon/drain
+	var kubeHelper *kubernetes.Helper
+	if kubeHelper, err = kubernetes.NewHelper(); err != nil {
+		return
+	}
+
+	var hostname string
+	if hostname, err = os.Hostname(); err != nil {
+		return
+	}
+
+	if err = kubeHelper.CordonAndDrain(hostname); err != nil {
+		return
+	}
+
+	return err
+}
+
+func upgradeRoot(url string) (err error) {
+	// Identify next root disk
+	var dev *probe.ProbedBlockDevice
+	if dev, err = probe.GetDevWithFileSystemLabel(constants.NextRootPartitionLabel()); err != nil {
+		return
+	}
+
+	// Should we handle anything around the disk/partition itself? maybe Format()
+	rootTarget := install.Target{
+		Label:          constants.NextRootPartitionLabel(),
+		MountPoint:     "/var/nextroot",
+		Device:         dev.Path,
+		FileSystemType: "xfs",
+		Force:          true,
+		PartitionName:  dev.Path,
+		Assets:         []*install.Asset{},
+	}
+
+	rootTarget.Assets = append(rootTarget.Assets, &install.Asset{
+		Source:      url + "/" + constants.RootfsAsset,
+		Destination: constants.RootfsAsset,
+	})
+
+	if err = rootTarget.Format(); err != nil {
+		return
+	}
+
+	// mount up 'next' root disk rw
+	mountpoint := mount.NewMountPoint(dev.Path, rootTarget.MountPoint, dev.SuperBlock.Type(), unix.MS_NOATIME, "")
+
+	if err = mount.WithRetry(mountpoint); err != nil {
+		return errors.Errorf("error mounting partitions: %v", err)
+	}
+
+	// Ensure we unmount the new rootfs in case of failure
+	// nolint: errcheck
+	defer mount.UnWithRetry(mountpoint)
+
+	// install assets
+	return rootTarget.Install()
+}
+
+func upgradeBoot(url string) error {
+	bootTarget := install.Target{
+		Label:      constants.BootPartitionLabel,
+		MountPoint: "/boot",
+		Assets:     []*install.Asset{},
+	}
+
+	// Kernel
+	bootTarget.Assets = append(bootTarget.Assets, &install.Asset{
+		Source:      url + "/" + constants.KernelAsset,
+		Destination: filepath.Join(constants.NextRootPartitionLabel(), constants.KernelAsset),
+	})
+
+	// Initramfs
+	bootTarget.Assets = append(bootTarget.Assets, &install.Asset{
+		Source:      url + "/" + constants.InitramfsAsset,
+		Destination: filepath.Join(constants.NextRootPartitionLabel(), constants.InitramfsAsset),
+	})
+
+	var err error
+	if err = bootTarget.Install(); err != nil {
+		return err
+	}
+
+	// TODO: Figure out a method to update kernel args
+	nextCmdline := kernel.NewCmdline(kernel.Cmdline().String())
+
+	// talos.root=
+	talosRoot := kernel.NewParameter(constants.KernelCurrentRoot)
+	talosRoot.Append(constants.NextRootPartitionLabel())
+	if root := nextCmdline.Get(constants.KernelCurrentRoot); root == nil {
+		nextCmdline.Append(constants.KernelCurrentRoot, *(talosRoot.First()))
+	} else {
+		nextCmdline.Set(constants.KernelCurrentRoot, talosRoot)
+	}
+
+	// initrd=
+	initParam := kernel.NewParameter("initrd")
+	initParam.Append(filepath.Join("/", constants.NextRootPartitionLabel(), constants.InitramfsAsset))
+	if initrd := nextCmdline.Get("initrd"); initrd == nil {
+		nextCmdline.Append("initrd", *(initParam.First()))
+	} else {
+		nextCmdline.Set("initrd", initParam)
+	}
+
+	// Create bootloader config
+	bootcfg := &syslinux.Cfg{
+		Default: constants.NextRootPartitionLabel(),
+		Labels: []*syslinux.Label{
+			{
+				Root:   constants.NextRootPartitionLabel(),
+				Kernel: filepath.Join("/", constants.NextRootPartitionLabel(), constants.KernelAsset),
+				Initrd: filepath.Join("/", constants.NextRootPartitionLabel(), constants.InitramfsAsset),
+				Append: nextCmdline.String(),
+			},
+			// TODO see if we can omit this section and/or pass in only the root(?)
+			// so we can make use of existing boot config
+			{
+				Root:   constants.CurrentRootPartitionLabel(),
+				Kernel: filepath.Join("/", constants.CurrentRootPartitionLabel(), constants.KernelAsset),
+				Initrd: filepath.Join("/", constants.CurrentRootPartitionLabel(), constants.InitramfsAsset),
+				Append: kernel.Cmdline().String(),
+			},
+		},
+	}
+
+	return syslinux.Install(constants.BootMountPoint, bootcfg)
+}