feat: add recovery API

This adds an API for recovering the self-hosted control plane. Signed-off-by: Andrew Rynhard <andrew@andrewrynhard.com>
2020-04-28 10:27:19 -07:00 · 2020-04-28 10:27:19 -07:00 · 56d7bf19fe
commit 56d7bf19fe
parent f59620473e
23 changed files with 885 additions and 233 deletions
--- a/.codecov.yml
+++ b/.codecov.yml
@ -2,7 +2,7 @@ coverage:
  status:
    project:
      default:
-        target: 28%
+        target: 27%
        threshold: 0.5%
        base: auto
    patch: off
--- a/api/machine/machine.pb.go
+++ b/api/machine/machine.pb.go
@ -29,6 +29,31 @@ var (
 // proto package needs to be updated.
 const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package

+type RecoverRequest_Source int32
+
+const (
+	RecoverRequest_ETCD      RecoverRequest_Source = 0
+	RecoverRequest_APISERVER RecoverRequest_Source = 1
+)
+
+var RecoverRequest_Source_name = map[int32]string{
+	0: "ETCD",
+	1: "APISERVER",
+}
+
+var RecoverRequest_Source_value = map[string]int32{
+	"ETCD":      0,
+	"APISERVER": 1,
+}
+
+func (x RecoverRequest_Source) String() string {
+	return proto.EnumName(RecoverRequest_Source_name, int32(x))
+}
+
+func (RecoverRequest_Source) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor_84b4f59d98cc997c, []int{5, 0}
+}
+
 // rpc reboot
 // The reboot message containing the reboot status.
 type Reboot struct {
@ -256,6 +281,136 @@ func (m *ResetResponse) GetMessages() []*Reset {
 	return nil
 }

+type RecoverRequest struct {
+	Source               RecoverRequest_Source `protobuf:"varint,1,opt,name=source,proto3,enum=machine.RecoverRequest_Source" json:"source,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}              `json:"-"`
+	XXX_unrecognized     []byte                `json:"-"`
+	XXX_sizecache        int32                 `json:"-"`
+}
+
+func (m *RecoverRequest) Reset()         { *m = RecoverRequest{} }
+func (m *RecoverRequest) String() string { return proto.CompactTextString(m) }
+func (*RecoverRequest) ProtoMessage()    {}
+func (*RecoverRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_84b4f59d98cc997c, []int{5}
+}
+
+func (m *RecoverRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_RecoverRequest.Unmarshal(m, b)
+}
+
+func (m *RecoverRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_RecoverRequest.Marshal(b, m, deterministic)
+}
+
+func (m *RecoverRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_RecoverRequest.Merge(m, src)
+}
+
+func (m *RecoverRequest) XXX_Size() int {
+	return xxx_messageInfo_RecoverRequest.Size(m)
+}
+
+func (m *RecoverRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_RecoverRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_RecoverRequest proto.InternalMessageInfo
+
+func (m *RecoverRequest) GetSource() RecoverRequest_Source {
+	if m != nil {
+		return m.Source
+	}
+	return RecoverRequest_ETCD
+}
+
+// The recover message containing the recover status.
+type Recover struct {
+	Metadata             *common.Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}         `json:"-"`
+	XXX_unrecognized     []byte           `json:"-"`
+	XXX_sizecache        int32            `json:"-"`
+}
+
+func (m *Recover) Reset()         { *m = Recover{} }
+func (m *Recover) String() string { return proto.CompactTextString(m) }
+func (*Recover) ProtoMessage()    {}
+func (*Recover) Descriptor() ([]byte, []int) {
+	return fileDescriptor_84b4f59d98cc997c, []int{6}
+}
+
+func (m *Recover) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Recover.Unmarshal(m, b)
+}
+
+func (m *Recover) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Recover.Marshal(b, m, deterministic)
+}
+
+func (m *Recover) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Recover.Merge(m, src)
+}
+
+func (m *Recover) XXX_Size() int {
+	return xxx_messageInfo_Recover.Size(m)
+}
+
+func (m *Recover) XXX_DiscardUnknown() {
+	xxx_messageInfo_Recover.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Recover proto.InternalMessageInfo
+
+func (m *Recover) GetMetadata() *common.Metadata {
+	if m != nil {
+		return m.Metadata
+	}
+	return nil
+}
+
+type RecoverResponse struct {
+	Messages             []*Recover `protobuf:"bytes,1,rep,name=messages,proto3" json:"messages,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}   `json:"-"`
+	XXX_unrecognized     []byte     `json:"-"`
+	XXX_sizecache        int32      `json:"-"`
+}
+
+func (m *RecoverResponse) Reset()         { *m = RecoverResponse{} }
+func (m *RecoverResponse) String() string { return proto.CompactTextString(m) }
+func (*RecoverResponse) ProtoMessage()    {}
+func (*RecoverResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_84b4f59d98cc997c, []int{7}
+}
+
+func (m *RecoverResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_RecoverResponse.Unmarshal(m, b)
+}
+
+func (m *RecoverResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_RecoverResponse.Marshal(b, m, deterministic)
+}
+
+func (m *RecoverResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_RecoverResponse.Merge(m, src)
+}
+
+func (m *RecoverResponse) XXX_Size() int {
+	return xxx_messageInfo_RecoverResponse.Size(m)
+}
+
+func (m *RecoverResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_RecoverResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_RecoverResponse proto.InternalMessageInfo
+
+func (m *RecoverResponse) GetMessages() []*Recover {
+	if m != nil {
+		return m.Messages
+	}
+	return nil
+}
+
 // rpc shutdown
 // The messages message containing the shutdown status.
 type Shutdown struct {
@ -269,7 +424,7 @@ func (m *Shutdown) Reset()         { *m = Shutdown{} }
 func (m *Shutdown) String() string { return proto.CompactTextString(m) }
 func (*Shutdown) ProtoMessage()    {}
 func (*Shutdown) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{5}
+	return fileDescriptor_84b4f59d98cc997c, []int{8}
 }

 func (m *Shutdown) XXX_Unmarshal(b []byte) error {
@ -312,7 +467,7 @@ func (m *ShutdownResponse) Reset()         { *m = ShutdownResponse{} }
 func (m *ShutdownResponse) String() string { return proto.CompactTextString(m) }
 func (*ShutdownResponse) ProtoMessage()    {}
 func (*ShutdownResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{6}
+	return fileDescriptor_84b4f59d98cc997c, []int{9}
 }

 func (m *ShutdownResponse) XXX_Unmarshal(b []byte) error {
@ -357,7 +512,7 @@ func (m *UpgradeRequest) Reset()         { *m = UpgradeRequest{} }
 func (m *UpgradeRequest) String() string { return proto.CompactTextString(m) }
 func (*UpgradeRequest) ProtoMessage()    {}
 func (*UpgradeRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{7}
+	return fileDescriptor_84b4f59d98cc997c, []int{10}
 }

 func (m *UpgradeRequest) XXX_Unmarshal(b []byte) error {
@ -408,7 +563,7 @@ func (m *Upgrade) Reset()         { *m = Upgrade{} }
 func (m *Upgrade) String() string { return proto.CompactTextString(m) }
 func (*Upgrade) ProtoMessage()    {}
 func (*Upgrade) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{8}
+	return fileDescriptor_84b4f59d98cc997c, []int{11}
 }

 func (m *Upgrade) XXX_Unmarshal(b []byte) error {
@ -458,7 +613,7 @@ func (m *UpgradeResponse) Reset()         { *m = UpgradeResponse{} }
 func (m *UpgradeResponse) String() string { return proto.CompactTextString(m) }
 func (*UpgradeResponse) ProtoMessage()    {}
 func (*UpgradeResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{9}
+	return fileDescriptor_84b4f59d98cc997c, []int{12}
 }

 func (m *UpgradeResponse) XXX_Unmarshal(b []byte) error {
@ -503,7 +658,7 @@ func (m *ServiceList) Reset()         { *m = ServiceList{} }
 func (m *ServiceList) String() string { return proto.CompactTextString(m) }
 func (*ServiceList) ProtoMessage()    {}
 func (*ServiceList) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{10}
+	return fileDescriptor_84b4f59d98cc997c, []int{13}
 }

 func (m *ServiceList) XXX_Unmarshal(b []byte) error {
@ -553,7 +708,7 @@ func (m *ServiceListResponse) Reset()         { *m = ServiceListResponse{} }
 func (m *ServiceListResponse) String() string { return proto.CompactTextString(m) }
 func (*ServiceListResponse) ProtoMessage()    {}
 func (*ServiceListResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{11}
+	return fileDescriptor_84b4f59d98cc997c, []int{14}
 }

 func (m *ServiceListResponse) XXX_Unmarshal(b []byte) error {
@ -599,7 +754,7 @@ func (m *ServiceInfo) Reset()         { *m = ServiceInfo{} }
 func (m *ServiceInfo) String() string { return proto.CompactTextString(m) }
 func (*ServiceInfo) ProtoMessage()    {}
 func (*ServiceInfo) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{12}
+	return fileDescriptor_84b4f59d98cc997c, []int{15}
 }

 func (m *ServiceInfo) XXX_Unmarshal(b []byte) error {
@ -663,7 +818,7 @@ func (m *ServiceEvents) Reset()         { *m = ServiceEvents{} }
 func (m *ServiceEvents) String() string { return proto.CompactTextString(m) }
 func (*ServiceEvents) ProtoMessage()    {}
 func (*ServiceEvents) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{13}
+	return fileDescriptor_84b4f59d98cc997c, []int{16}
 }

 func (m *ServiceEvents) XXX_Unmarshal(b []byte) error {
@ -708,7 +863,7 @@ func (m *ServiceEvent) Reset()         { *m = ServiceEvent{} }
 func (m *ServiceEvent) String() string { return proto.CompactTextString(m) }
 func (*ServiceEvent) ProtoMessage()    {}
 func (*ServiceEvent) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{14}
+	return fileDescriptor_84b4f59d98cc997c, []int{17}
 }

 func (m *ServiceEvent) XXX_Unmarshal(b []byte) error {
@ -768,7 +923,7 @@ func (m *ServiceHealth) Reset()         { *m = ServiceHealth{} }
 func (m *ServiceHealth) String() string { return proto.CompactTextString(m) }
 func (*ServiceHealth) ProtoMessage()    {}
 func (*ServiceHealth) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{15}
+	return fileDescriptor_84b4f59d98cc997c, []int{18}
 }

 func (m *ServiceHealth) XXX_Unmarshal(b []byte) error {
@ -833,7 +988,7 @@ func (m *ServiceStartRequest) Reset()         { *m = ServiceStartRequest{} }
 func (m *ServiceStartRequest) String() string { return proto.CompactTextString(m) }
 func (*ServiceStartRequest) ProtoMessage()    {}
 func (*ServiceStartRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{16}
+	return fileDescriptor_84b4f59d98cc997c, []int{19}
 }

 func (m *ServiceStartRequest) XXX_Unmarshal(b []byte) error {
@ -877,7 +1032,7 @@ func (m *ServiceStart) Reset()         { *m = ServiceStart{} }
 func (m *ServiceStart) String() string { return proto.CompactTextString(m) }
 func (*ServiceStart) ProtoMessage()    {}
 func (*ServiceStart) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{17}
+	return fileDescriptor_84b4f59d98cc997c, []int{20}
 }

 func (m *ServiceStart) XXX_Unmarshal(b []byte) error {
@ -927,7 +1082,7 @@ func (m *ServiceStartResponse) Reset()         { *m = ServiceStartResponse{} }
 func (m *ServiceStartResponse) String() string { return proto.CompactTextString(m) }
 func (*ServiceStartResponse) ProtoMessage()    {}
 func (*ServiceStartResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{18}
+	return fileDescriptor_84b4f59d98cc997c, []int{21}
 }

 func (m *ServiceStartResponse) XXX_Unmarshal(b []byte) error {
@ -970,7 +1125,7 @@ func (m *ServiceStopRequest) Reset()         { *m = ServiceStopRequest{} }
 func (m *ServiceStopRequest) String() string { return proto.CompactTextString(m) }
 func (*ServiceStopRequest) ProtoMessage()    {}
 func (*ServiceStopRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{19}
+	return fileDescriptor_84b4f59d98cc997c, []int{22}
 }

 func (m *ServiceStopRequest) XXX_Unmarshal(b []byte) error {
@ -1014,7 +1169,7 @@ func (m *ServiceStop) Reset()         { *m = ServiceStop{} }
 func (m *ServiceStop) String() string { return proto.CompactTextString(m) }
 func (*ServiceStop) ProtoMessage()    {}
 func (*ServiceStop) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{20}
+	return fileDescriptor_84b4f59d98cc997c, []int{23}
 }

 func (m *ServiceStop) XXX_Unmarshal(b []byte) error {
@ -1064,7 +1219,7 @@ func (m *ServiceStopResponse) Reset()         { *m = ServiceStopResponse{} }
 func (m *ServiceStopResponse) String() string { return proto.CompactTextString(m) }
 func (*ServiceStopResponse) ProtoMessage()    {}
 func (*ServiceStopResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{21}
+	return fileDescriptor_84b4f59d98cc997c, []int{24}
 }

 func (m *ServiceStopResponse) XXX_Unmarshal(b []byte) error {
@ -1107,7 +1262,7 @@ func (m *ServiceRestartRequest) Reset()         { *m = ServiceRestartRequest{} }
 func (m *ServiceRestartRequest) String() string { return proto.CompactTextString(m) }
 func (*ServiceRestartRequest) ProtoMessage()    {}
 func (*ServiceRestartRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{22}
+	return fileDescriptor_84b4f59d98cc997c, []int{25}
 }

 func (m *ServiceRestartRequest) XXX_Unmarshal(b []byte) error {
@ -1151,7 +1306,7 @@ func (m *ServiceRestart) Reset()         { *m = ServiceRestart{} }
 func (m *ServiceRestart) String() string { return proto.CompactTextString(m) }
 func (*ServiceRestart) ProtoMessage()    {}
 func (*ServiceRestart) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{23}
+	return fileDescriptor_84b4f59d98cc997c, []int{26}
 }

 func (m *ServiceRestart) XXX_Unmarshal(b []byte) error {
@ -1201,7 +1356,7 @@ func (m *ServiceRestartResponse) Reset()         { *m = ServiceRestartResponse{}
 func (m *ServiceRestartResponse) String() string { return proto.CompactTextString(m) }
 func (*ServiceRestartResponse) ProtoMessage()    {}
 func (*ServiceRestartResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{24}
+	return fileDescriptor_84b4f59d98cc997c, []int{27}
 }

 func (m *ServiceRestartResponse) XXX_Unmarshal(b []byte) error {
@ -1245,7 +1400,7 @@ func (m *StartRequest) Reset()         { *m = StartRequest{} }
 func (m *StartRequest) String() string { return proto.CompactTextString(m) }
 func (*StartRequest) ProtoMessage()    {}
 func (*StartRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{25}
+	return fileDescriptor_84b4f59d98cc997c, []int{28}
 }

 func (m *StartRequest) XXX_Unmarshal(b []byte) error {
@ -1289,7 +1444,7 @@ func (m *StartResponse) Reset()         { *m = StartResponse{} }
 func (m *StartResponse) String() string { return proto.CompactTextString(m) }
 func (*StartResponse) ProtoMessage()    {}
 func (*StartResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{26}
+	return fileDescriptor_84b4f59d98cc997c, []int{29}
 }

 func (m *StartResponse) XXX_Unmarshal(b []byte) error {
@ -1333,7 +1488,7 @@ func (m *StopRequest) Reset()         { *m = StopRequest{} }
 func (m *StopRequest) String() string { return proto.CompactTextString(m) }
 func (*StopRequest) ProtoMessage()    {}
 func (*StopRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{27}
+	return fileDescriptor_84b4f59d98cc997c, []int{30}
 }

 func (m *StopRequest) XXX_Unmarshal(b []byte) error {
@ -1377,7 +1532,7 @@ func (m *StopResponse) Reset()         { *m = StopResponse{} }
 func (m *StopResponse) String() string { return proto.CompactTextString(m) }
 func (*StopResponse) ProtoMessage()    {}
 func (*StopResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{28}
+	return fileDescriptor_84b4f59d98cc997c, []int{31}
 }

 func (m *StopResponse) XXX_Unmarshal(b []byte) error {
@ -1424,7 +1579,7 @@ func (m *CopyRequest) Reset()         { *m = CopyRequest{} }
 func (m *CopyRequest) String() string { return proto.CompactTextString(m) }
 func (*CopyRequest) ProtoMessage()    {}
 func (*CopyRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{29}
+	return fileDescriptor_84b4f59d98cc997c, []int{32}
 }

 func (m *CopyRequest) XXX_Unmarshal(b []byte) error {
@ -1475,7 +1630,7 @@ func (m *ListRequest) Reset()         { *m = ListRequest{} }
 func (m *ListRequest) String() string { return proto.CompactTextString(m) }
 func (*ListRequest) ProtoMessage()    {}
 func (*ListRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{30}
+	return fileDescriptor_84b4f59d98cc997c, []int{33}
 }

 func (m *ListRequest) XXX_Unmarshal(b []byte) error {
@ -1550,7 +1705,7 @@ func (m *FileInfo) Reset()         { *m = FileInfo{} }
 func (m *FileInfo) String() string { return proto.CompactTextString(m) }
 func (*FileInfo) ProtoMessage()    {}
 func (*FileInfo) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{31}
+	return fileDescriptor_84b4f59d98cc997c, []int{34}
 }

 func (m *FileInfo) XXX_Unmarshal(b []byte) error {
@ -1651,7 +1806,7 @@ func (m *Mounts) Reset()         { *m = Mounts{} }
 func (m *Mounts) String() string { return proto.CompactTextString(m) }
 func (*Mounts) ProtoMessage()    {}
 func (*Mounts) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{32}
+	return fileDescriptor_84b4f59d98cc997c, []int{35}
 }

 func (m *Mounts) XXX_Unmarshal(b []byte) error {
@ -1701,7 +1856,7 @@ func (m *MountsResponse) Reset()         { *m = MountsResponse{} }
 func (m *MountsResponse) String() string { return proto.CompactTextString(m) }
 func (*MountsResponse) ProtoMessage()    {}
 func (*MountsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{33}
+	return fileDescriptor_84b4f59d98cc997c, []int{36}
 }

 func (m *MountsResponse) XXX_Unmarshal(b []byte) error {
@ -1748,7 +1903,7 @@ func (m *MountStat) Reset()         { *m = MountStat{} }
 func (m *MountStat) String() string { return proto.CompactTextString(m) }
 func (*MountStat) ProtoMessage()    {}
 func (*MountStat) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{34}
+	return fileDescriptor_84b4f59d98cc997c, []int{37}
 }

 func (m *MountStat) XXX_Unmarshal(b []byte) error {
@ -1814,7 +1969,7 @@ func (m *Version) Reset()         { *m = Version{} }
 func (m *Version) String() string { return proto.CompactTextString(m) }
 func (*Version) ProtoMessage()    {}
 func (*Version) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{35}
+	return fileDescriptor_84b4f59d98cc997c, []int{38}
 }

 func (m *Version) XXX_Unmarshal(b []byte) error {
@ -1871,7 +2026,7 @@ func (m *VersionResponse) Reset()         { *m = VersionResponse{} }
 func (m *VersionResponse) String() string { return proto.CompactTextString(m) }
 func (*VersionResponse) ProtoMessage()    {}
 func (*VersionResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{36}
+	return fileDescriptor_84b4f59d98cc997c, []int{39}
 }

 func (m *VersionResponse) XXX_Unmarshal(b []byte) error {
@ -1919,7 +2074,7 @@ func (m *VersionInfo) Reset()         { *m = VersionInfo{} }
 func (m *VersionInfo) String() string { return proto.CompactTextString(m) }
 func (*VersionInfo) ProtoMessage()    {}
 func (*VersionInfo) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{37}
+	return fileDescriptor_84b4f59d98cc997c, []int{40}
 }

 func (m *VersionInfo) XXX_Unmarshal(b []byte) error {
@ -1998,7 +2153,7 @@ func (m *PlatformInfo) Reset()         { *m = PlatformInfo{} }
 func (m *PlatformInfo) String() string { return proto.CompactTextString(m) }
 func (*PlatformInfo) ProtoMessage()    {}
 func (*PlatformInfo) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{38}
+	return fileDescriptor_84b4f59d98cc997c, []int{41}
 }

 func (m *PlatformInfo) XXX_Unmarshal(b []byte) error {
@ -2055,7 +2210,7 @@ func (m *LogsRequest) Reset()         { *m = LogsRequest{} }
 func (m *LogsRequest) String() string { return proto.CompactTextString(m) }
 func (*LogsRequest) ProtoMessage()    {}
 func (*LogsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{39}
+	return fileDescriptor_84b4f59d98cc997c, []int{42}
 }

 func (m *LogsRequest) XXX_Unmarshal(b []byte) error {
@ -2126,7 +2281,7 @@ func (m *ReadRequest) Reset()         { *m = ReadRequest{} }
 func (m *ReadRequest) String() string { return proto.CompactTextString(m) }
 func (*ReadRequest) ProtoMessage()    {}
 func (*ReadRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_84b4f59d98cc997c, []int{40}
+	return fileDescriptor_84b4f59d98cc997c, []int{43}
 }

 func (m *ReadRequest) XXX_Unmarshal(b []byte) error {
@ -2159,11 +2314,15 @@ func (m *ReadRequest) GetPath() string {
 }

 func init() {
+	proto.RegisterEnum("machine.RecoverRequest_Source", RecoverRequest_Source_name, RecoverRequest_Source_value)
 	proto.RegisterType((*Reboot)(nil), "machine.Reboot")
 	proto.RegisterType((*RebootResponse)(nil), "machine.RebootResponse")
 	proto.RegisterType((*ResetRequest)(nil), "machine.ResetRequest")
 	proto.RegisterType((*Reset)(nil), "machine.Reset")
 	proto.RegisterType((*ResetResponse)(nil), "machine.ResetResponse")
+	proto.RegisterType((*RecoverRequest)(nil), "machine.RecoverRequest")
+	proto.RegisterType((*Recover)(nil), "machine.Recover")
+	proto.RegisterType((*RecoverResponse)(nil), "machine.RecoverResponse")
 	proto.RegisterType((*Shutdown)(nil), "machine.Shutdown")
 	proto.RegisterType((*ShutdownResponse)(nil), "machine.ShutdownResponse")
 	proto.RegisterType((*UpgradeRequest)(nil), "machine.UpgradeRequest")
@ -2205,100 +2364,106 @@ func init() {
 func init() { proto.RegisterFile("machine/machine.proto", fileDescriptor_84b4f59d98cc997c) }

 var fileDescriptor_84b4f59d98cc997c = []byte{
-	// 1484 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x73, 0xdb, 0x44,
-	0x14, 0x1e, 0x39, 0x89, 0x2f, 0xc7, 0x89, 0x13, 0xd4, 0x26, 0x11, 0x69, 0x7a, 0x13, 0x97, 0x76,
-	0x42, 0xeb, 0xa4, 0x29, 0x74, 0x80, 0x52, 0x98, 0x36, 0x29, 0x6d, 0xa7, 0x0d, 0x0d, 0x0a, 0xf0,
-	0xc0, 0x8b, 0x59, 0x5b, 0x6b, 0x79, 0xa7, 0x92, 0x56, 0x68, 0xd7, 0xe9, 0x84, 0xe1, 0x17, 0xf0,
-	0xca, 0x5b, 0x5f, 0xf9, 0x8f, 0x3c, 0x33, 0x7b, 0x95, 0x62, 0xd9, 0xa5, 0x9e, 0xe9, 0x93, 0x77,
-	0xcf, 0xf9, 0xf6, 0xdc, 0x75, 0xf6, 0xac, 0x61, 0x3d, 0x41, 0x83, 0x11, 0x49, 0xf1, 0xae, 0xfe,
-	0xed, 0x66, 0x39, 0xe5, 0xd4, 0x6d, 0xe8, 0xed, 0xd6, 0xa5, 0x88, 0xd2, 0x28, 0xc6, 0xbb, 0x92,
-	0xdc, 0x1f, 0x0f, 0x77, 0x71, 0x92, 0xf1, 0x33, 0x85, 0xda, 0xba, 0x3a, 0xc9, 0xe4, 0x24, 0xc1,
-	0x8c, 0xa3, 0x24, 0xd3, 0x80, 0x0b, 0x03, 0x9a, 0x24, 0x34, 0xdd, 0x55, 0x3f, 0x8a, 0xe8, 0xdf,
-	0x83, 0x7a, 0x80, 0xfb, 0x94, 0x72, 0xf7, 0x16, 0x34, 0x13, 0xcc, 0x51, 0x88, 0x38, 0xf2, 0x9c,
-	0x6b, 0xce, 0xcd, 0xf6, 0xfe, 0x5a, 0x57, 0x43, 0x8f, 0x34, 0x3d, 0xb0, 0x08, 0xff, 0x01, 0x74,
-	0xd4, 0xb9, 0x00, 0xb3, 0x8c, 0xa6, 0x0c, 0xbb, 0x9f, 0x89, 0xf3, 0x8c, 0xa1, 0x08, 0x33, 0xcf,
-	0xb9, 0xb6, 0x70, 0xb3, 0xbd, 0xbf, 0xda, 0x35, 0x7e, 0x68, 0xa8, 0x05, 0xf8, 0x8f, 0x60, 0x39,
-	0xc0, 0x0c, 0xf3, 0x00, 0xff, 0x3e, 0xc6, 0x8c, 0xbb, 0x5b, 0xd0, 0x8c, 0x72, 0x34, 0xc0, 0xc3,
-	0x71, 0x2c, 0x95, 0x37, 0x03, 0xbb, 0x77, 0x37, 0xa0, 0x9e, 0xcb, 0xf3, 0x5e, 0x4d, 0x72, 0xf4,
-	0xce, 0xff, 0x02, 0x96, 0xa4, 0x8c, 0x39, 0x2d, 0xbf, 0x0f, 0x2b, 0x5a, 0xb5, 0x36, 0x7c, 0xa7,
-	0x62, 0x78, 0xa7, 0x64, 0xb8, 0x40, 0x16, 0x76, 0x7f, 0x09, 0xcd, 0x93, 0xd1, 0x98, 0x87, 0xf4,
-	0x75, 0x3a, 0xa7, 0xda, 0x87, 0xb0, 0x66, 0x4e, 0x5a, 0xcd, 0xb7, 0x2b, 0x9a, 0x3f, 0xb0, 0x9a,
-	0x2d, 0xb8, 0x1c, 0xb4, 0xce, 0xcf, 0x59, 0x94, 0xa3, 0x10, 0x9b, 0xb0, 0x5d, 0x84, 0x25, 0x92,
-	0xa0, 0x08, 0x4b, 0xfd, 0xad, 0x40, 0x6d, 0x44, 0x30, 0xb3, 0x1c, 0x33, 0x9c, 0x9f, 0x62, 0x1d,
-	0x32, 0xbb, 0xf7, 0x9f, 0x41, 0x43, 0xcb, 0x98, 0xcf, 0x7e, 0x77, 0x0d, 0x16, 0xd0, 0xe0, 0x95,
-	0x94, 0xd7, 0x0a, 0xc4, 0xd2, 0xff, 0x0e, 0x56, 0xad, 0x39, 0xda, 0xa1, 0x5b, 0x15, 0x87, 0xd6,
-	0xac, 0x43, 0x06, 0x5b, 0xf8, 0x93, 0x40, 0xfb, 0x04, 0xe7, 0xa7, 0x64, 0x80, 0x5f, 0x10, 0x36,
-	0x67, 0x1a, 0xdd, 0x3d, 0x68, 0x32, 0x75, 0x98, 0x79, 0x35, 0xa9, 0xea, 0x62, 0x11, 0x3b, 0xc5,
-	0x78, 0x96, 0x0e, 0x69, 0x60, 0x51, 0xfe, 0x13, 0xb8, 0x50, 0x52, 0x67, 0x6d, 0xde, 0xab, 0xd8,
-	0x5c, 0x11, 0x24, 0xf1, 0x85, 0xdd, 0x7f, 0x3b, 0xd6, 0x70, 0xa1, 0xc2, 0xed, 0x40, 0x8d, 0x84,
-	0x3a, 0x05, 0x35, 0x12, 0x8a, 0xac, 0x30, 0x8e, 0x38, 0xd6, 0xc1, 0x52, 0x1b, 0xb7, 0x0b, 0x75,
-	0x7c, 0x8a, 0x53, 0xce, 0xbc, 0x05, 0xe9, 0xdc, 0xc6, 0xa4, 0x96, 0xc7, 0x92, 0x1b, 0x68, 0x94,
-	0xc0, 0x8f, 0x30, 0x8a, 0xf9, 0xc8, 0x5b, 0x9c, 0x8e, 0x7f, 0x2a, 0xb9, 0x81, 0x46, 0xf9, 0xdf,
-	0xc2, 0xca, 0x39, 0x41, 0xee, 0x6d, 0xab, 0x50, 0xb9, 0xb5, 0x3e, 0x55, 0xa1, 0xd1, 0xe7, 0xf7,
-	0x61, 0xb9, 0x4c, 0x17, 0x09, 0x4f, 0x58, 0xa4, 0xdd, 0x12, 0xcb, 0x19, 0x7e, 0xed, 0x40, 0xcd,
-	0xfa, 0xb4, 0xd5, 0x55, 0x4d, 0xa8, 0x6b, 0x9a, 0x50, 0xf7, 0x27, 0xd3, 0x84, 0x82, 0x1a, 0x67,
-	0xfe, 0x3f, 0x8e, 0x35, 0x52, 0x59, 0xef, 0x7a, 0xd0, 0x18, 0xa7, 0xaf, 0x52, 0xfa, 0x3a, 0xd5,
-	0xdf, 0xbd, 0xd9, 0x0a, 0x8e, 0xf2, 0xec, 0x4c, 0x17, 0xb1, 0xd9, 0xba, 0xd7, 0x61, 0x39, 0x46,
-	0x8c, 0xf7, 0x74, 0x42, 0xa4, 0xee, 0x56, 0xd0, 0x16, 0xb4, 0x23, 0x45, 0x72, 0xef, 0x83, 0xdc,
-	0xf6, 0x06, 0x23, 0x94, 0x46, 0x58, 0x47, 0xf0, 0x6d, 0xd6, 0x81, 0x80, 0x1f, 0x48, 0xb4, 0xff,
-	0x89, 0x2d, 0x94, 0x13, 0x8e, 0x72, 0xdb, 0xa3, 0x26, 0xd2, 0xec, 0x1f, 0xdb, 0x80, 0x49, 0xd8,
-	0x9c, 0xf5, 0xeb, 0xc2, 0x62, 0x8e, 0x59, 0xa6, 0x63, 0x29, 0xd7, 0xfe, 0x33, 0xb8, 0x78, 0x5e,
-	0xb1, 0x2e, 0xd1, 0x3b, 0x95, 0x12, 0xad, 0xe4, 0x52, 0x1d, 0x28, 0x6a, 0xf4, 0x63, 0x70, 0x2d,
-	0x87, 0x66, 0xb3, 0x5c, 0x78, 0x69, 0x0b, 0x59, 0xa0, 0xde, 0x83, 0x07, 0x4f, 0x4a, 0xa1, 0x13,
-	0x6a, 0xdf, 0xfd, 0x1b, 0x93, 0xf8, 0xc2, 0xfe, 0x1b, 0xb0, 0xae, 0x19, 0x81, 0xc8, 0xd0, 0xec,
-	0x2c, 0x04, 0xd0, 0x39, 0x0f, 0x7c, 0x0f, 0x5e, 0x1c, 0xc1, 0xc6, 0xa4, 0x72, 0xed, 0xc8, 0xdd,
-	0x8a, 0x23, 0x9b, 0x93, 0x8e, 0x98, 0x23, 0x85, 0x2f, 0x3e, 0x2c, 0xbf, 0xad, 0x90, 0xbe, 0xae,
-	0x79, 0x8e, 0x7f, 0x03, 0x56, 0xce, 0xe7, 0xdc, 0xd8, 0xe5, 0x14, 0x76, 0x49, 0xe0, 0x75, 0x68,
-	0xbf, 0x25, 0xa3, 0x12, 0xf2, 0xa9, 0xd0, 0x57, 0x8a, 0xfe, 0x2c, 0x51, 0x3b, 0xd0, 0x3e, 0xa0,
-	0xd9, 0x99, 0x11, 0x75, 0x09, 0x5a, 0x39, 0xa5, 0xbc, 0x97, 0x21, 0x3e, 0xd2, 0xd8, 0xa6, 0x20,
-	0x1c, 0x23, 0x3e, 0xf2, 0x43, 0x68, 0xab, 0xae, 0xa9, 0xb0, 0x42, 0xa4, 0xb8, 0x91, 0x8d, 0x48,
-	0x31, 0x40, 0x78, 0xd0, 0xc8, 0xf1, 0x60, 0x9c, 0x33, 0x73, 0xeb, 0x98, 0xad, 0x7b, 0x03, 0x56,
-	0xd5, 0x92, 0xd0, 0xb4, 0x17, 0xe2, 0x8c, 0x8f, 0xe4, 0x37, 0xbb, 0x14, 0x74, 0x2c, 0xf9, 0x50,
-	0x50, 0xfd, 0x7f, 0x1d, 0x68, 0x7e, 0x4f, 0x62, 0xd5, 0x56, 0xe7, 0xce, 0x63, 0x8a, 0x12, 0xd3,
-	0x9b, 0xe4, 0x5a, 0xd0, 0x18, 0xf9, 0x43, 0x35, 0x88, 0x85, 0x40, 0xae, 0x05, 0x2d, 0xa1, 0xa1,
-	0x6a, 0x09, 0x2b, 0x81, 0x5c, 0x8b, 0x0b, 0x33, 0xa1, 0x21, 0x19, 0x12, 0x1c, 0x7a, 0x4b, 0x12,
-	0x6b, 0xf7, 0xee, 0x3a, 0xd4, 0x09, 0xeb, 0x85, 0x24, 0xf7, 0xea, 0xd2, 0xa9, 0x25, 0xc2, 0x0e,
-	0x49, 0x2e, 0x7a, 0x21, 0xce, 0x73, 0x9a, 0x7b, 0x0d, 0xd5, 0x0b, 0xe5, 0x46, 0x08, 0x8f, 0x49,
-	0xfa, 0xca, 0x6b, 0x2a, 0x23, 0xc4, 0xda, 0xfd, 0x08, 0x56, 0x72, 0x1c, 0x23, 0x4e, 0x4e, 0x71,
-	0x4f, 0x5a, 0xd8, 0x92, 0xcc, 0x65, 0x43, 0xfc, 0x01, 0x25, 0xd8, 0xff, 0x0d, 0xea, 0x47, 0x74,
-	0x2c, 0xba, 0xf6, 0x7c, 0x5e, 0xdf, 0x54, 0x2d, 0xd9, 0x5c, 0x81, 0xae, 0x2d, 0x46, 0x29, 0xed,
-	0x84, 0x23, 0xae, 0xda, 0x34, 0x13, 0x03, 0x9b, 0xd2, 0xf0, 0x4e, 0x03, 0x9b, 0x86, 0x16, 0x35,
-	0xfc, 0x27, 0xb4, 0xac, 0x48, 0xf7, 0x0a, 0xc0, 0x90, 0xc4, 0x98, 0x9d, 0x31, 0x8e, 0x13, 0x5d,
-	0x03, 0x25, 0x8a, 0x8d, 0xbb, 0xc8, 0xc5, 0xa2, 0x8e, 0xfb, 0x36, 0xb4, 0xd0, 0x29, 0x22, 0x31,
-	0xea, 0xc7, 0x2a, 0x21, 0x8b, 0x41, 0x41, 0x70, 0x2f, 0x03, 0x24, 0x42, 0x3c, 0x0e, 0x7b, 0x34,
-	0x95, 0xb9, 0x69, 0x05, 0x2d, 0x4d, 0x79, 0x99, 0xfa, 0x6f, 0x1c, 0x68, 0xfc, 0x82, 0x65, 0xa1,
-	0xcc, 0x19, 0xa0, 0x2e, 0x34, 0x4e, 0xd5, 0x41, 0x69, 0x4d, 0xb9, 0xf1, 0x68, 0x81, 0x72, 0x4a,
-	0x30, 0x20, 0xd1, 0x6a, 0xb3, 0x18, 0xf1, 0x21, 0xcd, 0x13, 0x7d, 0xa7, 0x15, 0xad, 0xf6, 0x58,
-	0x33, 0xd4, 0x5c, 0x61, 0x60, 0x62, 0x0e, 0xd2, 0xa2, 0xde, 0x69, 0x0e, 0x32, 0xd8, 0x22, 0xb6,
-	0x7f, 0x39, 0xd0, 0x2e, 0x19, 0x23, 0x6e, 0x5e, 0x8e, 0xec, 0xcd, 0xcb, 0x51, 0x24, 0x28, 0x6c,
-	0x84, 0xcc, 0xf0, 0xc5, 0x46, 0x48, 0xd4, 0x5f, 0x7f, 0x4c, 0x62, 0xae, 0x2f, 0x3f, 0xb5, 0x11,
-	0x61, 0x8c, 0x68, 0xcf, 0x38, 0xac, 0xc3, 0x18, 0x51, 0x13, 0xba, 0x0e, 0xd4, 0x28, 0x93, 0x15,
-	0xde, 0x0a, 0x6a, 0x94, 0x89, 0x3c, 0xa1, 0x7c, 0x30, 0x92, 0x95, 0xdd, 0x0a, 0xe4, 0xda, 0xbf,
-	0x07, 0xcb, 0x65, 0x3f, 0xed, 0x77, 0xe5, 0x9c, 0xff, 0xae, 0xe4, 0x37, 0xa4, 0xbf, 0x35, 0xb1,
-	0x16, 0x57, 0x7b, 0xfb, 0x05, 0x8d, 0x98, 0xe9, 0x10, 0xdb, 0xd0, 0x12, 0x58, 0x96, 0xa1, 0x81,
-	0x39, 0x5c, 0x10, 0x74, 0xdb, 0xaa, 0xd9, 0x91, 0x69, 0x17, 0xea, 0x61, 0x4e, 0x4e, 0x71, 0x2e,
-	0xfd, 0xe9, 0xec, 0x6f, 0x9a, 0x94, 0x1e, 0xd0, 0x94, 0x23, 0x92, 0xe2, 0xfc, 0x50, 0xb2, 0x03,
-	0x0d, 0x13, 0x8f, 0x82, 0x21, 0x8d, 0x63, 0xfa, 0x5a, 0x7a, 0xd9, 0x0c, 0xf4, 0x4e, 0x44, 0x80,
-	0x23, 0x12, 0xf7, 0x62, 0x92, 0x62, 0xe5, 0xea, 0x52, 0xd0, 0x12, 0x94, 0x17, 0x82, 0x20, 0xba,
-	0x67, 0x80, 0x51, 0x58, 0x6a, 0x63, 0xa5, 0x6e, 0x27, 0xd7, 0xfb, 0x6f, 0x1a, 0xd0, 0x39, 0x52,
-	0xb9, 0xd2, 0x1d, 0xdd, 0xbd, 0x05, 0x8b, 0xa2, 0x51, 0xba, 0x45, 0xed, 0x94, 0xfa, 0xe6, 0xd6,
-	0xb2, 0xb1, 0xf5, 0x10, 0x71, 0xb4, 0xe7, 0xb8, 0x9f, 0x03, 0x3c, 0x1f, 0xf7, 0xf1, 0x80, 0xa6,
-	0x43, 0x12, 0xb9, 0x1b, 0x95, 0xa1, 0xe3, 0xb1, 0x78, 0xb4, 0x55, 0x4e, 0xdd, 0x81, 0x45, 0x39,
-	0x05, 0x17, 0x3a, 0x4a, 0xfd, 0x76, 0xab, 0x78, 0x17, 0x98, 0xf6, 0xb8, 0xe7, 0x08, 0xb3, 0x44,
-	0xc4, 0xcb, 0x47, 0x8a, 0x04, 0x54, 0x14, 0x7c, 0x65, 0x5b, 0xcc, 0x2c, 0x93, 0x36, 0x27, 0x3f,
-	0xff, 0xa2, 0x9c, 0x17, 0x45, 0xd4, 0x4a, 0x8a, 0x4a, 0x41, 0x9c, 0xa6, 0x48, 0x3f, 0x29, 0xff,
-	0x5f, 0xd1, 0xc4, 0x1b, 0xf2, 0x9e, 0x79, 0xd2, 0xad, 0x4f, 0xbc, 0xc0, 0xb4, 0xaa, 0x8d, 0x49,
-	0xb2, 0x3e, 0x77, 0x70, 0xfe, 0x25, 0x31, 0x4b, 0xef, 0xf6, 0xd4, 0xc1, 0xde, 0x08, 0xf9, 0xb1,
-	0x32, 0x49, 0x5c, 0x99, 0x75, 0xb7, 0x6b, 0x73, 0xae, 0xce, 0xe4, 0x6b, 0x91, 0xcf, 0x27, 0x46,
-	0xc4, 0xed, 0xe9, 0x63, 0x9b, 0x16, 0x77, 0x79, 0x06, 0x57, 0x0b, 0x7b, 0x7a, 0x7e, 0x58, 0xbb,
-	0x34, 0x75, 0x82, 0xd2, 0xa2, 0xb6, 0xa7, 0x33, 0xb5, 0xa4, 0x07, 0xa5, 0x57, 0xec, 0xac, 0x58,
-	0x7d, 0x58, 0x7d, 0x89, 0x9a, 0xe3, 0xdf, 0x14, 0x6f, 0xc8, 0xcd, 0xca, 0xf3, 0x4e, 0x1b, 0xe0,
-	0x55, 0x19, 0xfa, 0xf4, 0xfd, 0xa2, 0x95, 0xcf, 0xd2, 0xed, 0x55, 0x9a, 0xa5, 0x3e, 0xfc, 0xe8,
-	0x39, 0xac, 0x0e, 0x68, 0x62, 0xd9, 0x28, 0x23, 0x8f, 0x40, 0x7f, 0xac, 0x0f, 0x33, 0x72, 0xec,
-	0xfc, 0xba, 0x13, 0x11, 0x3e, 0x1a, 0xf7, 0x45, 0x49, 0xee, 0x72, 0x14, 0x53, 0x76, 0x5b, 0xdd,
-	0x49, 0x4c, 0xed, 0x76, 0x51, 0x46, 0xcc, 0xbf, 0x2b, 0xfd, 0xba, 0x54, 0x7b, 0xf7, 0xbf, 0x00,
-	0x00, 0x00, 0xff, 0xff, 0xe9, 0x7e, 0xca, 0xfb, 0x77, 0x11, 0x00, 0x00,
+	// 1573 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4b, 0x53, 0x1b, 0xc7,
+	0x16, 0xbe, 0x23, 0x84, 0x1e, 0x47, 0x20, 0xb8, 0x63, 0x03, 0x73, 0x31, 0x7e, 0xcd, 0x4d, 0x62,
+	0x17, 0xb1, 0x05, 0xc6, 0x09, 0x79, 0x38, 0x4e, 0xca, 0x06, 0x62, 0x53, 0x36, 0x31, 0x19, 0x1c,
+	0x2f, 0xb2, 0x51, 0x5a, 0x52, 0x4b, 0xea, 0x62, 0x66, 0x7a, 0x32, 0xdd, 0x92, 0x8b, 0x54, 0x7e,
+	0x41, 0xb6, 0xd9, 0x65, 0x9b, 0x3f, 0x95, 0x5f, 0x92, 0x75, 0xaa, 0x9f, 0x33, 0x68, 0x90, 0x83,
+	0xaa, 0xbc, 0x52, 0xf7, 0xe9, 0xaf, 0xcf, 0x7b, 0x4e, 0x9f, 0x23, 0x58, 0x89, 0x50, 0x77, 0x48,
+	0x62, 0xbc, 0xa5, 0x7f, 0x5b, 0x49, 0x4a, 0x39, 0x75, 0xab, 0x7a, 0xbb, 0x7e, 0x6d, 0x40, 0xe9,
+	0x20, 0xc4, 0x5b, 0x92, 0xdc, 0x19, 0xf5, 0xb7, 0x70, 0x94, 0xf0, 0x33, 0x85, 0x5a, 0xbf, 0x39,
+	0x79, 0xc8, 0x49, 0x84, 0x19, 0x47, 0x51, 0xa2, 0x01, 0x57, 0xba, 0x34, 0x8a, 0x68, 0xbc, 0xa5,
+	0x7e, 0x14, 0xd1, 0xdf, 0x85, 0x4a, 0x80, 0x3b, 0x94, 0x72, 0xf7, 0x1e, 0xd4, 0x22, 0xcc, 0x51,
+	0x0f, 0x71, 0xe4, 0x39, 0xb7, 0x9c, 0xbb, 0x8d, 0x9d, 0xe5, 0x96, 0x86, 0x1e, 0x69, 0x7a, 0x60,
+	0x11, 0xfe, 0x63, 0x68, 0xaa, 0x7b, 0x01, 0x66, 0x09, 0x8d, 0x19, 0x76, 0x3f, 0x16, 0xf7, 0x19,
+	0x43, 0x03, 0xcc, 0x3c, 0xe7, 0xd6, 0xdc, 0xdd, 0xc6, 0xce, 0x52, 0xcb, 0xd8, 0xa1, 0xa1, 0x16,
+	0xe0, 0x3f, 0x85, 0x85, 0x00, 0x33, 0xcc, 0x03, 0xfc, 0xf3, 0x08, 0x33, 0xee, 0xae, 0x43, 0x6d,
+	0x90, 0xa2, 0x2e, 0xee, 0x8f, 0x42, 0x29, 0xbc, 0x16, 0xd8, 0xbd, 0xbb, 0x0a, 0x95, 0x54, 0xde,
+	0xf7, 0x4a, 0xf2, 0x44, 0xef, 0xfc, 0x4f, 0x61, 0x5e, 0xf2, 0x98, 0x51, 0xf3, 0x47, 0xb0, 0xa8,
+	0x45, 0x6b, 0xc5, 0x37, 0x0b, 0x8a, 0x37, 0x73, 0x8a, 0x0b, 0x64, 0xa6, 0xf7, 0xa9, 0x30, 0xbb,
+	0x4b, 0xc7, 0x38, 0x35, 0x9a, 0xef, 0x42, 0x85, 0xd1, 0x51, 0xda, 0xc5, 0x52, 0x74, 0x73, 0xe7,
+	0x46, 0xee, 0x6e, 0x1e, 0xd8, 0x3a, 0x91, 0xa8, 0x40, 0xa3, 0xfd, 0xdb, 0x50, 0x51, 0x14, 0xb7,
+	0x06, 0xe5, 0x83, 0xd7, 0x7b, 0xfb, 0xcb, 0xff, 0x71, 0x17, 0xa1, 0xfe, 0xe4, 0xf8, 0xf0, 0xe4,
+	0x20, 0x78, 0x73, 0x10, 0x2c, 0x3b, 0xfe, 0x67, 0x50, 0xd5, 0x3c, 0x66, 0x34, 0xf1, 0x1b, 0x58,
+	0xb2, 0xc2, 0xb5, 0x91, 0xf7, 0x0a, 0x46, 0x2e, 0x17, 0x14, 0xcd, 0xcc, 0xfc, 0x1c, 0x6a, 0x27,
+	0xc3, 0x11, 0xef, 0xd1, 0xb7, 0xf1, 0x8c, 0xa2, 0x9f, 0xc0, 0xb2, 0xb9, 0x69, 0x65, 0xdf, 0x2f,
+	0xc8, 0xfe, 0xaf, 0x95, 0x6d, 0xc1, 0xf9, 0xdc, 0x68, 0xfe, 0x90, 0x0c, 0x52, 0xd4, 0xc3, 0xc6,
+	0xc7, 0x57, 0x61, 0x9e, 0x44, 0x68, 0xa0, 0x5c, 0x5c, 0x0f, 0xd4, 0x46, 0xe4, 0x4c, 0x92, 0x62,
+	0x86, 0xd3, 0x31, 0xd6, 0x99, 0x61, 0xf7, 0xfe, 0x21, 0x54, 0x35, 0x8f, 0xd9, 0xf4, 0x77, 0x97,
+	0x61, 0x0e, 0x75, 0x4f, 0x25, 0xbf, 0x7a, 0x20, 0x96, 0xc2, 0x99, 0x56, 0x9d, 0x4b, 0x38, 0xd3,
+	0x60, 0x33, 0x7b, 0x22, 0x68, 0x9c, 0xe0, 0x74, 0x4c, 0xba, 0xf8, 0x25, 0x61, 0x33, 0x66, 0xab,
+	0xbb, 0x0d, 0x35, 0xa6, 0x2e, 0x33, 0xaf, 0x24, 0x45, 0x5d, 0xcd, 0x7c, 0xa7, 0x0e, 0x0e, 0xe3,
+	0x3e, 0x0d, 0x2c, 0xca, 0x7f, 0x06, 0x57, 0x72, 0xe2, 0xac, 0xce, 0xdb, 0x05, 0x9d, 0x0b, 0x8c,
+	0x24, 0x3e, 0xd3, 0xfb, 0x77, 0xc7, 0x2a, 0x2e, 0x44, 0xb8, 0x4d, 0x28, 0x91, 0x9e, 0x0e, 0x41,
+	0x89, 0xf4, 0x44, 0x54, 0x18, 0x47, 0x1c, 0x6b, 0x67, 0xa9, 0x8d, 0xdb, 0x82, 0x0a, 0x1e, 0xe3,
+	0x98, 0x33, 0x6f, 0x4e, 0x1a, 0xb7, 0x3a, 0x29, 0xe5, 0x40, 0x9e, 0x06, 0x1a, 0x25, 0xf0, 0x43,
+	0x8c, 0x42, 0x3e, 0xf4, 0xca, 0x17, 0xe3, 0x9f, 0xcb, 0xd3, 0x40, 0xa3, 0xfc, 0xaf, 0x61, 0xf1,
+	0x1c, 0x23, 0xf7, 0xbe, 0x15, 0xa8, 0xcc, 0x5a, 0xb9, 0x50, 0xa0, 0x91, 0xe7, 0x77, 0x60, 0x21,
+	0x4f, 0x17, 0x01, 0x8f, 0xd8, 0x40, 0x9b, 0x25, 0x96, 0x53, 0xec, 0xda, 0x84, 0x92, 0xb5, 0x69,
+	0xbd, 0xa5, 0x6a, 0x6d, 0xcb, 0xd4, 0xda, 0xd6, 0x6b, 0x53, 0x6b, 0x83, 0x12, 0x67, 0xfe, 0x9f,
+	0x8e, 0x55, 0x52, 0x69, 0xef, 0x7a, 0x50, 0x1d, 0xc5, 0xa7, 0x31, 0x7d, 0x1b, 0xeb, 0xf2, 0x66,
+	0xb6, 0xe2, 0x44, 0x59, 0x76, 0xa6, 0x93, 0xd8, 0x6c, 0xdd, 0xdb, 0xb0, 0x10, 0x22, 0xc6, 0xdb,
+	0x3a, 0x20, 0x52, 0x76, 0x3d, 0x68, 0x08, 0xda, 0x91, 0x22, 0xb9, 0x8f, 0x40, 0x6e, 0xdb, 0xdd,
+	0x21, 0x8a, 0x07, 0x58, 0x7b, 0xf0, 0x5d, 0xda, 0x81, 0x80, 0xef, 0x49, 0xb4, 0xff, 0xa1, 0x4d,
+	0x94, 0x13, 0x8e, 0x52, 0x5b, 0x8a, 0x27, 0xc2, 0xec, 0x1f, 0x5b, 0x87, 0x49, 0xd8, 0x8c, 0xf9,
+	0xeb, 0x42, 0x39, 0xc5, 0x2c, 0xd1, 0xbe, 0x94, 0x6b, 0xff, 0x10, 0xae, 0x9e, 0x17, 0xac, 0x53,
+	0xf4, 0x41, 0x21, 0x45, 0x0b, 0xb1, 0x54, 0x17, 0xb2, 0x1c, 0xfd, 0x00, 0x5c, 0x7b, 0x42, 0x93,
+	0x69, 0x26, 0xbc, 0xb2, 0x89, 0x2c, 0x50, 0xef, 0xc1, 0x82, 0x67, 0x39, 0xd7, 0x09, 0xb1, 0x97,
+	0xff, 0xc6, 0x24, 0x3e, 0xd3, 0xff, 0x0e, 0xac, 0xe8, 0x83, 0x40, 0x44, 0x68, 0x7a, 0x14, 0x02,
+	0x68, 0x9e, 0x07, 0xbe, 0x07, 0x2b, 0x8e, 0x60, 0x75, 0x52, 0xb8, 0x36, 0xe4, 0x61, 0xc1, 0x90,
+	0xb5, 0x49, 0x43, 0xcc, 0x95, 0xcc, 0x16, 0x1f, 0x16, 0xde, 0x95, 0x48, 0x5f, 0x96, 0x3c, 0xc7,
+	0xbf, 0x03, 0x8b, 0xe7, 0x63, 0x6e, 0xf4, 0x72, 0x32, 0xbd, 0x24, 0xf0, 0x36, 0x34, 0xde, 0x11,
+	0x51, 0x09, 0xf9, 0x48, 0xc8, 0xcb, 0x79, 0x7f, 0x1a, 0xab, 0x4d, 0x68, 0xec, 0xd1, 0xe4, 0xcc,
+	0xb0, 0xba, 0x06, 0xf5, 0x94, 0x52, 0xde, 0x4e, 0x10, 0x1f, 0x6a, 0x6c, 0x4d, 0x10, 0x8e, 0x11,
+	0x1f, 0xfa, 0x3d, 0x68, 0xa8, 0xaa, 0xa9, 0xb0, 0x82, 0xa5, 0x68, 0x3c, 0x0c, 0x4b, 0xd1, 0x27,
+	0x79, 0x50, 0x4d, 0x71, 0x77, 0x94, 0x32, 0xf3, 0xea, 0x98, 0xad, 0x7b, 0x07, 0x96, 0xd4, 0x92,
+	0xd0, 0xb8, 0xdd, 0xc3, 0x09, 0x1f, 0xca, 0x6f, 0x76, 0x3e, 0x68, 0x5a, 0xf2, 0xbe, 0xa0, 0xfa,
+	0x7f, 0x3b, 0x50, 0xfb, 0x96, 0x84, 0xaa, 0xac, 0xce, 0x1c, 0xc7, 0x18, 0x45, 0xa6, 0x36, 0xc9,
+	0xb5, 0xa0, 0x31, 0xf2, 0x8b, 0x2a, 0x10, 0x73, 0x81, 0x5c, 0x0b, 0x5a, 0x44, 0x7b, 0xaa, 0x24,
+	0x2c, 0x06, 0x72, 0x2d, 0x1e, 0xcc, 0x88, 0xf6, 0x48, 0x9f, 0xe0, 0x9e, 0x37, 0x2f, 0xb1, 0x76,
+	0xef, 0xae, 0x40, 0x85, 0xb0, 0x76, 0x8f, 0xa4, 0x5e, 0x45, 0x1a, 0x35, 0x4f, 0xd8, 0x3e, 0x49,
+	0x45, 0x2d, 0xc4, 0x69, 0x4a, 0x53, 0xaf, 0xaa, 0x6a, 0xa1, 0xdc, 0x08, 0xe6, 0x21, 0x89, 0x4f,
+	0xbd, 0x9a, 0x52, 0x42, 0xac, 0xdd, 0xff, 0xc3, 0x62, 0x8a, 0x43, 0xc4, 0xc9, 0x18, 0xb7, 0xa5,
+	0x86, 0x75, 0x79, 0xb8, 0x60, 0x88, 0xdf, 0xa1, 0x08, 0xfb, 0x3f, 0x41, 0xe5, 0x88, 0x8e, 0x44,
+	0xd5, 0x9e, 0xcd, 0xea, 0xbb, 0xaa, 0x24, 0x9b, 0x27, 0xd0, 0xb5, 0xc9, 0x28, 0xb9, 0x9d, 0x70,
+	0xc4, 0x55, 0x99, 0x66, 0xa2, 0x2f, 0x55, 0x12, 0x2e, 0xd5, 0x97, 0x6a, 0x68, 0x96, 0xc3, 0xbf,
+	0x42, 0xdd, 0xb2, 0x74, 0x6f, 0x00, 0xf4, 0x49, 0x88, 0xd9, 0x19, 0xe3, 0x38, 0xd2, 0x39, 0x90,
+	0xa3, 0x58, 0xbf, 0x8b, 0x58, 0x94, 0xb5, 0xdf, 0x37, 0xa0, 0x8e, 0xc6, 0x88, 0x84, 0xa8, 0x13,
+	0xaa, 0x80, 0x94, 0x83, 0x8c, 0xe0, 0x5e, 0x07, 0x88, 0x04, 0x7b, 0xdc, 0x6b, 0xd3, 0x58, 0xc6,
+	0xa6, 0x1e, 0xd4, 0x35, 0xe5, 0x55, 0xec, 0xff, 0xe1, 0x40, 0xf5, 0x0d, 0x96, 0x89, 0x32, 0xa3,
+	0x83, 0x5a, 0x50, 0x1d, 0xab, 0x8b, 0x52, 0x9b, 0x7c, 0xe1, 0xd1, 0x0c, 0x65, 0x97, 0x60, 0x40,
+	0xa2, 0xd4, 0x26, 0x21, 0xe2, 0x7d, 0x9a, 0x46, 0xfa, 0x4d, 0xcb, 0x4a, 0xed, 0xb1, 0x3e, 0x50,
+	0x7d, 0x85, 0x81, 0x89, 0x3e, 0x48, 0xb3, 0xba, 0x54, 0x1f, 0x64, 0xb0, 0x99, 0x6f, 0x7f, 0x73,
+	0xa0, 0x91, 0x53, 0x46, 0xbc, 0xbc, 0x1c, 0xd9, 0x97, 0x97, 0xa3, 0x81, 0xa0, 0xb0, 0x21, 0x32,
+	0xcd, 0x17, 0x1b, 0x22, 0x91, 0x7f, 0x9d, 0x11, 0x09, 0xb9, 0x7e, 0xfc, 0xd4, 0x46, 0xb8, 0x71,
+	0x40, 0xdb, 0xc6, 0x60, 0xed, 0xc6, 0x01, 0x35, 0xae, 0x6b, 0x42, 0x89, 0x32, 0x99, 0xe1, 0xf5,
+	0xa0, 0x44, 0x99, 0x88, 0x13, 0x4a, 0xbb, 0x43, 0x99, 0xd9, 0xf5, 0x40, 0xae, 0xfd, 0x5d, 0x58,
+	0xc8, 0xdb, 0x69, 0xbf, 0x2b, 0xe7, 0xfc, 0x77, 0x25, 0xbf, 0x21, 0xfd, 0xad, 0x89, 0xb5, 0x78,
+	0xda, 0x1b, 0x2f, 0xe9, 0x80, 0x99, 0x0a, 0xb1, 0x01, 0x75, 0x81, 0x65, 0x09, 0xea, 0x9a, 0xcb,
+	0x19, 0x41, 0x97, 0xad, 0x92, 0x6d, 0x99, 0xb6, 0xa0, 0xd2, 0x4b, 0xc9, 0x18, 0xa7, 0xd2, 0x9e,
+	0xe6, 0xce, 0x9a, 0x09, 0xe9, 0x1e, 0x8d, 0x39, 0x22, 0x31, 0x4e, 0xf7, 0xe5, 0x71, 0xa0, 0x61,
+	0x62, 0xf6, 0xe9, 0xd3, 0x30, 0xa4, 0x6f, 0xa5, 0x95, 0xb5, 0x40, 0xef, 0x84, 0x07, 0x38, 0x22,
+	0x61, 0x3b, 0x24, 0x31, 0x56, 0xa6, 0xce, 0x07, 0x75, 0x41, 0x79, 0x29, 0x08, 0xa2, 0x7a, 0x06,
+	0x18, 0xf5, 0x72, 0x65, 0x2c, 0x57, 0xed, 0xe4, 0x7a, 0xe7, 0xaf, 0x2a, 0x34, 0x8f, 0x54, 0xac,
+	0x74, 0x45, 0x77, 0xef, 0x41, 0x59, 0x14, 0x4a, 0x37, 0xcb, 0x9d, 0x5c, 0xdd, 0x5c, 0x5f, 0x30,
+	0xba, 0xee, 0x23, 0x8e, 0xb6, 0x1d, 0xf7, 0x13, 0x80, 0x17, 0xa3, 0x0e, 0xee, 0xd2, 0xb8, 0x4f,
+	0x06, 0xee, 0x6a, 0xa1, 0xe9, 0x38, 0x10, 0xb3, 0x69, 0xe1, 0xd6, 0x03, 0x28, 0xcb, 0x2e, 0x38,
+	0x93, 0x91, 0xab, 0xb7, 0xeb, 0xd9, 0x5c, 0x60, 0xca, 0xe3, 0xb6, 0x23, 0xd4, 0x12, 0x1e, 0xcf,
+	0x5f, 0xc9, 0x02, 0x50, 0x10, 0xf0, 0x85, 0x2d, 0x31, 0xd3, 0x54, 0x5a, 0x9b, 0xfc, 0xfc, 0xb3,
+	0x74, 0x2e, 0x0b, 0xaf, 0xe5, 0x04, 0xe5, 0x9c, 0x78, 0x91, 0x20, 0x3d, 0x39, 0xff, 0xbb, 0xa0,
+	0x89, 0x51, 0x79, 0xd7, 0x4c, 0xae, 0x2b, 0x13, 0x83, 0xa6, 0x16, 0xb5, 0x3a, 0x49, 0xd6, 0xf7,
+	0xbe, 0xca, 0x06, 0xc2, 0xb5, 0x29, 0x63, 0xe6, 0xba, 0x57, 0x3c, 0xd0, 0xb7, 0xf7, 0xce, 0xcf,
+	0x21, 0xd3, 0xb4, 0xde, 0xb8, 0x70, 0x2c, 0x30, 0x4c, 0xbe, 0x2f, 0xf4, 0x21, 0x37, 0xa6, 0x75,
+	0x06, 0x5a, 0xa1, 0x9b, 0x53, 0xcf, 0x35, 0xcb, 0x17, 0x13, 0x0d, 0xe6, 0xc6, 0xc5, 0x4d, 0x9f,
+	0x66, 0x77, 0x7d, 0xca, 0xa9, 0x66, 0xf6, 0xfc, 0x7c, 0xab, 0x77, 0xed, 0xc2, 0xfe, 0x4b, 0xb3,
+	0xda, 0xb8, 0xf8, 0x50, 0x73, 0x7a, 0x9c, 0x9b, 0x81, 0xa7, 0xf9, 0xea, 0x7f, 0xc5, 0x39, 0x36,
+	0x17, 0x2b, 0x33, 0x81, 0xae, 0x15, 0x86, 0xc3, 0x42, 0xac, 0x26, 0x27, 0xcc, 0x47, 0xd9, 0x43,
+	0x30, 0x4d, 0xb6, 0x57, 0x28, 0xb5, 0xfa, 0xf2, 0xd3, 0x17, 0xb0, 0xd4, 0xa5, 0x91, 0x3d, 0x46,
+	0x09, 0x79, 0x0a, 0xfa, 0x53, 0x7f, 0x92, 0x90, 0x63, 0xe7, 0xc7, 0xcd, 0x01, 0xe1, 0xc3, 0x51,
+	0x47, 0x24, 0xf4, 0x16, 0x47, 0x21, 0x65, 0xf7, 0xd5, 0x8b, 0xc6, 0xd4, 0x6e, 0x0b, 0x25, 0xc4,
+	0xfc, 0x05, 0xd5, 0xa9, 0x48, 0xb1, 0x0f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x94, 0x09, 0xbe,
+	0xdb, 0x9c, 0x12, 0x00, 0x00,
 }

 // Reference imports to suppress errors if they are not otherwise used.
@ -2323,6 +2488,7 @@ type MachineServiceClient interface {
 	Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (MachineService_ReadClient, error)
 	Reboot(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*RebootResponse, error)
 	Reset(ctx context.Context, in *ResetRequest, opts ...grpc.CallOption) (*ResetResponse, error)
+	Recover(ctx context.Context, in *RecoverRequest, opts ...grpc.CallOption) (*RecoverResponse, error)
 	ServiceList(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*ServiceListResponse, error)
 	ServiceRestart(ctx context.Context, in *ServiceRestartRequest, opts ...grpc.CallOption) (*ServiceRestartResponse, error)
 	ServiceStart(ctx context.Context, in *ServiceStartRequest, opts ...grpc.CallOption) (*ServiceStartResponse, error)
@ -2527,6 +2693,15 @@ func (c *machineServiceClient) Reset(ctx context.Context, in *ResetRequest, opts
 	return out, nil
 }

+func (c *machineServiceClient) Recover(ctx context.Context, in *RecoverRequest, opts ...grpc.CallOption) (*RecoverResponse, error) {
+	out := new(RecoverResponse)
+	err := c.cc.Invoke(ctx, "/machine.MachineService/Recover", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *machineServiceClient) ServiceList(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*ServiceListResponse, error) {
 	out := new(ServiceListResponse)
 	err := c.cc.Invoke(ctx, "/machine.MachineService/ServiceList", in, out, opts...)
@ -2600,6 +2775,7 @@ type MachineServiceServer interface {
 	Read(*ReadRequest, MachineService_ReadServer) error
 	Reboot(context.Context, *empty.Empty) (*RebootResponse, error)
 	Reset(context.Context, *ResetRequest) (*ResetResponse, error)
+	Recover(context.Context, *RecoverRequest) (*RecoverResponse, error)
 	ServiceList(context.Context, *empty.Empty) (*ServiceListResponse, error)
 	ServiceRestart(context.Context, *ServiceRestartRequest) (*ServiceRestartResponse, error)
 	ServiceStart(context.Context, *ServiceStartRequest) (*ServiceStartResponse, error)
@ -2772,6 +2948,24 @@ func _MachineService_Reset_Handler(srv interface{}, ctx context.Context, dec fun
 	return interceptor(ctx, in, info, handler)
 }

+func _MachineService_Recover_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(RecoverRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(MachineServiceServer).Recover(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/machine.MachineService/Recover",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(MachineServiceServer).Recover(ctx, req.(*RecoverRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _MachineService_ServiceList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(empty.Empty)
 	if err := dec(in); err != nil {
@ -2914,6 +3108,10 @@ var _MachineService_serviceDesc = grpc.ServiceDesc{
 			MethodName: "Reset",
 			Handler:    _MachineService_Reset_Handler,
 		},
+		{
+			MethodName: "Recover",
+			Handler:    _MachineService_Recover_Handler,
+		},
 		{
 			MethodName: "ServiceList",
 			Handler:    _MachineService_ServiceList_Handler,
--- a/api/machine/machine.proto
+++ b/api/machine/machine.proto
@ -21,6 +21,7 @@ service MachineService {
  rpc Read(ReadRequest) returns (stream common.Data);
  rpc Reboot(google.protobuf.Empty) returns (RebootResponse);
  rpc Reset(ResetRequest) returns (ResetResponse);
+  rpc Recover(RecoverRequest) returns (RecoverResponse);
  rpc ServiceList(google.protobuf.Empty) returns (ServiceListResponse);
  rpc ServiceRestart(ServiceRestartRequest) returns (ServiceRestartResponse);
  rpc ServiceStart(ServiceStartRequest) returns (ServiceStartResponse);
@ -32,12 +33,8 @@ service MachineService {

 // rpc reboot
 // The reboot message containing the reboot status.
-message Reboot {
-  common.Metadata metadata = 1;
-}
-message RebootResponse {
-  repeated Reboot messages = 1;
-}
+message Reboot { common.Metadata metadata = 1; }
+message RebootResponse { repeated Reboot messages = 1; }

 // rpc reset
 message ResetRequest {
@ -46,21 +43,28 @@ message ResetRequest {
 }

 // The reset message containing the restart status.
-message Reset {
-  common.Metadata metadata = 1;
-}
-message ResetResponse {
-  repeated Reset messages = 1;
+message Reset { common.Metadata metadata = 1; }
+message ResetResponse { repeated Reset messages = 1; }
+
+// rpc recover
+
+message RecoverRequest {
+  enum Source {
+    ETCD = 0;
+    APISERVER = 1;
+  }
+
+  Source source = 1;
 }

+// The recover message containing the recover status.
+message Recover { common.Metadata metadata = 1; }
+message RecoverResponse { repeated Recover messages = 1; }
+
 // rpc shutdown
 // The messages message containing the shutdown status.
-message Shutdown {
-  common.Metadata metadata = 1;
-}
-message ShutdownResponse {
-  repeated Shutdown messages = 1;
-}
+message Shutdown { common.Metadata metadata = 1; }
+message ShutdownResponse { repeated Shutdown messages = 1; }

 // rpc upgrade
 message UpgradeRequest {
@ -72,18 +76,14 @@ message Upgrade {
  common.Metadata metadata = 1;
  string ack = 2;
 }
-message UpgradeResponse {
-  repeated Upgrade messages = 1;
-}
+message UpgradeResponse { repeated Upgrade messages = 1; }

 // rpc servicelist
 message ServiceList {
  common.Metadata metadata = 1;
  repeated ServiceInfo services = 2;
 }
-message ServiceListResponse {
-  repeated ServiceList messages = 1;
-}
+message ServiceListResponse { repeated ServiceList messages = 1; }

 message ServiceInfo {
  string id = 1;
@ -92,9 +92,7 @@ message ServiceInfo {
  ServiceHealth health = 4;
 }

-message ServiceEvents {
-  repeated ServiceEvent events = 1;
-}
+message ServiceEvents { repeated ServiceEvent events = 1; }

 message ServiceEvent {
  string msg = 1;
@ -110,41 +108,29 @@ message ServiceHealth {
 }

 // rpc servicestart
-message ServiceStartRequest {
-  string id = 1;
-}
+message ServiceStartRequest { string id = 1; }

 message ServiceStart {
  common.Metadata metadata = 1;
  string resp = 2;
 }
-message ServiceStartResponse {
-  repeated ServiceStart messages = 1;
-}
+message ServiceStartResponse { repeated ServiceStart messages = 1; }

-message ServiceStopRequest {
-  string id = 1;
-}
+message ServiceStopRequest { string id = 1; }

 message ServiceStop {
  common.Metadata metadata = 1;
  string resp = 2;
 }
-message ServiceStopResponse {
-  repeated ServiceStop messages = 1;
-}
+message ServiceStopResponse { repeated ServiceStop messages = 1; }

-message ServiceRestartRequest {
-  string id = 1;
-}
+message ServiceRestartRequest { string id = 1; }

 message ServiceRestart {
  common.Metadata metadata = 1;
  string resp = 2;
 }
-message ServiceRestartResponse {
-  repeated ServiceRestart messages = 1;
-}
+message ServiceRestartResponse { repeated ServiceRestart messages = 1; }

 message StartRequest {
  option deprecated = true;
@ -213,9 +199,7 @@ message Mounts {
  common.Metadata metadata = 1;
  repeated MountStat stats = 2;
 }
-message MountsResponse {
-  repeated Mounts messages = 1;
-}
+message MountsResponse { repeated Mounts messages = 1; }

 // The messages message containing the requested processes.
 message MountStat {
@ -231,9 +215,7 @@ message Version {
  PlatformInfo platform = 3;
 }

-message VersionResponse {
-  repeated Version messages = 1;
-}
+message VersionResponse { repeated Version messages = 1; }

 message VersionInfo {
  string tag = 1;
@ -260,6 +242,4 @@ message LogsRequest {
  int32 tail_lines = 5;
 }

-message ReadRequest {
-  string path = 1;
-}
+message ReadRequest { string path = 1; }
--- a/cmd/talosctl/cmd/talos/recover.go
+++ b/cmd/talosctl/cmd/talos/recover.go
@ -0,0 +1,64 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package talos
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"github.com/talos-systems/talos/api/machine"
+	"github.com/talos-systems/talos/pkg/client"
+)
+
+var (
+	recoverSource   string
+	apiserverString = strings.ToLower(machine.RecoverRequest_APISERVER.String())
+	etcdString      = strings.ToLower(machine.RecoverRequest_ETCD.String())
+)
+
+// recoverCmd represents the recover command
+var recoverCmd = &cobra.Command{
+	Use:   "recover",
+	Short: "Recover a control plane",
+	Long:  ``,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return WithClient(func(ctx context.Context, c *client.Client) error {
+			var source machine.RecoverRequest_Source
+
+			switch recoverSource {
+			case apiserverString:
+				source = machine.RecoverRequest_APISERVER
+			case etcdString:
+				source = machine.RecoverRequest_ETCD
+			default:
+				return fmt.Errorf("unknown recovery source: %q", recoverSource)
+			}
+
+			if err := c.Recover(ctx, source); err != nil {
+				return fmt.Errorf("error executing recovery: %s", err)
+			}
+
+			return nil
+		})
+	},
+}
+
+func init() {
+	recoverCmd.Flags().StringVarP(
+		&recoverSource,
+		"source",
+		"s",
+		apiserverString,
+		fmt.Sprintf(
+			"The data source for restoring the control plane manifests from (valid options are %q and %q)",
+			apiserverString,
+			etcdString),
+	)
+
+	addCommand(recoverCmd)
+}
--- a/docs/talosctl/talosctl.md
+++ b/docs/talosctl/talosctl.md
@ -37,6 +37,7 @@ A CLI for out-of-band management of Kubernetes nodes created by Talos
 * [talosctl processes](talosctl_processes.md)	 - List running processes
 * [talosctl read](talosctl_read.md)	 - Read a file on the machine
 * [talosctl reboot](talosctl_reboot.md)	 - Reboot a node
+* [talosctl recover](talosctl_recover.md)	 - Recover a control plane
 * [talosctl reset](talosctl_reset.md)	 - Reset a node
 * [talosctl restart](talosctl_restart.md)	 - Restart a process
 * [talosctl routes](talosctl_routes.md)	 - List network routes
--- a/docs/talosctl/talosctl_recover.md
+++ b/docs/talosctl/talosctl_recover.md
@ -0,0 +1,33 @@
+<!-- markdownlint-disable -->
+## talosctl recover
+
+Recover a control plane
+
+### Synopsis
+
+Recover a control plane
+
+```
+talosctl recover [flags]
+```
+
+### Options
+
+```
+  -h, --help            help for recover
+  -s, --source string   The data source for restoring the control plane manifests from (valid options are "apiserver" and "etcd") (default "apiserver")
+```
+
+### Options inherited from parent commands
+
+```
+      --context string       Context to be used in command
+  -e, --endpoints strings    override default endpoints in Talos configuration
+  -n, --nodes strings        target the specified nodes
+      --talosconfig string   The path to the Talos configuration file (default "/home/user/.talos/config")
+```
+
+### SEE ALSO
+
+* [talosctl](talosctl.md)	 - A CLI for out-of-band management of Kubernetes nodes created by Talos
+
--- a/go.mod
+++ b/go.mod
@ -22,7 +22,7 @@ require (
 	github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd
 	github.com/containernetworking/cni v0.7.2-0.20190807151350-8c6c47d1c7fc
 	github.com/containernetworking/plugins v0.8.5
-	github.com/coreos/etcd v3.3.18+incompatible
+	github.com/coreos/etcd v3.3.18+incompatible // indirect
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/docker/docker v1.13.1
 	github.com/docker/go-connections v0.4.0
@ -43,7 +43,7 @@ require (
 	github.com/insomniacslk/dhcp v0.0.0-20190814082028-393ae75a101b
 	github.com/jsimonetti/rtnetlink v0.0.0-20191223084007-1b9462860ac0
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/kubernetes-sigs/bootkube v0.14.1-0.20200421144856-831deff0541a
+	github.com/kubernetes-sigs/bootkube v0.14.1-0.20200501183829-d8e5fa33347a
 	github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7 // indirect
 	github.com/mdlayher/genetlink v1.0.0
 	github.com/mdlayher/netlink v1.0.0
@ -70,7 +70,7 @@ require (
 	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
 	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	golang.org/x/sys v0.0.0-20200107162124-548cf772de50
+	golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5
 	golang.org/x/text v0.3.2
 	google.golang.org/grpc v1.26.0
 	gopkg.in/freddierice/go-losetup.v1 v1.0.0-20170407175016-fc9adea44124
--- a/go.sum
+++ b/go.sum
@ -414,8 +414,8 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kubernetes-sigs/bootkube v0.14.1-0.20200416193108-93ff8fc5e457/go.mod h1:CIpoNLW4Lm9zNVFRgqQIylnbZi/x9TnulTEA8edC0O4=
-github.com/kubernetes-sigs/bootkube v0.14.1-0.20200421144856-831deff0541a h1:orpy6vS+mQUTgwDABeIrpppItV2cevBqsLkAD+P9y94=
-github.com/kubernetes-sigs/bootkube v0.14.1-0.20200421144856-831deff0541a/go.mod h1:tApYbSMrDhKWcKRjKyLXNYLaF2cxt9Xq5e8AavMj7gI=
+github.com/kubernetes-sigs/bootkube v0.14.1-0.20200501183829-d8e5fa33347a h1:HA7wERlEdha8KL+Ezpajon+W/H+hOUVZB2QmQs57Co0=
+github.com/kubernetes-sigs/bootkube v0.14.1-0.20200501183829-d8e5fa33347a/go.mod h1:tgR06vqFs3qxPEkOFmiMDy9UbQ0PMcee/FK6dOyiHFs=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@ -631,6 +631,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
+go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.etcd.io/etcd v3.3.13+incompatible h1:jCejD5EMnlGxFvcGRyEV4VGlENZc7oPQX6o0t7n3xbw=
 go.etcd.io/etcd v3.3.13+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
@ -646,10 +648,18 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A=
+go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo=
+go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
 golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@ -759,8 +769,8 @@ golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200107162124-548cf772de50 h1:YvQ10rzcqWXLlJZ3XCUoO25savxmscf4+SC+ZqiCHhA=
-golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -797,6 +807,8 @@ golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191010171213-8abd42400456/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191028194131-d78a1f2664a0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191105231337-689d0f08e67a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191111154804-8cb0d02132ec h1:poMzDCrMtA4fJ5uWN0ZsAvrOiMhgdHmTW6m6i4+Xu2I=
 golang.org/x/tools v0.0.0-20191111154804-8cb0d02132ec/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
--- a/internal/app/bootkube/main.go
+++ b/internal/app/bootkube/main.go
@ -17,6 +17,20 @@ import (
 	"github.com/talos-systems/talos/pkg/constants"
 )

+var (
+	configPath *string
+	strict     *bool
+)
+
+func init() {
+	log.SetFlags(log.Lshortfile | log.Ldate | log.Lmicroseconds | log.Ltime)
+
+	configPath = flag.String("config", "", "the path to the config")
+	strict = flag.Bool("strict", true, "require all manifests to cleanly apply")
+
+	flag.Parse()
+}
+
 func run() error {
 	defaultRequiredPods := []string{
 		"kube-system/pod-checkpointer",
@ -28,7 +42,7 @@ func run() error {
 	cfg := bootkube.Config{
 		AssetDir:        constants.AssetsDirectory,
 		PodManifestPath: constants.ManifestsDirectory,
-		Strict:          true,
+		Strict:          *strict,
 		RequiredPods:    defaultRequiredPods,
 	}

@ -60,9 +74,6 @@ func run() error {
 }

 func main() {
-	configPath := flag.String("config", "", "the path to the config")
-
-	flag.Parse()
 	util.InitLogs()

 	defer util.FlushLogs()
--- a/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go
+++ b/internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go
@ -181,6 +181,36 @@ func (s *Server) Reset(ctx context.Context, in *machine.ResetRequest) (reply *ma
 	return reply, nil
 }

+// Recover recovers the control plane.
+//
+// nolint: dupl
+func (s *Server) Recover(ctx context.Context, in *machine.RecoverRequest) (reply *machine.RecoverResponse, err error) {
+	log.Printf("recover request received")
+
+	if s.Controller.Runtime().Config().Machine().Type() == runtime.MachineTypeJoin {
+		return nil, fmt.Errorf("recover can only be performed on a control plane node")
+	}
+
+	go func() {
+		if err := s.Controller.Run(runtime.SequenceRecover, in); err != nil {
+			log.Println("recover failed:", err)
+
+			if err != runtime.ErrLocked {
+				// NB: Stopping the gRPC server will trigger machined's reboot mechanism.
+				s.server.GracefulStop()
+			}
+		}
+	}()
+
+	reply = &machine.RecoverResponse{
+		Messages: []*machine.Recover{
+			{},
+		},
+	}
+
+	return reply, nil
+}
+
 // ServiceList returns list of the registered services and their status
 func (s *Server) ServiceList(ctx context.Context, in *empty.Empty) (result *machine.ServiceListResponse, err error) {
 	services := system.Services(s.Controller.Runtime()).List()
--- a/internal/app/machined/pkg/runtime/sequencer.go
+++ b/internal/app/machined/pkg/runtime/sequencer.go
@ -28,6 +28,8 @@ const (
 	SequenceReset
 	// SequenceReboot is the reboot sequence.
 	SequenceReboot
+	// SequenceRecover is the recover sequence.
+	SequenceRecover
 	// SequenceNoop is the noop sequence.
 	SequenceNoop
 )
@ -40,15 +42,18 @@ const (
 	upgrade    = "upgrade"
 	reset      = "reset"
 	reboot     = "reboot"
+	recover    = "recover"
 	noop       = "noop"
 )

 // String returns the string representation of a `Sequence`.
 func (s Sequence) String() string {
-	return [...]string{boot, initialize, install, shutdown, upgrade, reset, reboot, noop}[s]
+	return [...]string{boot, initialize, install, shutdown, upgrade, reset, reboot, recover, noop}[s]
 }

 // ParseSequence returns a `Sequence` that matches the specified string.
+//
+// nolint: gocyclo
 func ParseSequence(s string) (seq Sequence, err error) {
 	switch s {
 	case boot:
@ -65,6 +70,8 @@ func ParseSequence(s string) (seq Sequence, err error) {
 		seq = SequenceReset
 	case reboot:
 		seq = SequenceReboot
+	case recover:
+		seq = SequenceRecover
 	case noop:
 		seq = SequenceNoop
 	default:
@ -81,6 +88,7 @@ type Sequencer interface {
 	Initialize(Runtime) []Phase
 	Install(Runtime) []Phase
 	Reboot(Runtime) []Phase
+	Recover(Runtime, *machine.RecoverRequest) []Phase
 	Reset(Runtime, *machine.ResetRequest) []Phase
 	Shutdown(Runtime) []Phase
 	Upgrade(Runtime, *machine.UpgradeRequest) []Phase
--- a/internal/app/machined/pkg/runtime/sequencer_test.go
+++ b/internal/app/machined/pkg/runtime/sequencer_test.go
@ -43,6 +43,11 @@ func TestSequence_String(t *testing.T) {
 			s:    SequenceReset,
 			want: "reset",
 		},
+		{
+			name: "recover",
+			s:    SequenceRecover,
+			want: "recover",
+		},
 	}

 	for _, tt := range tests {
@ -101,6 +106,12 @@ func TestParseSequence(t *testing.T) {
 			wantSeq: SequenceReset,
 			wantErr: false,
 		},
+		{
+			name:    "recover",
+			args:    args{"recover"},
+			wantSeq: SequenceRecover,
+			wantErr: false,
+		},
 		{
 			name:    "invalid",
 			args:    args{"invalid"},
--- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller.go
+++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller.go
@ -238,6 +238,7 @@ func (c *Controller) runTask(n int, f runtime.TaskSetupFunc, seq runtime.Sequenc
 	return nil
 }

+// nolint: gocyclo
 func (c *Controller) phases(seq runtime.Sequence, data interface{}) ([]runtime.Phase, error) {
 	var phases []runtime.Phase

@ -252,6 +253,17 @@ func (c *Controller) phases(seq runtime.Sequence, data interface{}) ([]runtime.P
 		phases = c.s.Shutdown(c.r)
 	case runtime.SequenceReboot:
 		phases = c.s.Reboot(c.r)
+	case runtime.SequenceRecover:
+		var (
+			in *machine.RecoverRequest
+			ok bool
+		)
+
+		if in, ok = data.(*machine.RecoverRequest); !ok {
+			return nil, runtime.ErrInvalidSequenceData
+		}
+
+		phases = c.s.Recover(c.r, in)
 	case runtime.SequenceUpgrade:
 		var (
 			in *machine.UpgradeRequest
--- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go
+++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go
@ -206,6 +206,15 @@ func (*Sequencer) Reboot(r runtime.Runtime) []runtime.Phase {
 	return phases
 }

+// Recover is the recover sequence.
+func (*Sequencer) Recover(r runtime.Runtime, in *machine.RecoverRequest) []runtime.Phase {
+	phases := PhaseList{}
+
+	phases = phases.Append(Recover)
+
+	return phases
+}
+
 // Reset is the reset sequence.
 func (*Sequencer) Reset(r runtime.Runtime, in *machine.ResetRequest) []runtime.Phase {
 	phases := PhaseList{}
--- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go
+++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go
@ -24,9 +24,12 @@ import (
 	"time"

 	"github.com/hashicorp/go-multierror"
+	"go.etcd.io/etcd/clientv3"
 	"golang.org/x/sys/unix"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"

+	"github.com/kubernetes-sigs/bootkube/pkg/recovery"
+
 	"github.com/talos-systems/talos/api/machine"
 	installer "github.com/talos-systems/talos/cmd/installer/pkg/install"
 	"github.com/talos-systems/talos/internal/app/machined/internal/install"
@ -40,6 +43,7 @@ import (
 	"github.com/talos-systems/talos/internal/pkg/etcd"
 	"github.com/talos-systems/talos/internal/pkg/kernel/kspp"
 	"github.com/talos-systems/talos/internal/pkg/kmsg"
+	"github.com/talos-systems/talos/internal/pkg/kubeconfig"
 	"github.com/talos-systems/talos/internal/pkg/mount"
 	"github.com/talos-systems/talos/pkg/blockdevice/probe"
 	"github.com/talos-systems/talos/pkg/blockdevice/util"
@ -1455,3 +1459,88 @@ func Install(seq runtime.Sequence, data interface{}) runtime.TaskExecutionFunc {
 		return nil
 	}
 }
+
+// Recover attempts to recover the control plane.
+//
+// nolint: gocyclo
+func Recover(seq runtime.Sequence, data interface{}) runtime.TaskExecutionFunc {
+	return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) {
+		var (
+			in *machine.RecoverRequest
+			ok bool
+		)
+
+		if in, ok = data.(*machine.RecoverRequest); !ok {
+			return runtime.ErrInvalidSequenceData
+		}
+
+		kubeconfigPath := "/etc/kubernetes/recovery.yaml"
+
+		var b bytes.Buffer
+
+		if err = kubeconfig.GenerateAdmin(r.Config().Cluster(), &b); err != nil {
+			return err
+		}
+
+		if err = ioutil.WriteFile(kubeconfigPath, b.Bytes(), 0600); err != nil {
+			return fmt.Errorf("failed to create recovery kubeconfig: %w", err)
+		}
+
+		// nolint: errcheck
+		defer os.Remove(kubeconfigPath)
+
+		var backend recovery.Backend
+
+		switch in.Source {
+		case machine.RecoverRequest_ETCD:
+			var client *clientv3.Client
+
+			client, err = etcd.NewClient([]string{"127.0.0.1:2379"})
+			if err != nil {
+				return err
+			}
+
+			backend = recovery.NewEtcdBackend(client, "/registry")
+
+		case machine.RecoverRequest_APISERVER:
+			backend, err = recovery.NewAPIServerBackend(kubeconfigPath)
+			if err != nil {
+				return err
+			}
+		}
+
+		as, err := recovery.Recover(context.Background(), backend, kubeconfigPath)
+		if err != nil {
+			return err
+		}
+
+		if err = os.MkdirAll(constants.AssetsDirectory, 0600); err != nil {
+			return err
+		}
+
+		if err = as.WriteFiles(constants.AssetsDirectory); err != nil {
+			return fmt.Errorf("failed to write recovered assets: %w", err)
+		}
+
+		svc := &services.Bootkube{Recover: true}
+
+		if r.Config().Machine().Type() == runtime.MachineTypeControlPlane {
+			system.Services(r).LoadAndStart(svc)
+		} else {
+			loaded := system.Services(r).Reload(svc)
+
+			if len(loaded) == 0 {
+				return fmt.Errorf("bootkube service is already running")
+			}
+
+			if err = system.Services(r).Start(svc.ID(r)); err != nil {
+				return fmt.Errorf("failed to start bootkube: %w", err)
+			}
+		}
+
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+		defer cancel()
+
+		return system.WaitForService(system.StateEventUp, svc.ID(r)).Wait(ctx)
+	}
+}
--- a/internal/app/machined/pkg/system/services/bootkube.go
+++ b/internal/app/machined/pkg/system/services/bootkube.go
@ -9,13 +9,14 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"strconv"
 	"time"

 	containerdapi "github.com/containerd/containerd"
 	"github.com/containerd/containerd/oci"
-	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"

 	"github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
 	"github.com/talos-systems/talos/internal/app/machined/pkg/system/events"
@ -30,6 +31,8 @@ import (
 // Bootkube implements the Service interface. It serves as the concrete type with
 // the required methods.
 type Bootkube struct {
+	Recover bool
+
 	provisioned bool
 }

@ -40,48 +43,50 @@ func (b *Bootkube) ID(r runtime.Runtime) string {

 // PreFunc implements the Service interface.
 func (b *Bootkube) PreFunc(ctx context.Context, r runtime.Runtime) (err error) {
-	client, err := etcd.NewClient([]string{"127.0.0.1:2379"})
-	if err != nil {
-		return err
-	}
+	if !b.Recover {
+		client, err := etcd.NewClient([]string{"127.0.0.1:2379"})
+		if err != nil {
+			return err
+		}

-	// nolint: errcheck
-	defer client.Close()
+		// nolint: errcheck
+		defer client.Close()

-	err = retry.Exponential(3*time.Minute, retry.WithUnits(50*time.Millisecond), retry.WithJitter(25*time.Millisecond)).Retry(func() error {
-		var resp *clientv3.GetResponse
+		err = retry.Exponential(3*time.Minute, retry.WithUnits(50*time.Millisecond), retry.WithJitter(25*time.Millisecond)).Retry(func() error {
+			var resp *clientv3.GetResponse

-		// limit single attempt to 15 seconds to allow for 12 attempts at least
-		attemptCtx, attemptCtxCancel := context.WithTimeout(ctx, 15*time.Second)
-		defer attemptCtxCancel()
+			// limit single attempt to 15 seconds to allow for 12 attempts at least
+			attemptCtx, attemptCtxCancel := context.WithTimeout(ctx, 15*time.Second)
+			defer attemptCtxCancel()

-		if resp, err = client.Get(clientv3.WithRequireLeader(attemptCtx), constants.InitializedKey); err != nil {
-			if errors.Is(err, rpctypes.ErrGRPCKeyNotFound) {
-				// no key set yet, treat as not provisioned yet
+			if resp, err = client.Get(clientv3.WithRequireLeader(attemptCtx), constants.InitializedKey); err != nil {
+				if errors.Is(err, rpctypes.ErrGRPCKeyNotFound) {
+					// no key set yet, treat as not provisioned yet
+					return nil
+				}
+
+				return retry.ExpectedError(err)
+			}
+
+			if len(resp.Kvs) == 0 {
+				// no key/values in the range, treat as not provisioned yet
 				return nil
 			}

-			return retry.ExpectedError(err)
+			if string(resp.Kvs[0].Value) == "true" {
+				b.provisioned = true
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			return fmt.Errorf("error querying cluster provisioned state in etcd: %w", err)
 		}

-		if len(resp.Kvs) == 0 {
-			// no key/values in the range, treat as not provisioned yet
+		if b.provisioned {
 			return nil
 		}
-
-		if string(resp.Kvs[0].Value) == "true" {
-			b.provisioned = true
-		}
-
-		return nil
-	})
-
-	if err != nil {
-		return fmt.Errorf("error querying cluster provisioned state in etcd: %w", err)
-	}
-
-	if b.provisioned {
-		return nil
 	}

 	importer := containerd.NewImporter(constants.SystemContainerdNamespace, containerd.WithContainerdAddress(constants.SystemContainerdAddress))
@ -150,6 +155,7 @@ func (b *Bootkube) Runner(r runtime.Runtime) (runner.Runner, error) {
 		ProcessArgs: []string{
 			"/bootkube",
 			"--config=" + constants.ConfigPath,
+			"--strict=" + strconv.FormatBool(!b.Recover),
 		},
 	}

--- a/internal/app/machined/pkg/system/system.go
+++ b/internal/app/machined/pkg/system/system.go
@ -14,6 +14,7 @@ import (
 	"github.com/hashicorp/go-multierror"

 	"github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
+	"github.com/talos-systems/talos/internal/app/machined/pkg/system/events"
 	"github.com/talos-systems/talos/internal/pkg/conditions"
 )

@ -83,6 +84,37 @@ func (s *singleton) Load(services ...Service) []string {
 	return ids
 }

+// Reload adds service to the list of services managed by the runner.
+//
+// Reload returns service IDs for each of the services.
+func (s *singleton) Reload(services ...Service) []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if s.terminating {
+		return nil
+	}
+
+	ids := make([]string, 0, len(services))
+
+	for _, service := range services {
+		id := service.ID(s.runtime)
+		ids = append(ids, id)
+
+		if svc, exists := s.state[id]; exists {
+			switch svc.GetState() {
+			case events.StateFailed, events.StateFinished, events.StateSkipped:
+				svcrunner := NewServiceRunner(service, s.runtime)
+				s.state[id] = svcrunner
+
+				return ids
+			}
+		}
+	}
+
+	return nil
+}
+
 // Start will invoke the service's Pre, Condition, and Type funcs. If the any
 // error occurs in the Pre or Condition invocations, it is up to the caller to
 // to restart the service.
@ -157,6 +189,15 @@ func (s *singleton) LoadAndStart(services ...Service) {
 	}
 }

+// ReloadAndStart combines Reload and Start into single call.
+func (s *singleton) ReloadAndStart(services ...Service) {
+	err := s.Start(s.Reload(services...)...)
+	if err != nil {
+		// should never happen
+		panic(err)
+	}
+}
+
 // Shutdown all the services
 func (s *singleton) Shutdown() {
 	s.mu.Lock()
--- a/internal/integration/api/recover.go
+++ b/internal/integration/api/recover.go
@ -0,0 +1,110 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration_api
+
+package api
+
+import (
+	"context"
+	"sort"
+	"testing"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/talos-systems/talos/api/machine"
+	"github.com/talos-systems/talos/internal/integration/base"
+	"github.com/talos-systems/talos/pkg/client"
+)
+
+type RecoverSuite struct {
+	base.K8sSuite
+
+	ctx       context.Context
+	ctxCancel context.CancelFunc
+}
+
+// SuiteName ...
+func (suite *RecoverSuite) SuiteName() string {
+	return "api.RecoverSuite"
+}
+
+// SetupTest ...
+func (suite *RecoverSuite) SetupTest() {
+	if testing.Short() {
+		suite.T().Skip("skipping in short mode")
+	}
+
+	// make sure we abort at some point in time, but give enough room for Recovers
+	suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 30*time.Minute)
+}
+
+// TearDownTest ...
+func (suite *RecoverSuite) TearDownTest() {
+	suite.ctxCancel()
+}
+
+// TestRecoverControlPlane removes the control plane components and attempts to recover them with the recover API.
+func (suite *RecoverSuite) TestRecoverControlPlane() {
+	if !suite.Capabilities().SupportsRecover {
+		suite.T().Skip("cluster doesn't support recovery")
+	}
+
+	if suite.Cluster == nil {
+		suite.T().Skip("without full cluster state recover test is not reliable (can't wait for cluster readiness)")
+	}
+
+	pods, err := suite.Clientset.CoreV1().Pods("kube-system").List(suite.ctx, metav1.ListOptions{
+		LabelSelector: "k8s-app in (kube-scheduler,kube-controller-manager)",
+	})
+
+	suite.Assert().NoError(err)
+
+	var eg errgroup.Group
+
+	for _, pod := range pods.Items {
+		pod := pod
+
+		eg.Go(func() error {
+			suite.T().Logf("Deleting %s", pod.GetName())
+
+			err := suite.Clientset.CoreV1().Pods(pod.GetNamespace()).Delete(suite.ctx, pod.GetName(), metav1.DeleteOptions{})
+
+			return err
+		})
+	}
+
+	suite.Assert().NoError(eg.Wait())
+
+	nodes := suite.DiscoverNodes()
+	suite.Require().NotEmpty(nodes)
+
+	sort.Strings(nodes)
+
+	node := nodes[0]
+
+	suite.T().Log("Recovering control plane")
+
+	ctx, ctxCancel := context.WithTimeout(suite.ctx, 5*time.Minute)
+	defer ctxCancel()
+
+	nodeCtx := client.WithNodes(ctx, node)
+
+	in := &machine.RecoverRequest{
+		Source: machine.RecoverRequest_APISERVER,
+	}
+
+	_, err = suite.Client.MachineClient.Recover(nodeCtx, in)
+
+	suite.Assert().NoError(err)
+
+	// NB: using `ctx` here to have client talking to init node by default
+	suite.AssertClusterHealthy(ctx)
+}
+
+func init() {
+	allSuites = append(allSuites, new(RecoverSuite))
+}
--- a/internal/integration/api/reset.go
+++ b/internal/integration/api/reset.go
@ -86,14 +86,18 @@ func (suite *ResetSuite) TestResetNodeByNode() {

 			var uptimeAfter float64

+			start := time.Now()
+
 			suite.Require().NoError(retry.Constant(3 * time.Minute).Retry(func() error {
+				since := time.Since(start)
+
 				uptimeAfter, err = suite.ReadUptime(nodeCtx)
 				if err != nil {
 					// API might be unresponsive during reboot
 					return retry.ExpectedError(err)
 				}

-				if uptimeAfter >= uptimeBefore {
+				if uptimeAfter >= uptimeBefore+since.Seconds() {
 					// uptime should go down after Reset, as it reboots the node
 					return retry.ExpectedError(fmt.Errorf("uptime didn't go down: before %f, after %f", uptimeBefore, uptimeAfter))
 				}
--- a/internal/integration/base/api.go
+++ b/internal/integration/base/api.go
@ -82,6 +82,7 @@ func (apiSuite *APISuite) DiscoverNodes() []string {
 type Capabilities struct {
 	RunsTalosKernel bool
 	SupportsReboot  bool
+	SupportsRecover bool
 }

 // Capabilities returns a set of capabilities to skip tests for different environments.
@ -97,6 +98,7 @@ func (apiSuite *APISuite) Capabilities() Capabilities {
 		default:
 			caps.RunsTalosKernel = true
 			caps.SupportsReboot = true
+			caps.SupportsRecover = true
 		}
 	}

--- a/internal/pkg/cluster/check/default.go
+++ b/internal/pkg/cluster/check/default.go
@ -47,7 +47,7 @@ func DefaultClusterChecks() []ClusterCheck {
 		},
 		// wait for HA k8s control plane
 		func(cluster ClusterInfo) conditions.Condition {
-			return conditions.PollingCondition("all master nodes to be part of k8s control plane", func(ctx context.Context) error {
+			return conditions.PollingCondition("all control plane components to be ready", func(ctx context.Context) error {
 				return K8sFullControlPlaneAssertion(ctx, cluster)
 			}, 2*time.Minute, 5*time.Second)
 		},
--- a/internal/pkg/cluster/check/kubernetes.go
+++ b/internal/pkg/cluster/check/kubernetes.go
@ -88,11 +88,36 @@ func K8sFullControlPlaneAssertion(ctx context.Context, cluster ClusterInfo) erro
 	sort.Strings(expectedNodes)
 	sort.Strings(actualNodes)

-	if reflect.DeepEqual(expectedNodes, actualNodes) {
-		return nil
+	if !reflect.DeepEqual(expectedNodes, actualNodes) {
+		return fmt.Errorf("expected %v nodes, but got %v nodes", expectedNodes, actualNodes)
 	}

-	return fmt.Errorf("expected %v nodes, but got %v nodes", expectedNodes, actualNodes)
+	// NB: We run the control plane check after node readiness check in order to
+	// ensure that all control plane nodes have been labeled with the master
+	// label.
+
+	daemonsets, err := clientset.AppsV1().DaemonSets("kube-system").List(ctx, metav1.ListOptions{
+		LabelSelector: "k8s-app in (kube-scheduler,kube-controller-manager)",
+	})
+	if err != nil {
+		return err
+	}
+
+	for _, ds := range daemonsets.Items {
+		if ds.Status.CurrentNumberScheduled != ds.Status.DesiredNumberScheduled {
+			return fmt.Errorf("expected current number scheduled for %s to be %d, got %d", ds.GetName(), ds.Status.DesiredNumberScheduled, ds.Status.CurrentNumberScheduled)
+		}
+
+		if ds.Status.NumberAvailable != ds.Status.DesiredNumberScheduled {
+			return fmt.Errorf("expected number available for %s to be %d, got %d", ds.GetName(), ds.Status.DesiredNumberScheduled, ds.Status.NumberAvailable)
+		}
+
+		if ds.Status.NumberReady != ds.Status.DesiredNumberScheduled {
+			return fmt.Errorf("expected number ready for %s to be %d, got %d", ds.GetName(), ds.Status.DesiredNumberScheduled, ds.Status.NumberReady)
+		}
+	}
+
+	return nil
 }

 // K8sAllNodesReadyAssertion checks whether all the nodes are Ready.
--- a/pkg/client/client.go
+++ b/pkg/client/client.go
@ -251,6 +251,12 @@ func (c *Client) Reboot(ctx context.Context) (err error) {
 	return
 }

+// Recover implements the proto.OSClient interface.
+func (c *Client) Recover(ctx context.Context, source machineapi.RecoverRequest_Source) (err error) {
+	_, err = c.MachineClient.Recover(ctx, &machineapi.RecoverRequest{Source: source})
+	return
+}
+
 // Shutdown implements the proto.OSClient interface.
 func (c *Client) Shutdown(ctx context.Context) (err error) {
 	_, err = c.MachineClient.Shutdown(ctx, &empty.Empty{})