fix: ignore sequence lock errors in machined

This prevents reboots when some actions triggers sequence while another
sequence is still running.

Fixes #2209

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>
This commit is contained in:
Andrey Smirnov 2020-07-21 00:17:50 +03:00 committed by talos-bot
parent 0aae950518
commit 74413b1393
4 changed files with 25 additions and 12 deletions

View File

@ -30,16 +30,19 @@ const _ = proto.ProtoPackageIsVersion4
type Code int32
const (
Code_FATAL Code = 0
Code_FATAL Code = 0
Code_LOCKED Code = 1
)
// Enum value maps for Code.
var (
Code_name = map[int32]string{
0: "FATAL",
1: "LOCKED",
}
Code_value = map[string]int32{
"FATAL": 0,
"FATAL": 0,
"LOCKED": 1,
}
)
@ -467,14 +470,15 @@ var file_common_common_proto_rawDesc = []byte{
0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x29, 0x0a, 0x08, 0x6d, 0x65, 0x73,
0x73, 0x61, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x63, 0x6f,
0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x73, 0x73,
0x61, 0x67, 0x65, 0x73, 0x2a, 0x11, 0x0a, 0x04, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x09, 0x0a, 0x05,
0x46, 0x41, 0x54, 0x41, 0x4c, 0x10, 0x00, 0x2a, 0x2a, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x74, 0x61,
0x69, 0x6e, 0x65, 0x72, 0x44, 0x72, 0x69, 0x76, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x4f,
0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x44, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x43, 0x52,
0x49, 0x10, 0x01, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x6d, 0x2f, 0x74, 0x61, 0x6c, 0x6f, 0x73, 0x2d, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x73, 0x2f,
0x74, 0x61, 0x6c, 0x6f, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e,
0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x61, 0x67, 0x65, 0x73, 0x2a, 0x1d, 0x0a, 0x04, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x09, 0x0a, 0x05,
0x46, 0x41, 0x54, 0x41, 0x4c, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x4f, 0x43, 0x4b, 0x45,
0x44, 0x10, 0x01, 0x2a, 0x2a, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
0x44, 0x72, 0x69, 0x76, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49,
0x4e, 0x45, 0x52, 0x44, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x43, 0x52, 0x49, 0x10, 0x01, 0x42,
0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x61,
0x6c, 0x6f, 0x73, 0x2d, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x73, 0x2f, 0x74, 0x61, 0x6c, 0x6f,
0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
}
var (

View File

@ -6,7 +6,10 @@ option go_package = "github.com/talos-systems/talos/api/common";
import "google/protobuf/any.proto";
enum Code { FATAL = 0; }
enum Code {
FATAL = 0;
LOCKED = 1;
}
message Error {
Code code = 1;

View File

@ -21,6 +21,7 @@ import (
"github.com/talos-systems/go-procfs/procfs"
"github.com/talos-systems/talos/api/common"
"github.com/talos-systems/talos/api/machine"
"github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
v1alpha1runtime "github.com/talos-systems/talos/internal/app/machined/pkg/runtime/v1alpha1"
@ -158,6 +159,11 @@ func main() {
for event := range events {
if msg, ok := event.Payload.(*machine.SequenceEvent); ok {
if msg.Error != nil {
if msg.Error.GetCode() == common.Code_LOCKED {
// ignore sequence lock errors, they're not fatal
continue
}
handle(fmt.Errorf("fatal sequencer error in %q sequence: %v", msg.GetSequence(), msg.GetError().String()))
}
}

View File

@ -106,7 +106,7 @@ func (c *Controller) Run(seq runtime.Sequence, data interface{}, setters ...runt
Sequence: seq.String(),
Action: machine.SequenceEvent_NOOP,
Error: &common.Error{
Code: common.Code_FATAL,
Code: common.Code_LOCKED,
Message: fmt.Sprintf("sequence not started: %s", runtime.ErrLocked.Error()),
},
})