test: add CLI integration test

This starts with a very simple test for `osctl version` using regexps as
output of the command depends a lot on current version.

We might use more of 'gold' matches for other commands potentially.

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

Dockerfile

@@ -344,7 +344,7 @@ ARG TAG
 ARG VERSION_PKG="github.com/talos-systems/talos/pkg/version"
 RUN --mount=type=cache,target=/.cache/go-build GOOS=linux GOARCH=amd64 go test -c \
     -ldflags "-s -w -X ${VERSION_PKG}.Name=Client -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" \
-    -tags integration,integration_api \
+    -tags integration,integration_api,integration_cli \
     ./internal/integration
 
 FROM scratch AS integration-test

internal/integration/api/version.go

@@ -22,18 +22,6 @@ func (suite *VersionSuite) SuiteName() string {
 	return "api.VersionSuite"
 }
 
-// SetupSuite ...
-func (suite *VersionSuite) SetupSuite() {
-	suite.InitClient()
-}
-
-// TearDownSuite ...
-func (suite *VersionSuite) TearDownSuite() {
-	if suite.Client != nil {
-		suite.Assert().NoError(suite.Client.Close())
-	}
-}
-
 // TestExpectedVersionMaster verifies master node version matches expected
 func (suite *VersionSuite) TestExpectedVersionMaster() {
 	v, err := suite.Client.Version(context.Background())

internal/integration/base/api.go

@@ -21,8 +21,8 @@ type APISuite struct {
 	Client *client.Client
 }
 
-// InitClient initializes Talos API client
-func (apiSuite *APISuite) InitClient() {
+// SetupSuite initializes Talos API client
+func (apiSuite *APISuite) SetupSuite() {
 	target, creds, err := client.NewClientTargetAndCredentialsFromConfig(apiSuite.TalosConfig)
 	apiSuite.Require().NoError(err)
 
@@ -33,3 +33,10 @@ func (apiSuite *APISuite) InitClient() {
 	apiSuite.Client, err = client.NewClient(creds, target, constants.OsdPort)
 	apiSuite.Require().NoError(err)
 }
+
+// TearDownSuite closes Talos API client
+func (apiSuite *APISuite) TearDownSuite() {
+	if apiSuite.Client != nil {
+		apiSuite.Assert().NoError(apiSuite.Client.Close())
+	}
+}

internal/integration/base/base.go

@@ -15,6 +15,8 @@ type TalosSuite struct {
 	TalosConfig string
 	// Version is the (expected) version of Talos tests are running against
 	Version string
+	// OsctlPath is path to osctl binary
+	OsctlPath string
 }
 
 // ConfiguredSuite expects config to be set before running

internal/integration/base/cli.go

@@ -0,0 +1,32 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration_cli
+
+package base
+
+import (
+	"os/exec"
+
+	"github.com/stretchr/testify/suite"
+)
+
+// CLISuite is a base suite for CLI tests
+type CLISuite struct {
+	suite.Suite
+	TalosSuite
+}
+
+// RunOsctl runs osctl binary with the options provided
+func (cliSuite *CLISuite) RunOsctl(args []string, options ...RunOption) {
+	if cliSuite.Target != "" {
+		args = append([]string{"--target", cliSuite.Target}, args...)
+	}
+
+	args = append([]string{"--talosconfig", cliSuite.TalosConfig}, args...)
+
+	cmd := exec.Command(cliSuite.OsctlPath, args...)
+
+	Run(&cliSuite.Suite, cmd, options...)
+}

internal/integration/base/run.go

@@ -0,0 +1,118 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration_cli
+
+package base
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+
+	"github.com/stretchr/testify/suite"
+)
+
+// RunOption configures options for Run
+type RunOption func(*runOptions)
+
+type runOptions struct {
+	shouldFail     bool
+	stderrNotEmpty bool
+	stdoutRegexps  []*regexp.Regexp
+}
+
+// ShouldFail tells Run command should fail.
+//
+// ShouldFail also sets StdErrNotEmpty.
+func ShouldFail() RunOption {
+	return func(opts *runOptions) {
+		opts.shouldFail = true
+	}
+}
+
+// ShouldSucceed tells Run command should succeed (that is default).
+func ShouldSucceed() RunOption {
+	return func(opts *runOptions) {
+		opts.shouldFail = true
+	}
+}
+
+// StderrNotEmpty tells run that stderr of the command should not be empty.
+func StdErrNotEmpty() RunOption {
+	return func(opts *runOptions) {
+		opts.stderrNotEmpty = true
+	}
+}
+
+// StdoutShouldMatch appends to the set of regexps stdout contents should match.
+func StdoutShouldMatch(r *regexp.Regexp) RunOption {
+	return func(opts *runOptions) {
+		opts.stdoutRegexps = append(opts.stdoutRegexps, r)
+	}
+}
+
+// Run executes command and asserts on its exit status/output
+func Run(suite *suite.Suite, cmd *exec.Cmd, options ...RunOption) {
+	var opts runOptions
+
+	for _, o := range options {
+		o(&opts)
+	}
+
+	var stdout, stderr bytes.Buffer
+
+	cmd.Stdin = nil
+	cmd.Stdout = &stdout
+	cmd.Stderr = io.MultiWriter(os.Stderr, &stderr)
+	cmd.Env = []string{}
+
+	// filter environment variables
+	for _, keyvalue := range os.Environ() {
+		index := strings.Index(keyvalue, "=")
+		if index < 0 {
+			continue
+		}
+		switch strings.ToUpper(keyvalue[:index]) {
+		case "PATH":
+			fallthrough
+		case "HOME":
+			fallthrough
+		case "USERNAME":
+			cmd.Env = append(cmd.Env, keyvalue)
+		}
+	}
+
+	suite.Require().NoError(cmd.Start())
+
+	err := cmd.Wait()
+
+	if err == nil {
+		if opts.shouldFail {
+			suite.Assert().NotNil(err, "command should have failed but it exited with zero exit code")
+		}
+	} else {
+		exitErr, ok := err.(*exec.ExitError)
+		if !ok {
+			suite.Require().Nil(err, "command failed to be run")
+		}
+
+		if !opts.shouldFail {
+			suite.Assert().Nil(exitErr, "command failed with exit code: %d", exitErr.ExitCode())
+		}
+	}
+
+	if opts.stderrNotEmpty {
+		suite.Assert().NotEmpty(stderr.String(), "stderr should be not empty")
+	} else {
+		suite.Assert().Empty(stderr.String(), "stderr should be empty")
+	}
+
+	for _, rx := range opts.stdoutRegexps {
+		suite.Assert().Regexp(rx, stdout.String())
+	}
+}

internal/integration/cli/cli.go

@@ -0,0 +1,19 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration
+
+// Package cli provides CLI (osctl) integration tests for Talos
+package cli
+
+import "github.com/stretchr/testify/suite"
+
+var allSuites []suite.TestingSuite
+
+// GetAllSuites returns all the suites for CLI test.
+//
+// Depending on build tags, this might return different lists.
+func GetAllSuites() []suite.TestingSuite {
+	return allSuites
+}

internal/integration/cli/version.go

@@ -0,0 +1,35 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration_cli
+
+package cli
+
+import (
+	"regexp"
+
+	"github.com/talos-systems/talos/internal/integration/base"
+)
+
+// VersionSuite verifies version API
+type VersionSuite struct {
+	base.CLISuite
+}
+
+// SuiteName ...
+func (suite *VersionSuite) SuiteName() string {
+	return "cli.VersionSuite"
+}
+
+// TestExpectedVersionMaster verifies master node version matches expected
+func (suite *VersionSuite) TestExpectedVersionMaster() {
+	suite.RunOsctl([]string{"version"},
+		base.StdoutShouldMatch(regexp.MustCompile(`Client:\n\s*Tag:\s*`+regexp.QuoteMeta(suite.Version))),
+		base.StdoutShouldMatch(regexp.MustCompile(`Server:\n\s*NODE:[^\n]+\n\s*Tag:\s*`+regexp.QuoteMeta(suite.Version))),
+	)
+}
+
+func init() {
+	allSuites = append(allSuites, new(VersionSuite))
+}

internal/integration/integration_test.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/talos-systems/talos/internal/integration/api"
 	"github.com/talos-systems/talos/internal/integration/base"
+	"github.com/talos-systems/talos/internal/integration/cli"
 	"github.com/talos-systems/talos/pkg/constants"
 	"github.com/talos-systems/talos/pkg/version"
 )
@@ -29,6 +30,7 @@ var (
 	talosConfig     string
 	target          string
 	expectedVersion string
+	osctlPath       string
 )
 
 func TestIntegration(t *testing.T) {
@@ -42,6 +44,7 @@ func TestIntegration(t *testing.T) {
 				Target:      target,
 				TalosConfig: talosConfig,
 				Version:     expectedVersion,
+				OsctlPath:   osctlPath,
 			})
 		}
 
@@ -72,6 +75,8 @@ func init() {
 	flag.StringVar(&talosConfig, "talos.config", defaultTalosConfig, "The path to the Talos configuration file")
 	flag.StringVar(&target, "talos.target", "", "target the specificed node")
 	flag.StringVar(&expectedVersion, "talos.version", version.Tag, "expected Talos version")
+	flag.StringVar(&osctlPath, "talos.osctlpath", "osctl", "The path to 'osctl' binary")
 
 	allSuites = append(allSuites, api.GetAllSuites()...)
+	allSuites = append(allSuites, cli.GetAllSuites()...)
 }