test: add integration test framework

This is just first steps and core foundation.

It can be used like:

```
make integration.test
osctl cluster create
build/integration.test -test.v
```

This should run the test against the Docker instance.

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

Dockerfile

@@ -336,6 +336,20 @@ ENV GO111MODULE on
 ARG TESTPKGS
 RUN --mount=type=cache,target=/root/.cache/go-build go test -v -count 1 -race ${TESTPKGS}
 
+# The integration-test target builds integration test binary.
+
+FROM base AS integration-test-build
+ARG SHA
+ARG TAG
+ARG VERSION_PKG="github.com/talos-systems/talos/pkg/version"
+RUN --mount=type=cache,target=/.cache/go-build GOOS=linux GOARCH=amd64 go test -c \
+    -ldflags "-s -w -X ${VERSION_PKG}.Name=Client -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" \
+    -tags integration,integration_api \
+    ./internal/integration
+
+FROM scratch AS integration-test
+COPY --from=integration-test-build /src/integration.test /integration-test
+
 # The lint target performs linting on the source code.
 
 FROM base AS lint

Makefile

@@ -272,7 +272,7 @@ container: buildkitd
 	@docker load < build/$@.tar
 
 .PHONY: basic-integration
-basic-integration:
+basic-integration: integration-test
 	@TAG=$(TAG) ./hack/test/$@.sh
 
 .PHONY: capi
@@ -300,6 +300,14 @@ unit-tests-race: buildkitd
 		--opt build-arg:TESTPKGS=$(TESTPKGS) \
 		$(COMMON_ARGS)
 
+.PHONY: integration-test
+integration-test: buildkitd
+	@$(BINDIR)/buildctl --addr $(BUILDKIT_HOST) \
+		build \
+		--output type=local,dest=bin \
+		--opt target=$@ \
+		$(COMMON_ARGS)
+
 .PHONY: fmt
 fmt:
 	@docker run --rm -it -v $(PWD):/src -w /src golang:$(GO_VERSION) bash -c "export GO111MODULE=on; export GOPROXY=https://proxy.golang.org; cd /tmp && go mod init tmp && go get mvdan.cc/gofumpt/gofumports && cd - && gofumports -w -local github.com/talos-systems/talos ."

hack/test/basic-integration.sh

@@ -10,6 +10,7 @@ export TALOSCONFIG="${TMP}/talosconfig"
 export KUBECONFIG="${TMP}/kubeconfig"
 export TIMEOUT=300
 export OSCTL="${PWD}/build/osctl-linux-amd64"
+export INTEGRATIONTEST="${PWD}/bin/integration-test"
 
 case $(uname -s) in
   Linux*)
@@ -34,6 +35,7 @@ run() {
          --entrypoint=bash \
          --mount type=bind,source=${TMP},target=${TMP} \
          -v ${OSCTL}:/bin/osctl:ro \
+         -v ${INTEGRATIONTEST}:/bin/integration-test:ro \
          -e KUBECONFIG=${KUBECONFIG} \
          -e TALOSCONFIG=${TALOSCONFIG} \
          k8s.gcr.io/hyperkube:${KUBERNETES_VERSION} -c "${1}"
@@ -89,3 +91,5 @@ run "osctl config target 10.5.0.3 && osctl -t 10.5.0.3 service etcd | grep Runni
 run "osctl config target 10.5.0.4 && osctl -t 10.5.0.4 service etcd | grep Running"
 run "osctl --target 10.5.0.2,10.5.0.3,10.5.0.4,10.5.0.5 containers"
 run "osctl --target 10.5.0.2,10.5.0.3,10.5.0.4,10.5.0.5 services"
+
+run "integration-test -test.v -talos.target 10.5.0.2"

internal/integration/api/api.go

@@ -0,0 +1,19 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration
+
+// Package api provides API integration tests for Talos
+package api
+
+import "github.com/stretchr/testify/suite"
+
+var allSuites []suite.TestingSuite
+
+// GetAllSuites returns all the suites for API test.
+//
+// Depending on build tags, this might return different lists.
+func GetAllSuites() []suite.TestingSuite {
+	return allSuites
+}

internal/integration/api/version.go

@@ -0,0 +1,47 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration_api
+
+package api
+
+import (
+	"context"
+
+	"github.com/talos-systems/talos/internal/integration/base"
+)
+
+// VersionSuite verifies version API
+type VersionSuite struct {
+	base.APISuite
+}
+
+// SuiteName ...
+func (suite *VersionSuite) SuiteName() string {
+	return "api.VersionSuite"
+}
+
+// SetupSuite ...
+func (suite *VersionSuite) SetupSuite() {
+	suite.InitClient()
+}
+
+// TearDownSuite ...
+func (suite *VersionSuite) TearDownSuite() {
+	if suite.Client != nil {
+		suite.Assert().NoError(suite.Client.Close())
+	}
+}
+
+// TestExpectedVersionMaster verifies master node version matches expected
+func (suite *VersionSuite) TestExpectedVersionMaster() {
+	v, err := suite.Client.Version(context.Background())
+	suite.Require().NoError(err)
+
+	suite.Assert().Equal(suite.Version, v.Response[0].Version.Tag)
+}
+
+func init() {
+	allSuites = append(allSuites, new(VersionSuite))
+}

internal/integration/base/api.go

@@ -0,0 +1,35 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration_api
+
+package base
+
+import (
+	"github.com/stretchr/testify/suite"
+
+	"github.com/talos-systems/talos/cmd/osctl/pkg/client"
+	"github.com/talos-systems/talos/pkg/constants"
+)
+
+// APISuite is a base suite for API tests
+type APISuite struct {
+	suite.Suite
+	TalosSuite
+
+	Client *client.Client
+}
+
+// InitClient initializes Talos API client
+func (apiSuite *APISuite) InitClient() {
+	target, creds, err := client.NewClientTargetAndCredentialsFromConfig(apiSuite.TalosConfig)
+	apiSuite.Require().NoError(err)
+
+	if apiSuite.Target != "" {
+		target = apiSuite.Target
+	}
+
+	apiSuite.Client, err = client.NewClient(creds, target, constants.OsdPort)
+	apiSuite.Require().NoError(err)
+}

internal/integration/base/base.go

@@ -0,0 +1,33 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration
+
+// Package base provides shared definition of base suites for tests
+package base
+
+// TalosSuite defines most common settings for integration test suites
+type TalosSuite struct {
+	// Target is address of master node, if not set config is used
+	Target string
+	// TalosConfig is a path to talosconfig
+	TalosConfig string
+	// Version is the (expected) version of Talos tests are running against
+	Version string
+}
+
+// ConfiguredSuite expects config to be set before running
+type ConfiguredSuite interface {
+	SetConfig(config TalosSuite)
+}
+
+// SetConfig implements ConfiguredSuite
+func (suite *TalosSuite) SetConfig(config TalosSuite) {
+	*suite = config
+}
+
+// NamedSuite interface provides names for test suites
+type NamedSuite interface {
+	SuiteName() string
+}

internal/integration/integration_test.go

@@ -0,0 +1,77 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration
+
+// Package integration_test contains core runners for integration tests
+package integration_test
+
+import (
+	"flag"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+
+	"github.com/talos-systems/talos/internal/integration/api"
+	"github.com/talos-systems/talos/internal/integration/base"
+	"github.com/talos-systems/talos/pkg/constants"
+	"github.com/talos-systems/talos/pkg/version"
+)
+
+// Accumulated list of all the suites to run
+var allSuites []suite.TestingSuite
+
+// Flag values
+var (
+	talosConfig     string
+	target          string
+	expectedVersion string
+)
+
+func TestIntegration(t *testing.T) {
+	if talosConfig == "" {
+		t.Error("--talos.config is not provided")
+	}
+
+	for _, s := range allSuites {
+		if configuredSuite, ok := s.(base.ConfiguredSuite); ok {
+			configuredSuite.SetConfig(base.TalosSuite{
+				Target:      target,
+				TalosConfig: talosConfig,
+				Version:     expectedVersion,
+			})
+		}
+
+		var suiteName string
+		if namedSuite, ok := s.(base.NamedSuite); ok {
+			suiteName = namedSuite.SuiteName()
+		}
+
+		t.Run(suiteName, func(tt *testing.T) {
+			suite.Run(tt, s) //nolint: scopelint
+		})
+	}
+}
+
+func init() {
+	var (
+		defaultTalosConfig string
+		ok                 bool
+	)
+
+	if defaultTalosConfig, ok = os.LookupEnv(constants.TalosConfigEnvVar); !ok {
+		home, err := os.UserHomeDir()
+		if err == nil {
+			defaultTalosConfig = path.Join(home, ".talos", "config")
+		}
+	}
+
+	flag.StringVar(&talosConfig, "talos.config", defaultTalosConfig, "The path to the Talos configuration file")
+	flag.StringVar(&target, "talos.target", "", "target the specificed node")
+	flag.StringVar(&expectedVersion, "talos.version", version.Tag, "expected Talos version")
+
+	allSuites = append(allSuites, api.GetAllSuites()...)
+}

internal/integration/version_test.go

@@ -0,0 +1,34 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build integration
+
+// Package integration_test contains core runners for integration tests
+package integration_test
+
+import (
+	"github.com/stretchr/testify/suite"
+
+	"github.com/talos-systems/talos/internal/integration/base"
+)
+
+// VersionSuite
+type VersionSuite struct {
+	suite.Suite
+	base.TalosSuite
+}
+
+func (suite *VersionSuite) SuiteName() string {
+	return "VersionSuite"
+}
+
+func (suite *VersionSuite) TestExpectedVersion() {
+	const versionRegex = `v([0-9]+)\.([0-9]+)\.([0-9]+)(-[0-9]+-[a-z]+\.[0-9]+)?(-.g[a-f0-9]+)?(-dirty)?`
+
+	suite.Assert().Regexp(versionRegex, suite.Version)
+}
+
+func init() {
+	allSuites = append(allSuites, new(VersionSuite))
+}