chore: re-enable e2e testing

This PR will re-enable e2e testing by using the new cluster api
bootstrap provider and various infra providers.

Signed-off-by: Spencer Smith <robertspencersmith@gmail.com>

.drone.yml

@@ -988,46 +988,6 @@ steps:
   depends_on:
   - image-gcp
 
-- name: e2e-integration-aws
-  pull: always
-  image: autonomy/build-container:latest
-  commands:
-  - make e2e-integration
-  environment:
-    BINDIR: /usr/local/bin
-    BUILDKIT_HOST: ${BUILDKIT_HOST=tcp://buildkitd.ci.svc:1234}
-    PLATFORM: aws
-  volumes:
-  - name: dockersock
-    path: /var/run
-  - name: dev
-    path: /dev
-  - name: tmp
-    path: /tmp
-  depends_on:
-  - capi
-  - push-image-aws
-
-- name: e2e-integration-azure
-  pull: always
-  image: autonomy/build-container:latest
-  commands:
-  - make e2e-integration
-  environment:
-    BINDIR: /usr/local/bin
-    BUILDKIT_HOST: ${BUILDKIT_HOST=tcp://buildkitd.ci.svc:1234}
-    PLATFORM: azure
-  volumes:
-  - name: dockersock
-    path: /var/run
-  - name: dev
-    path: /dev
-  - name: tmp
-    path: /tmp
-  depends_on:
-  - capi
-  - push-image-azure
-
 - name: e2e-integration-gcp
   pull: always
   image: autonomy/build-container:latest

Makefile

@@ -252,7 +252,7 @@ push-image-azure:
 
 .PHONY: push-image-gcp
 push-image-gcp:
-	@TAG=$(TAG) ./hack/test/gcp-setup.sh
+	@TAG=$(TAG) SHA=$(SHA) ./hack/test/gcp-setup.sh
 
 .PHONY: image-test
 image-test:
@@ -281,7 +281,7 @@ capi:
 
 .PHONY: e2e-integration
 e2e-integration:
-	@TAG=$(TAG) ./hack/test/$@.sh
+	@TAG=$(TAG) SHA=$(SHA) ./hack/test/$@.sh
 
 .PHONY: unit-tests
 unit-tests: buildkitd

hack/drone.jsonnet

@@ -258,8 +258,8 @@ local e2e_steps = default_steps + [
   push_image_aws,
   push_image_azure,
   push_image_gcp,
-  e2e_integration_aws,
-  e2e_integration_azure,
+  //e2e_integration_aws,
+  // e2e_integration_azure,
   e2e_integration_gcp,
 ];
 

hack/test/capi.sh

@@ -7,21 +7,25 @@ source ./hack/test/e2e-runner.sh
 
 ## Create tmp dir
 mkdir -p ${TMP}
+cp ${PWD}/hack/test/manifests/provider-components.yaml ${TMP}/provider-components.yaml
+
+## Template out gcp components
+apk add --no-cache gettext
+export GCP_B64ENCODED_CREDENTIALS=${GCE_SVC_ACCT} 
+cat ${PWD}/hack/test/manifests/capg-components.yaml| envsubst > ${TMP}/capg-components.yaml
+##Until next alpha release, keep a local copy of capg-components.yaml. 
+##They've got an incorrect image pull policy.
+##curl -L ${CAPG_COMPONENTS} | envsubst > ${TMP}/capg-components.yaml
 
 ## Drop in capi stuff
-sed "s/{{PACKET_AUTH_TOKEN}}/${PACKET_AUTH_TOKEN}/" ${PWD}/hack/test/manifests/provider-components.yaml > ${TMP}/provider-components.yaml
-
-sed -e "s#{{GCE_SVC_ACCT}}#${GCE_SVC_ACCT}#" \
-    -e "s#{{AZURE_SVC_ACCT}}#${AZURE_SVC_ACCT}#" \
-    -e "s#{{AWS_SVC_ACCT}}#${AWS_SVC_ACCT}#" ${PWD}/hack/test/manifests/capi-secrets.yaml > ${TMP}/capi-secrets.yaml
-
-e2e_run "kubectl apply -f ${TMP}/provider-components.yaml -f ${TMP}/capi-secrets.yaml"
+e2e_run "kubectl apply -f ${TMP}/provider-components.yaml"
+e2e_run "kubectl apply -f ${CAPI_COMPONENTS}"
+e2e_run "kubectl apply -f ${TMP}/capg-components.yaml"
 
 ## Wait for talosconfig in cm then dump it out
 e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
-         pod='pod/cluster-api-provider-talos-controller-manager-0'
-         until KUBECONFIG=${TMP}/kubeconfig kubectl wait --timeout=1s --for=condition=Ready -n ${CAPI_NS} \${pod}; do
+         until KUBECONFIG=${TMP}/kubeconfig kubectl wait --timeout=1s --for=condition=Ready -n ${CABPT_NS} pods --all; do
            [[ \$(date +%s) -gt \$timeout ]] && exit 1
-           echo 'Waiting to CAPT pod to be available...'
+           echo 'Waiting to CABPT pod to be available...'
            sleep 10
          done"

hack/test/e2e-integration.sh

@@ -6,14 +6,11 @@ source ./hack/test/e2e-runner.sh
 ## Create tmp dir
 mkdir -p ${TMPPLATFORM}
 
-NAME_PREFIX="talos-e2e-${TAG}-${PLATFORM}"
+NAME_PREFIX="talos-e2e-${SHA}-${PLATFORM}"
 
 ## Cleanup the platform resources upon any exit
 cleanup() {
-  e2e_run "KUBECONFIG=${TMP}/kubeconfig kubectl delete machine ${NAME_PREFIX}-master-0 ${NAME_PREFIX}-master-1 ${NAME_PREFIX}-master-2
-           KUBECONFIG=${TMP}/kubeconfig kubectl scale machinedeployment ${NAME_PREFIX}-workers --replicas=0
-           KUBECONFIG=${TMP}/kubeconfig kubectl delete machinedeployment ${NAME_PREFIX}-workers
-           KUBECONFIG=${TMP}/kubeconfig kubectl delete cluster ${NAME_PREFIX}"
+  e2e_run "KUBECONFIG=${TMP}/kubeconfig kubectl delete cluster ${NAME_PREFIX}"
 }
 
 trap cleanup EXIT
@@ -23,11 +20,25 @@ e2e_run "KUBECONFIG=${TMP}/kubeconfig kubectl apply -f ${TMPPLATFORM}/cluster.ya
 
 ## Wait for talosconfig in cm then dump it out
 e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
-         until KUBECONFIG=${TMP}/kubeconfig kubectl get cm -n ${CAPI_NS} ${NAME_PREFIX}-master-0; do
+         until [ -n \"\${STATUS_TALOSCONFIG}\" ]; do
            [[ \$(date +%s) -gt \$timeout ]] && exit 1
            sleep 10
+           STATUS_TALOSCONFIG=\$( KUBECONFIG=${TMP}/kubeconfig kubectl get talosconfig ${NAME_PREFIX}-controlplane-0 -o jsonpath='{.status.talosConfig}' ) 
          done
-         KUBECONFIG=${TMP}/kubeconfig kubectl get cm -n ${CAPI_NS} ${NAME_PREFIX}-master-0 -o jsonpath='{.data.talosconfig}' > ${TALOSCONFIG}"
+         echo \"\${STATUS_TALOSCONFIG}\" > ${TALOSCONFIG}"
+
+## Wait until we have an IP for master 0
+e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
+         until [ -n \"\${MASTER_0_IP}\" ]; do
+           [[ \$(date +%s) -gt \$timeout ]] && exit 1
+           sleep 10
+           MASTER_0_IP=\$( KUBECONFIG=${TMP}/kubeconfig kubectl get machine -o go-template --template='{{range .status.addresses}}{{if eq .type \"ExternalIP\"}}{{.address}}{{end}}{{end}}' ${NAME_PREFIX}-controlplane-0 )
+         done
+         echo \${MASTER_0_IP} > ${TMP}/master0ip"
+
+## Target master 0 for osctl
+e2e_run "MASTER_0_IP=\$( cat ${TMP}/master0ip )
+         /bin/osctl config target \${MASTER_0_IP}"
 
 ## Wait for kubeconfig from capi master-0
 e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
@@ -44,14 +55,6 @@ e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
            sleep 10
          done"
 
-## Wait for kube-proxy up
-e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
-         until kubectl get po -n kube-system -l k8s-app=kube-proxy -o go-template='{{ len .items }}' | grep ${NUM_NODES} > /dev/null; do
-           [[ \$(date +%s) -gt \$timeout ]] && exit 1
-           kubectl get po -n kube-system -l k8s-app=kube-proxy
-           sleep 10
-         done"
-
 ##  Wait for nodes ready
 e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
          until kubectl wait --timeout=1s --for=condition=ready=true --all nodes > /dev/null; do

hack/test/e2e-runner.sh

@@ -1,5 +1,5 @@
 # NB: There is a known bug that causes CRD scaling issues in 1.15 kubectl or later.
-export KUBERNETES_VERSION=v1.14.6
+export KUBERNETES_VERSION=v1.16.2
 export TALOS_IMG="docker.io/autonomy/talos:${TAG}"
 export TMP="/tmp/e2e"
 export TMPPLATFORM="${TMP}/${PLATFORM}"
@@ -10,17 +10,26 @@ export KUBECONFIG="${TMPPLATFORM}/kubeconfig"
 ## Long timeout due to provisioning times
 export TIMEOUT=9000
 
-## Total number of nodes we'll be waiting to come up (3 Masters + 3 Workers)
+## Total number of nodes we'll be waiting to come up (3 Masters, 3 Workers)
 export NUM_NODES=6
 
-## ClusterAPI Provider Talos (CAPT)
-export CAPT_VERSION="0.1.0-alpha.2"
-export PROVIDER_COMPONENTS="https://github.com/talos-systems/cluster-api-provider-talos/releases/download/v${CAPT_VERSION}/provider-components.yaml"
-export KUSTOMIZE_VERSION="1.0.11"
+## ClusterAPI Bootstrap Provider Talos (CABPT)
+export CABPT_VERSION="0.1.0-alpha.0"
+export CABPT_COMPONENTS="https://github.com/talos-systems/cluster-api-bootstrap-provider-talos/releases/download/v${CABPT_VERSION}/provider-components.yaml"
+
+## ClusterAPI (CAPI)
+export CAPI_VERSION="0.2.6"
+export CAPI_COMPONENTS="https://github.com/kubernetes-sigs/cluster-api/releases/download/v${CAPI_VERSION}/cluster-api-components.yaml"
+
+## ClusterAPI Provider GCP (CAPG)
+export CAPG_VERSION="0.2.0-alpha.2"
+export CAPG_COMPONENTS="https://github.com/kubernetes-sigs/cluster-api-provider-gcp/releases/download/v${CAPG_VERSION}/infrastructure-components.yaml"
+
+export KUSTOMIZE_VERSION="3.1.0"
 export KUSTOMIZE_URL="https://github.com/kubernetes-sigs/kustomize/releases/download/v${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64"
 export SONOBUOY_VERSION="0.16.1"
 export SONOBUOY_URL="https://github.com/heptio/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz"
-export CAPI_NS="cluster-api-provider-talos-system"
+export CABPT_NS="cabpt-system"
 
 e2e_run() {
   docker run \

hack/test/gcp-setup.sh

@@ -12,11 +12,11 @@ echo $GCE_SVC_ACCT | base64 -d > ${TMP}/svc-acct.json
 gcloud auth activate-service-account --key-file ${TMP}/svc-acct.json
 
 ## Push talos-gcp to storage bucket
-gsutil cp ./build/gcp.tar.gz gs://talos-e2e/gcp-${TAG}.tar.gz
+gsutil cp ./build/gcp.tar.gz gs://talos-e2e/gcp-${SHA}.tar.gz
 
 ## Create image from talos-gcp
-gcloud --quiet --project talos-testbed compute images delete talos-e2e-${TAG} || true ##Ignore error if image doesn't exist
-gcloud --quiet --project talos-testbed compute images create talos-e2e-${TAG} --source-uri gs://talos-e2e/gcp-${TAG}.tar.gz
+gcloud --quiet --project talos-testbed compute images delete talos-e2e-${SHA} || true ##Ignore error if image doesn't exist
+gcloud --quiet --project talos-testbed compute images create talos-e2e-${SHA} --source-uri gs://talos-e2e/gcp-${SHA}.tar.gz
 
 ## Setup the cluster YAML.
-sed "s/{{TAG}}/${TAG}/" ${PWD}/hack/test/manifests/gcp-cluster.yaml > ${TMP}/cluster.yaml
+sed -e "s/{{TAG}}/${SHA}/" ${PWD}/hack/test/manifests/gcp-cluster.yaml > ${TMP}/cluster.yaml

hack/test/manifests/capg-components.yaml

@@ -0,0 +1,728 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    control-plane: capa-controller-manager
+  name: capg-system
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  name: gcpclusters.infrastructure.cluster.x-k8s.io
+spec:
+  group: infrastructure.cluster.x-k8s.io
+  names:
+    categories:
+    - cluster-api
+    kind: GCPCluster
+    plural: gcpclusters
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: GCPCluster is the Schema for the gcpclusters API
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: GCPClusterSpec defines the desired state of GCPCluster
+          properties:
+            additionalLabels:
+              additionalProperties:
+                type: string
+              description: AdditionalLabels is an optional set of tags to add to GCP
+                resources managed by the GCP provider, in addition to the ones added
+                by default.
+              type: object
+            network:
+              description: NetworkSpec encapsulates all things related to GCP network.
+              properties:
+                autoCreateSubnetworks:
+                  description: "AutoCreateSubnetworks: When set to true, the VPC network
+                    is created in \"auto\" mode. When set to false, the VPC network
+                    is created in \"custom\" mode. \n An auto mode VPC network starts
+                    with one subnet per region. Each subnet has a predetermined range
+                    as described in Auto mode VPC network IP ranges. \n Defaults to
+                    true."
+                  type: boolean
+                loadBalancerBackendPort:
+                  description: Allow for configuration of load balancer backend (useful
+                    for changing apiserver port)
+                  format: int32
+                  type: integer
+                name:
+                  description: Name is the name of the network to be used.
+                  type: string
+                subnets:
+                  description: Subnets configuration.
+                  items:
+                    description: SubnetSpec configures an GCP Subnet.
+                    properties:
+                      cidrBlock:
+                        description: CidrBlock is the range of internal addresses
+                          that are owned by this subnetwork. Provide this property
+                          when you create the subnetwork. For example, 10.0.0.0/8
+                          or 192.168.0.0/16. Ranges must be unique and non-overlapping
+                          within a network. Only IPv4 is supported. This field can
+                          be set only at resource creation time.
+                        type: string
+                      description:
+                        description: Description is an optional description associated
+                          with the resource.
+                        type: string
+                      name:
+                        description: Name defines a unique identifier to reference
+                          this resource.
+                        type: string
+                      privateGoogleAccess:
+                        description: PrivateGoogleAccess defines whether VMs in this
+                          subnet can access Google services without assigning external
+                          IP addresses
+                        type: boolean
+                      region:
+                        description: Region is the name of the region where the Subnetwork
+                          resides.
+                        type: string
+                      routeTableId:
+                        description: 'EnableFlowLogs: Whether to enable flow logging
+                          for this subnetwork. If this field is not explicitly set,
+                          it will not appear in get listings. If not set the default
+                          behavior is to disable flow logging.'
+                        type: boolean
+                      secondaryCidrBlocks:
+                        additionalProperties:
+                          type: string
+                        description: SecondaryCidrBlocks defines secondary CIDR ranges,
+                          from which secondary IP ranges of a VM may be allocated
+                        type: object
+                    type: object
+                  type: array
+              type: object
+            project:
+              description: Project is the name of the project to deploy the cluster
+                to.
+              type: string
+            region:
+              description: The GCP Region the cluster lives in.
+              type: string
+          required:
+          - project
+          - region
+          type: object
+        status:
+          description: GCPClusterStatus defines the observed state of GCPCluster
+          properties:
+            apiEndpoints:
+              description: APIEndpoints represents the endpoints to communicate with
+                the control plane.
+              items:
+                description: APIEndpoint represents a reachable Kubernetes API endpoint.
+                properties:
+                  host:
+                    description: The hostname on which the API server is serving.
+                    type: string
+                  port:
+                    description: The port on which the API server is serving.
+                    type: integer
+                required:
+                - host
+                - port
+                type: object
+              type: array
+            network:
+              description: Network encapsulates GCP networking resources.
+              properties:
+                apiServerBackendService:
+                  description: APIServerBackendService is the full reference to the
+                    backend service created for the API Server.
+                  type: string
+                apiServerForwardingRule:
+                  description: APIServerForwardingRule is the full reference to the
+                    forwarding rule created for the API Server.
+                  type: string
+                apiServerHealthCheck:
+                  description: APIServerHealthCheck is the full reference to the health
+                    check created for the API Server.
+                  type: string
+                apiServerInstanceGroups:
+                  additionalProperties:
+                    type: string
+                  description: APIServerInstanceGroups is a map from zone to the full
+                    reference to the instance groups created for the control plane
+                    nodes created in the same zone.
+                  type: object
+                apiServerIpAddress:
+                  description: APIServerAddress is the IPV4 global address assigned
+                    to the load balancer created for the API Server.
+                  type: string
+                apiServerTargetProxy:
+                  description: APIServerTargetProxy is the full reference to the target
+                    proxy created for the API Server.
+                  type: string
+                firewallRules:
+                  additionalProperties:
+                    type: string
+                  description: FirewallRules is a map from the name of the rule to
+                    its full reference.
+                  type: object
+                selfLink:
+                  description: SelfLink is the link to the Network used for this cluster.
+                  type: string
+              type: object
+            ready:
+              description: Bastion Instance `json:"bastion,omitempty"`
+              type: boolean
+          required:
+          - ready
+          type: object
+      type: object
+  version: v1alpha2
+  versions:
+  - name: v1alpha2
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  name: gcpmachines.infrastructure.cluster.x-k8s.io
+spec:
+  group: infrastructure.cluster.x-k8s.io
+  names:
+    categories:
+    - cluster-api
+    kind: GCPMachine
+    plural: gcpmachines
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: GCPMachine is the Schema for the gcpmachines API
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: GCPMachineSpec defines the desired state of GCPMachine
+          properties:
+            additionalLabels:
+              additionalProperties:
+                type: string
+              description: AdditionalLabels is an optional set of tags to add to an
+                instance, in addition to the ones added by default by the GCP provider.
+                If both the GCPCluster and the GCPMachine specify the same tag name
+                with different values, the GCPMachine's value takes precedence.
+              type: object
+            additionalNetworkTags:
+              description: AdditionalNetworkTags is a list of network tags that should
+                be applied to the instance. These tags are set in addition to any
+                network tags defined at the cluster level or in the actuator.
+              items:
+                type: string
+              type: array
+            image:
+              description: Image is the full reference to a valid image to be used
+                for this machine. Takes precedence over ImageFamily.
+              type: string
+            imageFamily:
+              description: ImageFamily is the full reference to a valid image family
+                to be used for this machine.
+              type: string
+            instanceType:
+              description: 'InstanceType is the type of instance to create. Example:
+                n1.standard-2'
+              type: string
+            providerID:
+              description: ProviderID is the unique identifier as specified by the
+                cloud provider.
+              type: string
+            publicIP:
+              description: PublicIP specifies whether the instance should get a public
+                IP. Set this to true if you don't have a NAT instances or Cloud Nat
+                setup.
+              type: boolean
+            rootDeviceSize:
+              description: RootDeviceSize is the size of the root volume in GB. Defaults
+                to 30.
+              format: int64
+              type: integer
+            serviceAccounts:
+              description: 'ServiceAccount specifies the service account email and
+                which scopes to assign to the machine. Defaults to: email: "default",
+                scope: []{compute.CloudPlatformScope}'
+              properties:
+                email:
+                  description: 'Email: Email address of the service account.'
+                  type: string
+                scopes:
+                  description: 'Scopes: The list of scopes to be made available for
+                    this service account.'
+                  items:
+                    type: string
+                  type: array
+              type: object
+            subnet:
+              description: Subnet is a reference to the subnetwork to use for this
+                instance. If not specified, the first subnetwork retrieved from the
+                Cluster Region and Network is picked.
+              type: string
+            zone:
+              description: Zone is references the GCP zone to use for this instance.
+              type: string
+          required:
+          - instanceType
+          - zone
+          type: object
+        status:
+          description: GCPMachineStatus defines the observed state of GCPMachine
+          properties:
+            addresses:
+              description: Addresses contains the GCP instance associated addresses.
+              items:
+                description: NodeAddress contains information for the node's address.
+                properties:
+                  address:
+                    description: The node address.
+                    type: string
+                  type:
+                    description: Node address type, one of Hostname, ExternalIP or
+                      InternalIP.
+                    type: string
+                required:
+                - address
+                - type
+                type: object
+              type: array
+            errorMessage:
+              description: "ErrorMessage will be set in the event that there is a
+                terminal problem reconciling the Machine and will contain a more verbose
+                string suitable for logging and human consumption. \n This field should
+                not be set for transitive errors that a controller faces that are
+                expected to be fixed automatically over time (like service outages),
+                but instead indicate that something is fundamentally wrong with the
+                Machine's spec or the configuration of the controller, and that manual
+                intervention is required. Examples of terminal errors would be invalid
+                combinations of settings in the spec, values that are unsupported
+                by the controller, or the responsible controller itself being critically
+                misconfigured. \n Any transient errors that occur during the reconciliation
+                of Machines can be added as events to the Machine object and/or logged
+                in the controller's output."
+              type: string
+            errorReason:
+              description: "ErrorReason will be set in the event that there is a terminal
+                problem reconciling the Machine and will contain a succinct value
+                suitable for machine interpretation. \n This field should not be set
+                for transitive errors that a controller faces that are expected to
+                be fixed automatically over time (like service outages), but instead
+                indicate that something is fundamentally wrong with the Machine's
+                spec or the configuration of the controller, and that manual intervention
+                is required. Examples of terminal errors would be invalid combinations
+                of settings in the spec, values that are unsupported by the controller,
+                or the responsible controller itself being critically misconfigured.
+                \n Any transient errors that occur during the reconciliation of Machines
+                can be added as events to the Machine object and/or logged in the
+                controller's output."
+              type: string
+            instanceState:
+              description: InstanceStatus is the status of the GCP instance for this
+                machine.
+              type: string
+            ready:
+              description: Ready is true when the provider resource is ready.
+              type: boolean
+          type: object
+      type: object
+  version: v1alpha2
+  versions:
+  - name: v1alpha2
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  name: gcpmachinetemplates.infrastructure.cluster.x-k8s.io
+spec:
+  group: infrastructure.cluster.x-k8s.io
+  names:
+    categories:
+    - cluster-api
+    kind: GCPMachineTemplate
+    plural: gcpmachinetemplates
+  scope: Namespaced
+  validation:
+    openAPIV3Schema:
+      description: GCPMachineTemplate is the Schema for the gcpmachinetemplates API
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: GCPMachineTemplateSpec defines the desired state of GCPMachineTemplate
+          properties:
+            template:
+              description: GCPMachineTemplateResource describes the data needed to
+                create am GCPMachine from a template
+              properties:
+                spec:
+                  description: Spec is the specification of the desired behavior of
+                    the machine.
+                  properties:
+                    additionalLabels:
+                      additionalProperties:
+                        type: string
+                      description: AdditionalLabels is an optional set of tags to
+                        add to an instance, in addition to the ones added by default
+                        by the GCP provider. If both the GCPCluster and the GCPMachine
+                        specify the same tag name with different values, the GCPMachine's
+                        value takes precedence.
+                      type: object
+                    additionalNetworkTags:
+                      description: AdditionalNetworkTags is a list of network tags
+                        that should be applied to the instance. These tags are set
+                        in addition to any network tags defined at the cluster level
+                        or in the actuator.
+                      items:
+                        type: string
+                      type: array
+                    image:
+                      description: Image is the full reference to a valid image to
+                        be used for this machine. Takes precedence over ImageFamily.
+                      type: string
+                    imageFamily:
+                      description: ImageFamily is the full reference to a valid image
+                        family to be used for this machine.
+                      type: string
+                    instanceType:
+                      description: 'InstanceType is the type of instance to create.
+                        Example: n1.standard-2'
+                      type: string
+                    providerID:
+                      description: ProviderID is the unique identifier as specified
+                        by the cloud provider.
+                      type: string
+                    publicIP:
+                      description: PublicIP specifies whether the instance should
+                        get a public IP. Set this to true if you don't have a NAT
+                        instances or Cloud Nat setup.
+                      type: boolean
+                    rootDeviceSize:
+                      description: RootDeviceSize is the size of the root volume in
+                        GB. Defaults to 30.
+                      format: int64
+                      type: integer
+                    serviceAccounts:
+                      description: 'ServiceAccount specifies the service account email
+                        and which scopes to assign to the machine. Defaults to: email:
+                        "default", scope: []{compute.CloudPlatformScope}'
+                      properties:
+                        email:
+                          description: 'Email: Email address of the service account.'
+                          type: string
+                        scopes:
+                          description: 'Scopes: The list of scopes to be made available
+                            for this service account.'
+                          items:
+                            type: string
+                          type: array
+                      type: object
+                    subnet:
+                      description: Subnet is a reference to the subnetwork to use
+                        for this instance. If not specified, the first subnetwork
+                        retrieved from the Cluster Region and Network is picked.
+                      type: string
+                    zone:
+                      description: Zone is references the GCP zone to use for this
+                        instance.
+                      type: string
+                  required:
+                  - instanceType
+                  - zone
+                  type: object
+              required:
+              - spec
+              type: object
+          required:
+          - template
+          type: object
+      type: object
+  version: v1alpha2
+  versions:
+  - name: v1alpha2
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: capg-leader-election-role
+  namespace: capg-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - configmaps/status
+  verbs:
+  - get
+  - update
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: capg-manager-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - cluster.x-k8s.io
+  resources:
+  - clusters
+  - clusters/status
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - cluster.x-k8s.io
+  resources:
+  - machines
+  - machines/status
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - infrastructure.cluster.x-k8s.io
+  resources:
+  - gcpclusters
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - infrastructure.cluster.x-k8s.io
+  resources:
+  - gcpclusters/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - infrastructure.cluster.x-k8s.io
+  resources:
+  - gcpmachines
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - infrastructure.cluster.x-k8s.io
+  resources:
+  - gcpmachines/status
+  verbs:
+  - get
+  - patch
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: capg-proxy-role
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: capg-leader-election-rolebinding
+  namespace: capg-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: capg-leader-election-role
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: capg-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: capg-manager-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: capg-manager-role
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: capg-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: capg-proxy-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: capg-proxy-role
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: capg-system
+---
+apiVersion: v1
+data:
+  credentials.json: ${GCP_B64ENCODED_CREDENTIALS}
+kind: Secret
+metadata:
+  name: capg-manager-bootstrap-credentials
+  namespace: capg-system
+type: Opaque
+---
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    prometheus.io/port: "8443"
+    prometheus.io/scheme: https
+    prometheus.io/scrape: "true"
+  labels:
+    control-plane: capa-controller-manager
+  name: capg-controller-manager-metrics-service
+  namespace: capg-system
+spec:
+  ports:
+  - name: https
+    port: 8443
+    targetPort: https
+  selector:
+    control-plane: capa-controller-manager
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    control-plane: capa-controller-manager
+  name: capg-controller-manager
+  namespace: capg-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      control-plane: capa-controller-manager
+  template:
+    metadata:
+      labels:
+        control-plane: capa-controller-manager
+    spec:
+      containers:
+      - args:
+        - --enable-leader-election
+        env:
+        - name: GOOGLE_APPLICATION_CREDENTIALS
+          value: /home/.gcp/credentials.json
+        image: rsmitty/cluster-api-gcp-controller-amd64:latest
+        imagePullPolicy: Always
+        name: manager
+        volumeMounts:
+        - mountPath: /home/.gcp
+          name: credentials
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: credentials
+        secret:
+          secretName: capg-manager-bootstrap-credentials

hack/test/manifests/gcp-cluster.yaml

@@ -1,129 +1,222 @@
-apiVersion: cluster.k8s.io/v1alpha1
+## Cluster configs
+
+apiVersion: cluster.x-k8s.io/v1alpha2
 kind: Cluster
 metadata:
-  annotations: null
   name: talos-e2e-{{TAG}}-gcp
+  namespace: default
 spec:
   clusterNetwork:
     pods:
       cidrBlocks:
-        - 192.168.0.0/16
-    serviceDomain: cluster.local
-    services:
-      cidrBlocks:
-        - 10.96.0.0/12
-  providerSpec:
-    value:
-      apiVersion: talosproviderconfig/v1alpha1
-      kind: TalosClusterProviderSpec
-      platform:
-        config: |-
-          region: "us-central1"
-          project: "talos-testbed"
-        type: gce
-      controlplane:
-        count: 3
-        k8sversion: "1.16.1"
+      - 192.168.0.0/16
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
+    kind: GCPCluster
+    name: talos-e2e-{{TAG}}-gcp
+    namespace: default
 ---
-apiVersion: cluster.k8s.io/v1alpha1
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
+kind: GCPCluster
+metadata:
+  name: talos-e2e-{{TAG}}-gcp
+  namespace: default
+spec:
+  project: talos-testbed
+  region: us-central1
+---
+
+## Controlplane 0 configs
+
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha2
+kind: TalosConfig
+metadata:
+  name: talos-e2e-{{TAG}}-gcp-controlplane-0
+  labels:
+    cluster.x-k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
+spec:
+  machineType: init
+---
+apiVersion: cluster.x-k8s.io/v1alpha2
 kind: Machine
 metadata:
   labels:
-    cluster.k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
-    set: master
-  name: talos-e2e-{{TAG}}-gcp-master-0
+    cluster.x-k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
+    cluster.x-k8s.io/control-plane: "true"
+  name: talos-e2e-{{TAG}}-gcp-controlplane-0
+  namespace: default
 spec:
-  providerSpec:
-    value:
-      apiVersion: talosproviderconfig/v1alpha1
-      kind: TalosMachineProviderSpec
-      platform:
-        config: |-
-          zone: "us-central1-c"
-          project: "talos-testbed"
-          instances:
-            type:  "n1-standard-2"
-            image: "https://www.googleapis.com/compute/v1/projects/talos-testbed/global/images/talos-e2e-{{TAG}}"
-            disks:
-              size: 50
-        type: gce
+  bootstrap:
+    configRef:
+      apiVersion: bootstrap.cluster.x-k8s.io/v1alpha2
+      kind: TalosConfig
+      name: talos-e2e-{{TAG}}-gcp-controlplane-0
+      namespace: default
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
+    kind: GCPMachine
+    name: talos-e2e-{{TAG}}-gcp-controlplane-0
+    namespace: default
+  version: 1.16.1
 ---
-apiVersion: cluster.k8s.io/v1alpha1
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
+kind: GCPMachine
+metadata:
+  name: talos-e2e-{{TAG}}-gcp-controlplane-0
+  namespace: default
+spec:
+  instanceType: n1-standard-2
+  zone: us-central1-a
+  image: projects/talos-testbed/global/images/talos-e2e-{{TAG}}
+  serviceAccounts: {}
+  publicIP: true
+---
+
+## Controlplane 1 configs
+
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha2
+kind: TalosConfig
+metadata:
+  name: talos-e2e-{{TAG}}-gcp-controlplane-1
+  labels:
+    cluster.x-k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
+spec:
+  machineType: controlplane
+---
+apiVersion: cluster.x-k8s.io/v1alpha2
 kind: Machine
 metadata:
   labels:
-    cluster.k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
-    set: master
-  name: talos-e2e-{{TAG}}-gcp-master-1
+    cluster.x-k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
+    cluster.x-k8s.io/control-plane: "true"
+  name: talos-e2e-{{TAG}}-gcp-controlplane-1
+  namespace: default
 spec:
-  providerSpec:
-    value:
-      apiVersion: talosproviderconfig/v1alpha1
-      kind: TalosMachineProviderSpec
-      platform:
-        config: |-
-          zone: "us-central1-c"
-          project: "talos-testbed"
-          instances:
-            type:  "n1-standard-2"
-            image: "https://www.googleapis.com/compute/v1/projects/talos-testbed/global/images/talos-e2e-{{TAG}}"
-            disks:
-              size: 50
-        type: gce
+  bootstrap:
+    configRef:
+      apiVersion: bootstrap.cluster.x-k8s.io/v1alpha2
+      kind: TalosConfig
+      name: talos-e2e-{{TAG}}-gcp-controlplane-1
+      namespace: default
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
+    kind: GCPMachine
+    name: talos-e2e-{{TAG}}-gcp-controlplane-1
+    namespace: default
+  version: 1.16.1
 ---
-apiVersion: cluster.k8s.io/v1alpha1
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
+kind: GCPMachine
+metadata:
+  name: talos-e2e-{{TAG}}-gcp-controlplane-1
+  namespace: default
+spec:
+  instanceType: n1-standard-2
+  zone: us-central1-a
+  image: projects/talos-testbed/global/images/talos-e2e-{{TAG}}
+  serviceAccounts: {}
+  publicIP: true
+---
+
+## Controlplane 2 configs
+
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha2
+kind: TalosConfig
+metadata:
+  name: talos-e2e-{{TAG}}-gcp-controlplane-2
+  labels:
+    cluster.x-k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
+spec:
+  machineType: controlplane
+---
+apiVersion: cluster.x-k8s.io/v1alpha2
 kind: Machine
 metadata:
   labels:
-    cluster.k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
-    set: master
-  name: talos-e2e-{{TAG}}-gcp-master-2
+    cluster.x-k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
+    cluster.x-k8s.io/control-plane: "true"
+  name: talos-e2e-{{TAG}}-gcp-controlplane-2
+  namespace: default
 spec:
-  providerSpec:
-    value:
-      apiVersion: talosproviderconfig/v1alpha1
-      kind: TalosMachineProviderSpec
-      platform:
-        config: |-
-          zone: "us-central1-c"
-          project: "talos-testbed"
-          instances:
-            type:  "n1-standard-2"
-            image: "https://www.googleapis.com/compute/v1/projects/talos-testbed/global/images/talos-e2e-{{TAG}}"
-            disks:
-              size: 50
-        type: gce
+  bootstrap:
+    configRef:
+      apiVersion: bootstrap.cluster.x-k8s.io/v1alpha2
+      kind: TalosConfig
+      name: talos-e2e-{{TAG}}-gcp-controlplane-2
+      namespace: default
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
+    kind: GCPMachine
+    name: talos-e2e-{{TAG}}-gcp-controlplane-2
+    namespace: default
+  version: 1.16.1
 ---
-apiVersion: cluster.k8s.io/v1alpha1
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
+kind: GCPMachine
+metadata:
+  name: talos-e2e-{{TAG}}-gcp-controlplane-2
+  namespace: default
+spec:
+  instanceType: n1-standard-2
+  zone: us-central1-a
+  image: projects/talos-testbed/global/images/talos-e2e-{{TAG}}
+  serviceAccounts: {}
+  publicIP: true
+---
+
+## Worker deployment configs
+
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha2
+kind: TalosConfigTemplate
+metadata:
+  name: talos-e2e-{{TAG}}-gcp-workers
+  namespace: default
+spec:
+  template:
+    spec:
+      machineType: "join"
+---
+apiVersion: cluster.x-k8s.io/v1alpha2
 kind: MachineDeployment
 metadata:
   labels:
-    cluster.k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
-    set: worker
+    cluster.x-k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
+    nodepool: nodepool-0
   name: talos-e2e-{{TAG}}-gcp-workers
+  namespace: default
 spec:
   replicas: 3
   selector:
     matchLabels:
-      cluster.k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
-      set: worker
+      cluster.x-k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
+      nodepool: nodepool-0
   template:
     metadata:
       labels:
-        cluster.k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
-        set: worker
+        cluster.x-k8s.io/cluster-name: talos-e2e-{{TAG}}-gcp
+        nodepool: nodepool-0
     spec:
-      providerSpec:
-        value:
-          apiVersion: talosproviderconfig/v1alpha1
-          kind: TalosMachineProviderSpec
-          platform:
-            config: |-
-              zone: "us-central1-c"
-              project: "talos-testbed"
-              instances:
-                type:  "n1-standard-2"
-                image: "https://www.googleapis.com/compute/v1/projects/talos-testbed/global/images/talos-e2e-{{TAG}}"
-                disks:
-                  size: 50
-            type: gce
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1alpha2
+          kind: TalosConfigTemplate
+          name: talos-e2e-{{TAG}}-gcp-workers
+          namespace: default
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
+        kind: GCPMachineTemplate
+        name: talos-e2e-{{TAG}}-gcp-workers
+        namespace: default
+      version: 1.16.1
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2
+kind: GCPMachineTemplate
+metadata:
+  name: talos-e2e-{{TAG}}-gcp-workers
+  namespace: default
+spec:
+  template:
+    spec:
+      instanceType: n1-standard-2
+      zone: us-central1-a
+      image: projects/talos-testbed/global/images/talos-e2e-{{TAG}}