alt-orchestra/talos
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: add azure e2e testing

Browse Source 
This PR will allow us to run an azure e2e test in parallel with our
current GCE implementation.

Signed-off-by: Spencer Smith <robertspencersmith@gmail.com>
This commit is contained in:
Spencer Smith 2019-08-02 11:05:34 -04:00 committed by Spencer Smith
parent 53b1330c44
commit 9e02c77c0a
12 changed files with 347 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
.drone.yml
View File

@ -398,16 +398,63 @@ steps:
environment:
BUILDKIT_HOST: tcp://buildkitd.ci.svc:1234
BINDIR: /usr/local/bin
AZURE_SVC_ACCT:
from_secret: azure_svc_acct
GCE_SVC_ACCT:
from_secret: gce_svc_acct
PACKET_AUTH_TOKEN:
from_secret: packet_auth_token
TAG: latest
commands:
- until docker ps; do sleep 5; done
- make osctl-linux
- make osctl-cluster-create
- make capi-setup
volumes:
- name: dockersock
path: /var/run
- name: tmp
path: /tmp
- name: azure-image
image: autonomy/build-container:latest
pull: always
environment:
BUILDKIT_HOST: tcp://buildkitd.ci.svc:1234
BINDIR: /usr/local/bin
AZURE_SVC_ACCT:
from_secret: azure_svc_acct
TAG: latest
commands:
- until docker ps; do sleep 5; done
- make talos-azure
- ./hack/test/azure-setup.sh
volumes:
- name: dockersock
path: /var/run
- name: dev
path: /dev
- name: tmp
path: /tmp
- name: azure-e2e-integration
image: autonomy/build-container:latest
pull: always
environment:
BUILDKIT_HOST: tcp://buildkitd.ci.svc:1234
BINDIR: /usr/local/bin
PLATFORM: azure
commands:
- make e2e-integration
volumes:
- name: dockersock
path: /var/run
- name: tmp
path: /tmp
depends_on:
- azure-image
- osctl-cluster-create
- name: gce-image
image: autonomy/build-container:latest
pull: always
@ -418,6 +465,7 @@ steps:
from_secret: gce_svc_acct
TAG: latest
commands:
- until docker ps; do sleep 5; done
- make talos-gce
- ./hack/test/gce-setup.sh
volumes:
@ -425,8 +473,6 @@ steps:
path: /var/run
- name: dev
path: /dev
depends_on:
- osctl-cluster-create
- name: gce-e2e-integration
image: autonomy/build-container:latest
@ -434,16 +480,6 @@ steps:
environment:
BUILDKIT_HOST: tcp://buildkitd.ci.svc:1234
BINDIR: /usr/local/bin
PACKET_AUTH_TOKEN:
from_secret: packet_auth_token
PACKET_PROJECT_ID:
from_secret: packet_project_id
PACKET_PXE_SERVER:
from_secret: packet_pxe_server
GCE_SVC_ACCT:
from_secret: gce_svc_acct
AZURE_SVC_ACCT:
from_secret: azure_svc_acct
PLATFORM: gce
commands:
- make e2e-integration
@ -454,6 +490,7 @@ steps:
path: /tmp
depends_on:
- gce-image
- osctl-cluster-create
volumes:
- name: dockersock

5
Makefile
View File

@ -219,9 +219,12 @@ osctl-cluster-create:
basic-integration:
@TAG=$(TAG) ./hack/test/$@.sh
.PHONY: capi-setup
capi-setup:
@TAG=$(TAG) ./hack/test/$@.sh
.PHONY: e2e
e2e-integration:
## TODO(rsmitty): Bump this k8s version back up once the bug is fixed where kubectl can't scale crds
@TAG=$(TAG) ./hack/test/$@.sh
.PHONY: test

38
hack/test/azure-setup.sh Executable file
View File

@ -0,0 +1,38 @@
#!/bin/bash
set -eou pipefail
STORAGE_ACCOUNT=talostesting
STORAGE_CONTAINER=talostesting
GROUP=talos
TMP=/tmp/e2e
azcli_run() {
docker run \
--rm \
--interactive \
--entrypoint=bash \
--mount type=bind,source=${TMP},target=${TMP} \
mcr.microsoft.com/azure-cli -c "az login --service-principal --username ${CLIENT_ID} \
--password ${CLIENT_SECRET} --tenant ${TENANT_ID} > /dev/null && \
${1}"
}
## Setup svc acct vars
mkdir -p ${TMP}
echo ${AZURE_SVC_ACCT} | base64 -d > ${TMP}/svc-acct.json
CLIENT_ID="$( cat ${TMP}/svc-acct.json | jq -r '.clientId' )"
CLIENT_SECRET="$( cat ${TMP}/svc-acct.json | jq -r '.clientSecret' )"
TENANT_ID="$( cat ${TMP}/svc-acct.json | jq -r '.tenantId' )"
## Untar image
tar -C ${TMP} -xf ./build/talos-azure.tar.gz
## Login to azure, push blob, create image from blob
AZURE_STORAGE_CONNECTION_STRING=$( azcli_run "az storage account show-connection-string -n ${STORAGE_ACCOUNT} -g ${GROUP} -o tsv" )
azcli_run "AZURE_STORAGE_CONNECTION_STRING='${AZURE_STORAGE_CONNECTION_STRING}' az storage blob upload --container-name ${STORAGE_CONTAINER} -f ${TMP}/talos-azure.vhd -n talos-azure.vhd"
azcli_run "az image delete --name talos-e2e -g ${GROUP}"
azcli_run "az image create --name talos-e2e --source https://${STORAGE_ACCOUNT}.blob.core.windows.net/${STORAGE_CONTAINER}/talos-azure.vhd --os-type linux -g ${GROUP}"

25
hack/test/capi-setup.sh Executable file
View File

@ -0,0 +1,25 @@
#!/bin/bash
set -eou pipefail
source ./hack/test/e2e-runner.sh
## Create tmp dir
mkdir -p $TMP
## Drop in capi stuff
sed -i "s/{{PACKET_AUTH_TOKEN}}/${PACKET_AUTH_TOKEN}/" ${PWD}/hack/test/manifests/provider-components.yaml
sed -i "s#{{GCE_SVC_ACCT}}#${GCE_SVC_ACCT}#" ${PWD}/hack/test/manifests/capi-secrets.yaml
sed -i "s#{{AZURE_SVC_ACCT}}#${AZURE_SVC_ACCT}#" ${PWD}/hack/test/manifests/capi-secrets.yaml
e2e_run "kubectl apply -f /e2emanifests/provider-components.yaml -f /e2emanifests/capi-secrets.yaml"
## Wait for talosconfig in cm then dump it out
e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
until kubectl wait --timeout=1s --for=condition=Ready -n cluster-api-provider-talos-system pod/cluster-api-provider-talos-controller-manager-0
do
if [[ \$(date +%s) -gt \$timeout ]]
then
exit 1
fi
echo 'Waiting to CAPT pod to be available...'
sleep 10
done"

82
hack/test/e2e-integration.sh
View File

@ -1,5 +1,4 @@
#!/bin/bash
set -eou pipefail
source ./hack/test/e2e-runner.sh
@ -7,25 +6,90 @@ source ./hack/test/e2e-runner.sh
## Create tmp dir
mkdir -p $TMP
## Drop in capi stuff
sed -i "s/{{PACKET_AUTH_TOKEN}}/${PACKET_AUTH_TOKEN}/" ${PWD}/hack/test/manifests/provider-components.yaml
sed -i "s#{{GCE_SVC_ACCT}}#${GCE_SVC_ACCT}#" ${PWD}/hack/test/manifests/capi-secrets.yaml
sed -i "s#{{AZURE_SVC_ACCT}}#${AZURE_SVC_ACCT}#" ${PWD}/hack/test/manifests/capi-secrets.yaml
## Cleanup the platform resources upon any exit
cleanup() {
e2e_run "kubectl delete machine talos-e2e-${PLATFORM}-master-0 talos-e2e-${PLATFORM}-master-1 talos-e2e-${PLATFORM}-master-2
kubectl scale machinedeployment talos-e2e-${PLATFORM}-workers --replicas=0
kubectl delete machinedeployment talos-e2e-${PLATFORM}-workers
kubectl delete cluster talos-e2e-${PLATFORM}"
}
e2e_run "kubectl apply -f /e2emanifests/provider-components.yaml -f /e2emanifests/capi-secrets.yaml"
trap cleanup EXIT
## Download kustomize and template out capi cluster, then deploy it
e2e_run "kubectl apply -f /e2emanifests/${PLATFORM}-cluster.yaml"
## Wait for talosconfig in cm then dump it out
e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
until kubectl wait --timeout=1s --for=condition=Ready -n cluster-api-provider-talos-system pod/cluster-api-provider-talos-controller-manager-0
until kubectl get cm -n cluster-api-provider-talos-system talos-e2e-${PLATFORM}-master-0
do
if [[ \$(date +%s) -gt \$timeout ]]
then
exit 1
fi
sleep 10
done
kubectl get cm -n cluster-api-provider-talos-system talos-e2e-${PLATFORM}-master-0 -o jsonpath='{.data.talosconfig}' > ${TALOSCONFIG}-${PLATFORM}-capi"
## Wait for kubeconfig from capi master-0
e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
until /bin/osctl --talosconfig ${TALOSCONFIG}-${PLATFORM}-capi kubeconfig > ${KUBECONFIG}-${PLATFORM}-capi
do
if [[ \$(date +%s) -gt \$timeout ]]
then
exit 1
fi
echo 'Waiting to CAPT pod to be available...'
sleep 10
done"
./hack/test/e2e-platform.sh
## Wait for nodes to check in
e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
until KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl get nodes -o json | jq '.items | length' | grep ${NUM_NODES} >/dev/null
do
if [[ \$(date +%s) -gt \$timeout ]]
then
exit 1
fi
KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl get nodes -o wide
sleep 10
done"
## Apply psp and flannel
e2e_run "KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl apply -f /manifests/psp.yaml -f /manifests/flannel.yaml"
## Wait for kube-proxy up
e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
until KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl get po -n kube-system -l k8s-app=kube-proxy -o json | jq '.items | length' | grep ${NUM_NODES} > /dev/null
do
if [[ \$(date +%s) -gt \$timeout ]]
then
exit 1
fi
KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl get po -n kube-system -l k8s-app=kube-proxy
sleep 10
done"
## Wait for nodes ready
e2e_run "KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl wait --timeout=${TIMEOUT}s --for=condition=ready=true --all nodes"
## Verify that we have an HA controlplane
e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
until KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl get nodes -l node-role.kubernetes.io/master='' -o json | jq '.items | length' | grep 3 > /dev/null
do
if [[ \$(date +%s) -gt \$timeout ]]
then
exit 1
fi
KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl get nodes -l node-role.kubernetes.io/master='' -o json | jq '.items | length'
sleep 10
done"
## Download sonobuoy and run conformance
e2e_run "apt-get update && apt-get install wget
wget --quiet -O /tmp/sonobuoy.tar.gz ${SONOBUOY_URL}
tar -xf /tmp/sonobuoy.tar.gz -C /usr/local/bin
sonobuoy run --kubeconfig ${KUBECONFIG}-${PLATFORM}-capi --wait --skip-preflight --plugin e2e
results=\$(sonobuoy retrieve --kubeconfig ${KUBECONFIG}-${PLATFORM}-capi)
sonobuoy e2e --kubeconfig ${KUBECONFIG}-${PLATFORM}-capi \$results"
exit 0

87
hack/test/e2e-platform.sh
View File

@ -1,87 +0,0 @@
source ./hack/test/e2e-runner.sh
## Cleanup the platform resources upon any exit
cleanup() {
e2e_run "kubectl delete machine talos-e2e-${PLATFORM}-master-0 talos-e2e-${PLATFORM}-master-1 talos-e2e-${PLATFORM}-master-2
kubectl scale machinedeployment talos-e2e-${PLATFORM}-workers --replicas=0
kubectl delete machinedeployment talos-e2e-${PLATFORM}-workers
kubectl delete cluster talos-e2e-${PLATFORM}"
}
trap cleanup EXIT
## Download kustomize and template out capi cluster, then deploy it
e2e_run "kubectl apply -f /e2emanifests/${PLATFORM}-cluster.yaml"
## Wait for talosconfig in cm then dump it out
e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
until kubectl get cm -n cluster-api-provider-talos-system talos-e2e-${PLATFORM}-master-0
do
if [[ \$(date +%s) -gt \$timeout ]]
then
exit 1
fi
sleep 10
done
kubectl get cm -n cluster-api-provider-talos-system talos-e2e-${PLATFORM}-master-0 -o jsonpath='{.data.talosconfig}' > ${TALOSCONFIG}-${PLATFORM}-capi"
## Wait for kubeconfig from capi master-0
e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
until /bin/osctl --talosconfig ${TALOSCONFIG}-${PLATFORM}-capi kubeconfig > ${KUBECONFIG}-${PLATFORM}-capi
do
if [[ \$(date +%s) -gt \$timeout ]]
then
exit 1
fi
sleep 10
done"
## Wait for nodes to check in
e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
until KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl get nodes -o json | jq '.items | length' | grep ${NUM_NODES} >/dev/null
do
if [[ \$(date +%s) -gt \$timeout ]]
then
exit 1
fi
KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl get nodes -o wide
sleep 10
done"
## Apply psp and flannel
e2e_run "KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl apply -f /manifests/psp.yaml -f /manifests/flannel.yaml"
## Wait for kube-proxy up
e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
until KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl get po -n kube-system -l k8s-app=kube-proxy -o json | jq '.items | length' | grep ${NUM_NODES} > /dev/null
do
if [[ \$(date +%s) -gt \$timeout ]]
then
exit 1
fi
KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl get po -n kube-system -l k8s-app=kube-proxy
sleep 10
done"
## Wait for nodes ready
e2e_run "KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl wait --timeout=${TIMEOUT}s --for=condition=ready=true --all nodes"
## Verify that we have an HA controlplane
e2e_run "timeout=\$((\$(date +%s) + ${TIMEOUT}))
until KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl get nodes -l node-role.kubernetes.io/master='' -o json | jq '.items | length' | grep 3 > /dev/null
do
if [[ \$(date +%s) -gt \$timeout ]]
then
exit 1
fi
KUBECONFIG=${KUBECONFIG}-${PLATFORM}-capi kubectl get nodes -l node-role.kubernetes.io/master='' -o json | jq '.items | length'
sleep 10
done"
## Download sonobuoy and run conformance
e2e_run "apt-get update && apt-get install wget
wget --quiet -O /tmp/sonobuoy.tar.gz ${SONOBUOY_URL}
tar -xf /tmp/sonobuoy.tar.gz -C /usr/local/bin
sonobuoy run --kubeconfig ${KUBECONFIG}-${PLATFORM}-capi --wait --skip-preflight --plugin e2e
results=\$(sonobuoy retrieve --kubeconfig ${KUBECONFIG}-${PLATFORM}-capi)
sonobuoy e2e --kubeconfig ${KUBECONFIG}-${PLATFORM}-capi \$results"

5
hack/test/e2e-runner.sh
View File

@ -1,10 +1,11 @@
export KUBERNETES_VERSION=v1.14.4
export KUBERNETES_VERSION=v1.14.5
export TALOS_IMG="docker.io/autonomy/talos:${TAG}"
export TMP="/tmp/e2e"
export OSCTL="${PWD}/build/osctl-linux-amd64"
export TALOSCONFIG="${TMP}/talosconfig"
export KUBECONFIG="${TMP}/kubeconfig"
## Long timeout due to packet provisioning times
## Long timeout due to provisioning times
export TIMEOUT=9000
## Total number of nodes we'll be waiting to come up (3 Masters + 3 Workers)

2
hack/test/gce-setup.sh
View File

@ -14,5 +14,5 @@ tar -xf google-cloud-sdk-253.0.0-linux-x86_64.tar.gz
./google-cloud-sdk/bin/gsutil cp ./build/talos-gce.tar.gz gs://talos-e2e
## Create image from talos-gce
./google-cloud-sdk/bin/gcloud --quiet --project talos-testbed compute images delete talos-e2e
./google-cloud-sdk/bin/gcloud --quiet --project talos-testbed compute images delete talos-e2e || true ##Ignore error if image doesn't exist
./google-cloud-sdk/bin/gcloud --quiet --project talos-testbed compute images create talos-e2e --source-uri gs://talos-e2e/talos-gce.tar.gz

134
hack/test/manifests/azure-cluster.yaml Normal file
View File

@ -0,0 +1,134 @@
apiVersion: cluster.k8s.io/v1alpha1
kind: Cluster
metadata:
annotations: null
name: talos-e2e-azure
spec:
clusterNetwork:
pods:
cidrBlocks:
- 192.168.0.0/16
serviceDomain: cluster.local
services:
cidrBlocks:
- 10.96.0.0/12
providerSpec:
value:
apiVersion: talosproviderconfig/v1alpha1
kind: TalosClusterProviderSpec
masters:
ips:
- 23.99.218.95
- 23.99.220.43
- 23.99.225.139
---
apiVersion: cluster.k8s.io/v1alpha1
kind: Machine
metadata:
labels:
cluster.k8s.io/cluster-name: talos-e2e-azure
set: master
name: talos-e2e-azure-master-0
spec:
providerSpec:
value:
apiVersion: talosproviderconfig/v1alpha1
kind: TalosMachineProviderSpec
platform:
config: |-
location: "centralus"
resourcegroup: "talos"
instances:
type: "Standard_D2_v3"
image: "/subscriptions/64739c64-c063-4c9d-bf2c-d1191ed8befa/resourceGroups/talos/providers/Microsoft.Compute/images/talos"
network: "talos-vnet"
subnet: "default"
disks:
size: 10
type: azure
---
apiVersion: cluster.k8s.io/v1alpha1
kind: Machine
metadata:
labels:
cluster.k8s.io/cluster-name: talos-e2e-azure
set: master
name: talos-e2e-azure-master-1
spec:
providerSpec:
value:
apiVersion: talosproviderconfig/v1alpha1
kind: TalosMachineProviderSpec
platform:
config: |-
location: "centralus"
resourcegroup: "talos"
instances:
type: "Standard_D2_v3"
image: "/subscriptions/64739c64-c063-4c9d-bf2c-d1191ed8befa/resourceGroups/talos/providers/Microsoft.Compute/images/talos"
network: "talos-vnet"
subnet: "default"
disks:
size: 10
type: azure
---
apiVersion: cluster.k8s.io/v1alpha1
kind: Machine
metadata:
labels:
cluster.k8s.io/cluster-name: talos-e2e-azure
set: master
name: talos-e2e-azure-master-2
spec:
providerSpec:
value:
apiVersion: talosproviderconfig/v1alpha1
kind: TalosMachineProviderSpec
platform:
config: |-
location: "centralus"
resourcegroup: "talos"
instances:
type: "Standard_D2_v3"
image: "/subscriptions/64739c64-c063-4c9d-bf2c-d1191ed8befa/resourceGroups/talos/providers/Microsoft.Compute/images/talos"
network: "talos-vnet"
subnet: "default"
disks:
size: 10
type: azure
---
apiVersion: cluster.k8s.io/v1alpha1
kind: MachineDeployment
metadata:
labels:
cluster.k8s.io/cluster-name: talos-e2e-azure
set: worker
name: talos-e2e-azure-workers
spec:
replicas: 3
selector:
matchLabels:
cluster.k8s.io/cluster-name: talos-e2e-azure
set: worker
template:
metadata:
labels:
cluster.k8s.io/cluster-name: talos-e2e-azure
set: worker
spec:
providerSpec:
value:
apiVersion: talosproviderconfig/v1alpha1
kind: TalosMachineProviderSpec
platform:
config: |-
location: "centralus"
resourcegroup: "talos"
instances:
type: "Standard_D2_v3"
image: "/subscriptions/64739c64-c063-4c9d-bf2c-d1191ed8befa/resourceGroups/talos/providers/Microsoft.Compute/images/talos"
network: "talos-vnet"
subnet: "default"
disks:
size: 10
type: azure

10
hack/test/manifests/capi-secrets.yaml
View File

@ -11,4 +11,12 @@ metadata:
name: gce-credentials
namespace: cluster-api-provider-talos-system
data:
service-account.json: "{{GCE_SVC_ACCT}}"
service-account.json: "{{GCE_SVC_ACCT}}"
---
apiVersion: v1
kind: Secret
metadata:
name: azure-credentials
namespace: cluster-api-provider-talos-system
data:
service-account.json: "{{AZURE_SVC_ACCT}}"

13
hack/test/manifests/provider-components.yaml
View File

@ -182,6 +182,8 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: AZURE_AUTH_LOCATION
value: /.azure/service-account.json
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /.gce/service-account.json
- name: PACKET_AUTH_TOKEN
@ -195,15 +197,17 @@ spec:
protocol: TCP
resources:
limits:
cpu: 100m
memory: 30Mi
cpu: 1000m
memory: 1000Mi
requests:
cpu: 100m
memory: 20Mi
memory: 100Mi
volumeMounts:
- mountPath: /tmp/cert
name: cert
readOnly: true
- mountPath: /.azure
name: azure-credentials
- mountPath: /.gce
name: gce-credentials
- mountPath: /.aws
@ -216,6 +220,9 @@ spec:
secret:
defaultMode: 420
secretName: cluster-api-provider-talos-webhook-server-secret
- name: azure-credentials
secret:
secretName: azure-credentials
- name: gce-credentials
secret:
secretName: gce-credentials

2
hack/test/osctl-cluster-create.sh
View File

@ -2,7 +2,7 @@
set -eou pipefail
export KUBERNETES_VERSION=v1.15.0
export KUBERNETES_VERSION=v1.15.2
export TALOS_IMG="docker.io/autonomy/talos:${TAG}"
export TMP="/tmp/e2e"
export OSCTL="${PWD}/build/osctl-linux-amd64"