some changes

ansible/playbook.yaml

@@ -12,6 +12,7 @@
         - kubernetes{{ k8s_version }}-kubelet
         - kubernetes{{ k8s_version }}-crio
         - cri-tools{{ k8s_version }}
+        - wget
       update_cache: true
   - name: start and enable k8s services
     service:
@@ -27,7 +28,9 @@
   tasks:
   - name: run kubeadm init
     #shell: "kubeadm init --pod-network-cidr=10.244.0.0/16 --ignore-preflight-errors=FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml,FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml,FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml,FileAvailable--etc-kubernetes-manifests-etcd.yaml  | grep 'kubeadm join' -A 1 | sed 's|\\\\||' | xargs"
-    shell: "kubeadm init --pod-network-cidr=10.244.0.0/16 --image-repository=registry.altlinux.org/k8s-{{ branch }} --kubernetes-version=1.28.7 | grep 'kubeadm join' -A 1 | sed 's|\\\\||' | xargs"
+    shell: "kubeadm init --pod-network-cidr=10.244.0.0/16 --image-repository=registry.altlinux.org/k8s-{{ branch }} | grep 'kubeadm join' -A 1 | sed 's|\\\\||' | xargs"
+    #shell: "kubeadm init --pod-network-cidr=10.244.0.0/16 --image-repository=10.4.4.52:5000/k8s-{{ branch }} | grep 'kubeadm join' -A 1 | sed 's|\\\\||' | xargs"
+    #shell: "kubeadm init --pod-network-cidr=10.244.0.0/16 | grep 'kubeadm join' -A 1 | sed 's|\\\\||' | xargs"
     register: kubeadm_init
     changed_when: kubeadm_init.rc == 0
     failed_when: kubeadm_init.rc not in [0, 2]
@@ -55,7 +58,24 @@
 - name: Create network
   hosts: master
   tasks:
+  - name: download kube-flannel.yml
+    ansible.builtin.command: wget https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml
+  - name: replace upstream flannel-cni-plugin image with alt image
+    ansible.builtin.replace:
+      path: kube-flannel.yml
+      regexp: '^([ tab]+)image: docker.io/flannel/flannel-cni-plugin:.*'
+      replace: '\1image: registry.altlinux.org/k8s-{{ branch }}/flannel-cni-plugin:v1.4.0-flannel1'
+  #- name: replace upstream flannel-cni-plugin image with alt image
+    #ansible.builtin.replace:
+      #path: kube-flannel.yml
+      #regexp: '^([ tab]+)image: docker.io/flannel/(.*)'
+      #replace: '\1image: registry.altlinux.org/k8s-sisyphus/\2'
+  - name: replace upstream flannel image with alt image
+    ansible.builtin.replace:
+      path: kube-flannel.yml
+      regexp: '^([ tab]+)image: docker.io/flannel/flannel:.*'
+      replace: '\1image: registry.altlinux.org/k8s-{{ branch }}/flannel:v0.25.1'
   - name: create flannel network
-    command: kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml
+    ansible.builtin.command: kubectl apply -f kube-flannel.yml
     environment:
       KUBECONFIG: /etc/kubernetes/admin.conf

functions.py

@@ -3,6 +3,7 @@ import logging
 from proxmoxer import ProxmoxAPI
 
 logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
 
 def create_vm(
     proxmox: ProxmoxAPI,

main.py

@@ -1,3 +1,5 @@
+#!/usr/bin/python3
+
 import os
 import logging
 from typing import Callable
@@ -15,7 +17,7 @@ urllib3.disable_warnings()
 FORMAT = "%(asctime)s %(name)s %(levelname)s %(message)s"
 logging.basicConfig(format=FORMAT)
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
+logger.setLevel(logging.DEBUG)
 
 
 load_dotenv()
@@ -54,10 +56,15 @@ def main() -> None:
         'worker1': f'{prefix}-node1',
         'worker2': f'{prefix}-node2',
     }
-    vm_ids = {
+    # vm_ids = {
-        'master': 500,
+        # 'master': 500,
-        'worker1': 501,
+        # 'worker1': 501,
-        'worker2': 502,
+        # 'worker2': 502,
+    # }
+    vm_ids: dict[str, int] = {
+        'master': 510,
+        'worker1': 511,
+        'worker2': 512,
     }
 
     delete: str = os.environ['DELETE_VMS']

run.sh

@@ -1,28 +1,45 @@
-#!/bin/sh -exu
+#!/usr/bin/sh
 
-branch="${1:-"sisyphus"}"
+set -xeuo pipefail
-k8s_version="${2:-"1.28"}"
-task_number="${3:-}"
-ssh_options="-o IdentitiesOnly=yes -o StrictHostKeyChecking=no"
 
-# Create VMs
+function print_help() {
-python3 ./main.py
+	cat <<EOF
+usage:
+	$0 <branch> <k8s_version> <task_number>
+EOF
+}
 
-# let cloud-init update the system or apt won't be able to get the lock
+if [ "$#" -lt 3 ]; then
-# remove later
+	print_help
-sleep 80
+	exit 1
+fi
 
-# Send run/hosts to every vm
+K8S_VERSIONS="1.26 1.27 1.28 1.29 1.30"
-cat tmp/hosts | awk '{print $1}' | xargs -I _ rsync -e "ssh $ssh_options" tmp/hosts root@_:/etc/hosts
-# Restart network on every vm
-cat tmp/hosts | awk '{print $1}' | xargs -I _ ssh $ssh_options root@_ "systemctl restart network"
 
-ansible-playbook ansible/playbook.yaml -i tmp/generated_inventory.yaml \
+for k8s_version in $K8S_VERSIONS; do
-    -e task_number="$task_number" -e k8s_version="$k8s_version" -e branch="$branch"
+	branch="${1:-"sisyphus"}"
+	#k8s_version="${2:-"1.28"}"
+	task_number="${3:-}"
+	ssh_options="-o IdentitiesOnly=yes -o StrictHostKeyChecking=no"
 
-head -n 1 tmp/hosts | awk '{print $1}' | xargs -I _ rsync root@_:/etc/kubernetes/admin.conf ~/.kube/config
+	# Create VMs
+	python3 ./main.py
 
-kubectl apply -f - <<'EOF'
+	# let cloud-init update the system or apt won't be able to get the lock
+	# remove later
+	sleep 80
+
+	# Send run/hosts to every vm
+	cat tmp/hosts | awk '{print $1}' | xargs -I {} rsync -e "ssh $ssh_options" tmp/hosts root@{}:/etc/hosts
+	# Restart network on every vm
+	cat tmp/hosts | awk '{print $1}' | xargs -I {} ssh $ssh_options root@{} "systemctl restart network"
+
+	ansible-playbook ansible/playbook.yaml -i tmp/generated_inventory.yaml \
+		-e task_number="$task_number" -e k8s_version="$k8s_version" -e branch="$branch"
+
+	head -n 1 tmp/hosts | awk '{print $1}' | xargs -I {} rsync root@{}:/etc/kubernetes/admin.conf ~/.kube/config
+
+	kubectl apply -f - <<'EOF'
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -44,7 +61,7 @@ spec:
         - containerPort: 80
 EOF
 
-kubectl apply -f - <<'EOF'
+	kubectl apply -f - <<'EOF'
 apiVersion: v1
 kind: Service
 metadata:
@@ -61,10 +78,11 @@ spec:
     app: nginx
 EOF
 
-sleep 120
+	sleep 120
 
-head -n 1 tmp/hosts | awk '{print $1}' | xargs -I _ curl _:30007
+	head -n 1 tmp/hosts | awk '{print $1}' | xargs -I {} curl {}:30007
-kubectl exec $(kubectl get po | awk 'END{print $1}') -- nslookup nginx
+	kubectl exec $(kubectl get po | awk 'END{print $1}') -- nslookup nginx
 
-# Remove ip addresses from known_hosts
+	# Remove ip addresses from known_hosts
-cat tmp/hosts | awk '{print $1}' | xargs -I _ ssh-keygen -R _
+	cat tmp/hosts | awk '{print $1}' | xargs -I {} ssh-keygen -R {}
+done