feat: Add support for creating VMware images

This PR adds support for generating VMware compatible images as an ova. Signed-off-by: Brad Beam <brad.beam@talos-systems.com>
2019-09-13 15:10:47 -05:00 · 2019-09-13 15:10:47 -05:00 · ca76ccd4af
commit ca76ccd4af
parent 6e03adad06
8 changed files with 454 additions and 5 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -2871,6 +2871,24 @@ steps:
  depends_on:
  - installer

+- name: image-vmware
+  pull: always
+  image: autonomy/build-container:latest
+  commands:
+  - make image-vmware
+  environment:
+    BINDIR: /usr/local/bin
+    BUILDKIT_HOST: ${BUILDKIT_HOST=tcp://buildkitd.ci.svc:1234}
+  volumes:
+  - name: dockersock
+    path: /var/run
+  - name: dev
+    path: /dev
+  - name: tmp
+    path: /tmp
+  depends_on:
+  - installer
+
 - name: iso
  pull: always
  image: autonomy/build-container:latest
--- a/1
+++ b/1
@ -283,6 +283,7 @@ RUN apk --update add \
    util-linux \
    xfsprogs
 COPY hack/installer/entrypoint.sh /bin/entrypoint.sh
+COPY hack/installer/template.ovf /template.ovf
 COPY --from=kernel /vmlinuz /usr/install/vmlinuz
 COPY --from=rootfs /usr/lib/syslinux/ /usr/lib/syslinux
 COPY --from=initramfs /initramfs.xz /usr/install/initramfs.xz
--- a/16
+++ b/16
@ -217,6 +217,22 @@ image-gcp:
 	@tar -C $(PWD)/build -czf $(PWD)/build/gcp.tar.gz disk.raw
 	@rm -rf $(PWD)/build/disk.raw

+.PHONY: image-vmware
+image-vmware:
+	@docker run --rm -v /dev:/dev -v $(PWD)/build:/out \
+		--privileged $(DOCKER_ARGS) \
+		autonomy/installer:$(TAG) \
+		install \
+		-r \
+		-p vmware \
+		-u guestinfo \
+		-e console=tty0 \
+		-e earlyprintk=ttyS0,115200
+	@docker run --rm -v /dev:/dev -v $(PWD)/build:/out \
+		--privileged $(DOCKER_ARGS) \
+		autonomy/installer:$(TAG) \
+		ova
+
 .PHONY: push-image-aws
 push-image-aws:
 	@TAG=$(TAG) ./hack/test/aws-setup.sh
--- a/docs/website/content/v0.3/en/guides/cloud/vmware.md
+++ b/docs/website/content/v0.3/en/guides/cloud/vmware.md
@ -0,0 +1,219 @@
+---
+title: 'VMware'
+---
+
+## Creating a Cluster via the govc CLI
+
+In this guide we will create an HA Kubernetes cluster with 3 worker nodes.
+We will use the `govc` cli which can be downloaded [here](https://github.com/vmware/govmomi/tree/master/govc#installation).
+
+### Prereqs
+
+Prior to starting, it is important to have the following infrastructure in place and available:
+
+- DHCP server
+- Load Balancer or DNS address for cluster endpoint
+  - If using a load balancer, the most common setup is to balance `tcp/443` across the control plane nodes `tcp/6443`
+  - If using a DNS address, the A record should return back the addresses of the control plane nodes
+
+### Create the Machine Configuration Files
+
+#### Generating Base Configurations
+
+Using the DNS name or name of the loadbalancer used in the prereq steps, generate the base configuration files for the Talos machines:
+
+```bash
+$ osctl config generate talos-k8s-vmware-tutorial https://$LB_DNS_NAME
+created init.yaml
+created controlplane.yaml
+created join.yaml
+created talosconfig
+```
+
+**Note** If you are using a DNS record, you will want to specify the port for the API Server (`tcp/6443`)
+
+```bash
+$ osctl config generate talos-k8s-vmware-tutorial https://$DNS_NAME:6443
+created init.yaml
+created controlplane.yaml
+created join.yaml
+created talosconfig
+```
+
+At this point, you can modify the generated configs to your liking.
+
+#### Validate the Configuration Files
+
+```bash
+$ osctl validate --config init.yaml --mode cloud
+init.yaml is valid for cloud mode
+$ osctl validate --config controlplane.yaml --mode cloud
+controlplane.yaml is valid for cloud mode
+$ osctl validate --config join.yaml --mode cloud
+join.yaml is valid for cloud mode
+```
+
+### Set Environment Variables
+
+`govc` makes use of the following environment variables
+
+```bash
+export GOVC_URL=https://myvcenter
+export GOVC_USERNAME=root
+export GOVC_PASSWORD=Whywas6afraidof7,because789
+```
+
+If your vCenter installation makes use of self signed certificates, you'll want to have `GOVC_INSECURE=true` set.
+
+```bash
+ export GOVC_INSECURE=true
+```
+
+There are some additional variables that you may need to set
+
+```bash
+export GOVC_DATACENTER=datacenter1
+export GOVC_RESOURCE_POOL=cluster1/Resources
+export GOVC_DATASTORE=datastore1
+export GOVC_NETWORK="VM Network"
+```
+
+### Download the OVA
+
+A talos.ova asset is published with each release:
+
+```bash
+curl -LO https://github.com/talos-systems/talos/releases/download/v0.3.0-alpha.2/talos.ova
+```
+
+### Import the OVA into vCenter
+
+We'll need to repeat this step for each Talos node we want to create.
+In a typical HA setup, we'll have 3 control plane nodes and N workers.
+In the following example, we'll set up a HA control plane with two worker nodes.
+
+```bash
+govc import.ova -name talos-v0.3.0-alpha.2 $talos/build/talos.ova
+```
+
+#### Create the Bootstrap Node
+
+We'll clone the OVA to create the bootstrap node (our first control plane node).
+
+```bash
+govc vm.clone -on=false -vm talos-v0.3.0-alpha.2 control-plane-1
+```
+
+Talos makes use of the `guestinfo` facility of VMware to provide the machine/cluster configuration.
+This can be set using the `govc vm.change` command.
+To facilitate persistent storage using the vSphere cloud provider integration with Kubernetes, `disk.enableUUID=1` is used.
+
+```bash
+govc vm.change \
+  -e "guestinfo.talos.config=$(cat init.yaml | base64)" \
+  -e "disk.enableUUID=1" \
+  -vm /ha-datacenter/vm/control-plane-1
+```
+
+#### Update Hardware Resources for the Bootstrap Node
+
+- `-c` is used to configure the number of cpus
+- `-m` is used to configure the amount of memory (in MB)
+
+```bash
+govc vm.change \
+  -c 2 \
+  -m 4096 \
+  -vm /ha-datacenter/vm/control-plane-1
+```
+
+The following can be used to adjust the ephemeral disk size.
+
+```bash
+govc vm.disk.change -vm control-plane-1 -disk.name disk-1000-0 -size 10G
+```
+
+```bash
+govc vm.power -on control-plane-1
+```
+
+#### Create the Remaining Control Plane Nodes
+
+```bash
+govc vm.clone -on=false -vm talos-v0.3.0-alpha.2 control-plane-2
+govc vm.change \
+  -e "guestinfo.talos.config=$(base64 controlplane.yaml)" \
+  -e "disk.enableUUID=1" \
+  -vm /ha-datacenter/vm/control-plane-2
+govc vm.clone -on=false -vm talos-v0.3.0-alpha.2 control-plane-3
+govc vm.change \
+  -e "guestinfo.talos.config=$(base64 controlplane.yaml)" \
+  -e "disk.enableUUID=1" \
+  -vm /ha-datacenter/vm/control-plane-3
+```
+
+```bash
+govc vm.change \
+  -c 2 \
+  -m 4096 \
+  -vm /ha-datacenter/vm/control-plane-2
+govc vm.change \
+  -c 2 \
+  -m 4096 \
+  -vm /ha-datacenter/vm/control-plane-3
+```
+
+```bash
+govc vm.disk.change -vm control-plane-2 -disk.name disk-1000-0 -size 10G
+govc vm.disk.change -vm control-plane-3 -disk.name disk-1000-0 -size 10G
+```
+
+```bash
+govc vm.power -on control-plane-2
+govc vm.power -on control-plane-3
+```
+
+#### Update Settings for the Worker Nodes
+
+```bash
+govc vm.clone -on=false -vm talos-v0.3.0-alpha.2 worker-1
+govc vm.change \
+  -e "guestinfo.talos.config=$(base64 join.yaml)" \
+  -e "disk.enableUUID=1" \
+  -vm /ha-datacenter/vm/worker-1
+govc vm.clone -on=false -vm talos-v0.3.0-alpha.2 worker-2
+govc vm.change \
+  -e "guestinfo.talos.config=$(base64 join.yaml)" \
+  -e "disk.enableUUID=1" \
+  -vm /ha-datacenter/vm/worker-2
+```
+
+```bash
+govc vm.change \
+  -c 4 \
+  -m 8192 \
+  -vm /ha-datacenter/vm/worker-1
+govc vm.change \
+  -c 4 \
+  -m 8192 \
+  -vm /ha-datacenter/vm/worker-2
+```
+
+```bash
+govc vm.disk.change -vm worker-1 -disk.name disk-1000-0 -size 50G
+govc vm.disk.change -vm worker-2 -disk.name disk-1000-0 -size 50G
+```
+
+```bash
+govc vm.power -on worker-1
+govc vm.power -on worker-2
+```
+
+### Retrieve the `kubeconfig`
+
+At this point we can retrieve the admin `kubeconfig` by running:
+
+```bash
+osctl --talosconfig talosconfig config target <control plane 1 IP>
+osctl --talosconfig talosconfig kubeconfig > kubeconfig
+```
--- a/hack/drone.jsonnet
+++ b/hack/drone.jsonnet
@ -241,6 +241,7 @@ local image_aws = Step("image-aws", depends_on=[installer]);
 local image_azure = Step("image-azure", depends_on=[installer]);
 local image_digital_ocean = Step("image-digital-ocean", depends_on=[installer]);
 local image_gcp = Step("image-gcp", depends_on=[installer]);
+local image_vmware = Step("image-vmware", depends_on=[installer]);
 local capi = Step("capi", depends_on=[basic_integration], environment=creds_env_vars);
 local push_image_aws = Step("push-image-aws", depends_on=[image_aws], environment=creds_env_vars);
 local push_image_azure = Step("push-image-azure", depends_on=[image_azure], environment=creds_env_vars);
@ -353,6 +354,7 @@ local release_steps = default_steps + [
  image_azure,
  image_digital_ocean,
  image_gcp,
+  image_vmware,
  iso,
  release,
 ];
--- a/hack/installer/entrypoint.sh
+++ b/hack/installer/entrypoint.sh
@ -19,7 +19,7 @@ function setup_raw_disk(){
  if [[ -f ${TALOS_RAW} ]]; then
    rm ${TALOS_RAW}
  fi
-  dd if=/dev/zero of="${TALOS_RAW}" bs=1M count=0 seek=544
+  dd if=/dev/zero of="${TALOS_RAW}" bs=1M count=0 seek=${TALOS_RAW_SIZE}
  # NB: Since we use BLKRRPART to tell the kernel to re-read the partition
  # table, it is required to create a partitioned loop device. The BLKRRPART
  # command is meaningful only for partitionable devices.
@ -48,8 +48,16 @@ function create_iso() {
  isohybrid ${TALOS_ISO}
 }

-function create_vmdk() {
-  qemu-img convert -f raw -O vmdk ${TALOS_RAW} ${TALOS_VMDK}
+function create_ova() {
+  qemu-img convert -f raw -O vmdk -o compat6,subformat=streamOptimized,adapter_type=lsilogic ${TALOS_RAW} ${TALOS_VMDK}
+  # ovf creation
+  # reference format: https://www.dmtf.org/standards/ovf
+  img_size=$(stat -c %s ${TALOS_VMDK})
+  sed -e 's/{{FILESIZE}}/'${img_size}'/' \
+      -e 's/{{RAWSIZE}}/'${TALOS_RAW_SIZE}'/' /template.ovf > ${TALOS_OVF}
+  sha256sum ${TALOS_VMDK} ${TALOS_OVF} | awk '{ split($NF, filename, "/"); print "SHA256("filename[length(filename)]")= "$1 }' > ${TALOS_MF}
+  tar -cf ${TALOS_OVA} -C /out $(basename ${TALOS_OVF}) $(basename ${TALOS_MF}) $(basename ${TALOS_VMDK})
+  rm ${TALOS_VMDK} ${TALOS_OVF} ${TALOS_MF}
 }

 function cleanup {
@ -61,9 +69,13 @@ function usage() {
  printf "entrypoint.sh -p <platform> -u <userdata> [b|d|l|n]"
 }

+TALOS_RAW_SIZE=544
 TALOS_RAW="/out/talos.raw"
 TALOS_ISO="/out/talos.iso"
 TALOS_VMDK="/out/talos.vmdk"
+TALOS_OVF="/out/talos.ovf"
+TALOS_MF="/out/talos.mf"
+TALOS_OVA="/out/talos.ova"
 TALOS_PLATFORM="metal"
 TALOS_CONFIG="none"
 WITH_BOOTLOADER="true"
@ -123,8 +135,8 @@ case "$1" in
  iso)
    create_iso
    ;;
-  vmdk)
-    create_vmdk
+  ova)
+    create_ova
    ;;
  ami)
    shift
--- a/hack/installer/talos.vmx
+++ b/hack/installer/talos.vmx
@ -0,0 +1,80 @@
+.encoding = "UTF-8"
+RemoteDisplay.maxConnections = "-1"
+bios.bootRetry.delay = "10"
+cleanShutdown = "TRUE"
+config.version = "8"
+displayName = "talos"
+ethernet0.addressType = "generated"
+ethernet0.generatedAddress = "00:0c:29:27:3a:ac"
+ethernet0.generatedAddressOffset = "0"
+ethernet0.networkName = "VM Network"
+ethernet0.pciSlotNumber = "32"
+ethernet0.present = "TRUE"
+ethernet0.virtualDev = "vmxnet3"
+ethernet0.wakeOnPcktRcv = "FALSE"
+firmware = "efi"
+floppy0.present = "FALSE"
+guestOS = "otherlinux-64"
+hpet0.present = "TRUE"
+memSize = "4096"
+migrate.hostLog = "./talos-76a574f0.hlog"
+monitor.phys_bits_used = "43"
+numa.autosize.cookie = "20001"
+numa.autosize.vcpu.maxPerVirtualNode = "2"
+numvcpus = "2"
+nvram = "talos.nvram"
+pciBridge0.pciSlotNumber = "17"
+pciBridge0.present = "TRUE"
+pciBridge4.functions = "8"
+pciBridge4.pciSlotNumber = "21"
+pciBridge4.present = "TRUE"
+pciBridge4.virtualDev = "pcieRootPort"
+pciBridge5.functions = "8"
+pciBridge5.pciSlotNumber = "22"
+pciBridge5.present = "TRUE"
+pciBridge5.virtualDev = "pcieRootPort"
+pciBridge6.functions = "8"
+pciBridge6.pciSlotNumber = "23"
+pciBridge6.present = "TRUE"
+pciBridge6.virtualDev = "pcieRootPort"
+pciBridge7.functions = "8"
+pciBridge7.pciSlotNumber = "24"
+pciBridge7.present = "TRUE"
+pciBridge7.virtualDev = "pcieRootPort"
+powerType.suspend = "soft"
+sched.cpu.affinity = "all"
+sched.cpu.latencySensitivity = "normal"
+sched.cpu.min = "0"
+sched.cpu.shares = "normal"
+sched.cpu.units = "mhz"
+sched.mem.min = "0"
+sched.mem.minSize = "0"
+sched.mem.shares = "normal"
+sched.scsi0:0.shares = "normal"
+sched.scsi0:0.throughputCap = "off"
+sched.swap.derivedName = "/vmfs/volumes/5d7a8ba5-38ad6254-56c6-ac1f6b99be48/talos/talos-76a574f0.vswp"
+scsi0.pciSlotNumber = "16"
+scsi0.present = "TRUE"
+scsi0.virtualDev = "pvscsi"
+scsi0:0.deviceType = "scsi-hardDisk"
+scsi0:0.fileName = "../../build/talos.vmdk"
+scsi0:0.present = "TRUE"
+scsi0:0.redo = ""
+softPowerOff = "FALSE"
+svga.present = "TRUE"
+toolScripts.afterPowerOn = "TRUE"
+toolScripts.afterResume = "TRUE"
+toolScripts.beforePowerOff = "TRUE"
+toolScripts.beforeSuspend = "TRUE"
+tools.guest.desktop.autolock = "FALSE"
+tools.syncTime = "FALSE"
+tools.upgrade.policy = "manual"
+uuid.bios = "56 4d fe 2b 04 fe e8 f4-10 8b e4 42 0a 27 3a ac"
+uuid.location = "56 4d fe 2b 04 fe e8 f4-10 8b e4 42 0a 27 3a ac"
+vc.uuid = "52 18 fe 09 8a 1d 72 d7-91 bb ab d8 1b 6a 1e 7e"
+virtualHW.version = "13"
+vmci0.id = "170343084"
+vmci0.pciSlotNumber = "33"
+vmci0.present = "TRUE"
+vmotion.checkpointFBSize = "4194304"
+vmotion.checkpointSVGAPrimarySize = "4194304"
--- a/hack/installer/template.ovf
+++ b/hack/installer/template.ovf
@ -0,0 +1,101 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--Generated by VMware ovftool 4.3.0 (build-7948156), UTC time: 2019-10-31T01:41:10.540841Z-->
+<!-- Edited by Talos -->
+<Envelope vmw:buildId="build-7948156" xmlns="http://schemas.dmtf.org/ovf/envelope/1" xmlns:cim="http://schemas.dmtf.org/wbem/wscim/1/common" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1" xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:vmw="http://www.vmware.com/schema/ovf" xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <References>
+    <File ovf:href="talos.vmdk" ovf:id="file1" ovf:size="{{FILESIZE}}"/>
+  </References>
+  <DiskSection>
+    <Info>Virtual disk information</Info>
+    <Disk ovf:capacity="{{RAWSIZE}}" ovf:capacityAllocationUnits="byte * 2^20" ovf:diskId="vmdisk1" ovf:fileRef="file1" ovf:format="http://www.vmware.com/interfaces/specifications/vmdk.html#streamOptimized"/>
+  </DiskSection>
+  <NetworkSection>
+    <Info>The list of logical networks</Info>
+    <Network ovf:name="VM Network">
+      <Description>The VM Network network</Description>
+    </Network>
+  </NetworkSection>
+  <VirtualSystem ovf:id="vm">
+    <Info>A virtual machine</Info>
+    <Name>talos</Name>
+    <OperatingSystemSection ovf:id="101" vmw:osType="otherLinux64Guest">
+      <Info>The kind of installed guest operating system</Info>
+    </OperatingSystemSection>
+    <VirtualHardwareSection>
+      <Info>Virtual hardware requirements</Info>
+      <System>
+        <vssd:ElementName>Virtual Hardware Family</vssd:ElementName>
+        <vssd:InstanceID>0</vssd:InstanceID>
+        <vssd:VirtualSystemIdentifier>talos</vssd:VirtualSystemIdentifier>
+        <vssd:VirtualSystemType>vmx-13</vssd:VirtualSystemType>
+      </System>
+      <Item>
+        <rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
+        <rasd:Description>Number of Virtual CPUs</rasd:Description>
+        <rasd:ElementName>2 virtual CPU(s)</rasd:ElementName>
+        <rasd:InstanceID>1</rasd:InstanceID>
+        <rasd:ResourceType>3</rasd:ResourceType>
+        <rasd:VirtualQuantity>2</rasd:VirtualQuantity>
+      </Item>
+      <Item>
+        <rasd:AllocationUnits>byte * 2^20</rasd:AllocationUnits>
+        <rasd:Description>Memory Size</rasd:Description>
+        <rasd:ElementName>2048MB of memory</rasd:ElementName>
+        <rasd:InstanceID>2</rasd:InstanceID>
+        <rasd:ResourceType>4</rasd:ResourceType>
+        <rasd:VirtualQuantity>2048</rasd:VirtualQuantity>
+      </Item>
+      <Item>
+        <rasd:Address>0</rasd:Address>
+        <rasd:Description>SCSI Controller</rasd:Description>
+        <rasd:ElementName>scsiController0</rasd:ElementName>
+        <rasd:InstanceID>3</rasd:InstanceID>
+        <rasd:ResourceSubType>VirtualSCSI</rasd:ResourceSubType>
+        <rasd:ResourceType>6</rasd:ResourceType>
+      </Item>
+      <Item>
+        <rasd:AddressOnParent>0</rasd:AddressOnParent>
+        <rasd:ElementName>disk0</rasd:ElementName>
+        <rasd:HostResource>ovf:/disk/vmdisk1</rasd:HostResource>
+        <rasd:InstanceID>4</rasd:InstanceID>
+        <rasd:Parent>3</rasd:Parent>
+        <rasd:ResourceType>17</rasd:ResourceType>
+      </Item>
+      <Item>
+        <rasd:AddressOnParent>2</rasd:AddressOnParent>
+        <rasd:AutomaticAllocation>true</rasd:AutomaticAllocation>
+        <rasd:Connection>VM Network</rasd:Connection>
+        <rasd:Description>VmxNet3 ethernet adapter on &quot;VM Network&quot;</rasd:Description>
+        <rasd:ElementName>ethernet0</rasd:ElementName>
+        <rasd:InstanceID>5</rasd:InstanceID>
+        <rasd:ResourceSubType>VmxNet3</rasd:ResourceSubType>
+        <rasd:ResourceType>10</rasd:ResourceType>
+        <vmw:Config ovf:required="false" vmw:key="slotInfo.pciSlotNumber" vmw:value="32"/>
+        <vmw:Config ovf:required="false" vmw:key="wakeOnLanEnabled" vmw:value="false"/>
+        <vmw:Config ovf:required="false" vmw:key="connectable.allowGuestControl" vmw:value="false"/>
+      </Item>
+      <Item ovf:required="false">
+        <rasd:AutomaticAllocation>false</rasd:AutomaticAllocation>
+        <rasd:ElementName>video</rasd:ElementName>
+        <rasd:InstanceID>6</rasd:InstanceID>
+        <rasd:ResourceType>24</rasd:ResourceType>
+      </Item>
+      <Item ovf:required="false">
+        <rasd:AutomaticAllocation>false</rasd:AutomaticAllocation>
+        <rasd:ElementName>vmci</rasd:ElementName>
+        <rasd:InstanceID>7</rasd:InstanceID>
+        <rasd:ResourceSubType>vmware.vmci</rasd:ResourceSubType>
+        <rasd:ResourceType>1</rasd:ResourceType>
+      </Item>
+      <vmw:Config ovf:required="false" vmw:key="firmware" vmw:value="efi"/>
+      <vmw:Config ovf:required="false" vmw:key="tools.syncTimeWithHost" vmw:value="false"/>
+      <vmw:Config ovf:required="false" vmw:key="tools.afterPowerOn" vmw:value="true"/>
+      <vmw:Config ovf:required="false" vmw:key="tools.afterResume" vmw:value="true"/>
+      <vmw:Config ovf:required="false" vmw:key="tools.beforeGuestShutdown" vmw:value="true"/>
+      <vmw:Config ovf:required="false" vmw:key="tools.beforeGuestStandby" vmw:value="true"/>
+      <vmw:Config ovf:required="false" vmw:key="tools.toolsUpgradePolicy" vmw:value="manual"/>
+      <vmw:Config ovf:required="false" vmw:key="powerOpInfo.suspendType" vmw:value="soft"/>
+      <vmw:ExtraConfig ovf:required="false" vmw:key="nvram" vmw:value="talos.nvram"/>
+    </VirtualHardwareSection>
+  </VirtualSystem>
+</Envelope>