diff --git a/.semaphore/semaphore-runner.sh b/.semaphore/semaphore-runner.sh
index 6ccf271a82..ec95f65315 100755
--- a/.semaphore/semaphore-runner.sh
+++ b/.semaphore/semaphore-runner.sh
@@ -21,7 +21,7 @@ UBUNTU_RELEASE="$(lsb_release -cs)"
 create_container() {
     # Create autopkgtest LXC image; this sometimes fails with "Unable to fetch
     # GPG key from keyserver", so retry a few times with different keyservers.
-    for keyserver in "" "keys.gnupg.net" "keys.openpgp.org" "keyserver.ubuntu.com"; do
+    for keyserver in "keys.openpgp.org" "" "keyserver.ubuntu.com" "keys.gnupg.net"; do
         for retry in {1..5}; do
             sudo lxc-create -n "$CONTAINER" -t download -- -d "$DISTRO" -r "$RELEASE" -a "$ARCH" ${keyserver:+--keyserver "$keyserver"} && break 2
             sleep $((retry*retry))
@@ -36,8 +36,16 @@ create_container() {
     # enable source repositories so that apt-get build-dep works
     sudo lxc-attach -n "$CONTAINER" -- sh -ex <<EOF
 sed 's/^deb/deb-src/' /etc/apt/sources.list >> /etc/apt/sources.list.d/sources.list
-# wait until online
-while [ -z "\$(ip route list 0/0)" ]; do sleep 1; done
+# We might attach the console too soon
+while ! systemctl --quiet --wait is-system-running; do sleep 1; done
+# Manpages database trigger takes a lot of time and is not useful in a CI
+echo 'man-db man-db/auto-update boolean false' | debconf-set-selections
+# Speed up dpkg, image is thrown away after the test
+mkdir -p /etc/dpkg/dpkg.cfg.d/
+echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/unsafe_io
+# For some reason, it is necessary to run this manually or the interface won't be configured
+# Note that we avoid networkd, as some of the tests will break it later on
+dhclient
 apt-get -q --allow-releaseinfo-change update
 apt-get -y dist-upgrade
 apt-get install -y eatmydata
@@ -88,6 +96,10 @@ EOF
             rm -rf debian/patches
             # disable autopkgtests which are not for upstream
             sed -i '/# NOUPSTREAM/ q' debian/tests/control
+            # TODO: rebooting via autopkgtest-reboot seems to be broken, disable these tests for now
+            sed -i -n '1,/Tests: boot-and-services/p;/Tests: udev/,$p' debian/tests/control
+            sed -i '/Tests: boot-and-services/d' debian/tests/control
+            sed -i '/Tests: boot-smoke/,$d' debian/tests/control
             # enable more unit tests
             sed -i '/^CONFFLAGS =/ s/=/= --werror -Dtests=unsafe -Dsplit-usr=true -Dslow-tests=true -Dfuzz-tests=true -Dman=true /' debian/rules
             # no orig tarball
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index bd5d135e9e..07742337e8 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -7,7 +7,7 @@ name: Debian autopkgtest (LXC)
 agent:
   machine:
     type: e1-standard-2
-    os_image: ubuntu1804
+    os_image: ubuntu2004
 
 # Cancel any running or queued job for the same ref
 auto_cancel: