diff --git a/.github/workflows/mkosi.yml b/.github/workflows/mkosi.yml
index d2aa7f7b799..a4a7a761495 100644
--- a/.github/workflows/mkosi.yml
+++ b/.github/workflows/mkosi.yml
@@ -59,36 +59,43 @@ jobs:
             sanitizers: ""
             llvm: 0
             cflags: "-O2 -D_FORTIFY_SOURCE=3"
+            relabel: no
           - distro: debian
             release: testing
             sanitizers: ""
             llvm: 0
             cflags: "-Og"
+            relabel: no
           - distro: ubuntu
             release: noble
             sanitizers: ""
             llvm: 0
             cflags: "-Og"
+            relabel: no
           - distro: fedora
             release: "40"
             sanitizers: ""
             llvm: 0
             cflags: "-Og"
+            relabel: yes
           - distro: fedora
             release: rawhide
             sanitizers: address,undefined
             llvm: 1
             cflags: "-Og"
+            relabel: yes
           - distro: opensuse
             release: tumbleweed
             sanitizers: ""
             llvm: 0
             cflags: "-Og"
+            relabel: no
           - distro: centos
             release: "9"
             sanitizers: ""
             llvm: 0
             cflags: "-Og"
+            relabel: yes
 
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
@@ -141,6 +148,8 @@ jobs:
                 MESON_OPTIONS=--werror
                 LLVM=${{ matrix.llvm }}
 
+        SELinuxRelabel=${{ matrix.relabel }}
+
         [Host]
         QemuMem=4G
         # We build with debuginfo so there's no point in mounting the sources into the machine.
@@ -187,7 +196,7 @@ jobs:
             -Dvmspawn=enabled
 
     - name: Build image
-      run: meson compile -C build mkosi
+      run: sudo meson compile -C build mkosi
 
     - name: Run integration tests
       run: sudo --preserve-env meson test -C build --no-rebuild --suite integration-tests --print-errorlogs --no-stdsplit --num-processes "$(($(nproc) - 1))"
diff --git a/mkosi.conf b/mkosi.conf
index c90f5bfc774..96fb992497d 100644
--- a/mkosi.conf
+++ b/mkosi.conf
@@ -11,7 +11,6 @@ BuildDirectory=build/mkosi.builddir
 CacheDirectory=build/mkosi.cache
 
 [Content]
-SELinuxRelabel=no
 BuildSourcesEphemeral=yes
 Autologin=yes
 
@@ -24,6 +23,10 @@ ExtraTrees=
 Environment=
         SYSTEMD_REPART_OVERRIDE_FSTYPE_ROOT=%F
 
+# Disable relabeling by default as it only matters for TEST-06-SELINUX, takes a non-trivial amount of time
+# and results in lots of errors when building images as a regular user.
+SELinuxRelabel=no
+
 # Adding more kernel command line arguments is likely to hit the kernel command line limit (512 bytes) in
 # various scenarios. Consider adding support for a credential instead if possible and using that.
 KernelCommandLine=systemd.crash_shell
diff --git a/mkosi.conf.d/10-centos-fedora/mkosi.conf.d/10-selinux.conf b/mkosi.conf.d/10-centos-fedora/mkosi.conf.d/10-selinux.conf
index 9fe5509695f..0a388f3c081 100644
--- a/mkosi.conf.d/10-centos-fedora/mkosi.conf.d/10-selinux.conf
+++ b/mkosi.conf.d/10-centos-fedora/mkosi.conf.d/10-selinux.conf
@@ -1,20 +1,13 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
+# libselinux does not work in the slightest with /usr-only images so don't install the packages if we're
+# building a /usr-only image.
+
 [Match]
 Profile=!particle
 
 [Content]
-# libselinux does not work in the slightest with /usr-only images so don't install the packages if we're
-# building a /usr-only image.
 Packages=
         selinux-policy
         selinux-policy-targeted
         setools-console
-
-# We relabel on first boot instead of at build time because it is only possible to label without root
-# if the labels exist in the host system, and we want to be able to cross-build to other distributions.
-SELinuxRelabel=no
-
-InitrdPackages=
-        selinux-policy
-        selinux-policy-targeted
diff --git a/mkosi.extra/.autorelabel b/mkosi.extra/.autorelabel
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/mkosi.extra/usr/lib/systemd/system-preset/00-mkosi.preset b/mkosi.extra/usr/lib/systemd/system-preset/00-mkosi.preset
index c3640585e5c..5a15e6bcbbf 100644
--- a/mkosi.extra/usr/lib/systemd/system-preset/00-mkosi.preset
+++ b/mkosi.extra/usr/lib/systemd/system-preset/00-mkosi.preset
@@ -32,10 +32,10 @@ disable auditd.service
 # systemd-timesyncd is not enabled by default in the default systemd preset so enable it here instead.
 enable systemd-timesyncd.service
 
-# Skipped if selinux is not enabled, required for TEST-06-SELINUX.
-enable autorelabel.service
-
 # Enabled by default on OpenSUSE and not conditioned out in containers, so let's disable these here instead.
 disable iscsi.service
 disable iscsid.socket
 disable iscsiuio.socket
+
+# mkosi relabels the image itself so no need to do it on boot.
+disable selinux-autorelabel-mark.service
diff --git a/test/TEST-06-SELINUX/meson.build b/test/TEST-06-SELINUX/meson.build
index 110b65fbd7a..ea1a381471b 100644
--- a/test/TEST-06-SELINUX/meson.build
+++ b/test/TEST-06-SELINUX/meson.build
@@ -3,7 +3,7 @@
 integration_tests += [
         integration_test_template + {
                 'name' : fs.name(meson.current_source_dir()),
-                'cmdline' : integration_test_template['cmdline'] + ['systemd.wants=autorelabel.service', 'selinux=1', 'lsm=selinux'],
+                'cmdline' : integration_test_template['cmdline'] + ['selinux=1', 'lsm=selinux'],
                 # FIXME; Figure out why reboot sometimes hangs with 'linux' firmware.
                 # Use 'auto' to automatically fallback on non-uefi architectures.
                 'firmware' : 'auto',