diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index ac4746f454ed..effc9f33ade0 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -886,7 +886,7 @@ config RELOCATABLE
 
 config RANDOMIZE_BASE
 	bool "Randomize the address of the kernel image"
-	select ARM64_MODULE_PLTS
+	select ARM64_MODULE_PLTS if MODULES
 	select RELOCATABLE
 	help
 	  Randomizes the virtual address at which the kernel image is