diff --git a/arch/arm/cpu/armv7/am33xx/config.mk b/arch/arm/cpu/armv7/am33xx/config.mk
index ab9470820d..d4eb21ca14 100644
--- a/arch/arm/cpu/armv7/am33xx/config.mk
+++ b/arch/arm/cpu/armv7/am33xx/config.mk
@@ -26,7 +26,7 @@ endif
 else
 ifeq ($(CONFIG_TI_SECURE_DEVICE),y)
 ALL-$(CONFIG_QSPI_BOOT) += u-boot_HS_XIP_X-LOADER
-ALL-y	+= u-boot_HS.img
+ALL-$(CONFIG_SPL_LOAD_FIT) += u-boot_HS.img
 endif
 ALL-y	+= u-boot.img
 endif