diff --git a/Makefile b/Makefile
index 556cebdfc4..0ee0c70102 100644
--- a/Makefile
+++ b/Makefile
@@ -731,11 +731,6 @@ libs-y += drivers/
 libs-y += drivers/dma/
 libs-y += drivers/gpio/
 libs-y += drivers/i2c/
-libs-y += drivers/mtd/
-libs-$(CONFIG_CMD_NAND) += drivers/mtd/nand/raw/
-libs-y += drivers/mtd/onenand/
-libs-$(CONFIG_CMD_UBI) += drivers/mtd/ubi/
-libs-y += drivers/mtd/spi/
 libs-y += drivers/net/
 libs-y += drivers/net/phy/
 libs-y += drivers/power/ \
diff --git a/drivers/Makefile b/drivers/Makefile
index 0befeddfcb..0d1d6bd851 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -11,7 +11,7 @@ obj-$(CONFIG_$(SPL_TPL_)I2C_SUPPORT) += i2c/
 obj-$(CONFIG_$(SPL_TPL_)INPUT) += input/
 obj-$(CONFIG_$(SPL_TPL_)LED) += led/
 obj-$(CONFIG_$(SPL_TPL_)MMC_SUPPORT) += mmc/
-obj-$(CONFIG_$(SPL_TPL_)NAND_SUPPORT) += mtd/nand/raw/
+obj-y += mtd/
 obj-$(CONFIG_$(SPL_TPL_)PCH_SUPPORT) += pch/
 obj-$(CONFIG_$(SPL_TPL_)PCI) += pci/
 obj-$(CONFIG_$(SPL_TPL_)PHY) += phy/
@@ -19,7 +19,6 @@ obj-$(CONFIG_$(SPL_TPL_)PINCTRL) += pinctrl/
 obj-$(CONFIG_$(SPL_TPL_)RAM) += ram/
 obj-$(CONFIG_$(SPL_TPL_)RTC_SUPPORT) += rtc/
 obj-$(CONFIG_$(SPL_TPL_)SERIAL_SUPPORT) += serial/
-obj-$(CONFIG_$(SPL_TPL_)SPI_FLASH_SUPPORT) += mtd/spi/
 obj-$(CONFIG_$(SPL_TPL_)SPI_SUPPORT) += spi/
 obj-$(CONFIG_$(SPL_TPL_)TIMER) += timer/
 obj-$(CONFIG_$(SPL_TPL_)VIRTIO) += virtio/
@@ -42,9 +41,6 @@ obj-$(CONFIG_SPL_POWER_SUPPORT) += power/ power/pmic/
 obj-$(CONFIG_SPL_POWER_SUPPORT) += power/regulator/
 obj-$(CONFIG_SPL_POWER_DOMAIN) += power/domain/
 obj-$(CONFIG_SPL_DM_RESET) += reset/
-obj-$(CONFIG_SPL_MTD_SUPPORT) += mtd/
-obj-$(CONFIG_SPL_ONENAND_SUPPORT) += mtd/onenand/
-obj-$(CONFIG_SPL_UBI) += mtd/ubispl/
 obj-$(CONFIG_SPL_DMA_SUPPORT) += dma/
 obj-$(CONFIG_SPL_ETH_SUPPORT) += net/
 obj-$(CONFIG_SPL_ETH_SUPPORT) += net/phy/
@@ -102,6 +98,7 @@ obj-$(CONFIG_QE) += qe/
 obj-$(CONFIG_U_QE) += qe/
 obj-y += mailbox/
 obj-y += memory/
+obj-y += mtd/
 obj-y += pwm/
 obj-y += reset/
 obj-y += input/
diff --git a/drivers/mtd/Makefile b/drivers/mtd/Makefile
index c742bc671a..318788c5e2 100644
--- a/drivers/mtd/Makefile
+++ b/drivers/mtd/Makefile
@@ -3,21 +3,41 @@
 # (C) Copyright 2000-2007
 # Wolfgang Denk, DENX Software Engineering, wd@denx.de.
 
-ifneq (,$(findstring y,$(CONFIG_MTD)$(CONFIG_CMD_ONENAND)$(CONFIG_CMD_MTD)))
-obj-y += mtdcore.o mtd_uboot.o
-endif
-obj-$(CONFIG_DM_MTD) += mtd-uclass.o
-obj-$(CONFIG_MTD_PARTITIONS) += mtdpart.o
-obj-$(CONFIG_MTD_CONCAT) += mtdconcat.o
-obj-$(CONFIG_ALTERA_QSPI) += altera_qspi.o
-obj-$(CONFIG_FLASH_CFI_DRIVER) += cfi_flash.o
-obj-$(CONFIG_FLASH_CFI_MTD) += cfi_mtd.o
-obj-$(CONFIG_FLASH_CFI_LEGACY) += jedec_flash.o
-obj-$(CONFIG_MW_EEPROM) += mw_eeprom.o
-obj-$(CONFIG_FLASH_PIC32) += pic32_flash.o
-obj-$(CONFIG_ST_SMI) += st_smi.o
-obj-$(CONFIG_STM32_FLASH) += stm32_flash.o
-obj-$(CONFIG_RENESAS_RPC_HF) += renesas_rpc_hf.o
-obj-$(CONFIG_HBMC_AM654) += hbmc-am654.o
+mtd-$(CONFIG_MTD) += mtdcore.o mtd_uboot.o
+mtd-$(CONFIG_DM_MTD) += mtd-uclass.o
+mtd-$(CONFIG_MTD_PARTITIONS) += mtdpart.o
+mtd-$(CONFIG_MTD_CONCAT) += mtdconcat.o
+mtd-$(CONFIG_ALTERA_QSPI) += altera_qspi.o
+mtd-$(CONFIG_FLASH_CFI_DRIVER) += cfi_flash.o
+mtd-$(CONFIG_FLASH_CFI_MTD) += cfi_mtd.o
+mtd-$(CONFIG_FLASH_CFI_LEGACY) += jedec_flash.o
+mtd-$(CONFIG_MW_EEPROM) += mw_eeprom.o
+mtd-$(CONFIG_FLASH_PIC32) += pic32_flash.o
+mtd-$(CONFIG_ST_SMI) += st_smi.o
+mtd-$(CONFIG_STM32_FLASH) += stm32_flash.o
+mtd-$(CONFIG_RENESAS_RPC_HF) += renesas_rpc_hf.o
+mtd-$(CONFIG_HBMC_AM654) += hbmc-am654.o
 
+# U-Boot build
+ifeq ($(CONFIG_SPL_BUILD)$(CONFIG_TPL_BUILD),)
+
+ifneq ($(mtd-y),)
+obj-y += mtd.o
+endif
 obj-y += nand/
+obj-y += onenand/
+obj-y += spi/
+obj-$(CONFIG_MTD_UBI) += ubi/
+
+#SPL/TPL build
+else
+
+ifneq ($(mtd-y),)
+obj-$(CONFIG_SPL_MTD_SUPPORT) += mtd.o
+endif
+obj-$(CONFIG_$(SPL_TPL_)NAND_SUPPORT) += nand/
+obj-$(CONFIG_SPL_ONENAND_SUPPORT) += onenand/
+obj-$(CONFIG_$(SPL_TPL_)SPI_FLASH_SUPPORT) += spi/
+obj-$(CONFIG_SPL_UBI) += ubispl/
+
+endif
diff --git a/drivers/mtd/nand/Makefile b/drivers/mtd/nand/Makefile
index a358bc680e..96e186600a 100644
--- a/drivers/mtd/nand/Makefile
+++ b/drivers/mtd/nand/Makefile
@@ -1,5 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0+
 
+ifeq ($(CONFIG_SPL_BUILD)$(CONFIG_TPL_BUILD),)
 nandcore-objs := core.o bbt.o
 obj-$(CONFIG_MTD_NAND_CORE) += nandcore.o
+obj-$(CONFIG_MTD_RAW_NAND) += raw/
 obj-$(CONFIG_MTD_SPI_NAND) += spi/
+else
+obj-$(CONFIG_$(SPL_TPL_)NAND_SUPPORT) += raw/
+endif