diff --git a/README b/README
index fefa71c0a6..cac7978afa 100644
--- a/README
+++ b/README
@@ -3176,8 +3176,13 @@ CBFS (Coreboot Filesystem) support
 		This enables the RSA algorithm used for FIT image verification
 		in U-Boot. See doc/uImage.FIT/signature.txt for more information.
 
+		The Modular Exponentiation algorithm in RSA is implemented using
+		driver model. So CONFIG_DM needs to be enabled by default for this
+		library to function.
+
 		The signing part is build into mkimage regardless of this
-		option.
+		option. The software based modular exponentiation is built into
+		mkimage irrespective of this option.
 
 - bootcount support:
 		CONFIG_BOOTCOUNT_LIMIT
diff --git a/configs/am335x_boneblack_vboot_defconfig b/configs/am335x_boneblack_vboot_defconfig
index 5837a0a4da..51bf370364 100644
--- a/configs/am335x_boneblack_vboot_defconfig
+++ b/configs/am335x_boneblack_vboot_defconfig
@@ -4,3 +4,7 @@ CONFIG_SYS_EXTRA_OPTIONS="EMMC_BOOT,ENABLE_VBOOT"
 +S:CONFIG_TARGET_AM335X_EVM=y
 CONFIG_OF_CONTROL=y
 CONFIG_DEFAULT_DEVICE_TREE="am335x-boneblack"
+CONFIG_FIT=y
+CONFIG_FIT_VERBOSE=y
+CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/ids8313_defconfig b/configs/ids8313_defconfig
index 8479cd42f7..0950ec8b77 100644
--- a/configs/ids8313_defconfig
+++ b/configs/ids8313_defconfig
@@ -4,3 +4,4 @@ CONFIG_MPC83xx=y
 CONFIG_FIT=y
 CONFIG_FIT_SIGNATURE=y
 CONFIG_TARGET_IDS8313=y
+CONFIG_DM=y
diff --git a/configs/sandbox_defconfig b/configs/sandbox_defconfig
index 0111f25224..660063ebf3 100644
--- a/configs/sandbox_defconfig
+++ b/configs/sandbox_defconfig
@@ -3,4 +3,5 @@ CONFIG_OF_HOSTFILE=y
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
 CONFIG_DEFAULT_DEVICE_TREE="sandbox"
diff --git a/configs/zynq_microzed_defconfig b/configs/zynq_microzed_defconfig
index b9a6fe522a..8b985fe5a4 100644
--- a/configs/zynq_microzed_defconfig
+++ b/configs/zynq_microzed_defconfig
@@ -6,4 +6,5 @@ CONFIG_OF_CONTROL=y
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
 CONFIG_DEFAULT_DEVICE_TREE="zynq-microzed"
diff --git a/configs/zynq_zc70x_defconfig b/configs/zynq_zc70x_defconfig
index dc8aa84992..cceb32199d 100644
--- a/configs/zynq_zc70x_defconfig
+++ b/configs/zynq_zc70x_defconfig
@@ -7,3 +7,4 @@ CONFIG_DEFAULT_DEVICE_TREE="zynq-zc702"
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zc770_xm010_defconfig b/configs/zynq_zc770_xm010_defconfig
index 2f5fa8c59a..2935c0dff7 100644
--- a/configs/zynq_zc770_xm010_defconfig
+++ b/configs/zynq_zc770_xm010_defconfig
@@ -8,3 +8,4 @@ CONFIG_DEFAULT_DEVICE_TREE="zynq-zc770-xm010"
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zc770_xm012_defconfig b/configs/zynq_zc770_xm012_defconfig
index a92d495dfe..0401739652 100644
--- a/configs/zynq_zc770_xm012_defconfig
+++ b/configs/zynq_zc770_xm012_defconfig
@@ -8,3 +8,4 @@ CONFIG_DEFAULT_DEVICE_TREE="zynq-zc770-xm012"
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zc770_xm013_defconfig b/configs/zynq_zc770_xm013_defconfig
index 3a02f750c6..a95970a917 100644
--- a/configs/zynq_zc770_xm013_defconfig
+++ b/configs/zynq_zc770_xm013_defconfig
@@ -8,3 +8,4 @@ CONFIG_DEFAULT_DEVICE_TREE="zynq-zc770-xm013"
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zed_defconfig b/configs/zynq_zed_defconfig
index 1d816f68c4..0fbc41ab8a 100644
--- a/configs/zynq_zed_defconfig
+++ b/configs/zynq_zed_defconfig
@@ -7,3 +7,4 @@ CONFIG_DEFAULT_DEVICE_TREE="zynq-zed"
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/configs/zynq_zybo_defconfig b/configs/zynq_zybo_defconfig
index 9183629bfb..4e66760750 100644
--- a/configs/zynq_zybo_defconfig
+++ b/configs/zynq_zybo_defconfig
@@ -7,3 +7,4 @@ CONFIG_DEFAULT_DEVICE_TREE="zynq-zybo"
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_FIT_SIGNATURE=y
+CONFIG_DM=y
diff --git a/include/configs/am335x_evm.h b/include/configs/am335x_evm.h
index f9bc23b407..76ce7deb95 100644
--- a/include/configs/am335x_evm.h
+++ b/include/configs/am335x_evm.h
@@ -19,13 +19,11 @@
 #include <configs/ti_am335x_common.h>
 
 #ifndef CONFIG_SPL_BUILD
+#ifndef CONFIG_FIT
 # define CONFIG_FIT
+#endif
 # define CONFIG_TIMESTAMP
 # define CONFIG_LZO
-# ifdef CONFIG_ENABLE_VBOOT
-# define CONFIG_FIT_SIGNATURE
-# define CONFIG_RSA
-# endif
 #endif
 
 #define CONFIG_SYS_BOOTM_LEN		(16 << 20)
diff --git a/include/configs/sandbox.h b/include/configs/sandbox.h
index 6fd29b9c73..e9d3f3226b 100644
--- a/include/configs/sandbox.h
+++ b/include/configs/sandbox.h
@@ -23,7 +23,6 @@
 
 #define CONFIG_BOOTSTAGE
 #define CONFIG_BOOTSTAGE_REPORT
-#define CONFIG_DM
 #define CONFIG_CMD_DEMO
 #define CONFIG_CMD_DM
 #define CONFIG_DM_DEMO
diff --git a/lib/rsa/rsa-verify.c b/lib/rsa/rsa-verify.c
index f8bc086fd7..da45daffd3 100644
--- a/lib/rsa/rsa-verify.c
+++ b/lib/rsa/rsa-verify.c
@@ -12,6 +12,7 @@
 #include <asm/errno.h>
 #include <asm/types.h>
 #include <asm/unaligned.h>
+#include <dm.h>
 #else
 #include "fdt_host.h"
 #include "mkimage.h"
@@ -43,6 +44,9 @@ static int rsa_verify_key(struct key_prop *prop, const uint8_t *sig,
 	const uint8_t *padding;
 	int pad_len;
 	int ret;
+#if !defined(USE_HOSTCC)
+	struct udevice *mod_exp_dev;
+#endif
 
 	if (!prop || !sig || !hash || !algo)
 		return -EIO;
@@ -63,7 +67,17 @@ static int rsa_verify_key(struct key_prop *prop, const uint8_t *sig,
 
 	uint8_t buf[sig_len];
 
+#if !defined(USE_HOSTCC)
+	ret = uclass_get_device(UCLASS_MOD_EXP, 0, &mod_exp_dev);
+	if (ret) {
+		printf("RSA: Can't find Modular Exp implementation\n");
+		return -EINVAL;
+	}
+
+	ret = rsa_mod_exp(mod_exp_dev, sig, sig_len, prop, buf);
+#else
 	ret = rsa_mod_exp_sw(sig, sig_len, prop, buf);
+#endif
 	if (ret) {
 		debug("Error in Modular exponentation\n");
 		return ret;