diff --git a/common/spl/spl_fit.c b/common/spl/spl_fit.c
index f581a22421..365104fe02 100644
--- a/common/spl/spl_fit.c
+++ b/common/spl/spl_fit.c
@@ -619,9 +619,12 @@ int spl_load_simple_fit(struct spl_image_info *spl_image,
 	 * Booting a next-stage U-Boot may require us to append the FDT.
 	 * We allow this to fail, as the U-Boot image might embed its FDT.
 	 */
-	if (spl_image->os == IH_OS_U_BOOT)
-		spl_fit_append_fdt(spl_image, info, sector, fit,
-				   images, base_offset);
+	if (spl_image->os == IH_OS_U_BOOT) {
+		ret = spl_fit_append_fdt(spl_image, info, sector, fit,
+					 images, base_offset);
+		if (!IS_ENABLED(CONFIG_OF_EMBED) && ret < 0)
+			return ret;
+	}
 
 	firmware_node = node;
 	/* Now check if there are more images for us to load */