diff --git a/arch/arm/mach-at91/Makefile b/arch/arm/mach-at91/Makefile
index 30f2b49b69..49e14e8257 100644
--- a/arch/arm/mach-at91/Makefile
+++ b/arch/arm/mach-at91/Makefile
@@ -10,7 +10,7 @@ obj-$(CONFIG_AT91SAM9M10G45) += mpddrc.o spl_at91.o
 obj-$(CONFIG_AT91SAM9N12) += mpddrc.o spl_at91.o
 obj-$(CONFIG_AT91SAM9X5) += mpddrc.o spl_at91.o
 obj-$(CONFIG_SAMA5D3) += mpddrc.o spl_atmel.o
-obj-$(CONFIG_SAMA5D4) += mpddrc.o spl_atmel.o
+obj-$(CONFIG_SAMA5D4) += mpddrc.o spl_atmel.o matrix.o
 obj-y += spl.o
 endif
 
diff --git a/arch/arm/mach-at91/armv7/sama5d4_devices.c b/arch/arm/mach-at91/armv7/sama5d4_devices.c
index 76301d6302..52f486289d 100644
--- a/arch/arm/mach-at91/armv7/sama5d4_devices.c
+++ b/arch/arm/mach-at91/armv7/sama5d4_devices.c
@@ -10,7 +10,6 @@
 #include <asm/arch/at91_common.h>
 #include <asm/arch/at91_pmc.h>
 #include <asm/arch/clk.h>
-#include <asm/arch/sama5_matrix.h>
 #include <asm/arch/sama5_sfr.h>
 #include <asm/arch/sama5d4.h>
 
@@ -48,47 +47,6 @@ void at91_udp_hw_init(void)
 #endif
 
 #ifdef CONFIG_SPL_BUILD
-void matrix_init(void)
-{
-	struct atmel_matrix *h64mx = (struct atmel_matrix *)ATMEL_BASE_MATRIX0;
-	struct atmel_matrix *h32mx = (struct atmel_matrix *)ATMEL_BASE_MATRIX1;
-	int i;
-
-	/* Disable the write protect */
-	writel(ATMEL_MATRIX_WPMR_WPKEY & ~ATMEL_MATRIX_WPMR_WPEN, &h64mx->wpmr);
-	writel(ATMEL_MATRIX_WPMR_WPKEY & ~ATMEL_MATRIX_WPMR_WPEN, &h32mx->wpmr);
-
-	/* DDR port 1 ~ poart 7, slave number is: 4 ~ 10 */
-	for (i = 4; i <= 10; i++) {
-		writel(0x000f0f0f, &h64mx->ssr[i]);
-		writel(0x0000ffff, &h64mx->sassr[i]);
-		writel(0x0000000f, &h64mx->srtsr[i]);
-	}
-
-	/* CS3 */
-	writel(0x00c0c0c0, &h32mx->ssr[3]);
-	writel(0xff000000, &h32mx->sassr[3]);
-	writel(0xff000000, &h32mx->srtsr[3]);
-
-	/* NFC SRAM */
-	writel(0x00010101, &h32mx->ssr[4]);
-	writel(0x00000001, &h32mx->sassr[4]);
-	writel(0x00000001, &h32mx->srtsr[4]);
-
-	/* Configure Programmable Security peripherals on matrix 64 */
-	writel(readl(&h64mx->spselr[0]) | 0x00080000, &h64mx->spselr[0]);
-	writel(readl(&h64mx->spselr[1]) | 0x00180000, &h64mx->spselr[1]);
-	writel(readl(&h64mx->spselr[2]) | 0x00000008, &h64mx->spselr[2]);
-
-	/* Configure Programmable Security peripherals on matrix 32 */
-	writel(readl(&h32mx->spselr[0]) | 0xFFC00000, &h32mx->spselr[0]);
-	writel(readl(&h32mx->spselr[1]) | 0x60E3FFFF, &h32mx->spselr[1]);
-
-	/* Enable the write protect */
-	writel(ATMEL_MATRIX_WPMR_WPKEY | ATMEL_MATRIX_WPMR_WPEN, &h64mx->wpmr);
-	writel(ATMEL_MATRIX_WPMR_WPKEY | ATMEL_MATRIX_WPMR_WPEN, &h32mx->wpmr);
-}
-
 void redirect_int_from_saic_to_aic(void)
 {
 	struct atmel_sfr *sfr = (struct atmel_sfr *)ATMEL_BASE_SFR;
diff --git a/arch/arm/mach-at91/matrix.c b/arch/arm/mach-at91/matrix.c
new file mode 100644
index 0000000000..cf363861c6
--- /dev/null
+++ b/arch/arm/mach-at91/matrix.c
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2015 Atmel Corporation
+ *		      Wenyou Yang <wenyou.yang@atmel.com>
+ *
+ * SPDX-License-Identifier:	GPL-2.0+
+ */
+
+#include <common.h>
+#include <asm/io.h>
+#include <asm/arch/sama5_matrix.h>
+
+void matrix_init(void)
+{
+	struct atmel_matrix *h64mx = (struct atmel_matrix *)ATMEL_BASE_MATRIX0;
+	struct atmel_matrix *h32mx = (struct atmel_matrix *)ATMEL_BASE_MATRIX1;
+	int i;
+
+	/* Disable the write protect */
+	writel(ATMEL_MATRIX_WPMR_WPKEY & ~ATMEL_MATRIX_WPMR_WPEN, &h64mx->wpmr);
+	writel(ATMEL_MATRIX_WPMR_WPKEY & ~ATMEL_MATRIX_WPMR_WPEN, &h32mx->wpmr);
+
+	/* DDR port 1 ~ poart 7, slave number is: 4 ~ 10 */
+	for (i = 4; i <= 10; i++) {
+		writel(0x000f0f0f, &h64mx->ssr[i]);
+		writel(0x0000ffff, &h64mx->sassr[i]);
+		writel(0x0000000f, &h64mx->srtsr[i]);
+	}
+
+	/* CS3 */
+	writel(0x00c0c0c0, &h32mx->ssr[3]);
+	writel(0xff000000, &h32mx->sassr[3]);
+	writel(0xff000000, &h32mx->srtsr[3]);
+
+	/* NFC SRAM */
+	writel(0x00010101, &h32mx->ssr[4]);
+	writel(0x00000001, &h32mx->sassr[4]);
+	writel(0x00000001, &h32mx->srtsr[4]);
+
+	/* Configure Programmable Security peripherals on matrix 64 */
+	writel(readl(&h64mx->spselr[0]) | 0x00080000, &h64mx->spselr[0]);
+	writel(readl(&h64mx->spselr[1]) | 0x00180000, &h64mx->spselr[1]);
+	writel(readl(&h64mx->spselr[2]) | 0x00000008, &h64mx->spselr[2]);
+
+	/* Configure Programmable Security peripherals on matrix 32 */
+	writel(readl(&h32mx->spselr[0]) | 0xFFC00000, &h32mx->spselr[0]);
+	writel(readl(&h32mx->spselr[1]) | 0x60E3FFFF, &h32mx->spselr[1]);
+
+	/* Enable the write protect */
+	writel(ATMEL_MATRIX_WPMR_WPKEY | ATMEL_MATRIX_WPMR_WPEN, &h64mx->wpmr);
+	writel(ATMEL_MATRIX_WPMR_WPKEY | ATMEL_MATRIX_WPMR_WPEN, &h32mx->wpmr);
+}