diff --git a/configs/am57xx_hs_evm_usb_defconfig b/configs/am57xx_hs_evm_usb_defconfig
index 3b155cc202..2fa60ac517 100644
--- a/configs/am57xx_hs_evm_usb_defconfig
+++ b/configs/am57xx_hs_evm_usb_defconfig
@@ -6,6 +6,7 @@ CONFIG_TI_COMMON_CMD_OPTIONS=y
 CONFIG_SYS_MALLOC_F_LEN=0x2000
 CONFIG_SYS_SPI_U_BOOT_OFFS=0x40000
 CONFIG_DM_GPIO=y
+CONFIG_SPL_TEXT_BASE=0x40306D50
 CONFIG_OMAP54XX=y
 CONFIG_TI_SECURE_EMIF_REGION_START=0xbdb00000
 CONFIG_TI_SECURE_EMIF_TOTAL_REGION_SIZE=0x02000000
@@ -17,10 +18,10 @@ CONFIG_ENV_OFFSET_REDUND=0x280000
 CONFIG_SPL_SPI_FLASH_SUPPORT=y
 CONFIG_SPL_SPI_SUPPORT=y
 CONFIG_ARMV7_LPAE=y
-CONFIG_SPL_TEXT_BASE=0x40306D50
 CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT_IMAGE_POST_PROCESS=y
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_SPL_LOAD_FIT_ADDRESS=0x80200000
 CONFIG_SPL_FIT_IMAGE_POST_PROCESS=y
 CONFIG_OF_BOARD_SETUP=y
 CONFIG_USE_BOOTARGS=y
diff --git a/configs/avnet_ultrazedev_cc_v1_0_ultrazedev_som_v1_0_defconfig b/configs/avnet_ultrazedev_cc_v1_0_ultrazedev_som_v1_0_defconfig
index 030d28a5df..0ae610be56 100644
--- a/configs/avnet_ultrazedev_cc_v1_0_ultrazedev_som_v1_0_defconfig
+++ b/configs/avnet_ultrazedev_cc_v1_0_ultrazedev_som_v1_0_defconfig
@@ -13,6 +13,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_SPL_LOAD_FIT_ADDRESS=0x10000000
 CONFIG_BOOTDELAY=0
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_OS_BOOT=y
diff --git a/configs/dra7xx_evm_defconfig b/configs/dra7xx_evm_defconfig
index e4547d9dcc..393020fbd5 100644
--- a/configs/dra7xx_evm_defconfig
+++ b/configs/dra7xx_evm_defconfig
@@ -4,6 +4,7 @@ CONFIG_TI_COMMON_CMD_OPTIONS=y
 CONFIG_SYS_MALLOC_F_LEN=0x18000
 CONFIG_SYS_SPI_U_BOOT_OFFS=0x40000
 CONFIG_DM_GPIO=y
+CONFIG_SPL_TEXT_BASE=0x40300000
 CONFIG_OMAP54XX=y
 CONFIG_TARGET_DRA7XX_EVM=y
 CONFIG_NR_DRAM_BANKS=2
@@ -12,10 +13,10 @@ CONFIG_ENV_OFFSET_REDUND=0x280000
 CONFIG_SPL_SPI_FLASH_SUPPORT=y
 CONFIG_SPL_SPI_SUPPORT=y
 CONFIG_ARMV7_LPAE=y
-CONFIG_SPL_TEXT_BASE=0x40300000
 CONFIG_AHCI=y
 CONFIG_DISTRO_DEFAULTS=y
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_SPL_LOAD_FIT_ADDRESS=0x80200000
 CONFIG_OF_BOARD_SETUP=y
 CONFIG_USE_BOOTARGS=y
 CONFIG_BOOTARGS="androidboot.serialno=${serial#} console=ttyS0,115200 androidboot.console=ttyS0 androidboot.hardware=jacinto6evmboard"
diff --git a/configs/dra7xx_hs_evm_defconfig b/configs/dra7xx_hs_evm_defconfig
index c08bcce903..ee57b7997a 100644
--- a/configs/dra7xx_hs_evm_defconfig
+++ b/configs/dra7xx_hs_evm_defconfig
@@ -20,6 +20,7 @@ CONFIG_AHCI=y
 CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT_IMAGE_POST_PROCESS=y
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_SPL_LOAD_FIT_ADDRESS=0x80200000
 CONFIG_SPL_FIT_IMAGE_POST_PROCESS=y
 CONFIG_OF_BOARD_SETUP=y
 CONFIG_USE_BOOTARGS=y
diff --git a/configs/dra7xx_hs_evm_usb_defconfig b/configs/dra7xx_hs_evm_usb_defconfig
index 879c2b650b..5ea6e8bf07 100644
--- a/configs/dra7xx_hs_evm_usb_defconfig
+++ b/configs/dra7xx_hs_evm_usb_defconfig
@@ -6,6 +6,7 @@ CONFIG_TI_COMMON_CMD_OPTIONS=y
 CONFIG_SYS_MALLOC_F_LEN=0x18000
 CONFIG_SYS_SPI_U_BOOT_OFFS=0x40000
 CONFIG_DM_GPIO=y
+CONFIG_SPL_TEXT_BASE=0x40306D50
 CONFIG_OMAP54XX=y
 CONFIG_TI_SECURE_EMIF_REGION_START=0xbdb00000
 CONFIG_TI_SECURE_EMIF_TOTAL_REGION_SIZE=0x02000000
@@ -17,11 +18,11 @@ CONFIG_ENV_OFFSET_REDUND=0x280000
 CONFIG_SPL_SPI_FLASH_SUPPORT=y
 CONFIG_SPL_SPI_SUPPORT=y
 CONFIG_ARMV7_LPAE=y
-CONFIG_SPL_TEXT_BASE=0x40306D50
 CONFIG_AHCI=y
 CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT_IMAGE_POST_PROCESS=y
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_SPL_LOAD_FIT_ADDRESS=0x80200000
 CONFIG_SPL_FIT_IMAGE_POST_PROCESS=y
 CONFIG_OF_BOARD_SETUP=y
 CONFIG_USE_BOOTARGS=y
diff --git a/configs/j721e_evm_a72_defconfig b/configs/j721e_evm_a72_defconfig
index 4deb4e219f..fd7e4c8785 100644
--- a/configs/j721e_evm_a72_defconfig
+++ b/configs/j721e_evm_a72_defconfig
@@ -10,6 +10,7 @@ CONFIG_ENV_SIZE=0x20000
 CONFIG_ENV_OFFSET=0x680000
 CONFIG_SYS_SPI_U_BOOT_OFFS=0x280000
 CONFIG_DM_GPIO=y
+CONFIG_SPL_TEXT_BASE=0x80080000
 CONFIG_SPL_MMC_SUPPORT=y
 CONFIG_SPL_SERIAL_SUPPORT=y
 CONFIG_SPL_DRIVERS_MISC_SUPPORT=y
@@ -21,10 +22,10 @@ CONFIG_SPL_LIBDISK_SUPPORT=y
 CONFIG_SPL_SPI_FLASH_SUPPORT=y
 CONFIG_SPL_SPI_SUPPORT=y
 # CONFIG_PSCI_RESET is not set
-CONFIG_SPL_TEXT_BASE=0x80080000
 CONFIG_DISTRO_DEFAULTS=y
 # CONFIG_SYS_MALLOC_CLEAR_ON_INIT is not set
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_SPL_LOAD_FIT_ADDRESS=0x81000000
 CONFIG_BOOTCOMMAND="run findfdt; run envboot; run init_${boot}; run boot_rprocs; run get_kern_${boot}; run get_fdt_${boot}; run get_overlay_${boot}; run run_kern"
 CONFIG_SPL_BOARD_INIT=y
 CONFIG_SPL_SYS_MALLOC_SIMPLE=y
diff --git a/configs/j721e_evm_r5_defconfig b/configs/j721e_evm_r5_defconfig
index ee9217aee2..b2739ef426 100644
--- a/configs/j721e_evm_r5_defconfig
+++ b/configs/j721e_evm_r5_defconfig
@@ -10,6 +10,7 @@ CONFIG_TARGET_J721E_R5_EVM=y
 CONFIG_ENV_SIZE=0x20000
 CONFIG_SYS_SPI_U_BOOT_OFFS=0x80000
 CONFIG_DM_GPIO=y
+CONFIG_SPL_TEXT_BASE=0x41c00000
 CONFIG_SPL_MMC_SUPPORT=y
 CONFIG_SPL_SERIAL_SUPPORT=y
 CONFIG_SPL_DRIVERS_MISC_SUPPORT=y
@@ -18,9 +19,9 @@ CONFIG_SPL_FS_FAT=y
 CONFIG_SPL_LIBDISK_SUPPORT=y
 CONFIG_SPL_SPI_FLASH_SUPPORT=y
 CONFIG_SPL_SPI_SUPPORT=y
-CONFIG_SPL_TEXT_BASE=0x41c00000
 # CONFIG_SYS_MALLOC_CLEAR_ON_INIT is not set
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_SPL_LOAD_FIT_ADDRESS=0x80080000
 CONFIG_USE_BOOTCOMMAND=y
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_SPL_BOARD_INIT=y
diff --git a/configs/j721e_hs_evm_a72_defconfig b/configs/j721e_hs_evm_a72_defconfig
index ae540a26a4..3158abda64 100644
--- a/configs/j721e_hs_evm_a72_defconfig
+++ b/configs/j721e_hs_evm_a72_defconfig
@@ -9,6 +9,7 @@ CONFIG_TARGET_J721E_A72_EVM=y
 CONFIG_ENV_SIZE=0x20000
 CONFIG_ENV_OFFSET=0x680000
 CONFIG_SYS_SPI_U_BOOT_OFFS=0x280000
+CONFIG_SPL_TEXT_BASE=0x80080000
 CONFIG_SPL_MMC_SUPPORT=y
 CONFIG_SPL_SERIAL_SUPPORT=y
 CONFIG_SPL_DRIVERS_MISC_SUPPORT=y
@@ -20,11 +21,11 @@ CONFIG_SPL_LIBDISK_SUPPORT=y
 CONFIG_SPL_SPI_FLASH_SUPPORT=y
 CONFIG_SPL_SPI_SUPPORT=y
 # CONFIG_PSCI_RESET is not set
-CONFIG_SPL_TEXT_BASE=0x80080000
 CONFIG_DISTRO_DEFAULTS=y
 # CONFIG_SYS_MALLOC_CLEAR_ON_INIT is not set
 CONFIG_FIT_IMAGE_POST_PROCESS=y
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_SPL_LOAD_FIT_ADDRESS=0x81000000
 CONFIG_SPL_FIT_IMAGE_POST_PROCESS=y
 CONFIG_BOOTCOMMAND="run findfdt; run envboot; run init_${boot}; run boot_rprocs; run get_fit_${boot}; run get_overlaystring; run run_fit"
 CONFIG_SPL_BOARD_INIT=y
diff --git a/configs/j721e_hs_evm_r5_defconfig b/configs/j721e_hs_evm_r5_defconfig
index 51d5a3bb98..6f99898229 100644
--- a/configs/j721e_hs_evm_r5_defconfig
+++ b/configs/j721e_hs_evm_r5_defconfig
@@ -11,6 +11,7 @@ CONFIG_ENV_SIZE=0x20000
 CONFIG_ENV_OFFSET=0x680000
 CONFIG_SYS_SPI_U_BOOT_OFFS=0x80000
 CONFIG_DM_GPIO=y
+CONFIG_SPL_TEXT_BASE=0x41c00000
 CONFIG_SPL_MMC_SUPPORT=y
 CONFIG_SPL_SERIAL_SUPPORT=y
 CONFIG_SPL_DRIVERS_MISC_SUPPORT=y
@@ -20,9 +21,9 @@ CONFIG_SPL_FS_FAT=y
 CONFIG_SPL_LIBDISK_SUPPORT=y
 CONFIG_SPL_SPI_FLASH_SUPPORT=y
 CONFIG_SPL_SPI_SUPPORT=y
-CONFIG_SPL_TEXT_BASE=0x41c00000
 # CONFIG_SYS_MALLOC_CLEAR_ON_INIT is not set
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_SPL_LOAD_FIT_ADDRESS=0x80080000
 CONFIG_SPL_FIT_IMAGE_POST_PROCESS=y
 CONFIG_USE_BOOTCOMMAND=y
 # CONFIG_DISPLAY_CPUINFO is not set
diff --git a/configs/xilinx_zynq_virt_defconfig b/configs/xilinx_zynq_virt_defconfig
index 5da7440af2..770b8f50f5 100644
--- a/configs/xilinx_zynq_virt_defconfig
+++ b/configs/xilinx_zynq_virt_defconfig
@@ -14,6 +14,7 @@ CONFIG_FIT_SIGNATURE=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_SPL_FIT_PRINT=y
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_SPL_LOAD_FIT_ADDRESS=0x10000000
 CONFIG_LEGACY_IMAGE_FORMAT=y
 CONFIG_USE_PREBOOT=y
 CONFIG_SPL_STACK_R=y
diff --git a/configs/xilinx_zynqmp_virt_defconfig b/configs/xilinx_zynqmp_virt_defconfig
index 7886d5a38f..f66432adc7 100644
--- a/configs/xilinx_zynqmp_virt_defconfig
+++ b/configs/xilinx_zynqmp_virt_defconfig
@@ -13,6 +13,7 @@ CONFIG_DISTRO_DEFAULTS=y
 CONFIG_FIT=y
 CONFIG_FIT_VERBOSE=y
 CONFIG_SPL_LOAD_FIT=y
+CONFIG_SPL_LOAD_FIT_ADDRESS=0x10000000
 # CONFIG_DISPLAY_CPUINFO is not set
 CONFIG_BOARD_EARLY_INIT_R=y
 CONFIG_SPL_OS_BOOT=y
diff --git a/include/configs/am57xx_evm.h b/include/configs/am57xx_evm.h
index e94b7c8d21..2fe83714f8 100644
--- a/include/configs/am57xx_evm.h
+++ b/include/configs/am57xx_evm.h
@@ -41,7 +41,6 @@
 	DFU_ALT_INFO_QSPI
 #else
 #ifdef CONFIG_SPL_DFU
-#define CONFIG_SPL_LOAD_FIT_ADDRESS 0x80200000
 #define DFUARGS \
 	"dfu_bufsiz=0x10000\0" \
 	DFU_ALT_INFO_RAM
diff --git a/include/configs/ax25-ae350.h b/include/configs/ax25-ae350.h
index 34fc6b616e..4b3fc26127 100644
--- a/include/configs/ax25-ae350.h
+++ b/include/configs/ax25-ae350.h
@@ -12,12 +12,6 @@
 #define CONFIG_SPL_BSS_START_ADDR	0x04000000
 #define CONFIG_SPL_BSS_MAX_SIZE		0x00100000
 
-#ifndef CONFIG_XIP
-#define CONFIG_SPL_LOAD_FIT_ADDRESS	0x00200000
-#else
-#define CONFIG_SPL_LOAD_FIT_ADDRESS	0x80010000
-#endif
-
 #ifdef CONFIG_SPL_MMC_SUPPORT
 #define CONFIG_SYS_MMCSD_FS_BOOT_PARTITION	1
 #define CONFIG_SPL_FS_LOAD_PAYLOAD_NAME		"u-boot.itb"
diff --git a/include/configs/dra7xx_evm.h b/include/configs/dra7xx_evm.h
index 14cd82f5a3..cf7182d360 100644
--- a/include/configs/dra7xx_evm.h
+++ b/include/configs/dra7xx_evm.h
@@ -49,7 +49,6 @@
 
 #ifdef CONFIG_SPL_BUILD
 #ifdef CONFIG_SPL_DFU
-#define CONFIG_SPL_LOAD_FIT_ADDRESS 0x80200000
 #define DFUARGS \
 	"dfu_bufsiz=0x10000\0" \
 	DFU_ALT_INFO_RAM
diff --git a/include/configs/j721e_evm.h b/include/configs/j721e_evm.h
index 7488b661f5..7b917c2247 100644
--- a/include/configs/j721e_evm.h
+++ b/include/configs/j721e_evm.h
@@ -24,7 +24,6 @@
 #define CONFIG_SYS_INIT_SP_ADDR         (CONFIG_SPL_TEXT_BASE +	\
 					 CONFIG_SYS_K3_NON_SECURE_MSRAM_SIZE)
 /* Image load address in RAM for DFU boot*/
-#define CONFIG_SPL_LOAD_FIT_ADDRESS	0x81000000
 #else
 /*
  * Maximum size in memory allocated to the SPL BSS. Keep it as tight as
@@ -48,7 +47,6 @@
 #define CONFIG_SYS_SPL_MALLOC_START	0x84000000
 #define CONFIG_SYS_SPL_MALLOC_SIZE	SZ_16M
 /* Image load address in RAM for DFU boot*/
-#define CONFIG_SPL_LOAD_FIT_ADDRESS	0x80080000
 #endif
 
 #ifdef CONFIG_SYS_K3_SPL_ATF
diff --git a/include/configs/qemu-riscv.h b/include/configs/qemu-riscv.h
index 5fe9dcfc58..aed9a4ae99 100644
--- a/include/configs/qemu-riscv.h
+++ b/include/configs/qemu-riscv.h
@@ -16,8 +16,6 @@
 #define CONFIG_SYS_SPL_MALLOC_START	0x84100000
 #define CONFIG_SYS_SPL_MALLOC_SIZE	0x00100000
 
-#define CONFIG_SPL_LOAD_FIT_ADDRESS	0x80200000
-
 #endif
 
 #define CONFIG_SYS_SDRAM_BASE		0x80000000
diff --git a/include/configs/sifive-fu540.h b/include/configs/sifive-fu540.h
index 72c841eb9b..f720c0fbe4 100644
--- a/include/configs/sifive-fu540.h
+++ b/include/configs/sifive-fu540.h
@@ -20,8 +20,6 @@
 					 CONFIG_SPL_BSS_MAX_SIZE)
 #define CONFIG_SYS_SPL_MALLOC_SIZE	0x00100000
 
-#define CONFIG_SPL_LOAD_FIT_ADDRESS	0x84000000
-
 #define CONFIG_SPL_STACK	(0x08000000 + 0x001D0000 - \
 				 GENERATED_GBL_DATA_SIZE)
 
diff --git a/include/configs/xilinx_zynqmp.h b/include/configs/xilinx_zynqmp.h
index e868c13416..d98933b1f4 100644
--- a/include/configs/xilinx_zynqmp.h
+++ b/include/configs/xilinx_zynqmp.h
@@ -227,9 +227,6 @@
 /* ATF is my kernel image */
 #define CONFIG_SPL_FS_LOAD_KERNEL_NAME	"atf-uboot.ub"
 
-/* FIT load address for RAM boot */
-#define CONFIG_SPL_LOAD_FIT_ADDRESS	0x10000000
-
 /* MMC support */
 #ifdef CONFIG_MMC_SDHCI_ZYNQ
 # define CONFIG_SYS_MMCSD_FS_BOOT_PARTITION	1
diff --git a/include/configs/zynq-common.h b/include/configs/zynq-common.h
index 59e77f6ecc..3388b46dde 100644
--- a/include/configs/zynq-common.h
+++ b/include/configs/zynq-common.h
@@ -275,6 +275,4 @@
 #define CONFIG_SPL_BSS_START_ADDR	0x100000
 #define CONFIG_SPL_BSS_MAX_SIZE		0x100000
 
-#define CONFIG_SPL_LOAD_FIT_ADDRESS 0x10000000
-
 #endif /* __CONFIG_ZYNQ_COMMON_H */
diff --git a/scripts/config_whitelist.txt b/scripts/config_whitelist.txt
index 916768f361..53c0a740cf 100644
--- a/scripts/config_whitelist.txt
+++ b/scripts/config_whitelist.txt
@@ -1650,7 +1650,6 @@ CONFIG_SPL_GD_ADDR
 CONFIG_SPL_INIT_MINIMAL
 CONFIG_SPL_JR0_LIODN_NS
 CONFIG_SPL_JR0_LIODN_S
-CONFIG_SPL_LOAD_FIT_ADDRESS
 CONFIG_SPL_MAX_FOOTPRINT
 CONFIG_SPL_MAX_PEB_SIZE
 CONFIG_SPL_MAX_SIZE
