MA-16048 Fix imx8q u-boot hang

Address 0x8880_0000 is reserved for M4 image on imx8q, which
leaves limited memory region for the malloc pool. The avb
will consume much heap memory to verify the kernel and dtbo
image, memory conflicts may happen as the kernel/dtbo image
size is getting larger.

As the avb will load kernel/dtbo in every avb_slot_verify(), but
will only free the memory after both slots are checked(if needed).
And for trusty enabled platforms, extra heap memory will be used
to do the hash calculation.

This commit will free the slot memory once it's marked as unbootable
and will use fixed memory started from CONFIG_FASTBOOT_BUF_ADDR to
help store the data to do the hash calculation. With above change,
we get a chance to decrease the malloc pool size.

Test: boot on imx8qxp and imx8mn.

Change-Id: Id95b9becf6fa7fae4f120a4dc9e9a3ba90da578c
Signed-off-by: Ji Luo <ji.luo@nxp.com>

17 files changed, 22 insertions, 29 deletions

diff --git a/configs/imx8mq_aiy_android_defconfig b/configs/imx8mq_aiy_android_defconfig
index fc59c08be6..8a0819c739 100644
--- a/configs/imx8mq_aiy_android_defconfig
+++ b/configs/imx8mq_aiy_android_defconfig
@@ -54,7 +54,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x19000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/configs/imx8mq_aiy_android_trusty_defconfig b/configs/imx8mq_aiy_android_trusty_defconfig
index b6abbc5534..f90fb6e1ce 100644
--- a/configs/imx8mq_aiy_android_trusty_defconfig
+++ b/configs/imx8mq_aiy_android_trusty_defconfig
@@ -54,7 +54,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x19000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/configs/imx8mq_aiy_android_trusty_dual_defconfig b/configs/imx8mq_aiy_android_trusty_dual_defconfig
index 4987c0232f..10b13afab1 100644
--- a/configs/imx8mq_aiy_android_trusty_dual_defconfig
+++ b/configs/imx8mq_aiy_android_trusty_dual_defconfig
@@ -54,7 +54,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x19000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/configs/imx8mq_aiy_android_uuu_defconfig b/configs/imx8mq_aiy_android_uuu_defconfig
index 0f1a88dc28..63abbb4ee4 100644
--- a/configs/imx8mq_aiy_android_uuu_defconfig
+++ b/configs/imx8mq_aiy_android_uuu_defconfig
@@ -54,7 +54,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x19000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/configs/imx8mq_evk_android_defconfig b/configs/imx8mq_evk_android_defconfig
index 38705418ec..a6b2c0ee95 100644
--- a/configs/imx8mq_evk_android_defconfig
+++ b/configs/imx8mq_evk_android_defconfig
@@ -58,7 +58,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x40000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/configs/imx8mq_evk_android_dual_defconfig b/configs/imx8mq_evk_android_dual_defconfig
index 70dedd9ed0..f0f5abf32e 100644
--- a/configs/imx8mq_evk_android_dual_defconfig
+++ b/configs/imx8mq_evk_android_dual_defconfig
@@ -58,7 +58,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x40000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/configs/imx8mq_evk_android_uuu_defconfig b/configs/imx8mq_evk_android_uuu_defconfig
index 9bada1f002..892233c9c2 100644
--- a/configs/imx8mq_evk_android_uuu_defconfig
+++ b/configs/imx8mq_evk_android_uuu_defconfig
@@ -58,7 +58,7 @@ CONFIG_CMD_FASTBOOT=y
 CONFIG_ANDROID_BOOT_IMAGE=y
 CONFIG_FSL_FASTBOOT=y
 CONFIG_BCB_SUPPORT=y
-CONFIG_FASTBOOT_BUF_ADDR=0x42800000
+CONFIG_FASTBOOT_BUF_ADDR=0x44800000
 CONFIG_FASTBOOT_BUF_SIZE=0x40000000
 CONFIG_FASTBOOT_FLASH=y
 CONFIG_FASTBOOT_FLASH_MMC_DEV=0
diff --git a/include/configs/imx8mm_evk_android.h b/include/configs/imx8mm_evk_android.h
index ddfda6cba2..eeba699ad3 100644
--- a/include/configs/imx8mm_evk_android.h
+++ b/include/configs/imx8mm_evk_android.h
@@ -24,7 +24,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (96 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_USB_FUNCTION_FASTBOOT
@@ -36,7 +36,7 @@
 #define CONFIG_FSL_FASTBOOT
 #define CONFIG_ANDROID_RECOVERY
 
-#define CONFIG_FASTBOOT_BUF_ADDR   CONFIG_SYS_LOAD_ADDR
+#define CONFIG_FASTBOOT_BUF_ADDR   0x44800000
 #define CONFIG_FASTBOOT_BUF_SIZE   0x19000000
 
 #define CONFIG_CMD_BOOTA
diff --git a/include/configs/imx8mn_evk_android.h b/include/configs/imx8mn_evk_android.h
index 99d28186dc..88aec40005 100644
--- a/include/configs/imx8mn_evk_android.h
+++ b/include/configs/imx8mn_evk_android.h
@@ -24,7 +24,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (96 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_USB_FUNCTION_FASTBOOT
@@ -36,7 +36,7 @@
 #define CONFIG_FSL_FASTBOOT
 #define CONFIG_ANDROID_RECOVERY
 
-#define CONFIG_FASTBOOT_BUF_ADDR   CONFIG_SYS_LOAD_ADDR
+#define CONFIG_FASTBOOT_BUF_ADDR   0x44800000
 #define CONFIG_FASTBOOT_BUF_SIZE   0x19000000
 
 #define CONFIG_CMD_BOOTA
diff --git a/include/configs/imx8mq_aiy_android.h b/include/configs/imx8mq_aiy_android.h
index 7d1e6945fe..1f31be0096 100644
--- a/include/configs/imx8mq_aiy_android.h
+++ b/include/configs/imx8mq_aiy_android.h
@@ -29,7 +29,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (96 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_ANDROID_RECOVERY
diff --git a/include/configs/imx8mq_evk_android.h b/include/configs/imx8mq_evk_android.h
index 066fb15343..805b2c4540 100644
--- a/include/configs/imx8mq_evk_android.h
+++ b/include/configs/imx8mq_evk_android.h
@@ -23,7 +23,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (96 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_ANDROID_RECOVERY
diff --git a/include/configs/imx8qm_mek_android.h b/include/configs/imx8qm_mek_android.h
index 969c66f504..2446987df4 100644
--- a/include/configs/imx8qm_mek_android.h
+++ b/include/configs/imx8qm_mek_android.h
@@ -39,7 +39,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (86 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_USB_FUNCTION_FASTBOOT
diff --git a/include/configs/imx8qm_mek_android_auto.h b/include/configs/imx8qm_mek_android_auto.h
index e7b29276bb..760052b7be 100644
--- a/include/configs/imx8qm_mek_android_auto.h
+++ b/include/configs/imx8qm_mek_android_auto.h
@@ -42,7 +42,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (76 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_USB_FUNCTION_FASTBOOT
diff --git a/include/configs/imx8qxp_mek_android.h b/include/configs/imx8qxp_mek_android.h
index 838223ab47..19f567c42d 100644
--- a/include/configs/imx8qxp_mek_android.h
+++ b/include/configs/imx8qxp_mek_android.h
@@ -34,7 +34,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (86 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_USB_FUNCTION_FASTBOOT
diff --git a/include/configs/imx8qxp_mek_android_auto.h b/include/configs/imx8qxp_mek_android_auto.h
index 2394f791fc..bf445fdc29 100644
--- a/include/configs/imx8qxp_mek_android_auto.h
+++ b/include/configs/imx8qxp_mek_android_auto.h
@@ -41,7 +41,7 @@
 
 #ifdef CONFIG_SYS_MALLOC_LEN
 #undef CONFIG_SYS_MALLOC_LEN
-#define CONFIG_SYS_MALLOC_LEN           (76 * SZ_1M)
+#define CONFIG_SYS_MALLOC_LEN           (64 * SZ_1M)
 #endif
 
 #define CONFIG_USB_FUNCTION_FASTBOOT
diff --git a/lib/avb/fsl/fsl_avb_ab_flow.c b/lib/avb/fsl/fsl_avb_ab_flow.c
index 5637212157..091304b67c 100644
--- a/lib/avb/fsl/fsl_avb_ab_flow.c
+++ b/lib/avb/fsl/fsl_avb_ab_flow.c
@@ -1168,6 +1168,10 @@ AvbABFlowResult avb_ab_flow_fast(AvbABOps* ab_ops,
 				fsl_slot_set_unbootable(&ab_data.slots[target_slot]);
 				set_slot_unbootable = false;
 			}
+			if (slot_data[target_slot] != NULL) {
+				avb_slot_verify_data_free(slot_data[target_slot]);
+				slot_data[target_slot] = NULL;
+			}
 		}
 		/* switch to another slot */
 		target_slot = (target_slot == 1 ? 0 : 1);
diff --git a/lib/avb/libavb/avb_slot_verify.c b/lib/avb/libavb/avb_slot_verify.c
index 2ca4ff9bd3..f682007a5e 100644
--- a/lib/avb/libavb/avb_slot_verify.c
+++ b/lib/avb/libavb/avb_slot_verify.c
@@ -313,12 +313,7 @@ static AvbSlotVerifyResult load_and_verify_hash_partition(
     }
     uint32_t round_buf_size = ROUND(hash_desc.salt_len + hash_desc.image_size,
                                 ARCH_DMA_MINALIGN);
-    hash_buf = memalign(ARCH_DMA_MINALIGN, round_buf_size);
-    if (hash_buf == NULL) {
-        avb_error("failed to alloc memory!\n");
-        ret = AVB_SLOT_VERIFY_RESULT_ERROR_OOM;
-        goto out;
-    }
+    hash_buf = (void *)CONFIG_FASTBOOT_BUF_ADDR;
 
     avb_memcpy(hash_buf, desc_salt, hash_desc.salt_len);
     avb_memcpy(hash_buf + hash_desc.salt_len,
@@ -335,8 +330,6 @@ static AvbSlotVerifyResult load_and_verify_hash_partition(
     }
 
     digest = hash_out;
-    free(hash_buf);
-    hash_buf = NULL;
 #else
     AvbSHA256Ctx sha256_ctx;
     avb_sha256_init(&sha256_ctx);
@@ -399,10 +392,6 @@ out:
     free(hash_out);
     hash_out = NULL;
   }
-  if (hash_buf != NULL) {
-    free(hash_buf);
-    hash_buf = NULL;
-  }
 #endif
   /* If it worked and something was loaded, copy to slot_data. */
   if ((ret == AVB_SLOT_VERIFY_RESULT_OK || result_should_continue(ret)) &&