core: add boot_mem section for pager

With CFG_WITH_PAGER=y, a special section of memory is carved out by the
linker script to be used by the boot_mem*() functions during boot. The
amount of memory must cover the worst case, but the remaining unused
memory will be returned to be managed by the pager at end of boot in a
later patch. This enables boot configuration in the same way regardless
of CFG_DYN_CONFIG.

Add support in boot_mem_release_unused() and
boot_mem_release_tmp_alloc() to pass the unused memory to the pager.
This is needed in later patches with CFG_WITH_PAGER=y and
CFG_DYN_CONFIG=y.

Signed-off-by: Jens Wiklander <[email protected]>

Author: jenswi-linaro

core/arch/arm/kernel/boot.c

@@ -511,12 +511,6 @@ static void init_pager_runtime(unsigned long pageable_part)
 	assert(hashes);
 	asan_memcpy_unchecked(hashes, tmp_hashes, hash_size);
 
-	/*
-	 * The pager is about the be enabled below, eventual temporary boot
-	 * memory allocation must be removed now.
-	 */
-	boot_mem_release_tmp_alloc();
-
 	carve_out_asan_mem();
 
 	mm = nex_phys_mem_ta_alloc(pageable_size);
@@ -966,15 +960,6 @@ static void init_primary(unsigned long pageable_part)
 	core_mmu_save_mem_map();
 	core_mmu_init_phys_mem();
 	boot_mem_foreach_padding(add_padding_to_pool, NULL);
-	va = boot_mem_release_unused();
-	if (!IS_ENABLED(CFG_WITH_PAGER)) {
-		/*
-		 * We must update boot_cached_mem_end to reflect the memory
-		 * just unmapped by boot_mem_release_unused().
-		 */
-		assert(va && va <= boot_cached_mem_end);
-		boot_cached_mem_end = va;
-	}
 
 	if (IS_ENABLED(CFG_DYN_CONFIG)) {
 		/*
@@ -999,6 +984,23 @@ static void init_primary(unsigned long pageable_part)
 		init_pager_runtime(pageable_part);
 	}
 
+	va = boot_mem_release_unused();
+	if (IS_ENABLED(CFG_WITH_PAGER)) {
+		/*
+		 * Paging is activated, and anything beyond the start of
+		 * the released unused memory is managed by the pager.
+		 */
+		assert(va && va <= core_mmu_linear_map_end);
+		core_mmu_linear_map_end = va;
+	} else {
+		/*
+		 * We must update boot_cached_mem_end to reflect the memory
+		 * just unmapped by boot_mem_release_unused().
+		 */
+		assert(va && va <= boot_cached_mem_end);
+		boot_cached_mem_end = va;
+	}
+
 	/* Initialize canaries around the stacks */
 	thread_init_canaries();
 	thread_init_per_cpu();
@@ -1117,9 +1119,7 @@ void __weak boot_init_primary_runtime(void)
 				      ~THREAD_EXCP_NATIVE_INTR);
 		init_tee_runtime();
 	}
-
-	if (!IS_ENABLED(CFG_WITH_PAGER))
-		boot_mem_release_tmp_alloc();
+	boot_mem_release_tmp_alloc();
 }
 
 void __weak boot_init_primary_final(void)

core/arch/arm/kernel/entry_a32.S

@@ -534,23 +534,16 @@ shadow_stack_access_ok:
 	bl	boot_save_args
 	add	sp, sp, #(2 * 4)
 
+	ldr	r0, =__vcore_free_start
+	ldr	r2, =__vcore_free_end
 #ifdef CFG_WITH_PAGER
-	ldr	r0, =__init_end	/* pointer to boot_embdata */
-	ldr	r1, [r0]	/* struct boot_embdata::total_len */
-	add	r0, r0, r1
-	mov_imm	r1, 0xfff
-	add	r0, r0, r1	/* round up */
-	bic	r0, r0, r1	/* to next page */
-	mov_imm r1, (TEE_RAM_PH_SIZE + TEE_RAM_START)
-	mov	r2, r1
+	mov	r1, r2
 #else
-	ldr	r0, =__vcore_free_start
 	ldr	r1, =boot_embdata_ptr
 	ldr	r1, [r1]
 #ifdef CFG_DYN_CONFIG
 	sub	r1, r1, #THREAD_BOOT_INIT_TMP_ALLOC
 #endif
-	ldr	r2, =__vcore_free_end
 #endif
 	bl	boot_mem_init
 

core/arch/arm/kernel/entry_a64.S

@@ -388,22 +388,16 @@ clear_nex_bss:
 	mov	x4, xzr
 	bl	boot_save_args
 
+	adr_l	x0, __vcore_free_start
+	adr_l	x2, __vcore_free_end
 #ifdef CFG_WITH_PAGER
-	adr_l	x0, __init_end	/* pointer to boot_embdata */
-	ldr	w1, [x0]	/* struct boot_embdata::total_len */
-	add	x0, x0, x1
-	add	x0, x0, #0xfff	/* round up */
-	bic	x0, x0, #0xfff  /* to next page */
-	mov_imm x1, (TEE_RAM_PH_SIZE + TEE_RAM_START)
-	mov	x2, x1
+	mov	x1, x2
 #else
-	adr_l	x0, __vcore_free_start
 	adr_l	x1, boot_embdata_ptr
 	ldr	x1, [x1]
 #ifdef CFG_DYN_CONFIG
 	sub	x1, x1, #THREAD_BOOT_INIT_TMP_ALLOC
 #endif
-	adr_l	x2, __vcore_free_end;
 #endif
 	bl	boot_mem_init
 

core/arch/arm/kernel/kern.ld.S

@@ -244,10 +244,12 @@ SECTIONS
 		__bss_end = .;
 	}
 
-#ifdef CFG_DYN_CONFIG
+#if defined(CFG_DYN_CONFIG) && !defined(CFG_WITH_PAGER)
 	. = ALIGN(SMALL_PAGE_SIZE);
 	__flatmap_free_start = .;
-#else
+#endif
+
+#ifndef CFG_DYN_CONFIG
 	.heap1 (NOLOAD) : {
 		/*
 		 * We're keeping track of the padding added before the
@@ -285,10 +287,7 @@ SECTIONS
 		. = ALIGN(8);
 		__nozi_stack_end = .;
 	}
-#ifndef CFG_WITH_PAGER
-	. = ALIGN(SMALL_PAGE_SIZE);
-	__flatmap_free_start = .;
-#else
+#ifdef CFG_WITH_PAGER
 	.heap2 (NOLOAD) : {
 		__heap2_start = .;
 		/*
@@ -300,6 +299,27 @@ SECTIONS
 		. = ALIGN(SMALL_PAGE_SIZE);
 		__heap2_end = .;
 	}
+#endif
+#ifndef CFG_WITH_PAGER
+	. = ALIGN(SMALL_PAGE_SIZE);
+	__flatmap_free_start = .;
+#endif
+#endif /*!CFG_DYN_CONFIG*/
+
+#ifdef CFG_WITH_PAGER
+	/*
+	 * This memory is used by the boot_mem*() functions during boot.
+	 * Enough memory must be carved out to support the worst case
+	 * memory allocation, but the remaining unused memory will be
+	 * returned to be managed by the pager at end of boot. The tradeoff
+	 * is that the init code follows this section and must also fit in
+	 * the physical memory.
+	 */
+	.pager_boot_mem (NOLOAD): {
+		. += SMALL_PAGE_SIZE;
+	}
+	__flatmap_free_start = ADDR(.pager_boot_mem);
+	__flatmap_free_size = SIZEOF(.pager_boot_mem);
 
 	/* Start page aligned read-only memory */
 	__flatmap_unpg_rw_size = . - __flatmap_unpg_rw_start;
@@ -386,10 +406,11 @@ SECTIONS
 		(__pageable_end - __pageable_start) / 4096 * 32 +
 		SIZEOF(.rel) / 2 + SIZEOF(.rela) / 3 ,
 		"Too few free pages to initialize paging")
-
-
 #endif /*CFG_WITH_PAGER*/
-#endif /*CFG_DYN_CONFIG*/
+
+#ifndef CFG_WITH_PAGER
+	__flatmap_free_start = .;
+#endif
 
 #ifdef CFG_CORE_SANITIZE_KADDRESS
 	/*
@@ -483,11 +504,9 @@ __vcore_unpg_rw_start = __flatmap_unpg_rw_start;
 __vcore_unpg_rw_size = __flatmap_unpg_rw_size;
 __vcore_unpg_rw_end = __vcore_unpg_rw_start + __vcore_unpg_rw_size;
 
-#ifndef CFG_WITH_PAGER
 __vcore_free_start = __flatmap_free_start;
 __vcore_free_size = __flatmap_free_size;
 __vcore_free_end = __flatmap_free_start + __flatmap_free_size;
-#endif
 
 #ifdef CFG_NS_VIRTUALIZATION
 /* Nexus read-write memory */

core/include/kernel/linker.h

@@ -46,17 +46,10 @@ extern const uint8_t __extab_end[];
 
 #define VCORE_START_VA		((vaddr_t)__text_start)
 
-#ifndef CFG_WITH_PAGER
 #define VCORE_FREE_PA		((unsigned long)__vcore_free_start)
 #define VCORE_FREE_SZ		((size_t)(__vcore_free_end - \
 					  __vcore_free_start))
 #define VCORE_FREE_END_PA	((unsigned long)__vcore_free_end)
-#else
-/* No VCORE_FREE range in pager configuration since it uses all memory */
-#define VCORE_FREE_PA		PADDR_MAX
-#define VCORE_FREE_SZ		0
-#define VCORE_FREE_END_PA	PADDR_MAX
-#endif
 
 #define EMIT_SECTION_INFO_SYMBOLS(section_name) \
 	extern const uint8_t __vcore_ ## section_name ## _start[]; \

core/mm/boot_mem.c

@@ -14,6 +14,18 @@
 #include <string.h>
 #include <util.h>
 
+#ifdef CFG_WITH_PAGER
+#include <mm/tee_pager.h>
+#endif
+
+static inline void pager_add_pages(vaddr_t vaddr __maybe_unused,
+				   size_t npages __maybe_unused)
+{
+#ifdef CFG_WITH_PAGER
+	tee_pager_add_pages(vaddr, npages, true);
+#endif
+}
+
 /*
  * struct boot_mem_reloc - Pointers relocated in memory during boot
  * @ptrs: Array of relocation
@@ -109,7 +121,6 @@ static void *mem_alloc(struct boot_mem_desc *desc, size_t len, size_t align)
 	if (IS_ENABLED(CFG_CORE_SANITIZE_KADDRESS))
 		align = MAX(align, ASAN_BLOCK_SIZE);
 
-	runtime_assert(!IS_ENABLED(CFG_WITH_PAGER));
 	assert(desc && desc->mem_start && desc->mem_end);
 	assert(IS_POWER_OF_TWO(align));
 	va = ROUNDUP2(desc->mem_start, align);
@@ -301,18 +312,24 @@ vaddr_t boot_mem_release_unused(void)
 	     (size_t)(boot_mem_desc->orig_mem_end - boot_mem_desc->mem_end),
 	     boot_mem_desc->mem_end);
 
-	if (IS_ENABLED(CFG_WITH_PAGER))
+	va = ROUNDUP(boot_mem_desc->mem_start, SMALL_PAGE_SIZE);
+	tmp_va = ROUNDDOWN(boot_mem_desc->mem_end, SMALL_PAGE_SIZE);
+
+	if (IS_ENABLED(CFG_WITH_PAGER)) {
+		if (tmp_va > va) {
+			n = tmp_va - va;
+			DMSG("Releasing %zu bytes from va %#"PRIxVA, n, va);
+			pager_add_pages(va, n / SMALL_PAGE_SIZE);
+		}
 		goto out;
+	}
 
 	pa = vaddr_to_phys(ROUNDUP(boot_mem_desc->orig_mem_start,
 				   SMALL_PAGE_SIZE));
 	mm = nex_phys_mem_mm_find(pa);
 	if (!mm)
 		panic();
 
-	va = ROUNDUP(boot_mem_desc->mem_start, SMALL_PAGE_SIZE);
-
-	tmp_va = ROUNDDOWN(boot_mem_desc->mem_end, SMALL_PAGE_SIZE);
 	tmp_n = boot_mem_desc->orig_mem_end - tmp_va;
 	tmp_pa = vaddr_to_phys(tmp_va);
 
@@ -343,6 +360,7 @@ vaddr_t boot_mem_release_unused(void)
 void boot_mem_release_tmp_alloc(void)
 {
 	tee_mm_entry_t *mm = NULL;
+	vaddr_t tmp_va = 0;
 	vaddr_t va = 0;
 	paddr_t pa = 0;
 	size_t n = 0;
@@ -353,10 +371,15 @@ void boot_mem_release_tmp_alloc(void)
 	va = MAX(ROUNDDOWN(boot_mem_desc->mem_end, SMALL_PAGE_SIZE),
 		 ROUNDUP(boot_mem_desc->final_mem_start, SMALL_PAGE_SIZE));
 	if (IS_ENABLED(CFG_WITH_PAGER)) {
-		n = ROUNDDOWN(boot_mem_desc->orig_mem_end, SMALL_PAGE_SIZE) -
-		    va;
+		tmp_va = ROUNDDOWN(boot_mem_desc->orig_mem_end,
+				   SMALL_PAGE_SIZE);
+		if (tmp_va > va) {
+			n = tmp_va - va;
+			DMSG("Releasing %zu bytes from va %#"PRIxVA, n, va);
+			pager_add_pages(va, n / SMALL_PAGE_SIZE);
+		}
+
 		boot_mem_desc = NULL;
-		DMSG("Releasing %zu bytes from va %#"PRIxVA, n, va);
 		return;
 	}