2019-06-03 14:44:50 +09:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
2015-10-19 22:19:27 +09:00
|
|
|
/*
|
|
|
|
|
* Kernel page table mapping
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 2015 ARM Ltd.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifndef __ASM_KERNEL_PGTABLE_H
|
|
|
|
|
#define __ASM_KERNEL_PGTABLE_H
|
|
|
|
|
|
2016-07-02 00:53:00 +09:00
|
|
|
#include <asm/pgtable.h>
|
2016-03-30 21:25:47 +09:00
|
|
|
#include <asm/sparsemem.h>
|
2015-10-19 22:19:28 +09:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* The linear mapping and the start of memory are both 2M aligned (per
|
|
|
|
|
* the arm64 booting.txt requirements). Hence we can use section mapping
|
|
|
|
|
* with 4K (section size = 2M) but not with 16K (section size = 32M) or
|
|
|
|
|
* 64K (section size = 512M).
|
|
|
|
|
*/
|
|
|
|
|
#ifdef CONFIG_ARM64_4K_PAGES
|
|
|
|
|
#define ARM64_SWAPPER_USES_SECTION_MAPS 1
|
|
|
|
|
#else
|
|
|
|
|
#define ARM64_SWAPPER_USES_SECTION_MAPS 0
|
|
|
|
|
#endif
|
|
|
|
|
|
2015-10-19 22:19:27 +09:00
|
|
|
/*
|
|
|
|
|
* The idmap and swapper page tables need some space reserved in the kernel
|
|
|
|
|
* image. Both require pgd, pud (4 levels only) and pmd tables to (section)
|
|
|
|
|
* map the kernel. With the 64K page configuration, swapper and idmap need to
|
|
|
|
|
* map to pte level. The swapper also maps the FDT (see __create_page_tables
|
|
|
|
|
* for more information). Note that the number of ID map translation levels
|
|
|
|
|
* could be increased on the fly if system RAM is out of reach for the default
|
2015-10-19 22:19:30 +09:00
|
|
|
* VA range, so pages required to map highest possible PA are reserved in all
|
|
|
|
|
* cases.
|
2015-10-19 22:19:27 +09:00
|
|
|
*/
|
2015-10-19 22:19:28 +09:00
|
|
|
#if ARM64_SWAPPER_USES_SECTION_MAPS
|
2015-10-19 22:19:27 +09:00
|
|
|
#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1)
|
2015-10-19 22:19:30 +09:00
|
|
|
#define IDMAP_PGTABLE_LEVELS (ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT) - 1)
|
2015-10-19 22:19:28 +09:00
|
|
|
#else
|
|
|
|
|
#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS)
|
2015-10-19 22:19:30 +09:00
|
|
|
#define IDMAP_PGTABLE_LEVELS (ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT))
|
2015-10-19 22:19:27 +09:00
|
|
|
#endif
|
|
|
|
|
|
2018-01-11 19:11:59 +09:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* If KASLR is enabled, then an offset K is added to the kernel address
|
|
|
|
|
* space. The bottom 21 bits of this offset are zero to guarantee 2MB
|
|
|
|
|
* alignment for PA and VA.
|
|
|
|
|
*
|
|
|
|
|
* For each pagetable level of the swapper, we know that the shift will
|
|
|
|
|
* be larger than 21 (for the 4KB granule case we use section maps thus
|
|
|
|
|
* the smallest shift is actually 30) thus there is the possibility that
|
|
|
|
|
* KASLR can increase the number of pagetable entries by 1, so we make
|
|
|
|
|
* room for this extra entry.
|
|
|
|
|
*
|
|
|
|
|
* Note KASLR cannot increase the number of required entries for a level
|
|
|
|
|
* by more than one because it increments both the virtual start and end
|
|
|
|
|
* addresses equally (the extra entry comes from the case where the end
|
|
|
|
|
* address is just pushed over a boundary and the start address isn't).
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_RANDOMIZE_BASE
|
|
|
|
|
#define EARLY_KASLR (1)
|
|
|
|
|
#else
|
|
|
|
|
#define EARLY_KASLR (0)
|
|
|
|
|
#endif
|
|
|
|
|
|
arm64: head: avoid over-mapping in map_memory
commit 90268574a3e8a6b883bd802d702a2738577e1006 upstream.
The `compute_indices` and `populate_entries` macros operate on inclusive
bounds, and thus the `map_memory` macro which uses them also operates
on inclusive bounds.
We pass `_end` and `_idmap_text_end` to `map_memory`, but these are
exclusive bounds, and if one of these is sufficiently aligned (as a
result of kernel configuration, physical placement, and KASLR), then:
* In `compute_indices`, the computed `iend` will be in the page/block *after*
the final byte of the intended mapping.
* In `populate_entries`, an unnecessary entry will be created at the end
of each level of table. At the leaf level, this entry will map up to
SWAPPER_BLOCK_SIZE bytes of physical addresses that we did not intend
to map.
As we may map up to SWAPPER_BLOCK_SIZE bytes more than intended, we may
violate the boot protocol and map physical address past the 2MiB-aligned
end address we are permitted to map. As we map these with Normal memory
attributes, this may result in further problems depending on what these
physical addresses correspond to.
The final entry at each level may require an additional table at that
level. As EARLY_ENTRIES() calculates an inclusive bound, we allocate
enough memory for this.
Avoid the extraneous mapping by having map_memory convert the exclusive
end address to an inclusive end address by subtracting one, and do
likewise in EARLY_ENTRIES() when calculating the number of required
tables. For clarity, comments are updated to more clearly document which
boundaries the macros operate on. For consistency with the other
macros, the comments in map_memory are also updated to describe `vstart`
and `vend` as virtual addresses.
Fixes: 0370b31e4845 ("arm64: Extend early page table code to allow for larger kernels")
Cc: <stable@vger.kernel.org> # 4.16.x
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Will Deacon <will@kernel.org>
Acked-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20210823101253.55567-1-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2021-08-23 19:12:53 +09:00
|
|
|
#define EARLY_ENTRIES(vstart, vend, shift) \
|
|
|
|
|
((((vend) - 1) >> (shift)) - ((vstart) >> (shift)) + 1 + EARLY_KASLR)
|
2018-01-11 19:11:59 +09:00
|
|
|
|
|
|
|
|
#define EARLY_PGDS(vstart, vend) (EARLY_ENTRIES(vstart, vend, PGDIR_SHIFT))
|
|
|
|
|
|
|
|
|
|
#if SWAPPER_PGTABLE_LEVELS > 3
|
|
|
|
|
#define EARLY_PUDS(vstart, vend) (EARLY_ENTRIES(vstart, vend, PUD_SHIFT))
|
|
|
|
|
#else
|
|
|
|
|
#define EARLY_PUDS(vstart, vend) (0)
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#if SWAPPER_PGTABLE_LEVELS > 2
|
|
|
|
|
#define EARLY_PMDS(vstart, vend) (EARLY_ENTRIES(vstart, vend, SWAPPER_TABLE_SHIFT))
|
|
|
|
|
#else
|
|
|
|
|
#define EARLY_PMDS(vstart, vend) (0)
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#define EARLY_PAGES(vstart, vend) ( 1 /* PGDIR page */ \
|
|
|
|
|
+ EARLY_PGDS((vstart), (vend)) /* each PGDIR needs a next level page table */ \
|
|
|
|
|
+ EARLY_PUDS((vstart), (vend)) /* each PUD needs a next level page table */ \
|
|
|
|
|
+ EARLY_PMDS((vstart), (vend))) /* each PMD needs a next level page table */
|
arm64/mm: Separate boot-time page tables from swapper_pg_dir
Since the address of swapper_pg_dir is fixed for a given kernel image,
it is an attractive target for manipulation via an arbitrary write. To
mitigate this we'd like to make it read-only by moving it into the
rodata section.
We require that swapper_pg_dir is at a fixed offset from tramp_pg_dir
and reserved_ttbr0, so these will also need to move into rodata.
However, swapper_pg_dir is allocated along with some transient page
tables used for boot which we do not want to move into rodata.
As a step towards this, this patch separates the boot-time page tables
into a new init_pg_dir, and reduces swapper_pg_dir to the single page it
needs to be. This allows us to retain the relationship between
swapper_pg_dir, tramp_pg_dir, and swapper_pg_dir, while cleanly
separating these from the boot-time page tables.
The init_pg_dir holds all of the pgd/pud/pmd/pte levels needed during
boot, and all of these levels will be freed when we switch to the
swapper_pg_dir, which is initialized by the existing code in
paging_init(). Since we start off on the init_pg_dir, we no longer need
to allocate a transient page table in paging_init() in order to ensure
that swapper_pg_dir isn't live while we initialize it.
There should be no functional change as a result of this patch.
Signed-off-by: Jun Yao <yaojun8558363@gmail.com>
Reviewed-by: James Morse <james.morse@arm.com>
[Mark: place init_pg_dir after BSS, fold mm changes, commit message]
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2018-09-24 23:47:49 +09:00
|
|
|
#define INIT_DIR_SIZE (PAGE_SIZE * EARLY_PAGES(KIMAGE_VADDR + TEXT_OFFSET, _end))
|
2015-10-19 22:19:30 +09:00
|
|
|
#define IDMAP_DIR_SIZE (IDMAP_PGTABLE_LEVELS * PAGE_SIZE)
|
2015-10-19 22:19:27 +09:00
|
|
|
|
|
|
|
|
/* Initial memory map size */
|
2015-10-19 22:19:28 +09:00
|
|
|
#if ARM64_SWAPPER_USES_SECTION_MAPS
|
2015-10-19 22:19:27 +09:00
|
|
|
#define SWAPPER_BLOCK_SHIFT SECTION_SHIFT
|
|
|
|
|
#define SWAPPER_BLOCK_SIZE SECTION_SIZE
|
|
|
|
|
#define SWAPPER_TABLE_SHIFT PUD_SHIFT
|
2015-10-19 22:19:28 +09:00
|
|
|
#else
|
|
|
|
|
#define SWAPPER_BLOCK_SHIFT PAGE_SHIFT
|
|
|
|
|
#define SWAPPER_BLOCK_SIZE PAGE_SIZE
|
|
|
|
|
#define SWAPPER_TABLE_SHIFT PMD_SHIFT
|
2015-10-19 22:19:27 +09:00
|
|
|
#endif
|
|
|
|
|
|
2015-10-19 22:19:28 +09:00
|
|
|
/* The size of the initial kernel direct mapping */
|
|
|
|
|
#define SWAPPER_INIT_MAP_SIZE (_AC(1, UL) << SWAPPER_TABLE_SHIFT)
|
2015-10-19 22:19:27 +09:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Initial memory map attributes.
|
|
|
|
|
*/
|
2018-01-29 20:59:53 +09:00
|
|
|
#define SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
|
|
|
|
|
#define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
|
2015-10-19 22:19:27 +09:00
|
|
|
|
2015-10-19 22:19:28 +09:00
|
|
|
#if ARM64_SWAPPER_USES_SECTION_MAPS
|
2015-10-19 22:19:27 +09:00
|
|
|
#define SWAPPER_MM_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS)
|
2015-10-19 22:19:28 +09:00
|
|
|
#else
|
|
|
|
|
#define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS)
|
2015-10-19 22:19:27 +09:00
|
|
|
#endif
|
|
|
|
|
|
2016-02-16 21:52:42 +09:00
|
|
|
/*
|
|
|
|
|
* To make optimal use of block mappings when laying out the linear
|
|
|
|
|
* mapping, round down the base of physical memory to a size that can
|
|
|
|
|
* be mapped efficiently, i.e., either PUD_SIZE (4k granule) or PMD_SIZE
|
|
|
|
|
* (64k granule), or a multiple that can be mapped using contiguous bits
|
|
|
|
|
* in the page tables: 32 * PMD_SIZE (16k granule)
|
|
|
|
|
*/
|
2016-03-30 21:25:47 +09:00
|
|
|
#if defined(CONFIG_ARM64_4K_PAGES)
|
|
|
|
|
#define ARM64_MEMSTART_SHIFT PUD_SHIFT
|
|
|
|
|
#elif defined(CONFIG_ARM64_16K_PAGES)
|
|
|
|
|
#define ARM64_MEMSTART_SHIFT (PMD_SHIFT + 5)
|
2016-02-16 21:52:42 +09:00
|
|
|
#else
|
2016-03-30 21:25:47 +09:00
|
|
|
#define ARM64_MEMSTART_SHIFT PMD_SHIFT
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* sparsemem vmemmap imposes an additional requirement on the alignment of
|
|
|
|
|
* memstart_addr, due to the fact that the base of the vmemmap region
|
|
|
|
|
* has a direct correspondence, and needs to appear sufficiently aligned
|
|
|
|
|
* in the virtual address space.
|
|
|
|
|
*/
|
|
|
|
|
#if defined(CONFIG_SPARSEMEM_VMEMMAP) && ARM64_MEMSTART_SHIFT < SECTION_SIZE_BITS
|
|
|
|
|
#define ARM64_MEMSTART_ALIGN (1UL << SECTION_SIZE_BITS)
|
|
|
|
|
#else
|
|
|
|
|
#define ARM64_MEMSTART_ALIGN (1UL << ARM64_MEMSTART_SHIFT)
|
2016-02-16 21:52:42 +09:00
|
|
|
#endif
|
2015-10-19 22:19:27 +09:00
|
|
|
|
|
|
|
|
#endif /* __ASM_KERNEL_PGTABLE_H */
|