diff --git a/arch/s390/boot/boot.h b/arch/s390/boot/boot.h
index 8b50967f58048f2b80344e0902996ddf5adb4145..ae04e1c93764cb7bf343c23442909de6061dd5df 100644
--- a/arch/s390/boot/boot.h
+++ b/arch/s390/boot/boot.h
@@ -24,6 +24,7 @@ void __printf(1, 2) decompressor_printk(const char *fmt, ...);
extern const char kernel_version[];
extern unsigned long memory_limit;
+extern unsigned long vmalloc_size;
extern int vmalloc_size_set;
extern int kaslr_enabled;
diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c
index 3485a1d4e97bfc14235f37748ca31e5d0e1d6c53..bc1f973e656a0962a4bbf75d92d12971c8b95a80 100644
--- a/arch/s390/boot/ipl_parm.c
+++ b/arch/s390/boot/ipl_parm.c
@@ -12,13 +12,13 @@
#include "boot.h"
char __bootdata(early_command_line)[COMMAND_LINE_SIZE];
+int __bootdata(noexec_disabled);
+
+unsigned int __bootdata_preserved(zlib_dfltcc_support) = ZLIB_DFLTCC_FULL;
struct ipl_parameter_block __bootdata_preserved(ipl_block);
int __bootdata_preserved(ipl_block_valid);
-unsigned int __bootdata_preserved(zlib_dfltcc_support) = ZLIB_DFLTCC_FULL;
-
-unsigned long __bootdata(vmalloc_size) = VMALLOC_DEFAULT_SIZE;
-int __bootdata(noexec_disabled);
+unsigned long vmalloc_size = VMALLOC_DEFAULT_SIZE;
unsigned long memory_limit;
int vmalloc_size_set;
int kaslr_enabled;
diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c
index 61a8ac4067e513a6e8cea06a0b42759c55c76725..bbe4df6c2f8bf71773b55291008b689d6dddd086 100644
--- a/arch/s390/boot/startup.c
+++ b/arch/s390/boot/startup.c
@@ -5,6 +5,7 @@
#include <asm/sections.h>
#include <asm/cpu_mf.h>
#include <asm/setup.h>
+#include <asm/kasan.h>
#include <asm/kexec.h>
#include <asm/sclp.h>
#include <asm/diag.h>
@@ -15,6 +16,12 @@
extern char __boot_data_start[], __boot_data_end[];
extern char __boot_data_preserved_start[], __boot_data_preserved_end[];
unsigned long __bootdata_preserved(__kaslr_offset);
+unsigned long __bootdata_preserved(VMALLOC_START);
+unsigned long __bootdata_preserved(VMALLOC_END);
+struct page *__bootdata_preserved(vmemmap);
+unsigned long __bootdata_preserved(vmemmap_size);
+unsigned long __bootdata_preserved(MODULES_VADDR);
+unsigned long __bootdata_preserved(MODULES_END);
unsigned long __bootdata(ident_map_size);
u64 __bootdata_preserved(stfle_fac_list[16]);
@@ -172,6 +179,86 @@ static void setup_ident_map_size(unsigned long max_physmem_end)
#endif
}
+static void setup_kernel_memory_layout(void)
+{
+ bool vmalloc_size_verified = false;
+ unsigned long vmemmap_off;
+ unsigned long vspace_left;
+ unsigned long rte_size;
+ unsigned long pages;
+ unsigned long vmax;
+
+ pages = ident_map_size / PAGE_SIZE;
+ /* vmemmap contains a multiple of PAGES_PER_SECTION struct pages */
+ vmemmap_size = SECTION_ALIGN_UP(pages) * sizeof(struct page);
+
+ /* choose kernel address space layout: 4 or 3 levels. */
+ vmemmap_off = round_up(ident_map_size, _REGION3_SIZE);
+ if (IS_ENABLED(CONFIG_KASAN) ||
+ vmalloc_size > _REGION2_SIZE ||
+ vmemmap_off + vmemmap_size + vmalloc_size + MODULES_LEN > _REGION2_SIZE)
+ vmax = _REGION1_SIZE;
+ else
+ vmax = _REGION2_SIZE;
+
+ /* keep vmemmap_off aligned to a top level region table entry */
+ rte_size = vmax == _REGION1_SIZE ? _REGION2_SIZE : _REGION3_SIZE;
+ MODULES_END = vmax;
+ if (is_prot_virt_host()) {
+ /*
+ * forcing modules and vmalloc area under the ultravisor
+ * secure storage limit, so that any vmalloc allocation
+ * we do could be used to back secure guest storage.
+ */
+ adjust_to_uv_max(&MODULES_END);
+ }
+
+#ifdef CONFIG_KASAN
+ if (MODULES_END < vmax) {
+ /* force vmalloc and modules below kasan shadow */
+ MODULES_END = min(MODULES_END, KASAN_SHADOW_START);
+ } else {
+ /*
+ * leave vmalloc and modules above kasan shadow but make
+ * sure they don't overlap with it
+ */
+ vmalloc_size = min(vmalloc_size, vmax - KASAN_SHADOW_END - MODULES_LEN);
+ vmalloc_size_verified = true;
+ vspace_left = KASAN_SHADOW_START;
+ }
+#endif
+ MODULES_VADDR = MODULES_END - MODULES_LEN;
+ VMALLOC_END = MODULES_VADDR;
+
+ if (vmalloc_size_verified) {
+ VMALLOC_START = VMALLOC_END - vmalloc_size;
+ } else {
+ vmemmap_off = round_up(ident_map_size, rte_size);
+
+ if (vmemmap_off + vmemmap_size > VMALLOC_END ||
+ vmalloc_size > VMALLOC_END - vmemmap_off - vmemmap_size) {
+ /*
+ * allow vmalloc area to occupy up to 1/2 of
+ * the rest virtual space left.
+ */
+ vmalloc_size = min(vmalloc_size, VMALLOC_END / 2);
+ }
+ VMALLOC_START = VMALLOC_END - vmalloc_size;
+ vspace_left = VMALLOC_START;
+ }
+
+ pages = vspace_left / (PAGE_SIZE + sizeof(struct page));
+ pages = SECTION_ALIGN_UP(pages);
+ vmemmap_off = round_up(vspace_left - pages * sizeof(struct page), rte_size);
+ /* keep vmemmap left most starting from a fresh region table entry */
+ vmemmap_off = min(vmemmap_off, round_up(ident_map_size, rte_size));
+ /* take care that identity map is lower then vmemmap */
+ ident_map_size = min(ident_map_size, vmemmap_off);
+ vmemmap_size = SECTION_ALIGN_UP(ident_map_size / PAGE_SIZE) * sizeof(struct page);
+ VMALLOC_START = max(vmemmap_off + vmemmap_size, VMALLOC_START);
+ vmemmap = (struct page *)vmemmap_off;
+}
+
/*
* This function clears the BSS section of the decompressed Linux kernel and NOT the decompressor's.
*/
@@ -211,6 +298,7 @@ void startup_kernel(void)
parse_boot_command_line();
setup_ident_map_size(detect_memory());
setup_vmalloc_size();
+ setup_kernel_memory_layout();
random_lma = __kaslr_offset = 0;
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && kaslr_enabled) {
diff --git a/arch/s390/boot/uv.c b/arch/s390/boot/uv.c
index 87641dd65ccf915e8d11aa5062dcc0433f34eeb4..82b99b916243c470da82621ea6fc971c34e61724 100644
--- a/arch/s390/boot/uv.c
+++ b/arch/s390/boot/uv.c
@@ -44,3 +44,28 @@ void uv_query_info(void)
prot_virt_guest = 1;
#endif
}
+
+#if IS_ENABLED(CONFIG_KVM)
+static bool has_uv_sec_stor_limit(void)
+{
+ /*
+ * keep these conditions in line with setup_uv()
+ */
+ if (!is_prot_virt_host())
+ return false;
+
+ if (is_prot_virt_guest())
+ return false;
+
+ if (!test_facility(158))
+ return false;
+
+ return !!uv_info.max_sec_stor_addr;
+}
+
+void adjust_to_uv_max(unsigned long *vmax)
+{
+ if (has_uv_sec_stor_limit())
+ *vmax = min_t(unsigned long, *vmax, uv_info.max_sec_stor_addr);
+}
+#endif
diff --git a/arch/s390/include/asm/kasan.h b/arch/s390/include/asm/kasan.h
index 76f351bd66459f6742c15fc957082b3af308950e..2768d5db181f89992bdf2f1f9249aa03ad7db4c9 100644
--- a/arch/s390/include/asm/kasan.h
+++ b/arch/s390/include/asm/kasan.h
@@ -16,7 +16,6 @@
extern void kasan_early_init(void);
extern void kasan_copy_shadow_mapping(void);
extern void kasan_free_early_identity(void);
-extern unsigned long kasan_vmax;
/*
* Estimate kasan memory requirements, which it will reserve
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 9512f6820ead51c63ec0437713e6881ac0a003bc..5677be47326168b7f0ae2bff10cbb9516c36d7a8 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -17,6 +17,7 @@
#include <linux/page-flags.h>
#include <linux/radix-tree.h>
#include <linux/atomic.h>
+#include <asm/sections.h>
#include <asm/bug.h>
#include <asm/page.h>
#include <asm/uv.h>
@@ -86,16 +87,16 @@ extern unsigned long zero_page_mask;
* happen without trampolines and in addition the placement within a
* 2GB frame is branch prediction unit friendly.
*/
-extern unsigned long VMALLOC_START;
-extern unsigned long VMALLOC_END;
+extern unsigned long __bootdata_preserved(VMALLOC_START);
+extern unsigned long __bootdata_preserved(VMALLOC_END);
#define VMALLOC_DEFAULT_SIZE ((512UL << 30) - MODULES_LEN)
-extern struct page *vmemmap;
-extern unsigned long vmemmap_size;
+extern struct page *__bootdata_preserved(vmemmap);
+extern unsigned long __bootdata_preserved(vmemmap_size);
#define VMEM_MAX_PHYS ((unsigned long) vmemmap)
-extern unsigned long MODULES_VADDR;
-extern unsigned long MODULES_END;
+extern unsigned long __bootdata_preserved(MODULES_VADDR);
+extern unsigned long __bootdata_preserved(MODULES_END);
#define MODULES_VADDR MODULES_VADDR
#define MODULES_END MODULES_END
#define MODULES_LEN (1UL << 31)
diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h
index 29baab03f09199a49f0a525183960c8d2a8bea2d..a8b75da3c1b87908a382f71b2ad9c9dd1c079c96 100644
--- a/arch/s390/include/asm/setup.h
+++ b/arch/s390/include/asm/setup.h
@@ -89,7 +89,6 @@ extern unsigned int zlib_dfltcc_support;
extern int noexec_disabled;
extern unsigned long ident_map_size;
-extern unsigned long vmalloc_size;
/* The Write Back bit position in the physaddr is given by the SLPC PCI */
extern unsigned long mio_wb_bit_mask;
diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
index 146d01700a559e90ec935a217e886f86507d7bf4..9145ef983f38634fcf3f0e065c57f33503027f38 100644
--- a/arch/s390/kernel/setup.c
+++ b/arch/s390/kernel/setup.c
@@ -96,7 +96,6 @@ unsigned long int_hwcap = 0;
int __bootdata(noexec_disabled);
unsigned long __bootdata(ident_map_size);
-unsigned long __bootdata(vmalloc_size);
struct mem_detect_info __bootdata(mem_detect);
struct exception_table_entry *__bootdata_preserved(__start_dma_ex_table);
@@ -545,53 +544,10 @@ static void __init setup_resources(void)
#endif
}
-static void __init setup_ident_map_size(void)
+static void __init setup_memory_end(void)
{
- unsigned long vmax, tmp;
-
- /* Choose kernel address space layout: 3 or 4 levels. */
- tmp = ident_map_size / PAGE_SIZE;
- tmp = tmp * (sizeof(struct page) + PAGE_SIZE);
- if (tmp + vmalloc_size + MODULES_LEN <= _REGION2_SIZE)
- vmax = _REGION2_SIZE; /* 3-level kernel page table */
- else
- vmax = _REGION1_SIZE; /* 4-level kernel page table */
- /* module area is at the end of the kernel address space. */
- MODULES_END = vmax;
- if (is_prot_virt_host())
- adjust_to_uv_max(&MODULES_END);
-#ifdef CONFIG_KASAN
- vmax = _REGION1_SIZE;
- MODULES_END = kasan_vmax;
-#endif
- MODULES_VADDR = MODULES_END - MODULES_LEN;
- VMALLOC_END = MODULES_VADDR;
- VMALLOC_START = VMALLOC_END - vmalloc_size;
-
- /* Split remaining virtual space between 1:1 mapping & vmemmap array */
- tmp = VMALLOC_START / (PAGE_SIZE + sizeof(struct page));
- /* vmemmap contains a multiple of PAGES_PER_SECTION struct pages */
- tmp = SECTION_ALIGN_UP(tmp);
- tmp = VMALLOC_START - tmp * sizeof(struct page);
- tmp &= ~((vmax >> 11) - 1); /* align to page table level */
- tmp = min(tmp, 1UL << MAX_PHYSMEM_BITS);
- vmemmap = (struct page *) tmp;
-
- /* Take care that ident_map_size <= vmemmap */
- ident_map_size = min(ident_map_size, (unsigned long)vmemmap);
-#ifdef CONFIG_KASAN
- ident_map_size = min(ident_map_size, KASAN_SHADOW_START);
-#endif
- vmemmap_size = SECTION_ALIGN_UP(ident_map_size / PAGE_SIZE) * sizeof(struct page);
-#ifdef CONFIG_KASAN
- /* move vmemmap above kasan shadow only if stands in a way */
- if (KASAN_SHADOW_END > (unsigned long)vmemmap &&
- (unsigned long)vmemmap + vmemmap_size > KASAN_SHADOW_START)
- vmemmap = max(vmemmap, (struct page *)KASAN_SHADOW_END);
-#endif
- max_pfn = max_low_pfn = PFN_DOWN(ident_map_size);
memblock_remove(ident_map_size, ULONG_MAX);
-
+ max_pfn = max_low_pfn = PFN_DOWN(ident_map_size);
pr_notice("The maximum memory size is %luMB\n", ident_map_size >> 20);
}
@@ -1132,7 +1088,7 @@ void __init setup_arch(char **cmdline_p)
remove_oldmem();
setup_uv();
- setup_ident_map_size();
+ setup_memory_end();
setup_memory();
dma_contiguous_reserve(ident_map_size);
vmcp_cma_reserve();
diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c
index 370f664580af58aa4e6105623a81b5c49d94f890..6be2167943bb496e4ee0c51a88b4a13d07287ff6 100644
--- a/arch/s390/kernel/uv.c
+++ b/arch/s390/kernel/uv.c
@@ -52,7 +52,7 @@ void __init setup_uv(void)
unsigned long uv_stor_base;
/*
- * keep these conditions in line with kasan init code has_uv_sec_stor_limit()
+ * keep these conditions in line with has_uv_sec_stor_limit()
*/
if (!is_prot_virt_host())
return;
@@ -91,12 +91,6 @@ void __init setup_uv(void)
prot_virt_host = 0;
}
-void adjust_to_uv_max(unsigned long *vmax)
-{
- if (uv_info.max_sec_stor_addr)
- *vmax = min_t(unsigned long, *vmax, uv_info.max_sec_stor_addr);
-}
-
/*
* Requests the Ultravisor to pin the page in the shared state. This will
* cause an intercept when the guest attempts to unshare the pinned page.
diff --git a/arch/s390/mm/kasan_init.c b/arch/s390/mm/kasan_init.c
index db4d303aaaa9afdb0bc571c7dbbaf55d54e3a07c..a0fdc6dc5f9d0904a1a55f025396cd6a801156c2 100644
--- a/arch/s390/mm/kasan_init.c
+++ b/arch/s390/mm/kasan_init.c
@@ -13,7 +13,6 @@
#include <asm/setup.h>
#include <asm/uv.h>
-unsigned long kasan_vmax;
static unsigned long segment_pos __initdata;
static unsigned long segment_low __initdata;
static unsigned long pgalloc_pos __initdata;
@@ -251,28 +250,9 @@ static void __init kasan_early_detect_facilities(void)
}
}
-static bool __init has_uv_sec_stor_limit(void)
-{
- /*
- * keep these conditions in line with setup_uv()
- */
- if (!is_prot_virt_host())
- return false;
-
- if (is_prot_virt_guest())
- return false;
-
- if (!test_facility(158))
- return false;
-
- return !!uv_info.max_sec_stor_addr;
-}
-
void __init kasan_early_init(void)
{
- unsigned long untracked_mem_end;
unsigned long shadow_alloc_size;
- unsigned long vmax_unlimited;
unsigned long initrd_end;
unsigned long memsize;
unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO);
@@ -306,9 +286,6 @@ void __init kasan_early_init(void)
BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
crst_table_init((unsigned long *)early_pg_dir, _REGION2_ENTRY_EMPTY);
- untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE;
- if (has_uv_sec_stor_limit())
- kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr);
/* init kasan zero shadow */
crst_table_init((unsigned long *)kasan_early_shadow_p4d,
@@ -375,18 +352,18 @@ void __init kasan_early_init(void)
*/
/* populate kasan shadow (for identity mapping and zero page mapping) */
kasan_early_pgtable_populate(__sha(0), __sha(memsize), POPULATE_MAP);
- if (IS_ENABLED(CONFIG_MODULES))
- untracked_mem_end = kasan_vmax - MODULES_LEN;
if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
- untracked_mem_end = kasan_vmax - vmalloc_size - MODULES_LEN;
/* shallowly populate kasan shadow for vmalloc and modules */
- kasan_early_pgtable_populate(__sha(untracked_mem_end), __sha(kasan_vmax),
+ kasan_early_pgtable_populate(__sha(VMALLOC_START), __sha(MODULES_END),
POPULATE_SHALLOW);
}
/* populate kasan shadow for untracked memory */
- kasan_early_pgtable_populate(__sha(ident_map_size), __sha(untracked_mem_end),
+ kasan_early_pgtable_populate(__sha(ident_map_size),
+ IS_ENABLED(CONFIG_KASAN_VMALLOC) ?
+ __sha(VMALLOC_START) :
+ __sha(MODULES_VADDR),
POPULATE_ZERO_SHADOW);
- kasan_early_pgtable_populate(__sha(kasan_vmax), __sha(vmax_unlimited),
+ kasan_early_pgtable_populate(__sha(MODULES_END), __sha(_REGION1_SIZE),
POPULATE_ZERO_SHADOW);
/* memory allocated for identity mapping structs will be freed later */
pgalloc_freeable = pgalloc_pos;