From 447ad5c69ce2655a981cd30e4c1b511120b5f800 Mon Sep 17 00:00:00 2001 From: Fox Snowpatch Date: Wed, 28 Aug 2024 07:24:16 +0000 Subject: [PATCH] From patchwork series 421241 https://patchwork.ozlabs.org//project/linuxppc-dev/list/?series=421241 --- arch/arm/mm/mmap.c | 10 ++++++ arch/arm64/include/asm/processor.h | 34 +++++++++++++++--- arch/loongarch/mm/mmap.c | 11 ++++++ arch/mips/mm/mmap.c | 9 +++++ arch/parisc/include/uapi/asm/mman.h | 1 + arch/parisc/kernel/sys_parisc.c | 9 +++++ arch/powerpc/include/asm/task_size_64.h | 36 ++++++++++++++++---- arch/riscv/include/asm/processor.h | 32 ----------------- arch/s390/mm/mmap.c | 10 ++++++ arch/sh/mm/mmap.c | 10 ++++++ arch/sparc/kernel/sys_sparc_64.c | 8 +++++ arch/x86/kernel/sys_x86_64.c | 25 ++++++++++++-- fs/hugetlbfs/inode.c | 2 +- include/linux/sched/mm.h | 34 ++++++++++++++++-- include/uapi/asm-generic/mman-common.h | 1 + mm/mmap.c | 2 +- tools/arch/parisc/include/uapi/asm/mman.h | 1 + tools/include/uapi/asm-generic/mman-common.h | 1 + tools/testing/selftests/mm/Makefile | 1 + tools/testing/selftests/mm/map_below_hint.c | 29 ++++++++++++++++ 20 files changed, 216 insertions(+), 50 deletions(-) create mode 100644 tools/testing/selftests/mm/map_below_hint.c diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c index d65d0e6ed10ab..fa0c79447b78f 100644 --- a/arch/arm/mm/mmap.c +++ b/arch/arm/mm/mmap.c @@ -71,6 +71,8 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, info.length = len; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(info.high_limit, addr + len); info.align_mask = do_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; info.align_offset = pgoff << PAGE_SHIFT; return vm_unmapped_area(&info); @@ -122,6 +124,12 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, info.length = len; info.low_limit = FIRST_USER_ADDRESS; info.high_limit = mm->mmap_base; + if (flags & MAP_BELOW_HINT) + /* + * Subtract (STACK_TOP - mm->mmap_base) to get random + * offset defined in mmap_base() in mm/util.c + */ + info.high_limit = MIN(info.high_limit, (addr + len) - (STACK_TOP - mm->mmap_base)); info.align_mask = do_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); @@ -137,6 +145,8 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, info.flags = 0; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(info.high_limit, addr + len); addr = vm_unmapped_area(&info); } diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index f77371232d8c6..39aabb1619f66 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -92,12 +92,36 @@ #endif /* CONFIG_COMPAT */ #ifndef CONFIG_ARM64_FORCE_52BIT -#define arch_get_mmap_end(addr, len, flags) \ - (((addr) > DEFAULT_MAP_WINDOW) ? TASK_SIZE : DEFAULT_MAP_WINDOW) +#define arch_get_mmap_end(addr, len, flags) \ +({ \ + unsigned long mmap_end; \ + typeof(flags) _flags = (flags); \ + typeof(addr) _addr = (addr); \ + typeof(len) _len = (len); \ + if (_flags & MAP_BELOW_HINT && _addr != 0 && ((_addr + _len) > BIT(VA_BITS - 1))) \ + mmap_end = (_addr + _len); \ + else \ + mmap_end = ((_addr > DEFAULT_MAP_WINDOW) ? TASK_SIZE : DEFAULT_MAP_WINDOW); \ + mmap_end \ +}) + +#define arch_get_mmap_base(addr, len, base, flags) \ +({ \ + unsigned long mmap_base; \ + typeof(flags) _flags = (flags); \ + typeof(addr) _addr = (addr); \ + typeof(base) _base = (base); \ + typeof(len) _len = (len); \ + unsigned long rnd_gap = DEFAULT_MAP_WINDOW - (_base); \ + if (_flags & MAP_BELOW_HINT && _addr != 0 && ((_addr + _len) > BIT(VA_BITS - 1)))\ + mmap_base = (_addr + _len) - rnd_gap; \ + else \ + mmap_end = ((_addr > DEFAULT_MAP_WINDOW) ? \ + _base + TASK_SIZE - DEFAULT_MAP_WINDOW : \ + _base); \ + mmap_end \ +}) -#define arch_get_mmap_base(addr, base) ((addr > DEFAULT_MAP_WINDOW) ? \ - base + TASK_SIZE - DEFAULT_MAP_WINDOW :\ - base) #endif /* CONFIG_ARM64_FORCE_52BIT */ extern phys_addr_t arm64_dma_phys_limit; diff --git a/arch/loongarch/mm/mmap.c b/arch/loongarch/mm/mmap.c index 8890309851351..66a5badf849b8 100644 --- a/arch/loongarch/mm/mmap.c +++ b/arch/loongarch/mm/mmap.c @@ -70,6 +70,13 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; + if (flags & MAP_BELOW_HINT) + /* + * Subtract (STACK_TOP - mm->mmap_base) to get random + * offset defined in mmap_base() in mm/util.c + */ + info.high_limit = MIN(mm->mmap_base, + (addr + len) - (STACK_TOP - mm->mmap_base)) addr = vm_unmapped_area(&info); if (!(addr & ~PAGE_MASK)) @@ -84,7 +91,11 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, } info.low_limit = mm->mmap_base; + info.high_limit = TASK_SIZE; + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(info.high_limit, addr + len); + return vm_unmapped_area(&info); } diff --git a/arch/mips/mm/mmap.c b/arch/mips/mm/mmap.c index 7e11d7b587610..1595fda284cd8 100644 --- a/arch/mips/mm/mmap.c +++ b/arch/mips/mm/mmap.c @@ -79,6 +79,13 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; + if (flags & MAP_BELOW_HINT) + /* + * Subtract (STACK_TOP - mm->mmap_base) to get random + * offset defined in mmap_base() in mm/util.c + */ + info.high_limit = MIN(info.high_limit, + (addr + len) - (STACK_TOP - mm->mmap_base)); addr = vm_unmapped_area(&info); if (!(addr & ~PAGE_MASK)) @@ -94,6 +101,8 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(info.high_limit, addr + len); return vm_unmapped_area(&info); } diff --git a/arch/parisc/include/uapi/asm/mman.h b/arch/parisc/include/uapi/asm/mman.h index 68c44f99bc931..44925ef8ac44f 100644 --- a/arch/parisc/include/uapi/asm/mman.h +++ b/arch/parisc/include/uapi/asm/mman.h @@ -26,6 +26,7 @@ #define MAP_HUGETLB 0x80000 /* create a huge page mapping */ #define MAP_FIXED_NOREPLACE 0x100000 /* MAP_FIXED which doesn't unmap underlying mapping */ #define MAP_UNINITIALIZED 0 /* uninitialized anonymous mmap */ +#define MAP_BELOW_HINT 0x200000 /* give out address that is below (inclusive) hint address */ #define MS_SYNC 1 /* synchronous memory sync */ #define MS_ASYNC 2 /* sync memory asynchronously */ diff --git a/arch/parisc/kernel/sys_parisc.c b/arch/parisc/kernel/sys_parisc.c index f7722451276ef..feccb60cf7467 100644 --- a/arch/parisc/kernel/sys_parisc.c +++ b/arch/parisc/kernel/sys_parisc.c @@ -148,6 +148,13 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; + if (flags & MAP_BELOW_HINT) + /* + * Subtract (STACK_TOP - mm->mmap_base) to get random + * offset defined in mmap_base() in mm/util.c + */ + info.high_limit = MIN(info.high_limit, + (addr + len) - (STACK_TOP - mm->mmap_base)); addr = vm_unmapped_area(&info); if (!(addr & ~PAGE_MASK)) return addr; @@ -163,6 +170,8 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, info.low_limit = mm->mmap_base; info.high_limit = mmap_upper_limit(NULL); + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(info.high_limit, addr + len); return vm_unmapped_area(&info); } diff --git a/arch/powerpc/include/asm/task_size_64.h b/arch/powerpc/include/asm/task_size_64.h index 5a709951c9019..a37a5a81365df 100644 --- a/arch/powerpc/include/asm/task_size_64.h +++ b/arch/powerpc/include/asm/task_size_64.h @@ -72,12 +72,36 @@ #define STACK_TOP_MAX TASK_SIZE_USER64 #define STACK_TOP (is_32bit_task() ? STACK_TOP_USER32 : STACK_TOP_USER64) -#define arch_get_mmap_base(addr, base) \ - (((addr) > DEFAULT_MAP_WINDOW) ? (base) + TASK_SIZE - DEFAULT_MAP_WINDOW : (base)) +#define arch_get_mmap_base(addr, len, base, flags) \ +({ \ + unsigned long mmap_base; \ + typeof(flags) _flags = (flags); \ + typeof(addr) _addr = (addr); \ + typeof(base) _base = (base); \ + typeof(len) _len = (len); \ + unsigned long rnd_gap = DEFAULT_MAP_WINDOW - (_base); \ + if (_flags & MAP_BELOW_HINT && _addr != 0 && ((_addr + _len) > BIT(VA_BITS - 1)))\ + mmap_base = (_addr + _len) - rnd_gap; \ + else \ + mmap_end = ((_addr > DEFAULT_MAP_WINDOW) ? \ + _base + TASK_SIZE - DEFAULT_MAP_WINDOW : \ + _base); \ + mmap_end; \ +}) -#define arch_get_mmap_end(addr, len, flags) \ - (((addr) > DEFAULT_MAP_WINDOW) || \ - (((flags) & MAP_FIXED) && ((addr) + (len) > DEFAULT_MAP_WINDOW)) ? TASK_SIZE : \ - DEFAULT_MAP_WINDOW) +#define arch_get_mmap_end(addr, len, flags) \ +({ \ + unsigned long mmap_end; \ + typeof(flags) _flags = (flags); \ + typeof(addr) _addr = (addr); \ + typeof(len) _len = (len); \ + if (_flags & MAP_BELOW_HINT && _addr != 0 && ((_addr + _len) > BIT(VA_BITS - 1))) \ + mmap_end = (_addr + _len); \ + else \ + mmap_end = (((_addr) > DEFAULT_MAP_WINDOW) || \ + (((_flags) & MAP_FIXED) && ((_addr) + (_len) > DEFAULT_MAP_WINDOW))\ + ? TASK_SIZE : DEFAULT_MAP_WINDOW) \ + mmap_end; \ +}) #endif /* _ASM_POWERPC_TASK_SIZE_64_H */ diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 8702b8721a270..20b4ba7d32be9 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -14,38 +14,6 @@ #include -/* - * addr is a hint to the maximum userspace address that mmap should provide, so - * this macro needs to return the largest address space available so that - * mmap_end < addr, being mmap_end the top of that address space. - * See Documentation/arch/riscv/vm-layout.rst for more details. - */ -#define arch_get_mmap_end(addr, len, flags) \ -({ \ - unsigned long mmap_end; \ - typeof(addr) _addr = (addr); \ - if ((_addr) == 0 || is_compat_task() || \ - ((_addr + len) > BIT(VA_BITS - 1))) \ - mmap_end = STACK_TOP_MAX; \ - else \ - mmap_end = (_addr + len); \ - mmap_end; \ -}) - -#define arch_get_mmap_base(addr, base) \ -({ \ - unsigned long mmap_base; \ - typeof(addr) _addr = (addr); \ - typeof(base) _base = (base); \ - unsigned long rnd_gap = DEFAULT_MAP_WINDOW - (_base); \ - if ((_addr) == 0 || is_compat_task() || \ - ((_addr + len) > BIT(VA_BITS - 1))) \ - mmap_base = (_base); \ - else \ - mmap_base = (_addr + len) - rnd_gap; \ - mmap_base; \ -}) - #ifdef CONFIG_64BIT #define DEFAULT_MAP_WINDOW (UL(1) << (MMAP_VA_BITS - 1)) #define STACK_TOP_MAX TASK_SIZE_64 diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index 2067569465897..29e20351ca858 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -105,6 +105,8 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, info.length = len; info.low_limit = mm->mmap_base; info.high_limit = TASK_SIZE; + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(info.high_limit, addr + len); info.align_mask = get_align_mask(filp, flags); info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); @@ -143,6 +145,12 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned long ad info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; + if (flags & MAP_BELOW_HINT) + /* + * Subtract (STACK_TOP - mm->mmap_base) to get random + * offset defined in mmap_base() in mm/util.c + */ + info.high_limit = MIN(info.high_limit, addr + len) - (STACK_TOP - mm->mmap_base); info.align_mask = get_align_mask(filp, flags); info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); @@ -158,6 +166,8 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned long ad info.flags = 0; info.low_limit = TASK_UNMAPPED_BASE; info.high_limit = TASK_SIZE; + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(TASK_SIZE, addr + len); addr = vm_unmapped_area(&info); if (offset_in_page(addr)) return addr; diff --git a/arch/sh/mm/mmap.c b/arch/sh/mm/mmap.c index bee329d4149aa..867f6598b7d0a 100644 --- a/arch/sh/mm/mmap.c +++ b/arch/sh/mm/mmap.c @@ -91,6 +91,8 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, info.length = len; info.low_limit = TASK_UNMAPPED_BASE; info.high_limit = TASK_SIZE; + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(info.high_limit, addr + len); info.align_mask = do_colour_align ? (PAGE_MASK & shm_align_mask) : 0; info.align_offset = pgoff << PAGE_SHIFT; return vm_unmapped_area(&info); @@ -141,6 +143,12 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; + if (flags & MAP_BELOW_HINT) + /* + * Subtract (STACK_TOP - mm->mmap_base) to get random + * offset defined in mmap_base() in mm/util.c + */ + info.high_limit = MIN(info.high_limit, (addr + len) - (STACK_TOP - mm->mmap_base)); info.align_mask = do_colour_align ? (PAGE_MASK & shm_align_mask) : 0; info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); @@ -156,6 +164,8 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, info.flags = 0; info.low_limit = TASK_UNMAPPED_BASE; info.high_limit = TASK_SIZE; + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(info.high_limit, addr + len); addr = vm_unmapped_area(&info); } diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c index d9c3b34ca7447..b9ce9988551ab 100644 --- a/arch/sparc/kernel/sys_sparc_64.c +++ b/arch/sparc/kernel/sys_sparc_64.c @@ -129,6 +129,8 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi info.length = len; info.low_limit = TASK_UNMAPPED_BASE; info.high_limit = min(task_size, VA_EXCLUDE_START); + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(info.high_limit, addr + len); info.align_mask = do_color_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); @@ -137,6 +139,8 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi VM_BUG_ON(addr != -ENOMEM); info.low_limit = VA_EXCLUDE_END; info.high_limit = task_size; + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(info.high_limit, addr + len); addr = vm_unmapped_area(&info); } @@ -192,6 +196,8 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(info.high_limit, addr + len); info.align_mask = do_color_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; info.align_offset = pgoff << PAGE_SHIFT; addr = vm_unmapped_area(&info); @@ -207,6 +213,8 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, info.flags = 0; info.low_limit = TASK_UNMAPPED_BASE; info.high_limit = STACK_TOP32; + if (flags & MAP_BELOW_HINT) + info.high_limit = MIN(info.high_limit, addr + len); addr = vm_unmapped_area(&info); } diff --git a/arch/x86/kernel/sys_x86_64.c b/arch/x86/kernel/sys_x86_64.c index 01d7cd85ef978..fa16b38f37024 100644 --- a/arch/x86/kernel/sys_x86_64.c +++ b/arch/x86/kernel/sys_x86_64.c @@ -86,7 +86,7 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, return ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); } -static void find_start_end(unsigned long addr, unsigned long flags, +static void find_start_end(unsigned long addr, unsigned long len, unsigned long flags, unsigned long *begin, unsigned long *end) { if (!in_32bit_syscall() && (flags & MAP_32BIT)) { @@ -106,10 +106,14 @@ static void find_start_end(unsigned long addr, unsigned long flags, } *begin = get_mmap_base(1); + if (in_32bit_syscall()) *end = task_size_32bit(); else *end = task_size_64bit(addr > DEFAULT_MAP_WINDOW); + + if (flags & MAP_BELOW_HINT) + *end = MIN(*end, addr + len); } static inline unsigned long stack_guard_placement(vm_flags_t vm_flags) @@ -132,7 +136,7 @@ arch_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, unsigned l if (flags & MAP_FIXED) return addr; - find_start_end(addr, flags, &begin, &end); + find_start_end(addr, len, flags, &begin, &end); if (len > end) return -ENOMEM; @@ -166,6 +170,7 @@ arch_get_unmapped_area_topdown_vmflags(struct file *filp, unsigned long addr0, struct mm_struct *mm = current->mm; unsigned long addr = addr0; struct vm_unmapped_area_info info = {}; + unsigned long task_size, mmap_base; /* requested length too big for entire address space */ if (len > TASK_SIZE) @@ -198,7 +203,8 @@ arch_get_unmapped_area_topdown_vmflags(struct file *filp, unsigned long addr0, else info.low_limit = PAGE_SIZE; - info.high_limit = get_mmap_base(0); + mmap_base = get_mmap_base(0); + info.high_limit = mmap_base; info.start_gap = stack_guard_placement(vm_flags); /* @@ -210,6 +216,19 @@ arch_get_unmapped_area_topdown_vmflags(struct file *filp, unsigned long addr0, */ if (addr > DEFAULT_MAP_WINDOW && !in_32bit_syscall()) info.high_limit += TASK_SIZE_MAX - DEFAULT_MAP_WINDOW; + if (flags & MAP_BELOW_HINT) { +#ifdef CONFIG_HAVE_ARCH_COMPAT_MMAP_BASES + task_size = task_size_32bit(); +#else + task_size = task_size_64bit(0); +#endif + /* + * mmap_base is defined by PAGE_ALIGN(task_size - gap - rnd) so + * subtract out the task_size to isolate the gap + rnd. + */ + info.high_limit = MIN(info.high_limit, + (addr + len) - (task_size - mmap_base)); + } info.align_offset = pgoff << PAGE_SHIFT; if (filp) { diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 9f6cff356796c..05a52f85dba97 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -195,7 +195,7 @@ hugetlb_get_unmapped_area_topdown(struct file *file, unsigned long addr, info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; info.low_limit = PAGE_SIZE; - info.high_limit = arch_get_mmap_base(addr, current->mm->mmap_base); + info.high_limit = arch_get_mmap_base(addr, len, current->mm->mmap_base, flags); info.align_mask = PAGE_MASK & ~huge_page_mask(h); addr = vm_unmapped_area(&info); diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 91546493c43da..c350bb5ac0a2d 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -9,6 +9,7 @@ #include #include #include +#include /* * Routines for handling mm_structs @@ -170,11 +171,40 @@ static inline void mm_update_next_owner(struct mm_struct *mm) #ifdef CONFIG_MMU #ifndef arch_get_mmap_end -#define arch_get_mmap_end(addr, len, flags) (TASK_SIZE) +#define arch_get_mmap_end(addr, len, flags) \ +({ \ + unsigned long mmap_end; \ + typeof(flags) _flags = (flags); \ + typeof(addr) _addr = (addr); \ + typeof(len) _len = (len); \ + mmap_end = TASK_SIZE; \ + if (_flags & MAP_BELOW_HINT && _addr != 0) \ + mmap_end = MIN(mmap_end, _addr + _len); \ + mmap_end; \ +}) #endif #ifndef arch_get_mmap_base -#define arch_get_mmap_base(addr, base) (base) +/* + * rnd_gap is defined to be (STACK_TOP - _base) due to the definition of + * mmap_base in mm/util.c + * + * Assumes ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT, which all architectures that + * implement generic mmap use + */ +#define arch_get_mmap_base(addr, len, base, flags) \ +({ \ + unsigned long mmap_base; \ + typeof(flags) _flags = (flags); \ + typeof(addr) _addr = (addr); \ + typeof(base) _base = (base); \ + typeof(len) _len = (len); \ + unsigned long rnd_gap = STACK_TOP - _base; \ + mmap_base = _base; \ + if (_flags & MAP_BELOW_HINT && _addr != 0) \ + mmap_base = MIN(mmap_base, (_addr + _len) - rnd_gap); \ + mmap_base; \ +}) #endif extern void arch_pick_mmap_layout(struct mm_struct *mm, diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h index 6ce1f1ceb432c..03ac13d9aa376 100644 --- a/include/uapi/asm-generic/mman-common.h +++ b/include/uapi/asm-generic/mman-common.h @@ -32,6 +32,7 @@ #define MAP_UNINITIALIZED 0x4000000 /* For anonymous mmap, memory could be * uninitialized */ +#define MAP_BELOW_HINT 0x8000000 /* give out address that is below (inclusive) hint address */ /* * Flags for mlock diff --git a/mm/mmap.c b/mm/mmap.c index d0dfc85b209bb..27a7f2be3f688 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1861,7 +1861,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; info.low_limit = PAGE_SIZE; - info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); + info.high_limit = arch_get_mmap_base(addr, len, mm->mmap_base, flags); addr = vm_unmapped_area(&info); /* diff --git a/tools/arch/parisc/include/uapi/asm/mman.h b/tools/arch/parisc/include/uapi/asm/mman.h index 4cc88a642e106..297acc0f7b2ac 100644 --- a/tools/arch/parisc/include/uapi/asm/mman.h +++ b/tools/arch/parisc/include/uapi/asm/mman.h @@ -40,4 +40,5 @@ /* MAP_32BIT is undefined on parisc, fix it for perf */ #define MAP_32BIT 0 #define MAP_UNINITIALIZED 0 +#define MAP_BELOW_MAP 0x200000 #endif diff --git a/tools/include/uapi/asm-generic/mman-common.h b/tools/include/uapi/asm-generic/mman-common.h index 6ce1f1ceb432c..03ac13d9aa376 100644 --- a/tools/include/uapi/asm-generic/mman-common.h +++ b/tools/include/uapi/asm-generic/mman-common.h @@ -32,6 +32,7 @@ #define MAP_UNINITIALIZED 0x4000000 /* For anonymous mmap, memory could be * uninitialized */ +#define MAP_BELOW_HINT 0x8000000 /* give out address that is below (inclusive) hint address */ /* * Flags for mlock diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index 901e0d07765b6..0c652ff49d4b1 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -50,6 +50,7 @@ TEST_GEN_FILES += hugepage-shm TEST_GEN_FILES += hugepage-vmemmap TEST_GEN_FILES += khugepaged TEST_GEN_FILES += madv_populate +TEST_GEN_FILES += map_below_hint TEST_GEN_FILES += map_fixed_noreplace TEST_GEN_FILES += map_hugetlb TEST_GEN_FILES += map_populate diff --git a/tools/testing/selftests/mm/map_below_hint.c b/tools/testing/selftests/mm/map_below_hint.c new file mode 100644 index 0000000000000..305274c5af49f --- /dev/null +++ b/tools/testing/selftests/mm/map_below_hint.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test the MAP_BELOW_HINT mmap flag. + */ +#include +#include "../kselftest.h" + +#define ADDR 0x1000000UL +#define LENGTH (ADDR / 100) + +#define MAP_BELOW_HINT 0x8000000 /* Not defined in all libc */ + +/* + * Map memory with MAP_BELOW_HINT until no memory left. Ensure that all returned + * addresses are below the hint. + */ +int main(int argc, char **argv) +{ + void *addr; + + do { + addr = mmap((void *)ADDR, LENGTH, MAP_ANONYMOUS, MAP_BELOW_HINT, -1, 0); + } while (addr == MAP_FAILED && (unsigned long)addr <= ADDR); + + if (addr != MAP_FAILED && (unsigned long)addr > ADDR) + ksft_exit_fail_msg("mmap returned address above hint with MAP_BELOW_HINT\n"); + + ksft_test_result_pass("MAP_BELOW_HINT works\n"); +}