diff --git a/kernel/Makefile b/kernel/Makefile index fdd14261e..bfe02e84b 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -48,7 +48,7 @@ else # -Wno-missing-braces: Seems broken? At least it started spitting out weird ones for IPv6 code(IN6ADDR_ALL_NODES) # in onyx/net/ipv6.h CFLAGS:=$(CFLAGS) -Wno-null-pointer-arithmetic -Wno-unknown-attributes -Wno-error=unused-private-field \ - -Wno-missing-braces -Wno-c99-designator -Wthread-safety + -Wno-missing-braces -Wno-c99-designator -Wthread-safety -Wno-thread-safety-precise endif # NOTE: I don't think pointer-overflow works with kernel pointers diff --git a/kernel/arch/arm64/mmu.cpp b/kernel/arch/arm64/mmu.cpp index 3c438ad7d..b1acfaf94 100644 --- a/kernel/arch/arm64/mmu.cpp +++ b/kernel/arch/arm64/mmu.cpp @@ -414,15 +414,6 @@ bool __paging_change_perms(struct mm_address_space *mm, void *addr, int prot) return true; } -bool paging_change_perms(void *addr, int prot) -{ - struct mm_address_space *as = &kernel_address_space; - if ((unsigned long) addr < VM_HIGHER_HALF) - as = get_current_address_space(); - - return __paging_change_perms(as, addr, prot); -} - bool paging_write_protect(void *addr, struct mm_address_space *mm) { uint64_t *ptentry; @@ -598,15 +589,6 @@ void paging_free_page_tables(struct mm_address_space *mm) free_page(phys_to_page((unsigned long) mm->arch_mmu.top_pt)); } -unsigned long get_mapping_info(void *addr) -{ - struct mm_address_space *as = &kernel_address_space; - if ((unsigned long) addr < VM_HIGHER_HALF) - as = get_current_address_space(); - - return __get_mapping_info(addr, as); -} - unsigned long __get_mapping_info(void *addr, struct mm_address_space *as) { unsigned long *ppt_entry; @@ -1142,7 +1124,7 @@ static int arm64_mmu_fork(PML *parent_table, PML *child_table, unsigned int pt_l if (pt_level == PT_LEVEL || is_huge_page) { - const bool should_cow = old_region->vm_maptype == MAP_PRIVATE; + const bool should_cow = vma_private(old_region); child_table->entries[i] = pt_entry | (should_cow ? ARM64_MMU_READ_ONLY : 0); if (should_cow) { diff --git a/kernel/arch/riscv64/fpu.cpp b/kernel/arch/riscv64/fpu.cpp index 3d00144fd..b637b3799 100644 --- a/kernel/arch/riscv64/fpu.cpp +++ b/kernel/arch/riscv64/fpu.cpp @@ -106,6 +106,8 @@ static slab_cache *fpu_cache = nullptr; */ void fpu_init_cache() { + if (fpu_cache) + return; fpu_cache = kmem_cache_create("fpu-state", save_size, fpu_get_save_alignment(), 0, nullptr); if (!fpu_cache) panic("Out of memory allocating fpu state"); diff --git a/kernel/arch/riscv64/mmu.cpp b/kernel/arch/riscv64/mmu.cpp index 734bb9793..e391a5a60 100644 --- a/kernel/arch/riscv64/mmu.cpp +++ b/kernel/arch/riscv64/mmu.cpp @@ -53,9 +53,6 @@ static unsigned long vm_prots_to_mmu(unsigned int prots) #define RISCV_MMU_FLAGS_TO_SAVE_ON_MPROTECT \ (RISCV_MMU_GLOBAL | RISCV_MMU_USER | RISCV_MMU_ACCESSED | RISCV_MMU_DIRTY | RISCV_MMU_SPECIAL) -void *paging_map_phys_to_virt(struct mm_address_space *as, uint64_t virt, uint64_t phys, - uint64_t prot, struct vm_area_struct *vma); - static inline void __native_tlb_invalidate_page(void *addr) { __asm__ __volatile__("sfence.vma %0, zero" ::"r"(addr)); @@ -71,9 +68,6 @@ static inline bool pte_special(u64 pte) return pte & RISCV_MMU_SPECIAL; } -bool riscv_get_pt_entry(void *addr, uint64_t **entry_ptr, bool may_create_path, - struct mm_address_space *mm); - unsigned long allocated_page_tables = 0; PML *alloc_pt(void) @@ -205,59 +199,6 @@ void paging_init(void) riscv_pt_page_mapping(VERYHUGE512GB_SIZE) | flags; } -void *paging_map_phys_to_virt(struct mm_address_space *as, uint64_t virt, uint64_t phys, - uint64_t prot, struct vm_area_struct *vma) -{ - bool user = prot & VM_USER; - - if (!as) - { - as = user ? get_current_address_space() : &kernel_address_space; - assert(as != nullptr); - } - - uint64_t *ptentry; - - if (!riscv_get_pt_entry((void *) virt, &ptentry, true, as)) - return nullptr; - - uint64_t page_prots = vm_prots_to_mmu(prot); - bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()); - - if (special_mapping) - page_prots |= RISCV_MMU_SPECIAL; - - if (prot & VM_DONT_MAP_OVER && *ptentry & RISCV_MMU_VALID) - return (void *) virt; - - uint64_t old = *ptentry; - *ptentry = riscv_pt_page_mapping(phys) | page_prots; - if (pte_empty(old)) - { - increment_vm_stat(as, resident_set_size, PAGE_SIZE); - } - else - { - __native_tlb_invalidate_page((void *) PML_EXTRACT_ADDRESS(*ptentry)); - } - - if (!vma_is_pfnmap(vma)) - { - if (!pte_empty(old) && !pte_special(old)) - { - /* If old was a thing, decrement the mapcount */ - struct page *oldp = phys_to_page(PML_EXTRACT_ADDRESS(old)); - page_sub_mapcount(oldp); - } - - struct page *newp = phys_to_page(phys); - if (!special_mapping) - page_add_mapcount(newp); - } - - return (void *) virt; -} - bool pml_is_empty(const PML *pml) { for (int i = 0; i < 512; i++) @@ -269,52 +210,6 @@ bool pml_is_empty(const PML *pml) return true; } -struct pt_location -{ - PML *table; - unsigned int index; -}; - -bool riscv_get_pt_entry_with_ptables(void *addr, uint64_t **entry_ptr, struct mm_address_space *mm, - struct pt_location location[4]) -{ - unsigned long virt = (unsigned long) addr; - unsigned int indices[riscv_max_paging_levels]; - - for (unsigned int i = 0; i < riscv_paging_levels; i++) - { - indices[i] = (virt >> 12) >> (i * 9) & 0x1ff; - location[4 - 1 - i].index = indices[i]; - } - - PML *pml = (PML *) ((unsigned long) mm->arch_mmu.top_pt + PHYS_BASE); - unsigned int location_index = 0; - - for (unsigned int i = riscv_paging_levels; i != 1; i--) - { - uint64_t entry = pml->entries[indices[i - 1]]; - location[location_index].table = pml; - location[location_index++].index = indices[i - 1]; - - if (entry & RISCV_MMU_VALID) - { - void *page = (void *) PML_EXTRACT_ADDRESS(entry); - pml = (PML *) PHYS_TO_VIRT(page); - } - else - { - return false; - } - } - - location[location_index].table = pml; - location[location_index++].index = indices[0]; - - *entry_ptr = &pml->entries[indices[0]]; - - return true; -} - /** * @brief Clone the architecture specific part of an address space * @@ -355,92 +250,6 @@ static void dump_pt(PML *pt) printk("%016lx\n", entry); } -bool riscv_get_pt_entry(void *addr, uint64_t **entry_ptr, bool may_create_path, - struct mm_address_space *mm) -{ - unsigned long virt = (unsigned long) addr; - unsigned int indices[riscv_max_paging_levels]; - - addr_to_indices(virt, indices); - - PML *pml = (PML *) ((unsigned long) mm->arch_mmu.top_pt + PHYS_BASE); - - for (unsigned int i = riscv_paging_levels; i != 1; i--) - { - uint64_t entry = pml->entries[indices[i - 1]]; - if (entry & RISCV_MMU_VALID) - { - void *page = (void *) PML_EXTRACT_ADDRESS(entry); - pml = (PML *) PHYS_TO_VIRT(page); - } - else - { - if (!may_create_path) - return false; - - PML *pt = alloc_pt(); - - if (!pt) - return false; - increment_vm_stat(mm, page_tables_size, PAGE_SIZE); - - pml->entries[indices[i - 1]] = riscv_make_pt_entry_page_table(pt); - __asm__ __volatile__("sfence.vma zero, zero"); - - pml = (PML *) PHYS_TO_VIRT(pt); - } - } - - *entry_ptr = &pml->entries[indices[0]]; - - return true; -} - -bool __paging_change_perms(struct mm_address_space *mm, void *addr, int prot) -{ - MUST_HOLD_MUTEX(&mm->vm_lock); - - uint64_t *entry; - if (!riscv_get_pt_entry(addr, &entry, false, mm)) - { - return false; - } - - uint64_t pt_entry = *entry; - uint64_t perms = pt_entry & RISCV_MMU_FLAGS_TO_SAVE_ON_MPROTECT; - uint64_t page = PML_EXTRACT_ADDRESS(pt_entry); - - if (prot & VM_EXEC) - perms |= RISCV_MMU_EXECUTE; - if (prot & VM_WRITE) - perms |= RISCV_MMU_WRITE; - if (prot & VM_READ) - perms |= RISCV_MMU_VALID | RISCV_MMU_READ; - *entry = perms | page; - - return true; -} - -bool paging_change_perms(void *addr, int prot) -{ - struct mm_address_space *as = &kernel_address_space; - if ((unsigned long) addr < VM_HIGHER_HALF) - as = get_current_address_space(); - - return __paging_change_perms(as, addr, prot); -} - -bool paging_write_protect(void *addr, struct mm_address_space *mm) -{ - uint64_t *ptentry; - if (!riscv_get_pt_entry(addr, &ptentry, false, mm)) - return false; - - *ptentry = *ptentry & ~RISCV_MMU_WRITE; - - return true; -} - int is_invalid_arch_range(void *address, size_t pages) { unsigned long addr = (unsigned long) address; @@ -508,22 +317,6 @@ void paging_invalidate(void *page, size_t pages) } } -/** - * @brief Directly maps a page into the paging tables. - * - * @param as The target address space. - * @param virt The virtual address. - * @param phys The physical address of the page. - * @param prot Desired protection flags. - * @param vma VMA for this mapping (optional) - * @return NULL if out of memory, else virt. - */ -void *vm_map_page(struct mm_address_space *as, uint64_t virt, uint64_t phys, uint64_t prot, - struct vm_area_struct *vma) -{ - return paging_map_phys_to_virt(as, virt, phys, prot, vma); -} - void paging_free_pml2(PML *pml) { for (int i = 0; i < 512; i++) @@ -575,51 +368,6 @@ void paging_free_page_tables(struct mm_address_space *mm) free_page(phys_to_page((unsigned long) mm->arch_mmu.top_pt)); } -unsigned long get_mapping_info(void *addr) -{ - struct mm_address_space *as = &kernel_address_space; - if ((unsigned long) addr < VM_HIGHER_HALF) - as = get_current_address_space(); - - return __get_mapping_info(addr, as); -} - -unsigned long __get_mapping_info(void *addr, struct mm_address_space *as) -{ - unsigned long *ppt_entry; - // TODO: Recognize hugepages here - if (!riscv_get_pt_entry(addr, &ppt_entry, false, as)) - return PAGE_NOT_PRESENT; - - unsigned long pt_entry = *ppt_entry; - - unsigned long ret = 0; - - if (pt_entry & RISCV_MMU_VALID) - ret |= PAGE_PRESENT; - else - { - return PAGE_NOT_PRESENT; - } - - if (pt_entry & RISCV_MMU_USER) - ret |= PAGE_USER; - if (pt_entry & RISCV_MMU_WRITE) - ret |= PAGE_WRITABLE; - if (pt_entry & RISCV_MMU_EXECUTE) - ret |= PAGE_EXECUTABLE; - if (pt_entry & RISCV_MMU_DIRTY) - ret |= PAGE_DIRTY; - if (pt_entry & RISCV_MMU_ACCESSED) - ret |= PAGE_ACCESSED; - if (pt_entry & RISCV_MMU_GLOBAL) - ret |= PAGE_GLOBAL; - - ret |= PML_EXTRACT_ADDRESS(pt_entry); - - return ret; -} - /** * @brief Free the architecture dependent parts of the address space. * Called on address space destruction. @@ -651,158 +399,6 @@ void vm_save_current_mmu(struct mm_address_space *mm) mm->arch_mmu.top_pt = get_current_page_tables(); } -/** - * @brief Directly mprotect a page in the paging tables. - * Called by core MM code and should not be used outside of it. - * This function handles any edge cases like trying to re-apply write perms on - * a write-protected page. - * - * @param as The target address space. - * @param addr The virtual address of the page. - * @param old_prots The old protection flags. - * @param new_prots The new protection flags. - */ -void vm_mmu_mprotect_page(struct mm_address_space *as, void *addr, int old_prots, int new_prots) -{ - uint64_t *ptentry; - if (!riscv_get_pt_entry(addr, &ptentry, false, as)) - return; - - if (!*ptentry) - return; - - /* Make sure we don't accidentally mark a page as writable when - * it's write-protected and we're changing some other bits. - * For example: mprotect(PROT_EXEC) on a COW'd supposedly writable - * page would try to re-apply the writable permission. - */ - - /* In this function, we use the old_prots parameter to know whether it was a write-protected - * page. - */ - bool is_wp_page = !(*ptentry & RISCV_MMU_WRITE) && old_prots & VM_WRITE; - - if (is_wp_page) - { - new_prots &= ~VM_WRITE; - // printk("NOT VM_WRITING\n"); - } - - // printk("new prots: %x\n", new_prots); - - unsigned long paddr = PML_EXTRACT_ADDRESS(*ptentry); - - uint64_t page_prots = vm_prots_to_mmu(new_prots); - *ptentry = riscv_pt_page_mapping(paddr) | page_prots; -} - -class page_table_iterator -{ -private: - unsigned long curr_addr_; - size_t length_; - -public: - struct mm_address_space *as_; - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - bool debug; -#endif - - page_table_iterator(unsigned long virt, size_t len, struct mm_address_space *as) - : curr_addr_{virt}, length_{len}, as_{as} - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - , - debug{false} -#endif - - { - } - - size_t length() const - { - return length_; - } - - unsigned long curr_addr() const - { - return curr_addr_; - } - - void adjust_length(size_t size) - { - if (size > length_) - { - length_ = 0; - curr_addr_ += length_; - } - else - { - length_ -= size; - curr_addr_ += size; - } - } -}; - -struct tlb_invalidation_tracker -{ - unsigned long virt_start; - unsigned long virt_end; - bool is_started, is_flushed; - - explicit tlb_invalidation_tracker() : virt_start{}, virt_end{}, is_started{}, is_flushed{} - { - } - - void invalidate_tracker() - { - virt_start = 0xDEADDAD; - virt_end = 0xB0; - is_started = false; - is_flushed = false; - } - - void flush() - { - if (!is_started) - return; - - vm_invalidate_range(virt_start, (virt_end - virt_start) >> PAGE_SHIFT); - invalidate_tracker(); - } - - constexpr void init(unsigned long vaddr, size_t size) - { - is_started = true; - virt_start = vaddr; - virt_end = vaddr + size; - is_flushed = false; - } - - void add_page(unsigned long vaddr, size_t size) - { - /* If we've already started on a run of pages and this one is contiguous, just set the tail - */ - if (is_started && virt_end == vaddr) - { - virt_end = vaddr + size; - } - else - { - /* Else, try flushing if is_started == true and restart the page run */ - flush(); - init(vaddr, size); - } - } - - ~tlb_invalidation_tracker() - { - if (is_started && !is_flushed) - flush(); - } -}; - enum page_table_levels : unsigned int { PT_LEVEL, @@ -833,135 +429,6 @@ constexpr unsigned int addr_get_index(unsigned long virt, unsigned int pt_level) return (virt >> 12) >> (pt_level * 9) & 0x1ff; } -#define MMU_UNMAP_CAN_FREE_PML 1 -#define MMU_UNMAP_OK 0 - -static int riscv_mmu_unmap(PML *table, unsigned int pt_level, page_table_iterator &it, - struct vm_area_struct *vma) -{ - unsigned int index = addr_get_index(it.curr_addr(), pt_level); - - /* Get the size that each entry represents here */ - auto entry_size = level_to_entry_size(pt_level); - - tlb_invalidation_tracker invd_tracker; - unsigned int i; - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - if (it.debug) - { - printk("level %u - index %x\n", pt_level, index); - } -#endif - - for (i = index; i < PAGE_TABLE_ENTRIES && it.length(); i++) - { - auto &pt_entry = table->entries[i]; - bool is_pte_empty = pte_empty(pt_entry); - - if (is_pte_empty) - { - -#ifdef CONFIG_RISCV_MMU_UNMAP_DEBUG - if (it.debug) - printk("not present @ level %u\nentry size %lu\nlength %lu\n", pt_level, entry_size, - it.length()); -#endif - auto to_skip = entry_size - (it.curr_addr() & (entry_size - 1)); - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - if (it.debug) - { - printk("[level %u]: Skipping from %lx to %lx\n", pt_level, it.curr_addr(), - it.curr_addr() + to_skip); - } -#endif - - it.adjust_length(to_skip); - continue; - } - - bool is_huge_page = is_huge_page_level(pt_level) && pt_entry_is_huge(pt_entry); - - if (pt_level == PT_LEVEL || is_huge_page) - { - /* TODO: Handle huge page splitting */ - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - if (it.debug) - printk("Unmapping %lx\n", it.curr_addr()); -#endif - - unsigned long val = 0; - __atomic_exchange(&pt_entry, &val, &val, __ATOMIC_RELEASE); - - if (val & RISCV_MMU_ACCESSED) - invd_tracker.add_page(it.curr_addr(), entry_size); - - if (!vma_is_pfnmap(vma) && !pte_special(val)) - { - struct page *oldp = phys_to_page(PML_EXTRACT_ADDRESS(val)); - page_sub_mapcount(oldp); - } - - it.adjust_length(entry_size); - decrement_vm_stat(it.as_, resident_set_size, entry_size); - } - else - { - assert((pt_entry & RISCV_MMU_VALID) != 0); - PML *next_table = (PML *) PHYS_TO_VIRT(PML_EXTRACT_ADDRESS(pt_entry)); - int st = riscv_mmu_unmap(next_table, pt_level - 1, it, vma); - - if (st == MMU_UNMAP_CAN_FREE_PML) - { - auto page = phys_to_page(PML_EXTRACT_ADDRESS(pt_entry)); - - pt_entry = 0; - - COMPILER_BARRIER(); - - free_page(page); - __atomic_sub_fetch(&allocated_page_tables, 1, __ATOMIC_RELAXED); - decrement_vm_stat(it.as_, page_tables_size, PAGE_SIZE); - } - } - } - - /* We can know that the table is 100% empty if we ran through the table */ - bool unmapped_whole_table = index == 0 && i == PAGE_TABLE_ENTRIES; - - /* Don't bother to free the PML or even check if it's empty if we're the top paging structure */ - if (pt_level != riscv_paging_levels - 1 && (unmapped_whole_table || pml_is_empty(table))) - { - return MMU_UNMAP_CAN_FREE_PML; - } - -#if 0 - printk("nr entries %lu\n", nr_entries); - - printk("unmapping %lu\n", it.length()); -#endif - - return MMU_UNMAP_OK; -} - -int vm_mmu_unmap(struct mm_address_space *as, void *addr, size_t pages, struct vm_area_struct *vma) -{ - unsigned long virt = (unsigned long) addr; - size_t size = pages << PAGE_SHIFT; - - page_table_iterator it{virt, size, as}; - - PML *first_level = (PML *) PHYS_TO_VIRT(as->arch_mmu.top_pt); - - riscv_mmu_unmap(first_level, riscv_paging_levels - 1, it, vma); - - assert(it.length() == 0); - - return 0; -} - static inline bool is_higher_half(unsigned long address) { return address >= VM_HIGHER_HALF; @@ -1040,14 +507,12 @@ static void mmu_acct_page_table(PML *pt, page_table_levels level, mmu_acct &acct { acct.page_table_size += PAGE_SIZE; - for (const auto pte : pt->entries) + for (int i = 0; i < (level == PML4_LEVEL ? 256 : 512); i++) { + u64 pte = pt->entries[i]; if (pte_empty(pte)) continue; - if (!(pte & RISCV_MMU_USER)) - continue; - if (level != PT_LEVEL) { mmu_acct_page_table((PML *) PHYS_TO_VIRT(PML_EXTRACT_ADDRESS(pte)), @@ -1074,147 +539,3 @@ void mmu_verify_address_space_accounting(mm_address_space *as) assert(acct.page_table_size == as->page_tables_size); assert(acct.resident_set_size == as->resident_set_size); } - -static int riscv_mmu_fork(PML *parent_table, PML *child_table, unsigned int pt_level, - page_table_iterator &it, struct vm_area_struct *old_region) -{ - unsigned int index = addr_get_index(it.curr_addr(), pt_level); - - /* Get the size that each entry represents here */ - auto entry_size = level_to_entry_size(pt_level); - - unsigned int i; - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - if (it.debug) - { - printk("level %u - index %x\n", pt_level, index); - } -#endif - tlb_invalidation_tracker invd_tracker; - - for (i = index; i < PAGE_TABLE_ENTRIES && it.length(); i++) - { - const u64 pt_entry = parent_table->entries[i]; - bool pte_empty = pt_entry == 0; - - if (pte_empty) - { - -#ifdef CONFIG_X86_MMU_UNMAP_DEBUG - if (it.debug) - printk("not present @ level %u\nentry size %lu\nlength %lu\n", pt_level, entry_size, - it.length()); -#endif - auto to_skip = entry_size - (it.curr_addr() & (entry_size - 1)); - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - if (it.debug) - { - printk("[level %u]: Skipping from %lx to %lx\n", pt_level, it.curr_addr(), - it.curr_addr() + to_skip); - } -#endif - - it.adjust_length(to_skip); - continue; - } - - bool is_huge_page = is_huge_page_level(pt_level) && pt_entry_is_huge(pt_entry); - - if (pt_level == PT_LEVEL || is_huge_page) - { - const bool should_cow = old_region->vm_maptype == MAP_PRIVATE; - child_table->entries[i] = pt_entry & (should_cow ? ~RISCV_MMU_WRITE : ~0UL); - if (!vma_is_pfnmap(old_region) && !pte_special(pt_entry)) - page_add_mapcount(phys_to_page(PML_EXTRACT_ADDRESS(pt_entry))); - if (should_cow) - { - /* Write-protect the parent's page too. Make sure to invalidate the TLB if we - * downgraded permissions. - */ - __atomic_store_n(&parent_table->entries[i], pt_entry & ~RISCV_MMU_WRITE, - __ATOMIC_RELAXED); - - if (pt_entry & RISCV_MMU_WRITE) - invd_tracker.add_page(it.curr_addr(), entry_size); - } - - increment_vm_stat(it.as_, resident_set_size, entry_size); - it.adjust_length(entry_size); - } - else - { - assert((pt_entry & RISCV_MMU_VALID) != 0); - - PML *old = (PML *) PHYS_TO_VIRT(PML_EXTRACT_ADDRESS(pt_entry)); - PML *child_pt = (PML *) PHYS_TO_VIRT(PML_EXTRACT_ADDRESS(child_table->entries[i])); - - if (child_table->entries[i] != 0) - { - /* Allocate a new page table for the child process */ - PML *copy = (PML *) alloc_pt(); - if (!copy) - return -ENOMEM; - - increment_vm_stat(it.as_, page_tables_size, PAGE_SIZE); - - const unsigned long old_prots = pt_entry & RISCV_PAGING_PROT_BITS; - /* Set the PTE */ - child_table->entries[i] = (unsigned long) copy | old_prots; - child_pt = (PML *) PHYS_TO_VIRT(copy); - } - - int st = riscv_mmu_fork(old, child_pt, pt_level - 1, it, old_region); - - if (st < 0) - { - return st; - } - } - } - - return 0; -} - -/** - * @brief Fork MMU page tables - * - * @param old_region Old vm_area_struct - * @param addr_space Current address space - * @return 0 on success, negative error codes - */ -int mmu_fork_tables(struct vm_area_struct *old_region, struct mm_address_space *addr_space) -{ - page_table_iterator it{old_region->vm_start, vma_pages(old_region) << PAGE_SHIFT, addr_space}; - - return riscv_mmu_fork((PML *) PHYS_TO_VIRT(old_region->vm_mm->arch_mmu.top_pt), - (PML *) PHYS_TO_VIRT(addr_space->arch_mmu.top_pt), - riscv_paging_levels - 1, it, old_region); -} - -unsigned int mmu_get_clear_referenced(struct mm_address_space *mm, void *addr, struct page *page) -{ - scoped_lock g{mm->page_table_lock}; - - u64 *ptep; - if (!riscv_get_pt_entry(addr, &ptep, false, mm)) - return 0; - - u64 pte = READ_ONCE(*ptep); - u64 new_pte; - do - { - if (!(pte & RISCV_MMU_ACCESSED)) - return 0; - if (PML_EXTRACT_ADDRESS(pte) != (unsigned long) page_to_phys(page)) - return 0; - new_pte = pte & ~RISCV_MMU_ACCESSED; - } while (!__atomic_compare_exchange_n(ptep, &pte, new_pte, false, __ATOMIC_RELAXED, - __ATOMIC_RELAXED)); - /* Architectural note: We don't need to flush the TLB. Flushing the TLB is required by riscv if - * we want the A bit to be set again, but we can just wait for an unrelated TLB flush (e.g - * context switch) to do the job for us. A TLB shootdown is too much overhead for this purpose. - */ - return 1; -} diff --git a/kernel/arch/riscv64/scheduler.cpp b/kernel/arch/riscv64/scheduler.cpp index 7d4b047ae..2257f7b5e 100644 --- a/kernel/arch/riscv64/scheduler.cpp +++ b/kernel/arch/riscv64/scheduler.cpp @@ -1,11 +1,12 @@ /* - * Copyright (c) 2022 - 2023 Pedro Falcato + * Copyright (c) 2022 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * * SPDX-License-Identifier: GPL-2.0-only */ +#include #include #include #include @@ -191,8 +192,8 @@ int process_alloc_stack(struct stack_info *info) { void *ptr = vm_mmap(nullptr, info->length, PROT_WRITE | PROT_READ, MAP_ANON | MAP_PRIVATE, nullptr, 0); - if (!ptr) - return -ENOMEM; + if (IS_ERR(ptr)) + return PTR_ERR(ptr); info->base = ptr; info->top = reinterpret_cast((unsigned long) ptr + info->length); diff --git a/kernel/arch/x86_64/mmu.cpp b/kernel/arch/x86_64/mmu.cpp index 3ecf8d8a4..4a72602a7 100644 --- a/kernel/arch/x86_64/mmu.cpp +++ b/kernel/arch/x86_64/mmu.cpp @@ -45,7 +45,11 @@ unsigned long __x86_phys_base_limit = X86_PHYS_BASE_LIMIT_4L; unsigned long __x86_vm_higher_half = X86_VM_HIGHER_HALF_4L; unsigned long __x86_low_half_max = 0x00007fffffffffff; -static CONST_LA48 unsigned int x86_paging_levels = 4; +extern "C" +{ +CONST_LA48 unsigned int x86_paging_levels = 4; +} + static const unsigned int x86_max_paging_levels = 5; #define X86_CACHING_BITS(index) ((((index) &0x3) << 3) | (((index >> 2) & 1) << 7)) @@ -74,9 +78,6 @@ static const unsigned int x86_max_paging_levels = 5; X86_PAGING_DIRTY | X86_PAGING_WRITETHROUGH | X86_PAGING_PCD | X86_PAGING_PAT | \ X86_PAGING_SPECIAL) -static void *paging_map_phys_to_virt(struct mm_address_space *as, uint64_t virt, uint64_t phys, - uint64_t prot, struct vm_area_struct *vma); - __always_inline bool x86_is_pml5_enabled() { return x86_paging_levels == 5; @@ -97,8 +98,6 @@ static inline bool pte_special(u64 pte) return pte & X86_PAGING_SPECIAL; } -bool x86_get_pt_entry(void *addr, uint64_t **entry_ptr, struct mm_address_space *mm); - static inline uint64_t make_pml4e(uint64_t base, uint64_t avl, uint64_t pcd, uint64_t pwt, uint64_t us, uint64_t rw, uint64_t p) { @@ -228,10 +227,9 @@ void *x86_placement_map(unsigned long _phys) kernel_address_space.arch_mmu.cr3 = get_current_pgd(); /* Map two pages so memory that spans both pages can get accessed */ - paging_map_phys_to_virt(&kernel_address_space, placement_mappings_start, phys, - VM_READ | VM_WRITE, nullptr); - paging_map_phys_to_virt(&kernel_address_space, placement_mappings_start + PAGE_SIZE, - phys + PAGE_SIZE, VM_READ | VM_WRITE, nullptr); + vm_map_page(&kernel_address_space, placement_mappings_start, phys, VM_READ | VM_WRITE, nullptr); + vm_map_page(&kernel_address_space, placement_mappings_start + PAGE_SIZE, phys + PAGE_SIZE, + VM_READ | VM_WRITE, nullptr); __native_tlb_invalidate_page((void *) placement_mappings_start); __native_tlb_invalidate_page((void *) (placement_mappings_start + PAGE_SIZE)); return (void *) (placement_mappings_start + (_phys - phys)); @@ -281,6 +279,11 @@ void x86_setup_placement_mappings(void) } } +extern "C" +{ +unsigned int pgd_shift = 39, p4d_ptrs = 1; +} + NO_ASAN void paging_init(void) { @@ -290,6 +293,8 @@ void paging_init(void) __asm__ __volatile__("int3"); #else x86_paging_levels = 5; + pgd_shift = 48; + p4d_ptrs = 512; __x86_phys_base = X86_PHYS_BASE_5L; __x86_phys_base_limit = X86_PHYS_BASE_LIMIT_5L; __x86_vm_higher_half = X86_VM_HIGHER_HALF_5L; @@ -394,94 +399,6 @@ void paging_map_all_phys() __native_tlb_invalidate_page((void *) (virt + i * 0x40000000)); } -static void *paging_map_phys_to_virt(struct mm_address_space *as, uint64_t virt, uint64_t phys, - uint64_t prot, struct vm_area_struct *vma) -{ - bool user = prot & VM_USER; - const bool ispfnmap = vma_is_pfnmap(vma); - - if (!as) - { - as = user ? get_current_address_space() : &kernel_address_space; - assert(as != nullptr); - } - - scoped_lock g{as->page_table_lock}; - - unsigned int indices[x86_max_paging_levels]; - - /* Note: page table flags are different from page perms because a page table's - * permissions apply throughout the whole table. - * Because of that, the PT's flags are Present | Write | (possible User) - */ - uint64_t page_table_flags = - X86_PAGING_PRESENT | X86_PAGING_WRITE | (user ? X86_PAGING_USER : 0); - - x86_addr_to_indices(virt, indices); - - PML *pml = (PML *) PHYS_TO_VIRT(as->arch_mmu.cr3); - - for (unsigned int i = x86_paging_levels; i != 1; i--) - { - uint64_t entry = pml->entries[indices[i - 1]]; - if (entry & X86_PAGING_PRESENT) - { - void *page = (void *) PML_EXTRACT_ADDRESS(entry); - pml = (PML *) PHYS_TO_VIRT(page); - } - else - { - assert(entry == 0); - void *page = alloc_pt(); - if (!page) - return nullptr; - - increment_vm_stat(as, page_tables_size, PAGE_SIZE); - pml->entries[indices[i - 1]] = (uint64_t) page | page_table_flags; - pml = (PML *) PHYS_TO_VIRT(page); - } - } - - bool noexec = !(prot & VM_EXEC); - bool global = !user; - bool write = prot & VM_WRITE; - bool readable = prot & (VM_READ | VM_WRITE) || !noexec; - unsigned int cache_type = vm_prot_to_cache_type(prot); - uint8_t caching_bits = cache_to_paging_bits(cache_type); - bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()); - - uint64_t page_prots = (noexec ? X86_PAGING_NX : 0) | (global ? X86_PAGING_GLOBAL : 0) | - (user ? X86_PAGING_USER : 0) | (write ? X86_PAGING_WRITE : 0) | - X86_CACHING_BITS(caching_bits) | (readable ? X86_PAGING_PRESENT : 0) | - (special_mapping ? X86_PAGING_SPECIAL : 0); - - if (prot & VM_DONT_MAP_OVER && pml->entries[indices[0]] & X86_PAGING_PRESENT) - return (void *) virt; - - uint64_t old = pml->entries[indices[0]]; - - pml->entries[indices[0]] = phys | page_prots; - - if (x86_pte_empty(old)) - increment_vm_stat(as, resident_set_size, PAGE_SIZE); - - if (!ispfnmap) - { - if (!x86_pte_empty(old) && !pte_special(old)) - { - /* If old was a thing, decrement the mapcount */ - struct page *oldp = phys_to_page(PML_EXTRACT_ADDRESS(old)); - page_sub_mapcount(oldp); - } - - struct page *newp = phys_to_page(phys); - if (!special_mapping) - page_add_mapcount(newp); - } - - return (void *) virt; -} - bool pml_is_empty(const PML *pml) { for (int i = 0; i < 512; i++) @@ -493,52 +410,6 @@ bool pml_is_empty(const PML *pml) return true; } -struct pt_location -{ - PML *table; - unsigned int index; -}; - -bool x86_get_pt_entry_with_ptables(void *addr, uint64_t **entry_ptr, struct mm_address_space *mm, - struct pt_location location[4]) -{ - unsigned long virt = (unsigned long) addr; - unsigned int indices[x86_max_paging_levels]; - - for (unsigned int i = 0; i < x86_paging_levels; i++) - { - indices[i] = (virt >> 12) >> (i * 9) & 0x1ff; - location[4 - 1 - i].index = indices[i]; - } - - PML *pml = (PML *) ((unsigned long) mm->arch_mmu.cr3 + PHYS_BASE); - unsigned int location_index = 0; - - for (unsigned int i = x86_paging_levels; i != 1; i--) - { - uint64_t entry = pml->entries[indices[i - 1]]; - location[location_index].table = pml; - location[location_index++].index = indices[i - 1]; - - if (entry & X86_PAGING_PRESENT) - { - void *page = (void *) PML_EXTRACT_ADDRESS(entry); - pml = (PML *) PHYS_TO_VIRT(page); - } - else - { - return false; - } - } - - location[location_index].table = pml; - location[location_index++].index = indices[0]; - - *entry_ptr = &pml->entries[indices[0]]; - - return true; -} - /** * @brief Clone the architecture specific part of an address space * @@ -579,81 +450,6 @@ void paging_load_cr3(PML *pml) __asm__ __volatile__("movq %0, %%cr3" ::"r"(pml)); } -bool x86_get_pt_entry(void *addr, uint64_t **entry_ptr, struct mm_address_space *mm) -{ - unsigned long virt = (unsigned long) addr; - unsigned int indices[x86_max_paging_levels]; - - x86_addr_to_indices(virt, indices); - - PML *pml = (PML *) ((unsigned long) mm->arch_mmu.cr3 + PHYS_BASE); - - for (unsigned int i = x86_paging_levels; i != 1; i--) - { - uint64_t entry = pml->entries[indices[i - 1]]; - if (entry & X86_PAGING_PRESENT) - { - void *page = (void *) PML_EXTRACT_ADDRESS(entry); - pml = (PML *) PHYS_TO_VIRT(page); - } - else - { - return false; - } - } - - *entry_ptr = &pml->entries[indices[0]]; - - return true; -} - -bool __paging_change_perms(struct mm_address_space *mm, void *addr, int prot) -{ - scoped_lock g{mm->page_table_lock}; - - uint64_t *entry; - if (!x86_get_pt_entry(addr, &entry, mm)) - { - return false; - } - - uint64_t pt_entry = *entry; - uint64_t perms = pt_entry & X86_PAGING_FLAGS_TO_SAVE_ON_MPROTECT; - uint64_t page = PML_EXTRACT_ADDRESS(pt_entry); - - if (!(prot & VM_EXEC)) - perms |= X86_PAGING_NX; - if (prot & VM_WRITE) - perms |= X86_PAGING_WRITE; - if (prot & VM_READ) - perms |= X86_PAGING_PRESENT; - *entry = perms | page; - - return true; -} - -bool paging_change_perms(void *addr, int prot) -{ - struct mm_address_space *as = &kernel_address_space; - if ((unsigned long) addr < VM_HIGHER_HALF) - as = get_current_address_space(); - - return __paging_change_perms(as, addr, prot); -} - -bool paging_write_protect(void *addr, struct mm_address_space *mm) -{ - scoped_lock g{mm->page_table_lock}; - - uint64_t *ptentry; - if (!x86_get_pt_entry(addr, &ptentry, mm)) - return false; - - *ptentry = *ptentry & ~X86_PAGING_WRITE; - - return true; -} - int is_invalid_arch_range(void *address, size_t pages) { unsigned long addr = (unsigned long) address; @@ -749,22 +545,6 @@ void paging_invalidate(void *page, size_t pages) } } -/** - * @brief Directly maps a page into the paging tables. - * - * @param as The target address space. - * @param virt The virtual address. - * @param phys The physical address of the page. - * @param prot Desired protection flags. - * @param vma VMA for this mapping (optional) - * @return NULL if out of memory, else virt. - */ -void *vm_map_page(struct mm_address_space *as, uint64_t virt, uint64_t phys, uint64_t prot, - struct vm_area_struct *vma) -{ - return paging_map_phys_to_virt(as, virt, phys, prot, vma); -} - void paging_free_pml2(PML *pml) { for (int i = 0; i < 512; i++) @@ -846,171 +626,6 @@ void vm_save_current_mmu(struct mm_address_space *mm) mm->arch_mmu.cr3 = get_current_pgd(); } -/** - * @brief Directly mprotect a page in the paging tables. - * Called by core MM code and should not be used outside of it. - * This function handles any edge cases like trying to re-apply write perms on - * a write-protected page. - * - * @param as The target address space. - * @param addr The virtual address of the page. - * @param old_prots The old protection flags. - * @param new_prots The new protection flags. - */ -void vm_mmu_mprotect_page(struct mm_address_space *as, void *addr, int old_prots, int new_prots) -{ - scoped_lock g{as->page_table_lock}; - - uint64_t *ptentry; - if (!x86_get_pt_entry(addr, &ptentry, as)) - return; - - if (!*ptentry) - return; - - /* Make sure we don't accidentally mark a page as writable when - * it's write-protected and we're changing some other bits. - * For example: mprotect(PROT_EXEC) on a COW'd supposedly writable - * page would try to re-apply the writable permission. - */ - - /* In this function, we use the old_prots parameter to know whether it was a write-protected - * page. - */ - bool is_wp_page = !(*ptentry & X86_PAGING_WRITE) && old_prots & VM_WRITE; - - if (is_wp_page) - { - new_prots &= ~VM_WRITE; - // printk("NOT VM_WRITING\n"); - } - - // printk("new prots: %x\n", new_prots); - - unsigned long paddr = PML_EXTRACT_ADDRESS(*ptentry); - bool noexec = !(new_prots & VM_EXEC); - bool global = new_prots & VM_USER ? false : true; - bool user = new_prots & VM_USER ? true : false; - bool write = new_prots & VM_WRITE ? true : false; - bool readable = new_prots & (VM_READ | VM_WRITE) || !noexec; - - unsigned int cache_type = vm_prot_to_cache_type(new_prots); - uint8_t caching_bits = cache_to_paging_bits(cache_type); - - uint64_t page_prots = (noexec ? X86_PAGING_NX : 0) | (global ? X86_PAGING_GLOBAL : 0) | - (user ? X86_PAGING_USER : 0) | (write ? X86_PAGING_WRITE : 0) | - X86_CACHING_BITS(caching_bits) | (readable ? X86_PAGING_PRESENT : 0); - *ptentry = paddr | page_prots; -} - -class page_table_iterator -{ -private: - unsigned long curr_addr_; - size_t length_; - -public: - struct mm_address_space *as_; - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - bool debug; -#endif - - page_table_iterator(unsigned long virt, size_t len, struct mm_address_space *as) - : curr_addr_{virt}, length_{len}, as_{as} - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - , - debug{false} -#endif - - { - } - - size_t length() const - { - return length_; - } - - unsigned long curr_addr() const - { - return curr_addr_; - } - - void adjust_length(size_t size) - { - if (size > length_) - { - length_ = 0; - curr_addr_ += length_; - } - else - { - length_ -= size; - curr_addr_ += size; - } - } -}; - -struct tlb_invalidation_tracker -{ - unsigned long virt_start; - unsigned long virt_end; - bool is_started, is_flushed; - - explicit tlb_invalidation_tracker() : virt_start{}, virt_end{}, is_started{}, is_flushed{} - { - } - - void invalidate_tracker() - { - virt_start = 0xDEADDAD; - virt_end = 0xB0; - is_started = false; - is_flushed = false; - } - - void flush() - { - if (!is_started) - return; - - vm_invalidate_range(virt_start, (virt_end - virt_start) >> PAGE_SHIFT); - invalidate_tracker(); - } - - constexpr void init(unsigned long vaddr, size_t size) - { - is_started = true; - virt_start = vaddr; - virt_end = vaddr + size; - is_flushed = false; - } - - void add_page(unsigned long vaddr, size_t size) - { - /* If we've already started on a run of pages and this one is contiguous, just set the - * tail - */ - if (is_started && virt_end == vaddr) - { - virt_end = vaddr + size; - } - else - { - /* Else, try flushing if is_started == true and restart the page run */ - flush(); - init(vaddr, size); - } - } - - ~tlb_invalidation_tracker() - { - if (is_started && !is_flushed) - flush(); - } -}; - enum x86_page_table_levels : unsigned int { PT_LEVEL, @@ -1042,257 +657,6 @@ constexpr unsigned int addr_get_index(unsigned long virt, unsigned int pt_level) return (virt >> 12) >> (pt_level * 9) & 0x1ff; } -#define MMU_UNMAP_CAN_FREE_PML 1 -#define MMU_UNMAP_OK 0 - -static int x86_mmu_unmap(PML *table, unsigned int pt_level, page_table_iterator &it, - struct vm_area_struct *vma) -{ - const bool ispfnmap = vma_is_pfnmap(vma); - unsigned int index = addr_get_index(it.curr_addr(), pt_level); - - /* Get the size that each entry represents here */ - auto entry_size = level_to_entry_size(pt_level); - - tlb_invalidation_tracker invd_tracker; - unsigned int i; - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - if (it.debug) - { - printk("level %u - index %x\n", pt_level, index); - } -#endif - - for (i = index; i < PAGE_TABLE_ENTRIES && it.length(); i++) - { - auto &pt_entry = table->entries[i]; - bool pte_empty = x86_pte_empty(pt_entry); - - if (pte_empty) - { - -#ifdef CONFIG_X86_MMU_UNMAP_DEBUG - if (it.debug) - printk("not present @ level %u\nentry size %lu\nlength %lu\n", pt_level, entry_size, - it.length()); -#endif - auto to_skip = entry_size - (it.curr_addr() & (entry_size - 1)); - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - if (it.debug) - { - printk("[level %u]: Skipping from %lx to %lx\n", pt_level, it.curr_addr(), - it.curr_addr() + to_skip); - } -#endif - - it.adjust_length(to_skip); - continue; - } - - bool is_huge_page = is_huge_page_level(pt_level) && pt_entry & X86_PAGING_HUGE; - - if (pt_level == PT_LEVEL || is_huge_page) - { - /* TODO: Handle huge page splitting */ - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - if (it.debug) - printk("Unmapping %lx\n", it.curr_addr()); -#endif - - unsigned long val = 0; - __atomic_exchange(&pt_entry, &val, &val, __ATOMIC_RELEASE); - - if (val & X86_PAGING_ACCESSED) - invd_tracker.add_page(it.curr_addr(), entry_size); - - if (!ispfnmap && !pte_special(val)) - { - struct page *oldp = phys_to_page(PML_EXTRACT_ADDRESS(val)); - page_sub_mapcount(oldp); - } - - it.adjust_length(entry_size); - decrement_vm_stat(it.as_, resident_set_size, entry_size); - } - else - { - assert((pt_entry & X86_PAGING_PRESENT) != 0); - PML *next_table = (PML *) PHYS_TO_VIRT(PML_EXTRACT_ADDRESS(pt_entry)); - int st = x86_mmu_unmap(next_table, pt_level - 1, it, vma); - - if (st == MMU_UNMAP_CAN_FREE_PML) - { - auto page = phys_to_page(PML_EXTRACT_ADDRESS(pt_entry)); - - pt_entry = 0; - - COMPILER_BARRIER(); - - free_page(page); - __atomic_sub_fetch(&allocated_page_tables, 1, __ATOMIC_RELAXED); - decrement_vm_stat(it.as_, page_tables_size, PAGE_SIZE); - } - } - } - - /* We can know that the table is 100% empty if we ran through the table */ - bool unmapped_whole_table = index == 0 && i == PAGE_TABLE_ENTRIES; - - /* Don't bother to free the PML or even check if it's empty if we're the top paging - * structure */ - if (pt_level != x86_paging_levels - 1 && (unmapped_whole_table || pml_is_empty(table))) - { - return MMU_UNMAP_CAN_FREE_PML; - } - -#if 0 - printk("nr entries %lu\n", nr_entries); - - printk("unmapping %lu\n", it.length()); -#endif - - return MMU_UNMAP_OK; -} - -int vm_mmu_unmap(struct mm_address_space *as, void *addr, size_t pages, struct vm_area_struct *vma) -{ - unsigned long virt = (unsigned long) addr; - size_t size = pages << PAGE_SHIFT; - scoped_lock g{as->page_table_lock}; - - page_table_iterator it{virt, size, as}; - - PML *first_level = (PML *) PHYS_TO_VIRT(as->arch_mmu.cr3); - - x86_mmu_unmap(first_level, x86_paging_levels - 1, it, vma); - - assert(it.length() == 0); - - return 0; -} - -static int x86_mmu_fork(PML *parent_table, PML *child_table, unsigned int pt_level, - page_table_iterator &it, struct vm_area_struct *old_region) -{ - const bool ispfnmap = vma_is_pfnmap(old_region); - unsigned int index = addr_get_index(it.curr_addr(), pt_level); - - /* Get the size that each entry represents here */ - auto entry_size = level_to_entry_size(pt_level); - - unsigned int i; - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - if (it.debug) - { - printk("level %u - index %x\n", pt_level, index); - } -#endif - tlb_invalidation_tracker invd_tracker; - - for (i = index; i < PAGE_TABLE_ENTRIES && it.length(); i++) - { - const u64 pt_entry = parent_table->entries[i]; - bool pte_empty = x86_pte_empty(pt_entry); - - if (pte_empty) - { - -#ifdef CONFIG_X86_MMU_UNMAP_DEBUG - if (it.debug) - printk("not present @ level %u\nentry size %lu\nlength %lu\n", pt_level, entry_size, - it.length()); -#endif - auto to_skip = entry_size - (it.curr_addr() & (entry_size - 1)); - -#ifdef CONFIG_PT_ITERATOR_HAVE_DEBUG - if (it.debug) - { - printk("[level %u]: Skipping from %lx to %lx\n", pt_level, it.curr_addr(), - it.curr_addr() + to_skip); - } -#endif - - it.adjust_length(to_skip); - continue; - } - - bool is_huge_page = is_huge_page_level(pt_level) && pt_entry & X86_PAGING_HUGE; - - if (pt_level == PT_LEVEL || is_huge_page) - { - const bool should_cow = old_region->vm_maptype == MAP_PRIVATE; - child_table->entries[i] = pt_entry & (should_cow ? ~X86_PAGING_WRITE : ~0UL); - if (!ispfnmap && !pte_special(pt_entry)) - page_add_mapcount(phys_to_page(PML_EXTRACT_ADDRESS(pt_entry))); - if (should_cow) - { - /* Write-protect the parent's page too. Make sure to invalidate the TLB if we - * downgraded permissions. - */ - __atomic_store_n(&parent_table->entries[i], pt_entry & ~X86_PAGING_WRITE, - __ATOMIC_RELAXED); - - if (pt_entry & X86_PAGING_WRITE) - invd_tracker.add_page(it.curr_addr(), entry_size); - } - - increment_vm_stat(it.as_, resident_set_size, entry_size); - it.adjust_length(entry_size); - } - else - { - assert((pt_entry & X86_PAGING_PRESENT) != 0); - - PML *old = (PML *) PHYS_TO_VIRT(PML_EXTRACT_ADDRESS(pt_entry)); - PML *child_pt = (PML *) PHYS_TO_VIRT(PML_EXTRACT_ADDRESS(child_table->entries[i])); - - if (x86_pte_empty(child_table->entries[i])) - { - /* Allocate a new page table for the child process */ - PML *copy = (PML *) alloc_pt(); - if (!copy) - return -ENOMEM; - - increment_vm_stat(it.as_, page_tables_size, PAGE_SIZE); - - const unsigned long old_prots = pt_entry & X86_PAGING_PROT_BITS; - /* Set the PTE */ - child_table->entries[i] = (unsigned long) copy | old_prots; - child_pt = (PML *) PHYS_TO_VIRT(copy); - } - - int st = x86_mmu_fork(old, child_pt, pt_level - 1, it, old_region); - - if (st < 0) - { - return st; - } - } - } - - return 0; -} - -/** - * @brief Fork MMU page tables - * - * @param old_region Old vm_area_struct - * @param addr_space Current address space - * @return 0 on success, negative error codes - */ -int mmu_fork_tables(struct vm_area_struct *old_region, struct mm_address_space *addr_space) -{ - page_table_iterator it{old_region->vm_start, vma_pages(old_region) << PAGE_SHIFT, addr_space}; - - return x86_mmu_fork((PML *) PHYS_TO_VIRT(old_region->vm_mm->arch_mmu.cr3), - (PML *) PHYS_TO_VIRT(addr_space->arch_mmu.cr3), x86_paging_levels - 1, it, - old_region); -} - static inline bool is_higher_half(unsigned long address) { return address >= VM_HIGHER_HALF; @@ -1418,109 +782,6 @@ void x86_remap_top_pgd_to_top_pgd(unsigned long source, unsigned long dest) __native_tlb_invalidate_all(); } -unsigned long get_mapping_info(void *addr) -{ - struct mm_address_space *as = &kernel_address_space; - if ((unsigned long) addr < VM_HIGHER_HALF) - as = get_current_address_space(); - - return __get_mapping_info(addr, as); -} - -static inline unsigned long pte_to_mapping_info(unsigned long pt_entry, bool hugepage, - unsigned long offset) -{ - unsigned long ret = 0; - if (pt_entry & X86_PAGING_PRESENT) - ret |= PAGE_PRESENT; - else - { - return PAGE_NOT_PRESENT; - } - - if (pt_entry & X86_PAGING_USER) - ret |= PAGE_USER; - if (pt_entry & X86_PAGING_WRITE) - ret |= PAGE_WRITABLE; - if (!(pt_entry & X86_PAGING_NX)) - ret |= PAGE_EXECUTABLE; - if (pt_entry & X86_PAGING_DIRTY) - ret |= PAGE_DIRTY; - if (pt_entry & X86_PAGING_ACCESSED) - ret |= PAGE_ACCESSED; - if (pt_entry & X86_PAGING_GLOBAL) - ret |= PAGE_GLOBAL; - if (hugepage) - ret |= PAGE_HUGE; - - ret |= PML_EXTRACT_ADDRESS(pt_entry); - ret |= offset; - - return ret; -} - -unsigned long __get_mapping_info(void *addr, struct mm_address_space *as) -{ - // TODO: Should we lock here? May be slow. - const unsigned long virt = (unsigned long) addr; - unsigned int indices[x86_max_paging_levels]; - - x86_addr_to_indices(virt, indices); - - PML *pml = (PML *) PHYS_TO_VIRT(as->arch_mmu.cr3); - for (unsigned i = x86_paging_levels; i != 1; i--) - { - unsigned long entry = pml->entries[indices[i - 1]]; - void *page = (void *) PML_EXTRACT_ADDRESS(entry); - if (entry & X86_PAGING_PRESENT) - { - if (entry & X86_PAGING_HUGE && - (i == x86_paging_levels - 1 || i == x86_paging_levels - 2)) - { - // Calculate the offset inside the huge page by getting the size of each entry at - // this level and then masking the virtual address with it. We then chop off the - // PAGE_SIZE bits. - auto entry_size = level_to_entry_size(i - 1); - const auto offset = virt & (entry_size - 1) & -PAGE_SIZE; - return pte_to_mapping_info(entry, true, offset); - } - - pml = (PML *) PHYS_TO_VIRT(page); - } - else - { - return PAGE_NOT_PRESENT; - } - } - - return pte_to_mapping_info(pml->entries[indices[0]], false, 0); -} - -unsigned int mmu_get_clear_referenced(struct mm_address_space *mm, void *addr, struct page *page) -{ - scoped_lock g{mm->page_table_lock}; - - u64 *ptep; - if (!x86_get_pt_entry(addr, &ptep, mm)) - return 0; - - u64 pte = READ_ONCE(*ptep); - u64 new_pte; - do - { - if (!(pte & X86_PAGING_ACCESSED)) - return 0; - if (PML_EXTRACT_ADDRESS(pte) != (unsigned long) page_to_phys(page)) - return 0; - new_pte = pte & ~X86_PAGING_ACCESSED; - } while (!__atomic_compare_exchange_n(ptep, &pte, new_pte, false, __ATOMIC_RELAXED, - __ATOMIC_RELAXED)); - /* Architectural note: We don't need to flush the TLB. Flushing the TLB is required by x86 if we - * want the A bit to be set again, but we can just wait for an unrelated TLB flush (e.g context - * switch) to do the job for us. A TLB shootdown is too much overhead for this purpose. */ - return 1; -} - #ifdef CONFIG_KASAN unsigned long __x86_kasan_virt; diff --git a/kernel/arch/x86_64/process.cpp b/kernel/arch/x86_64/process.cpp index 6c9243dc3..1928ac007 100644 --- a/kernel/arch/x86_64/process.cpp +++ b/kernel/arch/x86_64/process.cpp @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -79,8 +80,8 @@ int process_alloc_stack(struct stack_info *info) { void *ptr = vm_mmap(nullptr, info->length, PROT_WRITE | PROT_READ, MAP_ANON | MAP_PRIVATE, nullptr, 0); - if (!ptr) - return -ENOMEM; + if (IS_ERR(ptr)) + return PTR_ERR(ptr); info->base = ptr; info->top = reinterpret_cast((unsigned long) ptr + info->length); diff --git a/kernel/arch/x86_64/vm.cpp b/kernel/arch/x86_64/vm.cpp index 4c5e141db..516974fda 100644 --- a/kernel/arch/x86_64/vm.cpp +++ b/kernel/arch/x86_64/vm.cpp @@ -61,7 +61,7 @@ u64 arch_vm_interpret_mmap_hint_flags(void *hint, int flags) bool arch_vm_validate_mmap_region(unsigned long start, unsigned long size, u64 flags) { // Check if we can indeed return this region - if (start > 0x00007fffffffffff || start + size > 0x00007fffffffffff) + if (start > 0x00007fffffffffff || start + size - 1 > 0x00007fffffffffff) { return flags & VM_FULL_ADDRESS_SPACE; } diff --git a/kernel/include/onyx/cpumask.h b/kernel/include/onyx/cpumask.h index 924231c3f..3e960227d 100644 --- a/kernel/include/onyx/cpumask.h +++ b/kernel/include/onyx/cpumask.h @@ -10,64 +10,53 @@ #define CONFIG_SMP_NR_CPUS 64 #endif -constexpr unsigned long cpumask_size_in_longs() +struct cpumask { - auto long_size_bits = sizeof(unsigned long) * 8; - auto size = CONFIG_SMP_NR_CPUS / long_size_bits; +#define LONG_SIZE_BITS __LONG_WIDTH__ +#define CPUMASK_SIZE CONFIG_SMP_NR_CPUS / LONG_SIZE_BITS + unsigned long mask[CPUMASK_SIZE]; - if (CONFIG_SMP_NR_CPUS % long_size_bits) - size++; - - return size; -} - -class cpumask -{ -private: - static constexpr unsigned long long_size_bits = sizeof(unsigned long) * 8; - unsigned long mask[cpumask_size_in_longs()]; - -public: +#ifdef __cplusplus explicit constexpr cpumask() : mask{} { } constexpr void set_cpu(unsigned long cpu) { - auto long_idx = cpu / long_size_bits; - auto bit_idx = cpu % long_size_bits; + auto long_idx = cpu / LONG_SIZE_BITS; + auto bit_idx = cpu % LONG_SIZE_BITS; mask[long_idx] |= (1UL << bit_idx); } void set_cpu_atomic(unsigned long cpu) { - auto long_idx = cpu / long_size_bits; - auto bit_idx = cpu % long_size_bits; + auto long_idx = cpu / LONG_SIZE_BITS; + auto bit_idx = cpu % LONG_SIZE_BITS; __atomic_or_fetch(&mask[long_idx], (1UL << bit_idx), __ATOMIC_RELAXED); } constexpr void remove_cpu(unsigned long cpu) { - auto long_idx = cpu / long_size_bits; - auto bit_idx = cpu % long_size_bits; + auto long_idx = cpu / LONG_SIZE_BITS; + auto bit_idx = cpu % LONG_SIZE_BITS; mask[long_idx] &= ~(1UL << bit_idx); } void remove_cpu_atomic(unsigned long cpu) { - auto long_idx = cpu / long_size_bits; - auto bit_idx = cpu % long_size_bits; + auto long_idx = cpu / LONG_SIZE_BITS; + auto bit_idx = cpu % LONG_SIZE_BITS; __atomic_and_fetch(&mask[long_idx], ~(1UL << bit_idx), __ATOMIC_RELAXED); } constexpr bool is_cpu_set(unsigned long cpu) const { - auto long_idx = cpu / long_size_bits; - auto bit_idx = cpu % long_size_bits; + auto long_idx = cpu / LONG_SIZE_BITS; + auto bit_idx = cpu % LONG_SIZE_BITS; return mask[long_idx] & (1UL << bit_idx); } @@ -81,7 +70,7 @@ class cpumask constexpr cpumask& operator|=(const cpumask& rhs) { - for (unsigned long i = 0; i < cpumask_size_in_longs(); i++) + for (unsigned long i = 0; i < CPUMASK_SIZE; i++) { mask[i] |= rhs.mask[i]; } @@ -99,7 +88,7 @@ class cpumask { cpumask m{*this}; - for (unsigned long i = 0; i < cpumask_size_in_longs(); i++) + for (unsigned long i = 0; i < CPUMASK_SIZE; i++) { m.mask[i] = ~m.mask[i]; } @@ -109,7 +98,7 @@ class cpumask constexpr cpumask& operator&=(const cpumask& rhs) { - for (unsigned long i = 0; i < cpumask_size_in_longs(); i++) + for (unsigned long i = 0; i < CPUMASK_SIZE; i++) { mask[i] &= rhs.mask[i]; } @@ -125,7 +114,7 @@ class cpumask constexpr cpumask& operator^=(const cpumask& rhs) { - for (unsigned long i = 0; i < cpumask_size_in_longs(); i++) + for (unsigned long i = 0; i < CPUMASK_SIZE; i++) { mask[i] ^= rhs.mask[i]; } @@ -143,7 +132,7 @@ class cpumask template void for_every_cpu(Callable c) { - for (unsigned long i = 0; i < cpumask_size_in_longs(); i++) + for (unsigned long i = 0; i < CPUMASK_SIZE; i++) { int curr_bit = -1; @@ -166,7 +155,7 @@ class cpumask /* The word == 0 case has already been dealt with before this */ curr_bit = __builtin_ffsl(word) - 1; - auto cpu = long_size_bits * i + curr_bit; + auto cpu = LONG_SIZE_BITS * i + curr_bit; if (!c(cpu)) return; @@ -208,6 +197,7 @@ class cpumask return true; } +#endif }; #endif diff --git a/kernel/include/onyx/err.h b/kernel/include/onyx/err.h new file mode 100644 index 000000000..2e059a5c0 --- /dev/null +++ b/kernel/include/onyx/err.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the GPLv2 License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef _ONYX_ERR_H +#define _ONYX_ERR_H + +#include + +#include + +#define IS_ERR_VALUE(x) unlikely((unsigned long) (void *) (x) >= (unsigned long) -MAX_ERRNO) +#define ERR_PTR(err) ((void *) (unsigned long) (err)) + +#define IS_ERR(x) IS_ERR_VALUE(x) +#define PTR_ERR(x) ((long) (x)) + +#endif diff --git a/kernel/include/onyx/maple_tree.h b/kernel/include/onyx/maple_tree.h new file mode 100644 index 000000000..47a26ee3d --- /dev/null +++ b/kernel/include/onyx/maple_tree.h @@ -0,0 +1,909 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef _LINUX_MAPLE_TREE_H +#define _LINUX_MAPLE_TREE_H +/* + * Maple Tree - An RCU-safe adaptive tree for storing ranges + * Copyright (c) 2018-2022 Oracle + * Authors: Liam R. Howlett + * Matthew Wilcox + */ + +#include +#include + +#include +#include +#include +/* #define CONFIG_MAPLE_RCU_DISABLED */ + +__BEGIN_CDECLS + +typedef unsigned int gfp_t; + +/* + * Allocated nodes are mutable until they have been inserted into the tree, + * at which time they cannot change their type until they have been removed + * from the tree and an RCU grace period has passed. + * + * Removed nodes have their ->parent set to point to themselves. RCU readers + * check ->parent before relying on the value that they loaded from the + * slots array. This lets us reuse the slots array for the RCU head. + * + * Nodes in the tree point to their parent unless bit 0 is set. + */ +#if defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) +/* 64bit sizes */ +#define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */ +#define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */ +#define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */ +#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1) +#else +/* 32bit sizes */ +#define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */ +#define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */ +#define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */ +#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2) +#endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */ + +#define MAPLE_NODE_MASK 255UL + +/* + * The node->parent of the root node has bit 0 set and the rest of the pointer + * is a pointer to the tree itself. No more bits are available in this pointer + * (on m68k, the data structure may only be 2-byte aligned). + * + * Internal non-root nodes can only have maple_range_* nodes as parents. The + * parent pointer is 256B aligned like all other tree nodes. When storing a 32 + * or 64 bit values, the offset can fit into 4 bits. The 16 bit values need an + * extra bit to store the offset. This extra bit comes from a reuse of the last + * bit in the node type. This is possible by using bit 1 to indicate if bit 2 + * is part of the type or the slot. + * + * Once the type is decided, the decision of an allocation range type or a range + * type is done by examining the immutable tree flag for the MAPLE_ALLOC_RANGE + * flag. + * + * Node types: + * 0x??1 = Root + * 0x?00 = 16 bit nodes + * 0x010 = 32 bit nodes + * 0x110 = 64 bit nodes + * + * Slot size and location in the parent pointer: + * type : slot location + * 0x??1 : Root + * 0x?00 : 16 bit values, type in 0-1, slot in 2-6 + * 0x010 : 32 bit values, type in 0-2, slot in 3-6 + * 0x110 : 64 bit values, type in 0-2, slot in 3-6 + */ + +/* + * This metadata is used to optimize the gap updating code and in reverse + * searching for gaps or any other code that needs to find the end of the data. + */ +struct maple_metadata +{ + unsigned char end; + unsigned char gap; +}; + +/* + * Leaf nodes do not store pointers to nodes, they store user data. Users may + * store almost any bit pattern. As noted above, the optimisation of storing an + * entry at 0 in the root pointer cannot be done for data which have the bottom + * two bits set to '10'. We also reserve values with the bottom two bits set to + * '10' which are below 4096 (ie 2, 6, 10 .. 4094) for internal use. Some APIs + * return errnos as a negative errno shifted right by two bits and the bottom + * two bits set to '10', and while choosing to store these values in the array + * is not an error, it may lead to confusion if you're testing for an error with + * mas_is_err(). + * + * Non-leaf nodes store the type of the node pointed to (enum maple_type in bits + * 3-6), bit 2 is reserved. That leaves bits 0-1 unused for now. + * + * In regular B-Tree terms, pivots are called keys. The term pivot is used to + * indicate that the tree is specifying ranges, Pivots may appear in the + * subtree with an entry attached to the value whereas keys are unique to a + * specific position of a B-tree. Pivot values are inclusive of the slot with + * the same index. + */ + +struct maple_range_64 +{ + struct maple_pnode *parent; + unsigned long pivot[MAPLE_RANGE64_SLOTS - 1]; + union { + void __rcu *slot[MAPLE_RANGE64_SLOTS]; + struct + { + void __rcu *pad[MAPLE_RANGE64_SLOTS - 1]; + struct maple_metadata meta; + }; + }; +}; + +/* + * At tree creation time, the user can specify that they're willing to trade off + * storing fewer entries in a tree in return for storing more information in + * each node. + * + * The maple tree supports recording the largest range of NULL entries available + * in this node, also called gaps. This optimises the tree for allocating a + * range. + */ +struct maple_arange_64 +{ + struct maple_pnode *parent; + unsigned long pivot[MAPLE_ARANGE64_SLOTS - 1]; + void __rcu *slot[MAPLE_ARANGE64_SLOTS]; + unsigned long gap[MAPLE_ARANGE64_SLOTS]; + struct maple_metadata meta; +}; + +struct maple_alloc +{ + unsigned long total; + unsigned char node_count; + unsigned int request_count; + struct maple_alloc *slot[MAPLE_ALLOC_SLOTS]; +}; + +struct maple_topiary +{ + struct maple_pnode *parent; + struct maple_enode *next; /* Overlaps the pivot */ +}; + +enum maple_type +{ + maple_dense, + maple_leaf_64, + maple_range_64, + maple_arange_64, +}; + +/** + * DOC: Maple tree flags + * + * * MT_FLAGS_ALLOC_RANGE - Track gaps in this tree + * * MT_FLAGS_USE_RCU - Operate in RCU mode + * * MT_FLAGS_HEIGHT_OFFSET - The position of the tree height in the flags + * * MT_FLAGS_HEIGHT_MASK - The mask for the maple tree height value + * * MT_FLAGS_LOCK_MASK - How the mt_lock is used + * * MT_FLAGS_LOCK_IRQ - Acquired irq-safe + * * MT_FLAGS_LOCK_BH - Acquired bh-safe + * * MT_FLAGS_LOCK_EXTERN - mt_lock is not used + * + * MAPLE_HEIGHT_MAX The largest height that can be stored + */ +#define MT_FLAGS_ALLOC_RANGE 0x01 +#define MT_FLAGS_USE_RCU 0x02 +#define MT_FLAGS_HEIGHT_OFFSET 0x02 +#define MT_FLAGS_HEIGHT_MASK 0x7C +#define MT_FLAGS_LOCK_MASK 0x300 +#define MT_FLAGS_LOCK_IRQ 0x100 +#define MT_FLAGS_LOCK_BH 0x200 +#define MT_FLAGS_LOCK_EXTERN 0x300 +#define MT_FLAGS_ALLOC_WRAPPED 0x0800 + +#define MAPLE_HEIGHT_MAX 31 + +#define MAPLE_NODE_TYPE_MASK 0x0F +#define MAPLE_NODE_TYPE_SHIFT 0x03 + +#define MAPLE_RESERVED_RANGE 4096 + +#ifdef CONFIG_LOCKDEP +typedef struct lockdep_map *lockdep_map_p; +#define mt_lock_is_held(mt) (!(mt)->ma_external_lock || lock_is_held((mt)->ma_external_lock)) + +#define mt_write_lock_is_held(mt) \ + (!(mt)->ma_external_lock || lock_is_held_type((mt)->ma_external_lock, 0)) + +#define mt_set_external_lock(mt, lock) (mt)->ma_external_lock = &(lock)->dep_map + +#define mt_on_stack(mt) (mt).ma_external_lock = NULL +#else +typedef struct +{ /* nothing */ + /* ONYX CHANGE - compat hack cuz empty structs are 1 sized in C++ */ + char empty[1]; +} lockdep_map_p; +#define mt_lock_is_held(mt) 1 +#define mt_write_lock_is_held(mt) 1 +#define mt_set_external_lock(mt, lock) \ + do \ + { \ + } while (0) +#define mt_on_stack(mt) \ + do \ + { \ + } while (0) +#endif + +/* + * If the tree contains a single entry at index 0, it is usually stored in + * tree->ma_root. To optimise for the page cache, an entry which ends in '00', + * '01' or '11' is stored in the root, but an entry which ends in '10' will be + * stored in a node. Bits 3-6 are used to store enum maple_type. + * + * The flags are used both to store some immutable information about this tree + * (set at tree creation time) and dynamic information set under the spinlock. + * + * Another use of flags are to indicate global states of the tree. This is the + * case with the MAPLE_USE_RCU flag, which indicates the tree is currently in + * RCU mode. This mode was added to allow the tree to reuse nodes instead of + * re-allocating and RCU freeing nodes when there is a single user. + */ +struct maple_tree +{ + union { + spinlock_t ma_lock; + lockdep_map_p ma_external_lock; + }; + unsigned int ma_flags; + void __rcu *ma_root; +}; + +/** + * MTREE_INIT() - Initialize a maple tree + * @name: The maple tree name + * @__flags: The maple tree flags + * + */ +#define MTREE_INIT(name, __flags) \ + { \ + .ma_lock = __SPIN_LOCK_UNLOCKED((name).ma_lock), .ma_flags = __flags, .ma_root = NULL, \ + } + +/** + * MTREE_INIT_EXT() - Initialize a maple tree with an external lock. + * @name: The tree name + * @__flags: The maple tree flags + * @__lock: The external lock + */ +#ifdef CONFIG_LOCKDEP +#define MTREE_INIT_EXT(name, __flags, __lock) \ + { \ + .ma_external_lock = &(__lock).dep_map, .ma_flags = (__flags), .ma_root = NULL, \ + } +#else +#define MTREE_INIT_EXT(name, __flags, __lock) MTREE_INIT(name, __flags) +#endif + +#define DEFINE_MTREE(name) struct maple_tree name = MTREE_INIT(name, 0) + +#define mtree_lock(mt) spin_lock((&(mt)->ma_lock)) +#define mtree_lock_nested(mas, subclass) spin_lock_nested((&(mt)->ma_lock), subclass) +#define mtree_unlock(mt) spin_unlock((&(mt)->ma_lock)) + +/* + * The Maple Tree squeezes various bits in at various points which aren't + * necessarily obvious. Usually, this is done by observing that pointers are + * N-byte aligned and thus the bottom log_2(N) bits are available for use. We + * don't use the high bits of pointers to store additional information because + * we don't know what bits are unused on any given architecture. + * + * Nodes are 256 bytes in size and are also aligned to 256 bytes, giving us 8 + * low bits for our own purposes. Nodes are currently of 4 types: + * 1. Single pointer (Range is 0-0) + * 2. Non-leaf Allocation Range nodes + * 3. Non-leaf Range nodes + * 4. Leaf Range nodes All nodes consist of a number of node slots, + * pivots, and a parent pointer. + */ + +struct maple_node +{ + union { + struct + { + struct maple_pnode *parent; + void __rcu *slot[MAPLE_NODE_SLOTS]; + }; + struct + { + void *pad; + struct rcu_head rcu; + struct maple_enode *piv_parent; + unsigned char parent_slot; + enum maple_type type; + unsigned char slot_len; + unsigned int ma_flags; + }; + struct maple_range_64 mr64; + struct maple_arange_64 ma64; + struct maple_alloc alloc; + }; +}; + +/* + * More complicated stores can cause two nodes to become one or three and + * potentially alter the height of the tree. Either half of the tree may need + * to be rebalanced against the other. The ma_topiary struct is used to track + * which nodes have been 'cut' from the tree so that the change can be done + * safely at a later date. This is done to support RCU. + */ +struct ma_topiary +{ + struct maple_enode *head; + struct maple_enode *tail; + struct maple_tree *mtree; +}; + +void *mtree_load(struct maple_tree *mt, unsigned long index); + +int mtree_insert(struct maple_tree *mt, unsigned long index, void *entry, gfp_t gfp); +int mtree_insert_range(struct maple_tree *mt, unsigned long first, unsigned long last, void *entry, + gfp_t gfp); +int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp, void *entry, unsigned long size, + unsigned long min, unsigned long max, gfp_t gfp); +int mtree_alloc_cyclic(struct maple_tree *mt, unsigned long *startp, void *entry, + unsigned long range_lo, unsigned long range_hi, unsigned long *next, + gfp_t gfp); +int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp, void *entry, + unsigned long size, unsigned long min, unsigned long max, gfp_t gfp); + +int mtree_store_range(struct maple_tree *mt, unsigned long first, unsigned long last, void *entry, + gfp_t gfp); +int mtree_store(struct maple_tree *mt, unsigned long index, void *entry, gfp_t gfp); +void *mtree_erase(struct maple_tree *mt, unsigned long index); + +int mtree_dup(struct maple_tree *mt, struct maple_tree *new_, gfp_t gfp); +int __mt_dup(struct maple_tree *mt, struct maple_tree *new_, gfp_t gfp); + +void mtree_destroy(struct maple_tree *mt); +void __mt_destroy(struct maple_tree *mt); + +/** + * mtree_empty() - Determine if a tree has any present entries. + * @mt: Maple Tree. + * + * Context: Any context. + * Return: %true if the tree contains only NULL pointers. + */ +static inline bool mtree_empty(const struct maple_tree *mt) +{ + return mt->ma_root == NULL; +} + +/* Advanced API */ + +/* + * Maple State Status + * ma_active means the maple state is pointing to a node and offset and can + * continue operating on the tree. + * ma_start means we have not searched the tree. + * ma_root means we have searched the tree and the entry we found lives in + * the root of the tree (ie it has index 0, length 1 and is the only entry in + * the tree). + * ma_none means we have searched the tree and there is no node in the + * tree for this entry. For example, we searched for index 1 in an empty + * tree. Or we have a tree which points to a full leaf node and we + * searched for an entry which is larger than can be contained in that + * leaf node. + * ma_pause means the data within the maple state may be stale, restart the + * operation + * ma_overflow means the search has reached the upper limit of the search + * ma_underflow means the search has reached the lower limit of the search + * ma_error means there was an error, check the node for the error number. + */ +enum maple_status +{ + ma_active, + ma_start, + ma_root, + ma_none, + ma_pause, + ma_overflow, + ma_underflow, + ma_error, +}; + +/* + * The maple state is defined in the struct ma_state and is used to keep track + * of information during operations, and even between operations when using the + * advanced API. + * + * If state->node has bit 0 set then it references a tree location which is not + * a node (eg the root). If bit 1 is set, the rest of the bits are a negative + * errno. Bit 2 (the 'unallocated slots' bit) is clear. Bits 3-6 indicate the + * node type. + * + * state->alloc either has a request number of nodes or an allocated node. If + * stat->alloc has a requested number of nodes, the first bit will be set (0x1) + * and the remaining bits are the value. If state->alloc is a node, then the + * node will be of type maple_alloc. maple_alloc has MAPLE_NODE_SLOTS - 1 for + * storing more allocated nodes, a total number of nodes allocated, and the + * node_count in this node. node_count is the number of allocated nodes in this + * node. The scaling beyond MAPLE_NODE_SLOTS - 1 is handled by storing further + * nodes into state->alloc->slot[0]'s node. Nodes are taken from state->alloc + * by removing a node from the state->alloc node until state->alloc->node_count + * is 1, when state->alloc is returned and the state->alloc->slot[0] is promoted + * to state->alloc. Nodes are pushed onto state->alloc by putting the current + * state->alloc into the pushed node's slot[0]. + * + * The state also contains the implied min/max of the state->node, the depth of + * this search, and the offset. The implied min/max are either from the parent + * node or are 0-oo for the root node. The depth is incremented or decremented + * every time a node is walked down or up. The offset is the slot/pivot of + * interest in the node - either for reading or writing. + * + * When returning a value the maple state index and last respectively contain + * the start and end of the range for the entry. Ranges are inclusive in the + * Maple Tree. + * + * The status of the state is used to determine how the next action should treat + * the state. For instance, if the status is ma_start then the next action + * should start at the root of the tree and walk down. If the status is + * ma_pause then the node may be stale data and should be discarded. If the + * status is ma_overflow, then the last action hit the upper limit. + * + */ +struct ma_state +{ + struct maple_tree *tree; /* The tree we're operating in */ + unsigned long index; /* The index we're operating on - range start */ + unsigned long last; /* The last index we're operating on - range end */ + struct maple_enode *node; /* The node containing this entry */ + unsigned long min; /* The minimum index of this node - implied pivot min */ + unsigned long max; /* The maximum index of this node - implied pivot max */ + struct maple_alloc *alloc; /* Allocated nodes for this operation */ + enum maple_status status; /* The status of the state (active, start, none, etc) */ + unsigned char depth; /* depth of tree descent during write */ + unsigned char offset; + unsigned char mas_flags; + unsigned char end; /* The end of the node */ +}; + +struct ma_wr_state +{ + struct ma_state *mas; + struct maple_node *node; /* Decoded mas->node */ + unsigned long r_min; /* range min */ + unsigned long r_max; /* range max */ + enum maple_type type; /* mas->node type */ + unsigned char offset_end; /* The offset where the write ends */ + unsigned long *pivots; /* mas->node->pivots pointer */ + unsigned long end_piv; /* The pivot at the offset end */ + void __rcu **slots; /* mas->node->slots pointer */ + void *entry; /* The entry to write */ + void *content; /* The existing entry that is being overwritten */ +}; + +#define mas_lock(mas) spin_lock(&((mas)->tree->ma_lock)) +#define mas_lock_nested(mas, subclass) spin_lock_nested(&((mas)->tree->ma_lock), subclass) +#define mas_unlock(mas) spin_unlock(&((mas)->tree->ma_lock)) + +/* + * Special values for ma_state.node. + * MA_ERROR represents an errno. After dropping the lock and attempting + * to resolve the error, the walk would have to be restarted from the + * top of the tree as the tree may have been modified. + */ +#define MA_ERROR(err) ((struct maple_enode *) (((unsigned long) err << 2) | 2UL)) + +// clang-format off +#define MA_STATE_INIT(mt, first, end) \ + { \ + .tree = mt, \ + .index = first, \ + .last = end, \ + .node = NULL, \ + .min = 0, \ + .max = ULONG_MAX, \ + .alloc = NULL, \ + .status = ma_start, \ + .mas_flags = 0, \ + } + +// clang-format on + +#define MA_STATE(name, mt, first, end) struct ma_state name = MA_STATE_INIT(mt, first, end) + +#define MA_WR_STATE(name, ma_state, wr_entry) \ + struct ma_wr_state name = { \ + .mas = ma_state, \ + .content = NULL, \ + .entry = wr_entry, \ + } + +#define MA_TOPIARY(name, tree) \ + struct ma_topiary name = { \ + .head = NULL, \ + .tail = NULL, \ + .mtree = tree, \ + } + +void *mas_walk(struct ma_state *mas); +void *mas_store(struct ma_state *mas, void *entry); +void *mas_erase(struct ma_state *mas); +int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp); +void mas_store_prealloc(struct ma_state *mas, void *entry); +void *mas_find(struct ma_state *mas, unsigned long max); +void *mas_find_range(struct ma_state *mas, unsigned long max); +void *mas_find_rev(struct ma_state *mas, unsigned long min); +void *mas_find_range_rev(struct ma_state *mas, unsigned long max); +int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp); +int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp, void *entry, + unsigned long range_lo, unsigned long range_hi, unsigned long *next, + gfp_t gfp); + +bool mas_nomem(struct ma_state *mas, gfp_t gfp); +void mas_pause(struct ma_state *mas); +void maple_tree_init(void); +void mas_destroy(struct ma_state *mas); +int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries); + +void *mas_prev(struct ma_state *mas, unsigned long min); +void *mas_prev_range(struct ma_state *mas, unsigned long max); +void *mas_next(struct ma_state *mas, unsigned long max); +void *mas_next_range(struct ma_state *mas, unsigned long max); + +int mas_empty_area(struct ma_state *mas, unsigned long min, unsigned long max, unsigned long size); +/* + * This finds an empty area from the highest address to the lowest. + * AKA "Topdown" version, + */ +int mas_empty_area_rev(struct ma_state *mas, unsigned long min, unsigned long max, + unsigned long size); + +static inline void mas_init(struct ma_state *mas, struct maple_tree *tree, unsigned long addr) +{ + memset(mas, 0, sizeof(struct ma_state)); + mas->tree = tree; + mas->index = mas->last = addr; + mas->max = ULONG_MAX; + mas->status = ma_start; + mas->node = NULL; +} + +static inline bool mas_is_active(struct ma_state *mas) +{ + return mas->status == ma_active; +} + +static inline bool mas_is_err(struct ma_state *mas) +{ + return mas->status == ma_error; +} + +/** + * mas_reset() - Reset a Maple Tree operation state. + * @mas: Maple Tree operation state. + * + * Resets the error or walk state of the @mas so future walks of the + * array will start from the root. Use this if you have dropped the + * lock and want to reuse the ma_state. + * + * Context: Any context. + */ +__always_inline void mas_reset(struct ma_state *mas) +{ + mas->status = ma_start; + mas->node = NULL; +} + +/** + * mas_for_each() - Iterate over a range of the maple tree. + * @__mas: Maple Tree operation state (maple_state) + * @__entry: Entry retrieved from the tree + * @__max: maximum index to retrieve from the tree + * + * When returned, mas->index and mas->last will hold the entire range for the + * entry. + * + * Note: may return the zero entry. + */ +#define mas_for_each(__mas, __entry, __max) while (((__entry) = mas_find((__mas), (__max))) != NULL) + +#if defined(__onyx__) +#define WARN_ON(x) (x) +#define BUG_ON(x) +#endif + +#ifdef CONFIG_DEBUG_MAPLE_TREE +enum mt_dump_format +{ + mt_dump_dec, + mt_dump_hex, +}; + +extern atomic_t maple_tree_tests_run; +extern atomic_t maple_tree_tests_passed; + +void mt_dump(const struct maple_tree *mt, enum mt_dump_format format); +void mas_dump(const struct ma_state *mas); +void mas_wr_dump(const struct ma_wr_state *wr_mas); +void mt_validate(struct maple_tree *mt); +void mt_cache_shrink(void); +#define MT_BUG_ON(__tree, __x) \ + do \ + { \ + atomic_inc(&maple_tree_tests_run); \ + if (__x) \ + { \ + pr_info("BUG at %s:%d (%u)\n", __func__, __LINE__, __x); \ + mt_dump(__tree, mt_dump_hex); \ + pr_info("Pass: %u Run:%u\n", atomic_read(&maple_tree_tests_passed), \ + atomic_read(&maple_tree_tests_run)); \ + dump_stack(); \ + } \ + else \ + { \ + atomic_inc(&maple_tree_tests_passed); \ + } \ + } while (0) + +#define MAS_BUG_ON(__mas, __x) \ + do \ + { \ + atomic_inc(&maple_tree_tests_run); \ + if (__x) \ + { \ + pr_info("BUG at %s:%d (%u)\n", __func__, __LINE__, __x); \ + mas_dump(__mas); \ + mt_dump((__mas)->tree, mt_dump_hex); \ + pr_info("Pass: %u Run:%u\n", atomic_read(&maple_tree_tests_passed), \ + atomic_read(&maple_tree_tests_run)); \ + dump_stack(); \ + } \ + else \ + { \ + atomic_inc(&maple_tree_tests_passed); \ + } \ + } while (0) + +#define MAS_WR_BUG_ON(__wrmas, __x) \ + do \ + { \ + atomic_inc(&maple_tree_tests_run); \ + if (__x) \ + { \ + pr_info("BUG at %s:%d (%u)\n", __func__, __LINE__, __x); \ + mas_wr_dump(__wrmas); \ + mas_dump((__wrmas)->mas); \ + mt_dump((__wrmas)->mas->tree, mt_dump_hex); \ + pr_info("Pass: %u Run:%u\n", atomic_read(&maple_tree_tests_passed), \ + atomic_read(&maple_tree_tests_run)); \ + dump_stack(); \ + } \ + else \ + { \ + atomic_inc(&maple_tree_tests_passed); \ + } \ + } while (0) + +#define MT_WARN_ON(__tree, __x) \ + ({ \ + int ret = !!(__x); \ + atomic_inc(&maple_tree_tests_run); \ + if (ret) \ + { \ + pr_info("WARN at %s:%d (%u)\n", __func__, __LINE__, __x); \ + mt_dump(__tree, mt_dump_hex); \ + pr_info("Pass: %u Run:%u\n", atomic_read(&maple_tree_tests_passed), \ + atomic_read(&maple_tree_tests_run)); \ + dump_stack(); \ + } \ + else \ + { \ + atomic_inc(&maple_tree_tests_passed); \ + } \ + unlikely(ret); \ + }) + +#define MAS_WARN_ON(__mas, __x) \ + ({ \ + int ret = !!(__x); \ + atomic_inc(&maple_tree_tests_run); \ + if (ret) \ + { \ + pr_info("WARN at %s:%d (%u)\n", __func__, __LINE__, __x); \ + mas_dump(__mas); \ + mt_dump((__mas)->tree, mt_dump_hex); \ + pr_info("Pass: %u Run:%u\n", atomic_read(&maple_tree_tests_passed), \ + atomic_read(&maple_tree_tests_run)); \ + dump_stack(); \ + } \ + else \ + { \ + atomic_inc(&maple_tree_tests_passed); \ + } \ + unlikely(ret); \ + }) + +#define MAS_WR_WARN_ON(__wrmas, __x) \ + ({ \ + int ret = !!(__x); \ + atomic_inc(&maple_tree_tests_run); \ + if (ret) \ + { \ + pr_info("WARN at %s:%d (%u)\n", __func__, __LINE__, __x); \ + mas_wr_dump(__wrmas); \ + mas_dump((__wrmas)->mas); \ + mt_dump((__wrmas)->mas->tree, mt_dump_hex); \ + pr_info("Pass: %u Run:%u\n", atomic_read(&maple_tree_tests_passed), \ + atomic_read(&maple_tree_tests_run)); \ + dump_stack(); \ + } \ + else \ + { \ + atomic_inc(&maple_tree_tests_passed); \ + } \ + unlikely(ret); \ + }) +#else +#define MT_BUG_ON(__tree, __x) BUG_ON(__x) +#define MAS_BUG_ON(__mas, __x) BUG_ON(__x) +#define MAS_WR_BUG_ON(__mas, __x) BUG_ON(__x) +#define MT_WARN_ON(__tree, __x) WARN_ON(__x) +#define MAS_WARN_ON(__mas, __x) WARN_ON(__x) +#define MAS_WR_WARN_ON(__mas, __x) WARN_ON(__x) +#endif /* CONFIG_DEBUG_MAPLE_TREE */ + +/** + * __mas_set_range() - Set up Maple Tree operation state to a sub-range of the + * current location. + * @mas: Maple Tree operation state. + * @start: New start of range in the Maple Tree. + * @last: New end of range in the Maple Tree. + * + * set the internal maple state values to a sub-range. + * Please use mas_set_range() if you do not know where you are in the tree. + */ +static inline void __mas_set_range(struct ma_state *mas, unsigned long start, unsigned long last) +{ + /* Ensure the range starts within the current slot */ + // Onyx patch - remove when WARN_ON is properly implemented. + // MAS_WARN_ON(mas, mas_is_active(mas) && (mas->index > start || mas->last < start)); + mas->index = start; + mas->last = last; +} + +/** + * mas_set_range() - Set up Maple Tree operation state for a different index. + * @mas: Maple Tree operation state. + * @start: New start of range in the Maple Tree. + * @last: New end of range in the Maple Tree. + * + * Move the operation state to refer to a different range. This will + * have the effect of starting a walk from the top; see mas_next() + * to move to an adjacent index. + */ +static inline void mas_set_range(struct ma_state *mas, unsigned long start, unsigned long last) +{ + mas_reset(mas); + __mas_set_range(mas, start, last); +} + +/** + * mas_set() - Set up Maple Tree operation state for a different index. + * @mas: Maple Tree operation state. + * @index: New index into the Maple Tree. + * + * Move the operation state to refer to a different index. This will + * have the effect of starting a walk from the top; see mas_next() + * to move to an adjacent index. + */ +static inline void mas_set(struct ma_state *mas, unsigned long index) +{ + + mas_set_range(mas, index, index); +} + +static inline bool mt_external_lock(const struct maple_tree *mt) +{ + return (mt->ma_flags & MT_FLAGS_LOCK_MASK) == MT_FLAGS_LOCK_EXTERN; +} + +/** + * mt_init_flags() - Initialise an empty maple tree with flags. + * @mt: Maple Tree + * @flags: maple tree flags. + * + * If you need to initialise a Maple Tree with special flags (eg, an + * allocation tree), use this function. + * + * Context: Any context. + */ +static inline void mt_init_flags(struct maple_tree *mt, unsigned int flags) +{ + mt->ma_flags = flags; + if (!mt_external_lock(mt)) + spin_lock_init(&mt->ma_lock); + rcu_assign_pointer(mt->ma_root, NULL); +} + +/** + * mt_init() - Initialise an empty maple tree. + * @mt: Maple Tree + * + * An empty Maple Tree. + * + * Context: Any context. + */ +static inline void mt_init(struct maple_tree *mt) +{ + mt_init_flags(mt, 0); +} + +static inline bool mt_in_rcu(struct maple_tree *mt) +{ +#ifdef CONFIG_MAPLE_RCU_DISABLED + return false; +#endif + return mt->ma_flags & MT_FLAGS_USE_RCU; +} + +/** + * mt_clear_in_rcu() - Switch the tree to non-RCU mode. + * @mt: The Maple Tree + */ +static inline void mt_clear_in_rcu(struct maple_tree *mt) +{ + if (!mt_in_rcu(mt)) + return; + + if (mt_external_lock(mt)) + { + // WARN_ON(!mt_lock_is_held(mt)); + mt->ma_flags &= ~MT_FLAGS_USE_RCU; + } + else + { + mtree_lock(mt); + mt->ma_flags &= ~MT_FLAGS_USE_RCU; + mtree_unlock(mt); + } +} + +/** + * mt_set_in_rcu() - Switch the tree to RCU safe mode. + * @mt: The Maple Tree + */ +static inline void mt_set_in_rcu(struct maple_tree *mt) +{ + if (mt_in_rcu(mt)) + return; + + if (mt_external_lock(mt)) + { + // WARN_ON(!mt_lock_is_held(mt)); + mt->ma_flags |= MT_FLAGS_USE_RCU; + } + else + { + mtree_lock(mt); + mt->ma_flags |= MT_FLAGS_USE_RCU; + mtree_unlock(mt); + } +} + +static inline unsigned int mt_height(const struct maple_tree *mt) +{ + return (mt->ma_flags & MT_FLAGS_HEIGHT_MASK) >> MT_FLAGS_HEIGHT_OFFSET; +} + +void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max); +void *mt_find_after(struct maple_tree *mt, unsigned long *index, unsigned long max); +void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min); +void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max); + +/** + * mt_for_each - Iterate over each entry starting at index until max. + * @__tree: The Maple Tree + * @__entry: The current entry + * @__index: The index to start the search from. Subsequently used as iterator. + * @__max: The maximum limit for @index + * + * This iterator skips all entries, which resolve to a NULL pointer, + * e.g. entries which has been reserved with XA_ZERO_ENTRY. + */ +#define mt_for_each(__tree, __entry, __index, __max) \ + for (__entry = mt_find(__tree, &(__index), __max); __entry; \ + __entry = mt_find_after(__tree, &(__index), __max)) + +__END_CDECLS + +#endif /*_LINUX_MAPLE_TREE_H */ diff --git a/kernel/include/onyx/mm/amap.h b/kernel/include/onyx/mm/amap.h deleted file mode 100644 index a5da4de9d..000000000 --- a/kernel/include/onyx/mm/amap.h +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (c) 2023 Pedro Falcato - * This file is part of Onyx, and is released under the terms of the GPLv2 License - * check LICENSE at the root directory for more information - * - * SPDX-License-Identifier: GPL-2.0-only - */ -#ifndef _ONYX_MM_AMAP_H -#define _ONYX_MM_AMAP_H - -#include -#include -#include - -struct amap -{ - radix_tree am_map; - unsigned long am_refc; - size_t am_size; /* size looks redundant? */ - struct spinlock am_lock; - - template - void for_range(Callable c, unsigned long start, unsigned long end = -1ul) - { - radix_tree::cursor cursor = radix_tree::cursor::from_range(&am_map, start, end); - - while (!cursor.is_end()) - { - struct page *page = (struct page *) cursor.get(); - if (!c(page, cursor.current_idx())) - break; - cursor.advance(); - } - } -}; - -/** - * @brief Allocate a new anonymous memory map - * - * @param size Size of the amap - * @return struct amap* - */ -struct amap *amap_alloc(size_t size); - -/** - * @brief Free an amap - * - * @param amap amap to free - */ -void amap_free(struct amap *amap); - -__always_inline void amap_ref(struct amap *amap) -{ - __atomic_add_fetch(&amap->am_refc, 1, __ATOMIC_ACQUIRE); -} - -__always_inline void amap_unref(struct amap *amap) -{ - if (__atomic_sub_fetch(&amap->am_refc, 1, __ATOMIC_RELEASE) == 0) - amap_free(amap); -} - -/** - * @brief Add a page to an amap - * - * @param amap Amap to add to - * @param page Page to add - * @param region Region to which the amap belongs - * @param pgoff Page offset (in pfn, shifted right by PAGE_SHIFT) - * @param nocopy Don't copy if we find an old page - * @return 0 on success, negative error codes - */ -int amap_add(struct amap *amap, struct page *page, struct vm_area_struct *region, size_t pgoff, - bool nocopy); -/** - * @brief Add a page to an amap - * - * @param amap Amap to add to - * @param page Page to add - * @param region Region to which the amap belongs - * @param pgoff Page offset (in pfn, shifted right by PAGE_SHIFT) - * @return 0 on success, negative error codes - */ -__always_inline int amap_ref_and_add(struct amap *amap, struct page *page, - struct vm_area_struct *region, size_t pgoff) -{ - page_ref(page); - return amap_add(amap, page, region, pgoff, false); -} - -/** - * @brief Get a page from the amap - * - * @param amap Amap to lookup from - * @param pgoff Page offset (in pfn, shifted right by PAGE_SHIFT) - * @return struct page in the amap, or NULL - */ -struct page *amap_get(struct amap *amap, size_t pgoff); - -/** - * @brief Split an amap into two - * - * @param amap Original amap - * @param region Region to which the amap belongs - * @param pgoff Page offset for the new amap - * @return New amap, or NULL - */ -struct amap *amap_split(struct amap *amap, struct vm_area_struct *region, size_t pgoff); - -/** - * @brief Truncate an amap - * - * @param amap Amap - * @param region Region to which the amap belongs - * @param new_pgsize New size, in pages - * @return 0 on success, negative error codes - */ -int amap_truncate(struct amap *amap, struct vm_area_struct *region, size_t new_pgsize); - -/** - * @brief Punch a hole through an amap - * - * @param amap Amap - * @param region Region to which the amap belongs - * @param first_pg First pfn of the hole - * @param end_pg End of the hole - * @return 0 on success, negative error codes - */ -int amap_punch_hole(struct amap *amap, struct vm_area_struct *region, size_t first_pg, - size_t end_pg); - -#endif diff --git a/kernel/include/onyx/mm/pgtable-nop4d.h b/kernel/include/onyx/mm/pgtable-nop4d.h new file mode 100644 index 000000000..6a8f7bd80 --- /dev/null +++ b/kernel/include/onyx/mm/pgtable-nop4d.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the GPLv2 License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef _ONYX_PGTABLE_NOP4D_H +#define _ONYX_PGTABLE_NOP4D_H + +#define PTRS_PER_P4D 1 + +static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long addr) +{ + return (p4d_t *) pgd; +} + +static inline bool pgd_none(pgd_t pgd) +{ + return false; +} + +static inline bool pgd_present(pgd_t pgd) +{ + return true; +} + +static inline bool p4d_folded(void) +{ + return true; +} + +#endif diff --git a/kernel/include/onyx/mm/slab.h b/kernel/include/onyx/mm/slab.h index 11cb8fbaa..03e5e446d 100644 --- a/kernel/include/onyx/mm/slab.h +++ b/kernel/include/onyx/mm/slab.h @@ -67,6 +67,10 @@ __BEGIN_CDECLS #define KMEM_CACHE_HWALIGN (1 << 0) #define KMEM_CACHE_VMALLOC (1 << 1) #define KMEM_CACHE_NOPCPU (1 << 2) +/* Panic if kmem_cache_create fails */ +#define KMEM_CACHE_PANIC (1 << 3) + +#define SLAB_PANIC KMEM_CACHE_PANIC /** * @brief Create a slab cache @@ -159,6 +163,28 @@ void kmem_cache_print_slab_info_kasan(void *mem, struct slab *slab); */ void slab_shrink_caches(unsigned long target_freep); +/** + * @brief Allocate objects in bulk + * Allocate slab objects in bulk, while avoiding relocking as much as we can. + * + * @param cache Slab cache + * @param gfp_flags GFP flags + * @param nr Number of objects desired + * @param res Array of results (output parameter) + * @return 0 on error (ENOMEM), or the number of objects allocated + */ +size_t kmem_cache_alloc_bulk(struct slab_cache *cache, unsigned int gfp_flags, size_t nr, + void **res); + +/** + * @brief Free objects in bulk + * Free objects in bulk, avoiding relocking and doing as much as we can, in batches. + * @param cache Slab cache + * @param size Number of objects to free + * @param ptrs Pointers to free (NULL is tolerated) + */ +void kmem_cache_free_bulk(struct slab_cache *cache, size_t size, void **ptrs); + __END_CDECLS #endif diff --git a/kernel/include/onyx/mm/vm_object.h b/kernel/include/onyx/mm/vm_object.h index 73bff8ad3..54b341480 100644 --- a/kernel/include/onyx/mm/vm_object.h +++ b/kernel/include/onyx/mm/vm_object.h @@ -287,6 +287,14 @@ bool vm_obj_remove_page(struct vm_object *obj, struct page *page); long vm_obj_get_page_references(struct vm_object *obj, struct page *page, unsigned int *vm_flags); +/** + * @brief Removes a mapping from the VMO + * Does not take the lock + * @param vmo The target vm object + * @param vma The VMA + */ +void vmo_remove_mapping_locked(struct vm_object *vmo, struct vm_area_struct *vma); + __END_CDECLS #endif diff --git a/kernel/include/onyx/mm_address_space.h b/kernel/include/onyx/mm_address_space.h index 80b70f492..24aed1079 100644 --- a/kernel/include/onyx/mm_address_space.h +++ b/kernel/include/onyx/mm_address_space.h @@ -8,45 +8,68 @@ #ifndef _ONYX_MM_ADDRESS_SPACE_H #define _ONYX_MM_ADDRESS_SPACE_H -#include +#include + +#include +#include +#include #include +#ifdef __cplusplus +#include + +#include +// clang-format off +#define CPP_DFLINIT {} +// clang-format on +#else +#define CPP_DFLINIT +#endif + /** * @brief An mm_address_space represents an address space inside the kernel and stores * all kinds of relevant data on it, like the owner process, a tree of vm_area_structs, locks * various statistics, etc. * */ -struct mm_address_space : public refcountable +struct mm_address_space +#ifdef __cplusplus + : public refcountable +#endif { +#ifndef __cplusplus + void *__vtable; + unsigned long refc; +#endif /* Virtual address space WAVL tree */ - struct bst_root region_tree; - unsigned long start{}; - unsigned long end{}; - mutex vm_lock{}; + struct maple_tree region_tree; + unsigned long start CPP_DFLINIT; + unsigned long end CPP_DFLINIT; + struct mutex vm_lock CPP_DFLINIT; /* mmap(2) base */ - void *mmap_base{}; + void *mmap_base CPP_DFLINIT; /* Process' brk */ - void *brk{}; + void *brk CPP_DFLINIT; - size_t virtual_memory_size{}; - size_t resident_set_size{}; - size_t shared_set_size{}; - size_t page_faults{}; - size_t page_tables_size{}; + size_t virtual_memory_size CPP_DFLINIT; + size_t resident_set_size CPP_DFLINIT; + size_t shared_set_size CPP_DFLINIT; + size_t page_faults CPP_DFLINIT; + size_t page_tables_size CPP_DFLINIT; - arch_mm_address_space arch_mmu{}; + struct arch_mm_address_space arch_mmu CPP_DFLINIT; // The active mask keeps track of where the address space is running. // This serves as an optimisation when doing a TLB shootdown, as it lets us // limit the shootdowns to CPUs where the address space is active instead of every CPU. - cpumask active_mask{}; + struct cpumask active_mask CPP_DFLINIT; - spinlock page_table_lock{}; + struct spinlock page_table_lock CPP_DFLINIT; +#ifdef __cplusplus mm_address_space &operator=(mm_address_space &&as) { start = as.start; @@ -66,7 +89,7 @@ struct mm_address_space : public refcountable constexpr mm_address_space() { spinlock_init(&page_table_lock); - bst_root_initialize(®ion_tree); + region_tree = MTREE_INIT(region_tree, MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN); } /** @@ -88,6 +111,7 @@ struct mm_address_space : public refcountable * */ ~mm_address_space() override; +#endif }; #define increment_vm_stat(as, name, amount) __sync_add_and_fetch(&as->name, amount) diff --git a/kernel/include/onyx/page.h b/kernel/include/onyx/page.h index 2d4cebc47..b5d69b0ca 100644 --- a/kernel/include/onyx/page.h +++ b/kernel/include/onyx/page.h @@ -169,17 +169,25 @@ struct page *page_add_page_late(void *paddr); #define __GFP_IO (1 << 11) #define __GFP_FS (1 << 12) #define __GFP_NO_INSTRUMENT (1 << 13) +#define __GFP_NOWARN (1 << 14) +#define __GFP_NOWAIT (1 << 15) #define __GFP_MAY_RECLAIM (__GFP_DIRECT_RECLAIM | __GFP_WAKE_PAGEDAEMON) #define GFP_KERNEL (__GFP_MAY_RECLAIM | __GFP_IO) #define GFP_ATOMIC (__GFP_ATOMIC | __GFP_WAKE_PAGEDAEMON) #define GFP_NOIO (__GFP_MAY_RECLAIM) #define GFP_NOFS (__GFP_MAY_RECLAIM | __GFP_IO) +#define GFP_NOWAIT (__GFP_WAKE_PAGEDAEMON | __GFP_NOWAIT | __GFP_NOWARN) static inline bool __page_should_zero(unsigned long flags) { return !(flags & PAGE_ALLOC_NO_ZERO); } +static inline bool gfpflags_allow_blocking(unsigned int gfp_flags) +{ + return !!(gfp_flags & __GFP_DIRECT_RECLAIM); +} + #define page_should_zero(x) likely(__page_should_zero(x)) struct page *alloc_pages(unsigned int order, unsigned long flags); diff --git a/kernel/include/onyx/paging.h b/kernel/include/onyx/paging.h index 34c5764b6..fee240233 100644 --- a/kernel/include/onyx/paging.h +++ b/kernel/include/onyx/paging.h @@ -12,9 +12,12 @@ #include #include +#include #include +__BEGIN_CDECLS + #define PHYS_BASE (0xffffd00000000000) #define PHYS_BASE_LIMIT (0xffffd08000000000) @@ -87,7 +90,19 @@ void paging_protect_kernel(void); void paging_free_page_tables(struct mm_address_space *mm); bool paging_write_protect(void *addr, struct mm_address_space *mm); int vm_mmu_unmap(struct mm_address_space *as, void *addr, size_t pages, struct vm_area_struct *vma); -void *paging_unmap(void *memory); + +/** + * @brief Directly maps a page into the paging tables. + * + * @param as The target address space. + * @param virt The virtual address. + * @param phys The physical address of the page. + * @param prot Desired protection flags. + * @param vma VMA for this mapping (optional) + * @return NULL if out of memory, else virt. + */ +void *vm_map_page(struct mm_address_space *as, uint64_t virt, uint64_t phys, uint64_t prot, + struct vm_area_struct *vma); #ifdef __x86_64__ @@ -122,4 +137,6 @@ unsigned long __get_mapping_info(void *addr, struct mm_address_space *as); struct page; unsigned int mmu_get_clear_referenced(struct mm_address_space *mm, void *addr, struct page *page); +__END_CDECLS + #endif diff --git a/kernel/include/onyx/rcupdate.h b/kernel/include/onyx/rcupdate.h index 607dab7e1..74e743123 100644 --- a/kernel/include/onyx/rcupdate.h +++ b/kernel/include/onyx/rcupdate.h @@ -12,6 +12,8 @@ #define rcu_read_lock() sched_disable_preempt() #define rcu_read_unlock() sched_enable_preempt() +__BEGIN_CDECLS + struct rcu_head { struct rcu_head *next; @@ -34,6 +36,8 @@ void rcu_do_quiesc(); */ void rcu_work(); +__END_CDECLS + #define rcu_dereference(ptr) __atomic_load_n(&(ptr), __ATOMIC_RELAXED) #define rcu_assign_pointer(ptr, val) \ diff --git a/kernel/include/onyx/riscv/include/platform/pgtable.h b/kernel/include/onyx/riscv/include/platform/pgtable.h new file mode 100644 index 000000000..383acc9a3 --- /dev/null +++ b/kernel/include/onyx/riscv/include/platform/pgtable.h @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the GPLv2 License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef _ONYX_PGTABLE_ARCH_H +#define _ONYX_PGTABLE_ARCH_H + +#include + +#include +#include +#include +#include + +__BEGIN_CDECLS + +typedef u64 pgdval_t; +typedef u64 p4dval_t; +typedef u64 pudval_t; +typedef u64 pmdval_t; +typedef u64 pteval_t; +typedef u64 pgprotval_t; + +#define PTE_GET_ADDR(n) ((n >> 10) << 12) +#define _PAGE_PRESENT (1 << 0) +#define _PAGE_READ (1 << 1) +#define _PAGE_WRITE (1 << 2) +#define _PAGE_EXEC (1 << 3) +#define _PAGE_USER (1 << 4) +#define _PAGE_GLOBAL (1 << 5) +#define _PAGE_ACCESSED (1 << 6) +#define _PAGE_DIRTY (1 << 7) +/* Use one of the ignored bits as SPECIAL. This will annotate zero page mappings (so we don't + * increment mapcount on zero_page and thus blow it up). add_mapcount and sub_mapcount will not be + * called on these struct pages. */ +#define _PAGE_SPECIAL (1 << 8) + +#define _PAGE_HUGE (_PAGE_WRITE | _PAGE_EXEC | _PAGE_READ) + +typedef struct pgd +{ + pgdval_t pgd; +} pgd_t; + +typedef struct p4d +{ + p4dval_t p4d; +} p4d_t; + +typedef struct pud +{ + pudval_t pud; +} pud_t; + +typedef struct pmd +{ + pmdval_t pmd; +} pmd_t; + +typedef struct pte +{ + pteval_t pte; +} pte_t; + +typedef struct pgprot +{ + pgprotval_t pgprot; +} pgprot_t; + +#define PTRS_PER_PGD 512 +#define PGD_SHIFT 39 + +#define PTRS_PER_P4D 1 +#define P4D_SHIFT 39 + +#define PTRS_PER_PUD 512 +#define PUD_SHIFT 30 + +#define PTRS_PER_PMD 512 +#define PMD_SHIFT 21 + +#define PTRS_PER_PTE 512 +#define PTE_SHIFT 12 + +#define __tovirt(x) (void *) (((uintptr_t) (x)) + PHYS_BASE) + +static inline unsigned long pgd_index(unsigned long addr) +{ + return (addr >> PGD_SHIFT) & (PTRS_PER_PGD - 1); +} + +static inline pgd_t *pgd_offset(struct mm_address_space *mm, unsigned long addr) +{ + return (pgd_t *) __tovirt(mm->arch_mmu.top_pt) + pgd_index(addr); +} + +#define pgd_val(x) ((x).pgd) +#define p4d_val(x) ((x).p4d) +#define pud_val(x) ((x).pud) +#define pmd_val(x) ((x).pmd) +#define pte_val(x) ((x).pte) +#define pgprot_val(x) ((x).pgprot) + +#define __pgd(x) ((pgd_t){(x)}) +#define __p4d(x) ((p4d_t){(x)}) +#define __pud(x) ((pud_t){(x)}) +#define __pmd(x) ((pmd_t){(x)}) +#define __pte(x) ((pte_t){(x)}) +#define __pgprot(x) ((pgprot_t){(x)}) + +static inline unsigned long p4d_index(unsigned long addr) +{ + return (addr >> P4D_SHIFT) & (PTRS_PER_P4D - 1); +} + +static inline unsigned long pgd_addr(pgd_t pgd) +{ + return PTE_GET_ADDR(pgd_val(pgd)); +} + +static inline unsigned long pud_index(unsigned long addr) +{ + return (addr >> PUD_SHIFT) & (PTRS_PER_PUD - 1); +} + +static inline unsigned long p4d_addr(p4d_t pgd) +{ + return PTE_GET_ADDR(p4d_val(pgd)); +} + +static inline pud_t *pud_offset(p4d_t *p4d, unsigned long addr) +{ + return (pud_t *) __tovirt(p4d_addr(*p4d)) + pud_index(addr); +} + +static inline unsigned long pmd_index(unsigned long addr) +{ + return (addr >> PMD_SHIFT) & (PTRS_PER_PMD - 1); +} + +static inline unsigned long pud_addr(pud_t pgd) +{ + return PTE_GET_ADDR(pud_val(pgd)); +} + +static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) +{ + return (pmd_t *) __tovirt(pud_addr(*pud)) + pmd_index(addr); +} + +static inline unsigned long pte_index(unsigned long addr) +{ + return (addr >> PTE_SHIFT) & (PTRS_PER_PTE - 1); +} + +static inline unsigned long pmd_addr(pmd_t pgd) +{ + return PTE_GET_ADDR(pmd_val(pgd)); +} + +static inline pte_t *pte_offset(pmd_t *pmd, unsigned long addr) +{ + return (pte_t *) __tovirt(pmd_addr(*pmd)) + pte_index(addr); +} + +static inline unsigned long pte_addr(pte_t pgd) +{ + return PTE_GET_ADDR(pte_val(pgd)); +} + +static inline bool p4d_none(p4d_t p4d) +{ + return p4d_val(p4d) == 0; +} + +static inline bool pud_none(pud_t pud) +{ + return pud_val(pud) == 0; +} + +static inline bool pmd_none(pmd_t pmd) +{ + return pmd_val(pmd) == 0; +} + +static inline bool pte_none(pte_t pte) +{ + return pte_val(pte) == 0; +} + +static inline bool p4d_present(p4d_t p4d) +{ + return p4d_val(p4d) & _PAGE_PRESENT; +} + +static inline bool pud_present(pud_t pud) +{ + return pud_val(pud) & _PAGE_PRESENT; +} + +static inline bool pmd_present(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_PRESENT; +} + +static inline bool pte_present(pte_t pte) +{ + return pte_val(pte) & _PAGE_PRESENT; +} + +#define KERNEL_PGTBL (_PAGE_PRESENT) +#define USER_PGTBL (KERNEL_PGTBL) + +static inline pte_t pte_mkpte(u64 phys, pgprot_t prot) +{ + return __pte(((phys >> PAGE_SHIFT) << 10) | pgprot_val(prot)); +} + +static inline pmd_t pmd_mkpmd(u64 phys, pgprot_t prot) +{ + return __pmd(((phys >> PAGE_SHIFT) << 10) | pgprot_val(prot)); +} + +static inline pud_t pud_mkpud(u64 phys, pgprot_t prot) +{ + return __pud(((phys >> PAGE_SHIFT) << 10) | pgprot_val(prot)); +} + +static inline p4d_t p4d_mkp4d(u64 phys, pgprot_t prot) +{ + return __p4d(((phys >> PAGE_SHIFT) << 10) | pgprot_val(prot)); +} + +static inline pgd_t pgd_mkpgd(u64 phys, pgprot_t prot) +{ + return __pgd(((phys >> PAGE_SHIFT) << 10) | pgprot_val(prot)); +} + +static inline bool pte_special(pte_t pte) +{ + return pte_val(pte) & _PAGE_SPECIAL; +} + +static inline bool pte_accessed(pte_t pte) +{ + return pte_val(pte) & _PAGE_ACCESSED; +} + +static inline bool pte_user(pte_t pte) +{ + return pte_val(pte) & _PAGE_USER; +} + +static inline bool pte_write(pte_t pte) +{ + return pte_val(pte) & _PAGE_WRITE; +} + +static inline bool pte_exec(pte_t pte) +{ + return pte_val(pte) & _PAGE_EXEC; +} + +static inline bool pte_dirty(pte_t pte) +{ + return pte_val(pte) & _PAGE_DIRTY; +} + +static inline bool pte_global(pte_t pte) +{ + return pte_val(pte) & _PAGE_GLOBAL; +} + +static void set_pgd(pgd_t *pgd, pgd_t val) +{ + WRITE_ONCE(pgd_val(*pgd), pgd_val(val)); +} +#define set_pgd set_pgd + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_ACCESSED); +} + +/* PML4-level hugepages not supported on x86, for now... */ +#define ARCH_HUGE_PUD_SUPPORT 1 +#define ARCH_HUGE_PMD_SUPPORT 1 + +static inline bool pud_huge(pud_t pud) +{ + return pud_val(pud) & _PAGE_HUGE; +} + +static inline bool pmd_huge(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_HUGE; +} + +static inline bool pud_user(pud_t pud) +{ + return pud_val(pud) & _PAGE_USER; +} + +static inline bool pud_write(pud_t pud) +{ + return pud_val(pud) & _PAGE_WRITE; +} + +static inline bool pud_exec(pud_t pud) +{ + return pud_val(pud) & _PAGE_EXEC; +} + +static inline bool pud_dirty(pud_t pud) +{ + return pud_val(pud) & _PAGE_DIRTY; +} + +static inline bool pud_accessed(pud_t pud) +{ + return pud_val(pud) & _PAGE_ACCESSED; +} + +static inline bool pud_global(pud_t pud) +{ + return pud_val(pud) & _PAGE_GLOBAL; +} + +static inline bool pmd_user(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_USER; +} + +static inline bool pmd_write(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_WRITE; +} + +static inline bool pmd_exec(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_EXEC; +} + +static inline bool pmd_dirty(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_DIRTY; +} + +static inline bool pmd_accessed(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_ACCESSED; +} + +static inline bool pmd_global(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_GLOBAL; +} + +#define pud_folded() (0) +#define pmd_folded() (0) + +static inline pte_t pte_wrprotect(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_WRITE); +} + +static inline pgprot_t calc_pgprot(u64 phys, u64 prots) +{ + bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()); + pgprotval_t page_prots = (prots & VM_EXEC ? _PAGE_EXEC : 0) | + (prots & VM_WRITE ? _PAGE_WRITE : 0) | + (prots & (VM_READ | VM_WRITE) ? _PAGE_READ : 0) | + (prots & VM_USER ? _PAGE_USER : _PAGE_GLOBAL) | _PAGE_PRESENT | + (special_mapping ? _PAGE_SPECIAL : 0); + + if (!(prots & (VM_READ | VM_WRITE | VM_EXEC))) + page_prots &= ~_PAGE_PRESENT; + + return __pgprot(page_prots); +} + +#include + +__END_CDECLS + +#endif diff --git a/kernel/include/onyx/spinlock.h b/kernel/include/onyx/spinlock.h index c747e6cf9..3023ecd4d 100644 --- a/kernel/include/onyx/spinlock.h +++ b/kernel/include/onyx/spinlock.h @@ -95,4 +95,9 @@ static inline void spin_unlock(struct spinlock *lock) __RELEASE(lock) #define MUST_HOLD_LOCK(lock) assert(spin_lock_held(lock) != false) +typedef struct spinlock spinlock_t; +#define spin_lock_init(s) spinlock_init(s) + +#define __SPIN_LOCK_UNLOCKED(name) (spinlock_t) STATIC_SPINLOCK_INIT + #endif diff --git a/kernel/include/onyx/utils.h b/kernel/include/onyx/utils.h index fd762bf64..3f67dae79 100644 --- a/kernel/include/onyx/utils.h +++ b/kernel/include/onyx/utils.h @@ -20,7 +20,19 @@ void *copy_page_to_page(void *p1, void *p2); #define containerof_null_safe(ptr, type, member) \ ((ptr) == NULL ? NULL : container_of(ptr, type, member)) #ifndef __cplusplus -#define min(x, y) (x < y ? x : y) +#define min(x, y) \ + ({ \ + __auto_type __x = x; \ + __auto_type __y = y; \ + __x < __y ? __x : __y; \ + }) + +#define max(x, y) \ + ({ \ + __auto_type __x = x; \ + __auto_type __y = y; \ + __x > __y ? __x : __y; \ + }) #else template diff --git a/kernel/include/onyx/vm.h b/kernel/include/onyx/vm.h index acfc2bd3a..ec0ab3f35 100644 --- a/kernel/include/onyx/vm.h +++ b/kernel/include/onyx/vm.h @@ -16,16 +16,13 @@ #include #include +#include #include #include #include #include #include -#ifdef __cplusplus -#include -#endif - #include #include #include @@ -50,16 +47,17 @@ __BEGIN_CDECLS #define VM_TYPE_FILE_BACKED (5) #define VM_TYPE_MODULE (6) -#define VM_WRITE (1 << 0) -#define VM_EXEC (1 << 1) -#define VM_USER (1 << 2) -#define VM_NOCACHE (1 << 3) -#define VM_WRITETHROUGH (1 << 4) -#define VM_WC (1 << 5) -#define VM_WP (1 << 6) -#define VM_DONT_MAP_OVER (1 << 7) -#define VM_READ (1 << 8) +#define VM_READ (1 << 0) +#define VM_WRITE (1 << 1) +#define VM_EXEC (1 << 2) +#define VM_USER (1 << 3) +#define VM_NOCACHE (1 << 4) +#define VM_WRITETHROUGH (1 << 5) +#define VM_WC (1 << 6) +#define VM_WP (1 << 7) +#define VM_DONT_MAP_OVER (1 << 8) #define VM_NOFLUSH (1 << 9) +#define VM_SHARED (1 << 10) /* Internal flags used by the mm code */ #define __VM_CACHE_TYPE_REGULAR 0 @@ -89,11 +87,9 @@ static inline unsigned long vm_prot_to_cache_type(uint64_t prot) #define PHYS_TO_VIRT(x) (void *) ((uintptr_t) (x) + PHYS_BASE) -#define VM_PFNMAP (1 << 1) -#define VM_USING_MAP_SHARED_OPT (1 << 2) +#define VM_PFNMAP (1 << 1) struct vm_object; -struct amap; struct fault_info; struct vm_pf_context @@ -133,18 +129,16 @@ struct vm_area_struct unsigned long vm_end; union { - struct bst_node vm_tree_node; + /* TODO: Can we union this with something else? */ struct list_head vm_detached_node; }; int vm_flags; - int vm_maptype; struct mm_address_space *vm_mm; const struct vm_operations *vm_ops; struct file *vm_file; off_t vm_offset; struct vm_object *vm_obj; - struct amap *vm_amap; struct interval_tree_node vm_objhead; }; @@ -153,6 +147,16 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; } +static inline bool vma_shared(const struct vm_area_struct *vma) +{ + return vma->vm_flags & VM_SHARED; +} + +static inline bool vma_private(const struct vm_area_struct *vma) +{ + return !vma_shared(vma); +} + #define VM_OK 0x0 #define VM_SIGBUS SIGBUS #define VM_SIGSEGV SIGSEGV @@ -379,7 +383,7 @@ ssize_t user_memset(void *data, int val, size_t len); * @param is_file_backed True if file backed. * @return 0 on success, negative for errors. */ -int vm_area_struct_setup_backing(struct vm_area_struct *region, size_t pages, bool is_file_backed); +int vma_setup_backing(struct vm_area_struct *region, size_t pages, bool is_file_backed); /** * @brief Updates the memory map's ranges. @@ -431,14 +435,6 @@ void *map_pages_to_vaddr(void *virt, void *phys, size_t size, size_t flags); void *__map_pages_to_vaddr(struct mm_address_space *as, void *virt, void *phys, size_t size, size_t flags); -/** - * @brief Determines if a mapping is shared. - * - * @param region A pointer to the vm_area_struct. - * @return True if shared, false if not. - */ -bool is_mapping_shared(struct vm_area_struct *region); - /** * @brief Determines if a mapping is file backed. * @@ -483,7 +479,7 @@ struct file; * @param flags The mapping flags (see MAP_* as in mmap(2)). * @param file An optional pointer to a file, if it is a file mapping. * @param off The offset into the file, if it is a file mapping. - * @return A pointer to the new memory mapping, or NULL if it failed (errno is set). + * @return A pointer to the new memory mapping, or an ERR_PTR on error. */ void *vm_mmap(void *addr, size_t length, int prot, int flags, struct file *file, off_t off); @@ -504,19 +500,6 @@ void vm_invalidate_range(unsigned long addr, size_t pages); struct process; -/** - * @brief Directly maps a page into the paging tables. - * - * @param as The target address space. - * @param virt The virtual address. - * @param phys The physical address of the page. - * @param prot Desired protection flags. - * @param vma VMA for this mapping (optional) - * @return NULL if out of memory, else virt. - */ -void *vm_map_page(struct mm_address_space *as, uint64_t virt, uint64_t phys, uint64_t prot, - struct vm_area_struct *vma); - /** * @brief Allocates a new mapping and maps a list of pages. * @@ -799,6 +782,9 @@ static inline bool vma_is_pfnmap(struct vm_area_struct *vma) return vma == NULL; } +void vm_do_mmu_mprotect(struct mm_address_space *as, void *address, size_t nr_pgs, int old_prots, + int new_prots); + __END_CDECLS #ifdef __cplusplus @@ -813,8 +799,11 @@ template inline void vm_for_every_region(mm_address_space &as, Callable func) { vm_area_struct *entry; - bst_for_every_entry(&as.region_tree, entry, vm_area_struct, vm_tree_node) + unsigned long index = 0; + void *entry_; + mt_for_each(&as.region_tree, entry_, index, -1UL) { + entry = (vm_area_struct *) entry_; if (!func(entry)) break; } diff --git a/kernel/include/onyx/x86/include/platform/pgtable.h b/kernel/include/onyx/x86/include/platform/pgtable.h new file mode 100644 index 000000000..c09c0e261 --- /dev/null +++ b/kernel/include/onyx/x86/include/platform/pgtable.h @@ -0,0 +1,429 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the GPLv2 License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef _ONYX_PGTABLE_ARCH_H +#define _ONYX_PGTABLE_ARCH_H + +#include + +#include +#include +#include +#include +#include + +__BEGIN_CDECLS + +typedef u64 pgdval_t; +typedef u64 p4dval_t; +typedef u64 pudval_t; +typedef u64 pmdval_t; +typedef u64 pteval_t; +typedef u64 pgprotval_t; + +extern unsigned int x86_paging_levels; + +#define X86_ADDR_MASK 0x0ffffffffffff000 +#define _PAGE_PRESENT (1 << 0) +#define _PAGE_WRITE (1 << 1) +#define _PAGE_USER (1 << 2) +#define _PAGE_WRITETHROUGH (1 << 3) +#define _PAGE_PCD (1 << 4) +#define _PAGE_ACCESSED (1 << 5) +#define _PAGE_DIRTY (1 << 6) +#define _PAGE_PAT (1 << 7) +#define _PAGE_HUGE (1 << 7) +#define _PAGE_GLOBAL (1 << 8) +/* Use one of the ignored bits as SPECIAL. This will annotate zero page mappings (so we don't + * increment mapcount on zero_page and thus blow it up). add_mapcount and sub_mapcount will not be + * called on these struct pages. */ +#define _PAGE_SPECIAL (1 << 9) +#define _PAGE_NX (1UL << 63) + +typedef struct pgd +{ + pgdval_t pgd; +} pgd_t; + +typedef struct p4d +{ + p4dval_t p4d; +} p4d_t; + +typedef struct pud +{ + pudval_t pud; +} pud_t; + +typedef struct pmd +{ + pmdval_t pmd; +} pmd_t; + +typedef struct pte +{ + pteval_t pte; +} pte_t; + +typedef struct pgprot +{ + pgprotval_t pgprot; +} pgprot_t; + +extern int pgd_shift, p4d_ptrs; + +#define PTRS_PER_PGD 512 +#define PGD_SHIFT pgd_shift + +#define PTRS_PER_P4D p4d_ptrs +#define P4D_SHIFT 39 + +#define PTRS_PER_PUD 512 +#define PUD_SHIFT 30 + +#define PTRS_PER_PMD 512 +#define PMD_SHIFT 21 + +#define PTRS_PER_PTE 512 +#define PTE_SHIFT 12 + +#define __tovirt(x) (void *) (((uintptr_t) (x)) + PHYS_BASE) + +static inline bool pml5_present(void) +{ + return x86_paging_levels == 5; +} + +static inline unsigned long pgd_index(unsigned long addr) +{ + return (addr >> PGD_SHIFT) & (PTRS_PER_PGD - 1); +} + +static inline pgd_t *pgd_offset(struct mm_address_space *mm, unsigned long addr) +{ + return (pgd_t *) __tovirt(mm->arch_mmu.cr3) + pgd_index(addr); +} + +#define pgd_val(x) ((x).pgd) +#define p4d_val(x) ((x).p4d) +#define pud_val(x) ((x).pud) +#define pmd_val(x) ((x).pmd) +#define pte_val(x) ((x).pte) +#define pgprot_val(x) ((x).pgprot) + +#define __pgd(x) ((pgd_t){(x)}) +#define __p4d(x) ((p4d_t){(x)}) +#define __pud(x) ((pud_t){(x)}) +#define __pmd(x) ((pmd_t){(x)}) +#define __pte(x) ((pte_t){(x)}) +#define __pgprot(x) ((pgprot_t){(x)}) + +static inline unsigned long p4d_index(unsigned long addr) +{ + return (addr >> P4D_SHIFT) & (PTRS_PER_P4D - 1); +} + +static inline unsigned long pgd_addr(pgd_t pgd) +{ + return pgd_val(pgd) & X86_ADDR_MASK; +} + +static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long addr) +{ + if (!pml5_present()) + return (p4d_t *) pgd; + return (p4d_t *) __tovirt(pgd_addr(*pgd)) + p4d_index(addr); +} + +static inline unsigned long pud_index(unsigned long addr) +{ + return (addr >> PUD_SHIFT) & (PTRS_PER_PUD - 1); +} + +static inline unsigned long p4d_addr(p4d_t pgd) +{ + return p4d_val(pgd) & X86_ADDR_MASK; +} + +static inline pud_t *pud_offset(p4d_t *p4d, unsigned long addr) +{ + return (pud_t *) __tovirt(p4d_addr(*p4d)) + pud_index(addr); +} + +static inline unsigned long pmd_index(unsigned long addr) +{ + return (addr >> PMD_SHIFT) & (PTRS_PER_PMD - 1); +} + +static inline unsigned long pud_addr(pud_t pgd) +{ + return pud_val(pgd) & X86_ADDR_MASK; +} + +static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) +{ + return (pmd_t *) __tovirt(pud_addr(*pud)) + pmd_index(addr); +} + +static inline unsigned long pte_index(unsigned long addr) +{ + return (addr >> PTE_SHIFT) & (PTRS_PER_PTE - 1); +} + +static inline unsigned long pmd_addr(pmd_t pgd) +{ + return pmd_val(pgd) & X86_ADDR_MASK; +} + +static inline pte_t *pte_offset(pmd_t *pmd, unsigned long addr) +{ + return (pte_t *) __tovirt(pmd_addr(*pmd)) + pte_index(addr); +} + +static inline unsigned long pte_addr(pte_t pgd) +{ + return pte_val(pgd) & X86_ADDR_MASK; +} + +static inline bool pgd_none(pgd_t pgd) +{ + if (!pml5_present()) + return false; + return pgd_val(pgd) == 0; +} + +static inline bool p4d_none(p4d_t p4d) +{ + return p4d_val(p4d) == 0; +} + +static inline bool pud_none(pud_t pud) +{ + return pud_val(pud) == 0; +} + +static inline bool pmd_none(pmd_t pmd) +{ + return pmd_val(pmd) == 0; +} + +static inline bool pte_none(pte_t pte) +{ + return pte_val(pte) == 0; +} + +static inline bool pgd_present(pgd_t pgd) +{ + if (!pml5_present()) + return true; + return pgd_val(pgd) & _PAGE_PRESENT; +} + +static inline bool p4d_present(p4d_t p4d) +{ + return p4d_val(p4d) & _PAGE_PRESENT; +} + +static inline bool pud_present(pud_t pud) +{ + return pud_val(pud) & _PAGE_PRESENT; +} + +static inline bool pmd_present(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_PRESENT; +} + +static inline bool pte_present(pte_t pte) +{ + return pte_val(pte) & _PAGE_PRESENT; +} + +#define KERNEL_PGTBL (_PAGE_PRESENT | _PAGE_WRITE | _PAGE_GLOBAL) +#define USER_PGTBL (KERNEL_PGTBL | _PAGE_USER) + +static inline pte_t pte_mkpte(u64 phys, pgprot_t prot) +{ + return __pte(phys | pgprot_val(prot)); +} + +static inline pmd_t pmd_mkpmd(u64 phys, pgprot_t prot) +{ + return __pmd(phys | pgprot_val(prot)); +} + +static inline pud_t pud_mkpud(u64 phys, pgprot_t prot) +{ + return __pud(phys | pgprot_val(prot)); +} + +static inline p4d_t p4d_mkp4d(u64 phys, pgprot_t prot) +{ + return __p4d(phys | pgprot_val(prot)); +} + +static inline pgd_t pgd_mkpgd(u64 phys, pgprot_t prot) +{ + return __pgd(phys | pgprot_val(prot)); +} + +static inline bool pte_special(pte_t pte) +{ + return pte_val(pte) & _PAGE_SPECIAL; +} + +static inline bool pte_accessed(pte_t pte) +{ + return pte_val(pte) & _PAGE_ACCESSED; +} + +static inline bool pte_user(pte_t pte) +{ + return pte_val(pte) & _PAGE_USER; +} + +static inline bool pte_write(pte_t pte) +{ + return pte_val(pte) & _PAGE_WRITE; +} + +static inline bool pte_exec(pte_t pte) +{ + return !(pte_val(pte) & _PAGE_NX); +} + +static inline bool pte_dirty(pte_t pte) +{ + return pte_val(pte) & _PAGE_DIRTY; +} + +static inline bool pte_global(pte_t pte) +{ + return pte_val(pte) & _PAGE_GLOBAL; +} + +static void set_pgd(pgd_t *pgd, pgd_t val) +{ + WRITE_ONCE(pgd_val(*pgd), pgd_val(val)); +} +#define set_pgd set_pgd + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_ACCESSED); +} + +/* PML4-level hugepages not supported on x86, for now... */ +#define ARCH_HUGE_PUD_SUPPORT 1 +#define ARCH_HUGE_PMD_SUPPORT 1 + +static inline bool pud_huge(pud_t pud) +{ + return pud_val(pud) & _PAGE_HUGE; +} + +static inline bool pmd_huge(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_HUGE; +} + +static inline bool pud_user(pud_t pud) +{ + return pud_val(pud) & _PAGE_USER; +} + +static inline bool pud_write(pud_t pud) +{ + return pud_val(pud) & _PAGE_WRITE; +} + +static inline bool pud_exec(pud_t pud) +{ + return !(pud_val(pud) & _PAGE_NX); +} + +static inline bool pud_dirty(pud_t pud) +{ + return pud_val(pud) & _PAGE_DIRTY; +} + +static inline bool pud_accessed(pud_t pud) +{ + return pud_val(pud) & _PAGE_ACCESSED; +} + +static inline bool pud_global(pud_t pud) +{ + return pud_val(pud) & _PAGE_GLOBAL; +} + +static inline bool pmd_user(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_USER; +} + +static inline bool pmd_write(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_WRITE; +} + +static inline bool pmd_exec(pmd_t pmd) +{ + return !(pmd_val(pmd) & _PAGE_NX); +} + +static inline bool pmd_dirty(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_DIRTY; +} + +static inline bool pmd_accessed(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_ACCESSED; +} + +static inline bool pmd_global(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_GLOBAL; +} + +static inline bool p4d_folded(void) +{ + return !pml5_present(); +} + +#define pud_folded() (0) +#define pmd_folded() (0) + +static inline pte_t pte_wrprotect(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_WRITE); +} + +#define X86_CACHING_BITS(index) ((((index) &0x3) << 3) | (((index >> 2) & 1) << 7)) + +static inline pgprot_t calc_pgprot(u64 phys, u64 prot) +{ + bool user = prot & VM_USER; + bool noexec = !(prot & VM_EXEC); + bool global = !user; + bool write = prot & VM_WRITE; + bool readable = prot & (VM_READ | VM_WRITE) || !noexec; + unsigned int cache_type = vm_prot_to_cache_type(prot); + uint8_t caching_bits = cache_to_paging_bits(cache_type); + bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()); + + pgprotval_t page_prots = (noexec ? _PAGE_NX : 0) | (global ? _PAGE_GLOBAL : 0) | + (user ? _PAGE_USER : 0) | (write ? _PAGE_WRITE : 0) | + X86_CACHING_BITS(caching_bits) | (readable ? _PAGE_PRESENT : 0) | + (special_mapping ? _PAGE_SPECIAL : 0); + return __pgprot(page_prots); +} + +__END_CDECLS + +#endif diff --git a/kernel/include/onyx/x86/pat.h b/kernel/include/onyx/x86/pat.h index 8a8c0fe68..f00733ff1 100644 --- a/kernel/include/onyx/x86/pat.h +++ b/kernel/include/onyx/x86/pat.h @@ -11,6 +11,8 @@ #include +#include + #define PAT_UNCACHEABLE 0 #define PAT_WC 1 #define PAT_WT 4 @@ -19,8 +21,11 @@ #define PAT_UNCACHED 7 #define PAT_NR_ENTRIES 8 +__BEGIN_CDECLS uint8_t cache_to_paging_bits(uint8_t type); void pat_init(void); +__END_CDECLS + #endif diff --git a/kernel/include/onyx/xarray.h b/kernel/include/onyx/xarray.h new file mode 100644 index 000000000..a365e7de9 --- /dev/null +++ b/kernel/include/onyx/xarray.h @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the GPLv2 License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only + */ +/* + * eXtensible Arrays + * Copyright (c) 2017 Microsoft Corporation + * Author: Matthew Wilcox + * + * See Documentation/core-api/xarray.rst for how to use the XArray. + */ +#ifndef _ONYX_XARRAY_H +#define _ONYX_XARRAY_H + +#include + +#include + +#include +/* + * xa_mk_internal() - Create an internal entry. + * @v: Value to turn into an internal entry. + * + * Internal entries are used for a number of purposes. Entries 0-255 are + * used for sibling entries (only 0-62 are used by the current code). 256 + * is used for the retry entry. 257 is used for the reserved / zero entry. + * Negative internal entries are used to represent errnos. Node pointers + * are also tagged as internal entries in some situations. + * + * Context: Any context. + * Return: An XArray internal entry corresponding to this value. + */ +static inline void *xa_mk_internal(unsigned long v) +{ + return (void *) ((v << 2) | 2); +} + +/* + * xa_to_internal() - Extract the value from an internal entry. + * @entry: XArray entry. + * + * Context: Any context. + * Return: The value which was stored in the internal entry. + */ +static inline unsigned long xa_to_internal(const void *entry) +{ + return (unsigned long) entry >> 2; +} + +/* + * xa_is_internal() - Is the entry an internal entry? + * @entry: XArray entry. + * + * Context: Any context. + * Return: %true if the entry is an internal entry. + */ +static inline bool xa_is_internal(const void *entry) +{ + return ((unsigned long) entry & 3) == 2; +} + +#define XA_ZERO_ENTRY xa_mk_internal(257) + +/** + * xa_is_zero() - Is the entry a zero entry? + * @entry: Entry retrieved from the XArray + * + * The normal API will return NULL as the contents of a slot containing + * a zero entry. You can only see zero entries by using the advanced API. + * + * Return: %true if the entry is a zero entry. + */ +static inline bool xa_is_zero(const void *entry) +{ + return unlikely(entry == XA_ZERO_ENTRY); +} + +/** + * xa_is_err() - Report whether an XArray operation returned an error + * @entry: Result from calling an XArray function + * + * If an XArray operation cannot complete an operation, it will return + * a special value indicating an error. This function tells you + * whether an error occurred; xa_err() tells you which error occurred. + * + * Context: Any context. + * Return: %true if the entry indicates an error. + */ +static inline bool xa_is_err(const void *entry) +{ + return unlikely(xa_is_internal(entry) && entry >= xa_mk_internal(-MAX_ERRNO)); +} + +/** + * xa_err() - Turn an XArray result into an errno. + * @entry: Result from calling an XArray function. + * + * If an XArray operation cannot complete an operation, it will return + * a special pointer value which encodes an errno. This function extracts + * the errno from the pointer value, or returns 0 if the pointer does not + * represent an errno. + * + * Context: Any context. + * Return: A negative errno or 0. + */ +static inline int xa_err(void *entry) +{ + /* xa_to_internal() would not do sign extension. */ + if (xa_is_err(entry)) + return (long) entry >> 2; + return 0; +} + +/** + * xa_to_value() - Get value stored in an XArray entry. + * @entry: XArray entry. + * + * Context: Any context. + * Return: The value stored in the XArray entry. + */ +static inline unsigned long xa_to_value(const void *entry) +{ + return (unsigned long) entry >> 1; +} + +/** + * xa_is_value() - Determine if an entry is a value. + * @entry: XArray entry. + * + * Context: Any context. + * Return: True if the entry is a value, false if it is a pointer. + */ +static inline bool xa_is_value(const void *entry) +{ + return (unsigned long) entry & 1; +} + +static inline bool xa_is_node(const void *entry) +{ + return xa_is_internal(entry) && (unsigned long) entry > 4096; +} + +#define XA_RETRY_ENTRY xa_mk_internal(256) + +/** + * xa_is_retry() - Is the entry a retry entry? + * @entry: Entry retrieved from the XArray + * + * Return: %true if the entry is a retry entry. + */ +static inline bool xa_is_retry(const void *entry) +{ + return unlikely(entry == XA_RETRY_ENTRY); +} + +/** + * xa_is_advanced() - Is the entry only permitted for the advanced API? + * @entry: Entry to be stored in the XArray. + * + * Return: %true if the entry cannot be stored by the normal API. + */ +static inline bool xa_is_advanced(const void *entry) +{ + return xa_is_internal(entry) && (entry <= XA_RETRY_ENTRY); +} + +#endif diff --git a/kernel/include/uapi/errno.h b/kernel/include/uapi/errno.h index fb521e31f..459a6eaa8 100644 --- a/kernel/include/uapi/errno.h +++ b/kernel/include/uapi/errno.h @@ -144,4 +144,6 @@ #define ERFKILL 132 #define EHWPOISON 133 +#define MAX_ERRNO 4095 + #endif diff --git a/kernel/kernel/Makefile b/kernel/kernel/Makefile index 78d30417f..a5eb0b156 100644 --- a/kernel/kernel/Makefile +++ b/kernel/kernel/Makefile @@ -4,7 +4,8 @@ kern-y+= arc4random.o binfmt.o compression.o copy.o cppnew.o cpprt.o crc32.o dev power_management.o proc_event.o process.o pid.o ptrace.o random.o ref.o signal.o \ smp.o spinlock.o symbol.o tasklet.o time.o timer.o utils.o wait_queue.o \ worker.o cred.o list.o softirq.o cputime.o rlimit.o handle.o ctor.o internal_abi.o ssp.o \ - cmdline.o syscall_thunk.o vdso.o sysinfo.o memstream.o perf.o radix.o rcupdate.o iovec_iter.o + cmdline.o syscall_thunk.o vdso.o sysinfo.o memstream.o perf.o radix.o rcupdate.o iovec_iter.o \ + maple_tree.o kern-$(CONFIG_UBSAN)+= ubsan.o diff --git a/kernel/kernel/binfmt/elf.cpp b/kernel/kernel/binfmt/elf.cpp index caa9ca492..d0133d0b4 100644 --- a/kernel/kernel/binfmt/elf.cpp +++ b/kernel/kernel/binfmt/elf.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 - 2022 Pedro Falcato + * Copyright (c) 2017 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -202,9 +203,9 @@ static void *elf_load(struct binfmt_args *args, elf_ehdr *header) { base = vm_mmap(nullptr, vm_size_to_pages(needed_size) << PAGE_SHIFT, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, nullptr, 0); - if (!base) + if (IS_ERR(base)) { - errno = ENOMEM; + errno = PTR_ERR(base); goto error1; } } @@ -286,9 +287,11 @@ static void *elf_load(struct binfmt_args *args, elf_ehdr *header) // printk("mmaping [%lx, %lx]\n", aligned_address, aligned_address + (pages << // PAGE_SHIFT)); - if (!vm_mmap((void *) addr, pages << PAGE_SHIFT, prot, MAP_PRIVATE | MAP_FIXED, fd, - offset)) + void *res = vm_mmap((void *) addr, pages << PAGE_SHIFT, prot, MAP_PRIVATE | MAP_FIXED, + fd, offset); + if (IS_ERR(res)) { + errno = PTR_ERR(res); goto error2; } @@ -319,10 +322,11 @@ static void *elf_load(struct binfmt_args *args, elf_ehdr *header) if (zero_pages_len % PAGE_SIZE) zero_pages++; - if (!vm_mmap(zero_pages_base, zero_pages << PAGE_SHIFT, prot, - MAP_PRIVATE | MAP_FIXED | MAP_ANON, nullptr, 0)) + res = vm_mmap(zero_pages_base, zero_pages << PAGE_SHIFT, prot, + MAP_PRIVATE | MAP_FIXED | MAP_ANON, nullptr, 0); + if (IS_ERR(res)) { - errno = ENOMEM; + errno = PTR_ERR(res); goto error2; } } diff --git a/kernel/kernel/fs/filemap.cpp b/kernel/kernel/fs/filemap.cpp index bb25f1a4b..e1483a261 100644 --- a/kernel/kernel/fs/filemap.cpp +++ b/kernel/kernel/fs/filemap.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -585,17 +584,18 @@ int filemap_fdatasync(struct inode *inode, unsigned long start, unsigned long en static int filemap_mkwrite_private(struct vm_pf_context *ctx, struct page *page) NO_THREAD_SAFETY_ANALYSIS { - struct vm_area_struct *region = ctx->entry; struct page *newp = nullptr; - unsigned long pgoff = (ctx->vpage - region->vm_start) >> PAGE_SHIFT; /* write-fault, let's CoW the page */ - /* Lazily allocate the vm_amap struct */ - if (!region->vm_amap) + if (page_flag_set(page, PAGE_FLAG_ANON) && page_mapcount(page) == 1) { - region->vm_amap = amap_alloc(vma_pages(region) << PAGE_SHIFT); - if (!region->vm_amap) - return -ENOMEM; + /* If this is an anon page *and* mapcount = 1, avoid allocating a new page. Since mapcount = + * 1 (AND *ANON*), no one else can grab a ref. */ + /* TODO: We might be able to explore this - we may avoid the TLB shootdown and just change + * prots, but it would require significant code refactoring as-is. */ + ctx->page = page; + page_ref(page); + return 0; } /* Allocate a brand new page and copy the old page */ @@ -605,14 +605,6 @@ static int filemap_mkwrite_private(struct vm_pf_context *ctx, page_set_anon(newp); copy_page_to_page(page_to_phys(newp), page_to_phys(page)); - - if (amap_add(region->vm_amap, newp, region, pgoff, true) < 0) - { - free_page(newp); - return -ENOMEM; - } - - page_unref(page); ctx->page = newp; return 0; } @@ -651,31 +643,27 @@ static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS struct inode *ino = region->vm_file->f_ino; int st = 0; unsigned long pgoff = (ctx->vpage - region->vm_start) >> PAGE_SHIFT; - bool amap = true; - bool newp = false; bool needs_invalidate = false; /* We need to lock the page in case we're mapping it (that is, it's either a read-fault on * a private region, or any fault on a MAP_SHARED). */ - bool locked = (region->vm_maptype == MAP_PRIVATE && !ctx->info->write) || - region->vm_maptype == MAP_SHARED; + bool locked = (vma_private(region) && !ctx->info->write) || vma_shared(region); /* Permission checks have already been handled before .fault() */ - if (region->vm_amap) + + /* If a page was present, use that as the CoW source */ + if (vma_private(region) && ctx->mapping_info & PAGE_PRESENT) { - /* Check if the amap has any kind of page. It's possible we may need to CoW that */ - page = amap_get(region->vm_amap, pgoff); - if (page) - locked = false; + page = phys_to_page(MAPPING_INFO_PADDR(ctx->mapping_info)); + DCHECK(info->write && !(ctx->mapping_info & PAGE_WRITABLE)); } if (!page) { unsigned long fileoff = (region->vm_offset >> PAGE_SHIFT) + pgoff; - amap = false; - if (ino->i_size <= fileoff) + if (ino->i_size <= (fileoff << PAGE_SHIFT)) { - info->error_info = VM_SIGBUS; + info->signal = VM_SIGBUS; return -EIO; } @@ -687,8 +675,6 @@ static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS goto err; } - (void) amap; - #ifdef FILEMAP_PARANOID if (ctx->mapping_info & PAGE_PRESENT) { @@ -707,12 +693,10 @@ static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS } else { - if (region->vm_maptype == MAP_PRIVATE) + if (vma_private(region)) { DCHECK(!locked); st = filemap_mkwrite_private(ctx, page); - if (st == 0) - newp = true; } else st = filemap_mkwrite_shared(ctx, page); @@ -730,9 +714,6 @@ static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS ctx->entry)) goto enomem; - /* TODO: Hmm... Do we want to invalidate the TLB when doing CoW? We don't actually need to do - * that. We could just take the spurious fault ezpz, and it would possibly be more efficient on - * IPI shootdown architectures? */ if (needs_invalidate) vm_invalidate_range(ctx->vpage, 1); @@ -742,8 +723,7 @@ static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS */ if (locked) unlock_page(page); - if (!newp) - page_unref(page); + page_unref(page); return 0; enomem: st = -ENOMEM; @@ -751,7 +731,7 @@ static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS info->error_info = VM_SIGSEGV; if (locked && page) unlock_page(page); - if (page && !newp) + if (page) page_unref(page); return st; } diff --git a/kernel/kernel/fs/tmpfs.cpp b/kernel/kernel/fs/tmpfs.cpp index 51e2f788a..826691cba 100644 --- a/kernel/kernel/fs/tmpfs.cpp +++ b/kernel/kernel/fs/tmpfs.cpp @@ -229,7 +229,8 @@ struct file_ops tmpfs_fops = {.read = nullptr, .writepage = tmpfs_writepage, .prepare_write = tmpfs_prepare_write, .read_iter = filemap_read_iter, - .write_iter = filemap_write_iter}; + .write_iter = filemap_write_iter, + .fsyncdata = filemap_writepages}; static void tmpfs_free_page(struct vm_object *vmo, struct page *page) { diff --git a/kernel/kernel/fs/zero.cpp b/kernel/kernel/fs/zero.cpp index 4e6023cc4..178ba8b7c 100644 --- a/kernel/kernel/fs/zero.cpp +++ b/kernel/kernel/fs/zero.cpp @@ -32,7 +32,7 @@ void *zero_mmap(struct vm_area_struct *area, struct file *node) { vm_make_anon(area); - if (vm_area_struct_setup_backing(area, vma_pages(area), false) < 0) + if (vma_setup_backing(area, vma_pages(area), false) < 0) return nullptr; return (void *) area->vm_start; diff --git a/kernel/kernel/kcov.cpp b/kernel/kernel/kcov.cpp index e398ae2ab..40e145253 100644 --- a/kernel/kernel/kcov.cpp +++ b/kernel/kernel/kcov.cpp @@ -292,7 +292,7 @@ void *kcov_mmap(struct vm_area_struct *area, struct file *node) { if (area->vm_offset != 0) return errno = EINVAL, nullptr; - if (area->vm_maptype != MAP_SHARED) + if (!vma_shared(area)) return errno = EINVAL, nullptr; auto data = (struct kcov_data *) node->private_data; diff --git a/kernel/kernel/maple_tree.c b/kernel/kernel/maple_tree.c new file mode 100644 index 000000000..09427eee6 --- /dev/null +++ b/kernel/kernel/maple_tree.c @@ -0,0 +1,7883 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Maple Tree implementation + * Copyright (c) 2018-2022 Oracle Corporation + * Authors: Liam R. Howlett + * Matthew Wilcox + * Copyright (c) 2023 ByteDance + * Author: Peng Zhang + * Copyright (c) 2024 Pedro Falcato + */ + +/* + * DOC: Interesting implementation details of the Maple Tree + * + * Each node type has a number of slots for entries and a number of slots for + * pivots. In the case of dense nodes, the pivots are implied by the position + * and are simply the slot index + the minimum of the node. + * + * In regular B-Tree terms, pivots are called keys. The term pivot is used to + * indicate that the tree is specifying ranges. Pivots may appear in the + * subtree with an entry attached to the value whereas keys are unique to a + * specific position of a B-tree. Pivot values are inclusive of the slot with + * the same index. + * + * + * The following illustrates the layout of a range64 nodes slots and pivots. + * + * + * Slots -> | 0 | 1 | 2 | ... | 12 | 13 | 14 | 15 | + * ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬ + * │ │ │ │ │ │ │ │ └─ Implied maximum + * │ │ │ │ │ │ │ └─ Pivot 14 + * │ │ │ │ │ │ └─ Pivot 13 + * │ │ │ │ │ └─ Pivot 12 + * │ │ │ │ └─ Pivot 11 + * │ │ │ └─ Pivot 2 + * │ │ └─ Pivot 1 + * │ └─ Pivot 0 + * └─ Implied minimum + * + * Slot contents: + * Internal (non-leaf) nodes contain pointers to other nodes. + * Leaf nodes contain entries. + * + * The location of interest is often referred to as an offset. All offsets have + * a slot, but the last offset has an implied pivot from the node above (or + * UINT_MAX for the root node. + * + * Ranges complicate certain write activities. When modifying any of + * the B-tree variants, it is known that one entry will either be added or + * deleted. When modifying the Maple Tree, one store operation may overwrite + * the entire data set, or one half of the tree, or the middle half of the tree. + * + */ + +/* Exported from linux commit d67978318827d06f1c0fa4c31343a279e9df6fde */ +#include +#include +#include + +#include +// ONYX PATCH +/* #include */ +#define EXPORT_SYMBOL_GPL(s) + +#include +#include +#include +#include + +#define MA_ROOT_PARENT 1 + +#pragma GCC diagnostic ignored "-Wunused-value" + +/* + * Maple state flags + * * MA_STATE_BULK - Bulk insert mode + * * MA_STATE_REBALANCE - Indicate a rebalance during bulk insert + * * MA_STATE_PREALLOC - Preallocated nodes, WARN_ON allocation + */ +#define MA_STATE_BULK 1 +#define MA_STATE_REBALANCE 2 +#define MA_STATE_PREALLOC 4 + +#define ma_parent_ptr(x) ((struct maple_pnode *) (x)) +#define mas_tree_parent(x) ((unsigned long) (x->tree) | MA_ROOT_PARENT) +#define ma_mnode_ptr(x) ((struct maple_node *) (x)) +#define ma_enode_ptr(x) ((struct maple_enode *) (x)) +static struct slab_cache *maple_node_cache; + +#ifdef CONFIG_DEBUG_MAPLE_TREE +static const unsigned long mt_max[] = { + [maple_dense] = MAPLE_NODE_SLOTS, + [maple_leaf_64] = ULONG_MAX, + [maple_range_64] = ULONG_MAX, + [maple_arange_64] = ULONG_MAX, +}; +#define mt_node_max(x) mt_max[mte_node_type(x)] +#endif + +static const unsigned char mt_slots[] = { + [maple_dense] = MAPLE_NODE_SLOTS, + [maple_leaf_64] = MAPLE_RANGE64_SLOTS, + [maple_range_64] = MAPLE_RANGE64_SLOTS, + [maple_arange_64] = MAPLE_ARANGE64_SLOTS, +}; +#define mt_slot_count(x) mt_slots[mte_node_type(x)] + +static const unsigned char mt_pivots[] = { + [maple_dense] = 0, + [maple_leaf_64] = MAPLE_RANGE64_SLOTS - 1, + [maple_range_64] = MAPLE_RANGE64_SLOTS - 1, + [maple_arange_64] = MAPLE_ARANGE64_SLOTS - 1, +}; +#define mt_pivot_count(x) mt_pivots[mte_node_type(x)] + +static const unsigned char mt_min_slots[] = { + [maple_dense] = MAPLE_NODE_SLOTS / 2, + [maple_leaf_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, + [maple_range_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, + [maple_arange_64] = (MAPLE_ARANGE64_SLOTS / 2) - 1, +}; +#define mt_min_slot_count(x) mt_min_slots[mte_node_type(x)] + +#define MAPLE_BIG_NODE_SLOTS (MAPLE_RANGE64_SLOTS * 2 + 2) +#define MAPLE_BIG_NODE_GAPS (MAPLE_ARANGE64_SLOTS * 2 + 1) + +struct maple_big_node +{ + struct maple_pnode *parent; + unsigned long pivot[MAPLE_BIG_NODE_SLOTS - 1]; + union { + struct maple_enode *slot[MAPLE_BIG_NODE_SLOTS]; + struct + { + unsigned long padding[MAPLE_BIG_NODE_GAPS]; + unsigned long gap[MAPLE_BIG_NODE_GAPS]; + }; + }; + unsigned char b_end; + enum maple_type type; +}; + +/* + * The maple_subtree_state is used to build a tree to replace a segment of an + * existing tree in a more atomic way. Any walkers of the older tree will hit a + * dead node and restart on updates. + */ +struct maple_subtree_state +{ + struct ma_state *orig_l; /* Original left side of subtree */ + struct ma_state *orig_r; /* Original right side of subtree */ + struct ma_state *l; /* New left side of subtree */ + struct ma_state *m; /* New middle of subtree (rare) */ + struct ma_state *r; /* New right side of subtree */ + struct ma_topiary *free; /* nodes to be freed */ + struct ma_topiary *destroy; /* Nodes to be destroyed (walked and freed) */ + struct maple_big_node *bn; +}; + +#define trace_ma_write(...) +#define trace_ma_op(...) +#define trace_ma_read(...) +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) +#define EXPORT_SYMBOL(s) +#define WARN_ON_ONCE(x) (x) + +#ifdef CONFIG_KASAN_STACK +/* Prevent mas_wr_bnode() from exceeding the stack frame limit */ +#define noinline_for_kasan noinline_for_stack +#else +#define noinline_for_kasan inline +#endif + +/* Functions */ +static inline struct maple_node *mt_alloc_one(gfp_t gfp) +{ + return kmem_cache_alloc(maple_node_cache, gfp); +} + +static inline int mt_alloc_bulk(gfp_t gfp, size_t size, void **nodes) +{ + return kmem_cache_alloc_bulk(maple_node_cache, gfp, size, nodes); +} + +static inline void mt_free_one(struct maple_node *node) +{ + kmem_cache_free(maple_node_cache, node); +} + +static inline void mt_free_bulk(size_t size, void __rcu **nodes) +{ + kmem_cache_free_bulk(maple_node_cache, size, (void **) nodes); +} + +static void mt_free_rcu(struct rcu_head *head) +{ + struct maple_node *node = container_of(head, struct maple_node, rcu); + + kmem_cache_free(maple_node_cache, node); +} + +/* + * ma_free_rcu() - Use rcu callback to free a maple node + * @node: The node to free + * + * The maple tree uses the parent pointer to indicate this node is no longer in + * use and will be freed. + */ +static void ma_free_rcu(struct maple_node *node) +{ + WARN_ON(node->parent != ma_parent_ptr(node)); + call_rcu(&node->rcu, mt_free_rcu); +} + +static void mas_set_height(struct ma_state *mas) +{ + unsigned int new_flags = mas->tree->ma_flags; + + new_flags &= ~MT_FLAGS_HEIGHT_MASK; + MAS_BUG_ON(mas, mas->depth > MAPLE_HEIGHT_MAX); + new_flags |= mas->depth << MT_FLAGS_HEIGHT_OFFSET; + mas->tree->ma_flags = new_flags; +} + +static unsigned int mas_mt_height(struct ma_state *mas) +{ + return mt_height(mas->tree); +} + +static inline unsigned int mt_attr(struct maple_tree *mt) +{ + return mt->ma_flags & ~MT_FLAGS_HEIGHT_MASK; +} + +__always_inline enum maple_type mte_node_type(const struct maple_enode *entry) +{ + return ((unsigned long) entry >> MAPLE_NODE_TYPE_SHIFT) & MAPLE_NODE_TYPE_MASK; +} + +__always_inline bool ma_is_dense(const enum maple_type type) +{ + return type < maple_leaf_64; +} + +__always_inline bool ma_is_leaf(const enum maple_type type) +{ + return type < maple_range_64; +} + +__always_inline bool mte_is_leaf(const struct maple_enode *entry) +{ + return ma_is_leaf(mte_node_type(entry)); +} + +/* + * We also reserve values with the bottom two bits set to '10' which are + * below 4096 + */ +__always_inline bool mt_is_reserved(const void *entry) +{ + return ((unsigned long) entry < MAPLE_RESERVED_RANGE) && xa_is_internal(entry); +} + +__always_inline void mas_set_err(struct ma_state *mas, long err) +{ + mas->node = MA_ERROR(err); + mas->status = ma_error; +} + +__always_inline bool mas_is_ptr(const struct ma_state *mas) +{ + return mas->status == ma_root; +} + +__always_inline bool mas_is_start(const struct ma_state *mas) +{ + return mas->status == ma_start; +} + +__always_inline bool mas_is_none(const struct ma_state *mas) +{ + return mas->status == ma_none; +} + +__always_inline bool mas_is_paused(const struct ma_state *mas) +{ + return mas->status == ma_pause; +} + +__always_inline bool mas_is_overflow(struct ma_state *mas) +{ + return mas->status == ma_overflow; +} + +static inline bool mas_is_underflow(struct ma_state *mas) +{ + return mas->status == ma_underflow; +} + +__always_inline struct maple_node *mte_to_node(const struct maple_enode *entry) +{ + return (struct maple_node *) ((unsigned long) entry & ~MAPLE_NODE_MASK); +} + +/* + * mte_to_mat() - Convert a maple encoded node to a maple topiary node. + * @entry: The maple encoded node + * + * Return: a maple topiary pointer + */ +static inline struct maple_topiary *mte_to_mat(const struct maple_enode *entry) +{ + return (struct maple_topiary *) ((unsigned long) entry & ~MAPLE_NODE_MASK); +} + +/* + * mas_mn() - Get the maple state node. + * @mas: The maple state + * + * Return: the maple node (not encoded - bare pointer). + */ +static inline struct maple_node *mas_mn(const struct ma_state *mas) +{ + return mte_to_node(mas->node); +} + +/* + * mte_set_node_dead() - Set a maple encoded node as dead. + * @mn: The maple encoded node. + */ +static inline void mte_set_node_dead(struct maple_enode *mn) +{ + mte_to_node(mn)->parent = ma_parent_ptr(mte_to_node(mn)); + smp_wmb(); /* Needed for RCU */ +} + +/* Bit 1 indicates the root is a node */ +#define MAPLE_ROOT_NODE 0x02 +/* maple_type stored bit 3-6 */ +#define MAPLE_ENODE_TYPE_SHIFT 0x03 +/* Bit 2 means a NULL somewhere below */ +#define MAPLE_ENODE_NULL 0x04 + +static inline struct maple_enode *mt_mk_node(const struct maple_node *node, enum maple_type type) +{ + return (void *) ((unsigned long) node | (type << MAPLE_ENODE_TYPE_SHIFT) | MAPLE_ENODE_NULL); +} + +static inline void *mte_mk_root(const struct maple_enode *node) +{ + return (void *) ((unsigned long) node | MAPLE_ROOT_NODE); +} + +static inline void *mte_safe_root(const struct maple_enode *node) +{ + return (void *) ((unsigned long) node & ~MAPLE_ROOT_NODE); +} + +static inline void *mte_set_full(const struct maple_enode *node) +{ + return (void *) ((unsigned long) node & ~MAPLE_ENODE_NULL); +} + +static inline void *mte_clear_full(const struct maple_enode *node) +{ + return (void *) ((unsigned long) node | MAPLE_ENODE_NULL); +} + +static inline bool mte_has_null(const struct maple_enode *node) +{ + return (unsigned long) node & MAPLE_ENODE_NULL; +} + +__always_inline bool ma_is_root(struct maple_node *node) +{ + return ((unsigned long) node->parent & MA_ROOT_PARENT); +} + +__always_inline bool mte_is_root(const struct maple_enode *node) +{ + return ma_is_root(mte_to_node(node)); +} + +static inline bool mas_is_root_limits(const struct ma_state *mas) +{ + return !mas->min && mas->max == ULONG_MAX; +} + +__always_inline bool mt_is_alloc(struct maple_tree *mt) +{ + return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE); +} + +/* + * The Parent Pointer + * Excluding root, the parent pointer is 256B aligned like all other tree nodes. + * When storing a 32 or 64 bit values, the offset can fit into 5 bits. The 16 + * bit values need an extra bit to store the offset. This extra bit comes from + * a reuse of the last bit in the node type. This is possible by using bit 1 to + * indicate if bit 2 is part of the type or the slot. + * + * Note types: + * 0x??1 = Root + * 0x?00 = 16 bit nodes + * 0x010 = 32 bit nodes + * 0x110 = 64 bit nodes + * + * Slot size and alignment + * 0b??1 : Root + * 0b?00 : 16 bit values, type in 0-1, slot in 2-7 + * 0b010 : 32 bit values, type in 0-2, slot in 3-7 + * 0b110 : 64 bit values, type in 0-2, slot in 3-7 + */ + +#define MAPLE_PARENT_ROOT 0x01 + +#define MAPLE_PARENT_SLOT_SHIFT 0x03 +#define MAPLE_PARENT_SLOT_MASK 0xF8 + +#define MAPLE_PARENT_16B_SLOT_SHIFT 0x02 +#define MAPLE_PARENT_16B_SLOT_MASK 0xFC + +#define MAPLE_PARENT_RANGE64 0x06 +#define MAPLE_PARENT_RANGE32 0x04 +#define MAPLE_PARENT_NOT_RANGE16 0x02 + +/* + * mte_parent_shift() - Get the parent shift for the slot storage. + * @parent: The parent pointer cast as an unsigned long + * Return: The shift into that pointer to the star to of the slot + */ +static inline unsigned long mte_parent_shift(unsigned long parent) +{ + /* Note bit 1 == 0 means 16B */ + if (likely(parent & MAPLE_PARENT_NOT_RANGE16)) + return MAPLE_PARENT_SLOT_SHIFT; + + return MAPLE_PARENT_16B_SLOT_SHIFT; +} + +/* + * mte_parent_slot_mask() - Get the slot mask for the parent. + * @parent: The parent pointer cast as an unsigned long. + * Return: The slot mask for that parent. + */ +static inline unsigned long mte_parent_slot_mask(unsigned long parent) +{ + /* Note bit 1 == 0 means 16B */ + if (likely(parent & MAPLE_PARENT_NOT_RANGE16)) + return MAPLE_PARENT_SLOT_MASK; + + return MAPLE_PARENT_16B_SLOT_MASK; +} + +/* + * mas_parent_type() - Return the maple_type of the parent from the stored + * parent type. + * @mas: The maple state + * @enode: The maple_enode to extract the parent's enum + * Return: The node->parent maple_type + */ +static inline enum maple_type mas_parent_type(struct ma_state *mas, struct maple_enode *enode) +{ + unsigned long p_type; + + p_type = (unsigned long) mte_to_node(enode)->parent; + if (WARN_ON(p_type & MAPLE_PARENT_ROOT)) + return 0; + + p_type &= MAPLE_NODE_MASK; + p_type &= ~mte_parent_slot_mask(p_type); + switch (p_type) + { + case MAPLE_PARENT_RANGE64: /* or MAPLE_PARENT_ARANGE64 */ + if (mt_is_alloc(mas->tree)) + return maple_arange_64; + return maple_range_64; + } + + return 0; +} + +/* + * mas_set_parent() - Set the parent node and encode the slot + * @enode: The encoded maple node. + * @parent: The encoded maple node that is the parent of @enode. + * @slot: The slot that @enode resides in @parent. + * + * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the + * parent type. + */ +static inline void mas_set_parent(struct ma_state *mas, struct maple_enode *enode, + const struct maple_enode *parent, unsigned char slot) +{ + unsigned long val = (unsigned long) parent; + unsigned long shift; + unsigned long type; + enum maple_type p_type = mte_node_type(parent); + + MAS_BUG_ON(mas, p_type == maple_dense); + MAS_BUG_ON(mas, p_type == maple_leaf_64); + + switch (p_type) + { + case maple_range_64: + case maple_arange_64: + shift = MAPLE_PARENT_SLOT_SHIFT; + type = MAPLE_PARENT_RANGE64; + break; + default: + case maple_dense: + case maple_leaf_64: + shift = type = 0; + break; + } + + val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */ + val |= (slot << shift) | type; + mte_to_node(enode)->parent = ma_parent_ptr(val); +} + +/* + * mte_parent_slot() - get the parent slot of @enode. + * @enode: The encoded maple node. + * + * Return: The slot in the parent node where @enode resides. + */ +__always_inline unsigned int mte_parent_slot(const struct maple_enode *enode) +{ + unsigned long val = (unsigned long) mte_to_node(enode)->parent; + + if (unlikely(val & MA_ROOT_PARENT)) + return 0; + + /* + * Okay to use MAPLE_PARENT_16B_SLOT_MASK as the last bit will be lost + * by shift if the parent shift is MAPLE_PARENT_SLOT_SHIFT + */ + return (val & MAPLE_PARENT_16B_SLOT_MASK) >> mte_parent_shift(val); +} + +/* + * mte_parent() - Get the parent of @node. + * @node: The encoded maple node. + * + * Return: The parent maple node. + */ +__always_inline struct maple_node *mte_parent(const struct maple_enode *enode) +{ + return (void *) ((unsigned long) (mte_to_node(enode)->parent) & ~MAPLE_NODE_MASK); +} + +/* + * ma_dead_node() - check if the @enode is dead. + * @enode: The encoded maple node + * + * Return: true if dead, false otherwise. + */ +__always_inline bool ma_dead_node(const struct maple_node *node) +{ + struct maple_node *parent; + + /* Do not reorder reads from the node prior to the parent check */ + smp_rmb(); + parent = (void *) ((unsigned long) node->parent & ~MAPLE_NODE_MASK); + return (parent == node); +} + +/* + * mte_dead_node() - check if the @enode is dead. + * @enode: The encoded maple node + * + * Return: true if dead, false otherwise. + */ +__always_inline bool mte_dead_node(const struct maple_enode *enode) +{ + struct maple_node *parent, *node; + + node = mte_to_node(enode); + /* Do not reorder reads from the node prior to the parent check */ + smp_rmb(); + parent = mte_parent(enode); + return (parent == node); +} + +/* + * mas_allocated() - Get the number of nodes allocated in a maple state. + * @mas: The maple state + * + * The ma_state alloc member is overloaded to hold a pointer to the first + * allocated node or to the number of requested nodes to allocate. If bit 0 is + * set, then the alloc contains the number of requested nodes. If there is an + * allocated node, then the total allocated nodes is in that node. + * + * Return: The total number of nodes allocated + */ +static inline unsigned long mas_allocated(const struct ma_state *mas) +{ + if (!mas->alloc || ((unsigned long) mas->alloc & 0x1)) + return 0; + + return mas->alloc->total; +} + +/* + * mas_set_alloc_req() - Set the requested number of allocations. + * @mas: the maple state + * @count: the number of allocations. + * + * The requested number of allocations is either in the first allocated node, + * located in @mas->alloc->request_count, or directly in @mas->alloc if there is + * no allocated node. Set the request either in the node or do the necessary + * encoding to store in @mas->alloc directly. + */ +static inline void mas_set_alloc_req(struct ma_state *mas, unsigned long count) +{ + if (!mas->alloc || ((unsigned long) mas->alloc & 0x1)) + { + if (!count) + mas->alloc = NULL; + else + mas->alloc = (struct maple_alloc *) (((count) << 1U) | 1U); + return; + } + + mas->alloc->request_count = count; +} + +/* + * mas_alloc_req() - get the requested number of allocations. + * @mas: The maple state + * + * The alloc count is either stored directly in @mas, or in + * @mas->alloc->request_count if there is at least one node allocated. Decode + * the request count if it's stored directly in @mas->alloc. + * + * Return: The allocation request count. + */ +static inline unsigned int mas_alloc_req(const struct ma_state *mas) +{ + if ((unsigned long) mas->alloc & 0x1) + return (unsigned long) (mas->alloc) >> 1; + else if (mas->alloc) + return mas->alloc->request_count; + return 0; +} + +/* + * ma_pivots() - Get a pointer to the maple node pivots. + * @node - the maple node + * @type - the node type + * + * In the event of a dead node, this array may be %NULL + * + * Return: A pointer to the maple node pivots + */ +static inline unsigned long *ma_pivots(struct maple_node *node, enum maple_type type) +{ + switch (type) + { + case maple_arange_64: + return node->ma64.pivot; + case maple_range_64: + case maple_leaf_64: + return node->mr64.pivot; + case maple_dense: + return NULL; + } + return NULL; +} + +/* + * ma_gaps() - Get a pointer to the maple node gaps. + * @node - the maple node + * @type - the node type + * + * Return: A pointer to the maple node gaps + */ +static inline unsigned long *ma_gaps(struct maple_node *node, enum maple_type type) +{ + switch (type) + { + case maple_arange_64: + return node->ma64.gap; + case maple_range_64: + case maple_leaf_64: + case maple_dense: + return NULL; + } + return NULL; +} + +/* + * mas_safe_pivot() - get the pivot at @piv or mas->max. + * @mas: The maple state + * @pivots: The pointer to the maple node pivots + * @piv: The pivot to fetch + * @type: The maple node type + * + * Return: The pivot at @piv within the limit of the @pivots array, @mas->max + * otherwise. + */ +__always_inline unsigned long mas_safe_pivot(const struct ma_state *mas, unsigned long *pivots, + unsigned char piv, enum maple_type type) +{ + if (piv >= mt_pivots[type]) + return mas->max; + + return pivots[piv]; +} + +/* + * mas_safe_min() - Return the minimum for a given offset. + * @mas: The maple state + * @pivots: The pointer to the maple node pivots + * @offset: The offset into the pivot array + * + * Return: The minimum range value that is contained in @offset. + */ +static inline unsigned long mas_safe_min(struct ma_state *mas, unsigned long *pivots, + unsigned char offset) +{ + if (likely(offset)) + return pivots[offset - 1] + 1; + + return mas->min; +} + +/* + * mte_set_pivot() - Set a pivot to a value in an encoded maple node. + * @mn: The encoded maple node + * @piv: The pivot offset + * @val: The value of the pivot + */ +static inline void mte_set_pivot(struct maple_enode *mn, unsigned char piv, unsigned long val) +{ + struct maple_node *node = mte_to_node(mn); + enum maple_type type = mte_node_type(mn); + + BUG_ON(piv >= mt_pivots[type]); + switch (type) + { + case maple_range_64: + case maple_leaf_64: + node->mr64.pivot[piv] = val; + break; + case maple_arange_64: + node->ma64.pivot[piv] = val; + break; + case maple_dense: + break; + } +} + +/* + * ma_slots() - Get a pointer to the maple node slots. + * @mn: The maple node + * @mt: The maple node type + * + * Return: A pointer to the maple node slots + */ +static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt) +{ + switch (mt) + { + case maple_arange_64: + return mn->ma64.slot; + case maple_range_64: + case maple_leaf_64: + return mn->mr64.slot; + case maple_dense: + return mn->slot; + } + + return NULL; +} + +#define lockdep_is_held(l) 1 + +static inline bool mt_write_locked(const struct maple_tree *mt) +{ + return mt_external_lock(mt) ? mt_write_lock_is_held(mt) : lockdep_is_held(&mt->ma_lock); +} + +__always_inline bool mt_locked(const struct maple_tree *mt) +{ + return mt_external_lock(mt) ? mt_lock_is_held(mt) : lockdep_is_held(&mt->ma_lock); +} + +#define rcu_dereference_check(p, locked) rcu_dereference(p) +#define rcu_dereference_protected(p, locked) rcu_dereference(p) + +__always_inline void *mt_slot(const struct maple_tree *mt, void __rcu **slots, unsigned char offset) +{ + return rcu_dereference_check(slots[offset], mt_locked(mt)); +} + +__always_inline void *mt_slot_locked(struct maple_tree *mt, void __rcu **slots, + unsigned char offset) +{ + return rcu_dereference_protected(slots[offset], mt_write_locked(mt)); +} +/* + * mas_slot_locked() - Get the slot value when holding the maple tree lock. + * @mas: The maple state + * @slots: The pointer to the slots + * @offset: The offset into the slots array to fetch + * + * Return: The entry stored in @slots at the @offset. + */ +__always_inline void *mas_slot_locked(struct ma_state *mas, void __rcu **slots, + unsigned char offset) +{ + return mt_slot_locked(mas->tree, slots, offset); +} + +/* + * mas_slot() - Get the slot value when not holding the maple tree lock. + * @mas: The maple state + * @slots: The pointer to the slots + * @offset: The offset into the slots array to fetch + * + * Return: The entry stored in @slots at the @offset + */ +__always_inline void *mas_slot(struct ma_state *mas, void __rcu **slots, unsigned char offset) +{ + return mt_slot(mas->tree, slots, offset); +} + +/* + * mas_root() - Get the maple tree root. + * @mas: The maple state. + * + * Return: The pointer to the root of the tree + */ +__always_inline void *mas_root(struct ma_state *mas) +{ + return rcu_dereference_check(mas->tree->ma_root, mt_locked(mas->tree)); +} + +static inline void *mt_root_locked(struct maple_tree *mt) +{ + return rcu_dereference_protected(mt->ma_root, mt_write_locked(mt)); +} + +/* + * mas_root_locked() - Get the maple tree root when holding the maple tree lock. + * @mas: The maple state. + * + * Return: The pointer to the root of the tree + */ +static inline void *mas_root_locked(struct ma_state *mas) +{ + return mt_root_locked(mas->tree); +} + +static inline struct maple_metadata *ma_meta(struct maple_node *mn, enum maple_type mt) +{ + switch (mt) + { + case maple_arange_64: + return &mn->ma64.meta; + default: + return &mn->mr64.meta; + } +} + +/* + * ma_set_meta() - Set the metadata information of a node. + * @mn: The maple node + * @mt: The maple node type + * @offset: The offset of the highest sub-gap in this node. + * @end: The end of the data in this node. + */ +static inline void ma_set_meta(struct maple_node *mn, enum maple_type mt, unsigned char offset, + unsigned char end) +{ + struct maple_metadata *meta = ma_meta(mn, mt); + + meta->gap = offset; + meta->end = end; +} + +/* + * mt_clear_meta() - clear the metadata information of a node, if it exists + * @mt: The maple tree + * @mn: The maple node + * @type: The maple node type + * @offset: The offset of the highest sub-gap in this node. + * @end: The end of the data in this node. + */ +static inline void mt_clear_meta(struct maple_tree *mt, struct maple_node *mn, enum maple_type type) +{ + struct maple_metadata *meta; + unsigned long *pivots; + void __rcu **slots; + void *next; + + switch (type) + { + case maple_range_64: + pivots = mn->mr64.pivot; + if (unlikely(pivots[MAPLE_RANGE64_SLOTS - 2])) + { + slots = mn->mr64.slot; + next = mt_slot_locked(mt, slots, MAPLE_RANGE64_SLOTS - 1); + if (unlikely((mte_to_node(next) && mte_node_type(next)))) + return; /* no metadata, could be node */ + } + /* fallthrough */ + case maple_arange_64: + meta = ma_meta(mn, type); + break; + default: + return; + } + + meta->gap = 0; + meta->end = 0; +} + +/* + * ma_meta_end() - Get the data end of a node from the metadata + * @mn: The maple node + * @mt: The maple node type + */ +static inline unsigned char ma_meta_end(struct maple_node *mn, enum maple_type mt) +{ + struct maple_metadata *meta = ma_meta(mn, mt); + + return meta->end; +} + +/* + * ma_meta_gap() - Get the largest gap location of a node from the metadata + * @mn: The maple node + */ +static inline unsigned char ma_meta_gap(struct maple_node *mn) +{ + return mn->ma64.meta.gap; +} + +/* + * ma_set_meta_gap() - Set the largest gap location in a nodes metadata + * @mn: The maple node + * @mn: The maple node type + * @offset: The location of the largest gap. + */ +static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, unsigned char offset) +{ + + struct maple_metadata *meta = ma_meta(mn, mt); + + meta->gap = offset; +} + +/* + * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes. + * @mat - the ma_topiary, a linked list of dead nodes. + * @dead_enode - the node to be marked as dead and added to the tail of the list + * + * Add the @dead_enode to the linked list in @mat. + */ +static inline void mat_add(struct ma_topiary *mat, struct maple_enode *dead_enode) +{ + mte_set_node_dead(dead_enode); + mte_to_mat(dead_enode)->next = NULL; + if (!mat->tail) + { + mat->tail = mat->head = dead_enode; + return; + } + + mte_to_mat(mat->tail)->next = dead_enode; + mat->tail = dead_enode; +} + +static void mt_free_walk(struct rcu_head *head); +static void mt_destroy_walk(struct maple_enode *enode, struct maple_tree *mt, bool free); +/* + * mas_mat_destroy() - Free all nodes and subtrees in a dead list. + * @mas - the maple state + * @mat - the ma_topiary linked list of dead nodes to free. + * + * Destroy walk a dead list. + */ +static void mas_mat_destroy(struct ma_state *mas, struct ma_topiary *mat) +{ + struct maple_enode *next; + struct maple_node *node; + bool in_rcu = mt_in_rcu(mas->tree); + + while (mat->head) + { + next = mte_to_mat(mat->head)->next; + node = mte_to_node(mat->head); + mt_destroy_walk(mat->head, mas->tree, !in_rcu); + if (in_rcu) + call_rcu(&node->rcu, mt_free_walk); + mat->head = next; + } +} +/* + * mas_descend() - Descend into the slot stored in the ma_state. + * @mas - the maple state. + * + * Note: Not RCU safe, only use in write side or debug code. + */ +static inline void mas_descend(struct ma_state *mas) +{ + enum maple_type type; + unsigned long *pivots; + struct maple_node *node; + void __rcu **slots; + + node = mas_mn(mas); + type = mte_node_type(mas->node); + pivots = ma_pivots(node, type); + slots = ma_slots(node, type); + + if (mas->offset) + mas->min = pivots[mas->offset - 1] + 1; + mas->max = mas_safe_pivot(mas, pivots, mas->offset, type); + mas->node = mas_slot(mas, slots, mas->offset); +} + +/* + * mte_set_gap() - Set a maple node gap. + * @mn: The encoded maple node + * @gap: The offset of the gap to set + * @val: The gap value + */ +static inline void mte_set_gap(const struct maple_enode *mn, unsigned char gap, unsigned long val) +{ + switch (mte_node_type(mn)) + { + default: + break; + case maple_arange_64: + mte_to_node(mn)->ma64.gap[gap] = val; + break; + } +} + +/* + * mas_ascend() - Walk up a level of the tree. + * @mas: The maple state + * + * Sets the @mas->max and @mas->min to the correct values when walking up. This + * may cause several levels of walking up to find the correct min and max. + * May find a dead node which will cause a premature return. + * Return: 1 on dead node, 0 otherwise + */ +static int mas_ascend(struct ma_state *mas) +{ + struct maple_enode *p_enode; /* parent enode. */ + struct maple_enode *a_enode; /* ancestor enode. */ + struct maple_node *a_node; /* ancestor node. */ + struct maple_node *p_node; /* parent node. */ + unsigned char a_slot; + enum maple_type a_type; + unsigned long min, max; + unsigned long *pivots; + bool set_max = false, set_min = false; + + a_node = mas_mn(mas); + if (ma_is_root(a_node)) + { + mas->offset = 0; + return 0; + } + + p_node = mte_parent(mas->node); + if (unlikely(a_node == p_node)) + return 1; + + a_type = mas_parent_type(mas, mas->node); + mas->offset = mte_parent_slot(mas->node); + a_enode = mt_mk_node(p_node, a_type); + + /* Check to make sure all parent information is still accurate */ + if (p_node != mte_parent(mas->node)) + return 1; + + mas->node = a_enode; + + if (mte_is_root(a_enode)) + { + mas->max = ULONG_MAX; + mas->min = 0; + return 0; + } + + min = 0; + max = ULONG_MAX; + if (!mas->offset) + { + min = mas->min; + set_min = true; + } + + if (mas->max == ULONG_MAX) + set_max = true; + + do + { + p_enode = a_enode; + a_type = mas_parent_type(mas, p_enode); + a_node = mte_parent(p_enode); + a_slot = mte_parent_slot(p_enode); + a_enode = mt_mk_node(a_node, a_type); + pivots = ma_pivots(a_node, a_type); + + if (unlikely(ma_dead_node(a_node))) + return 1; + + if (!set_min && a_slot) + { + set_min = true; + min = pivots[a_slot - 1] + 1; + } + + if (!set_max && a_slot < mt_pivots[a_type]) + { + set_max = true; + max = pivots[a_slot]; + } + + if (unlikely(ma_dead_node(a_node))) + return 1; + + if (unlikely(ma_is_root(a_node))) + break; + + } while (!set_min || !set_max); + + mas->max = max; + mas->min = min; + return 0; +} + +/* + * mas_pop_node() - Get a previously allocated maple node from the maple state. + * @mas: The maple state + * + * Return: A pointer to a maple node. + */ +static inline struct maple_node *mas_pop_node(struct ma_state *mas) +{ + struct maple_alloc *ret, *node = mas->alloc; + unsigned long total = mas_allocated(mas); + unsigned int req = mas_alloc_req(mas); + + /* nothing or a request pending. */ + if (WARN_ON(!total)) + return NULL; + + if (total == 1) + { + /* single allocation in this ma_state */ + mas->alloc = NULL; + ret = node; + goto single_node; + } + + if (node->node_count == 1) + { + /* Single allocation in this node. */ + mas->alloc = node->slot[0]; + mas->alloc->total = node->total - 1; + ret = node; + goto new_head; + } + node->total--; + ret = node->slot[--node->node_count]; + node->slot[node->node_count] = NULL; + +single_node: +new_head: + if (req) + { + req++; + mas_set_alloc_req(mas, req); + } + + memset(ret, 0, sizeof(*ret)); + return (struct maple_node *) ret; +} + +/* + * mas_push_node() - Push a node back on the maple state allocation. + * @mas: The maple state + * @used: The used maple node + * + * Stores the maple node back into @mas->alloc for reuse. Updates allocated and + * requested node count as necessary. + */ +static inline void mas_push_node(struct ma_state *mas, struct maple_node *used) +{ + struct maple_alloc *reuse = (struct maple_alloc *) used; + struct maple_alloc *head = mas->alloc; + unsigned long count; + unsigned int requested = mas_alloc_req(mas); + + count = mas_allocated(mas); + + reuse->request_count = 0; + reuse->node_count = 0; + if (count && (head->node_count < MAPLE_ALLOC_SLOTS)) + { + head->slot[head->node_count++] = reuse; + head->total++; + goto done; + } + + reuse->total = 1; + if ((head) && !((unsigned long) head & 0x1)) + { + reuse->slot[0] = head; + reuse->node_count = 1; + reuse->total += head->total; + } + + mas->alloc = reuse; +done: + if (requested > 1) + mas_set_alloc_req(mas, requested - 1); +} + +/* + * mas_alloc_nodes() - Allocate nodes into a maple state + * @mas: The maple state + * @gfp: The GFP Flags + */ +static inline void mas_alloc_nodes(struct ma_state *mas, gfp_t gfp) +{ + struct maple_alloc *node; + unsigned long allocated = mas_allocated(mas); + unsigned int requested = mas_alloc_req(mas); + unsigned int count; + void **slots = NULL; + unsigned int max_req = 0; + + if (!requested) + return; + + mas_set_alloc_req(mas, 0); + if (mas->mas_flags & MA_STATE_PREALLOC) + { + if (allocated) + return; + BUG_ON(!allocated); + WARN_ON(!allocated); + } + + if (!allocated || mas->alloc->node_count == MAPLE_ALLOC_SLOTS) + { + node = (struct maple_alloc *) mt_alloc_one(gfp); + if (!node) + goto nomem_one; + + if (allocated) + { + node->slot[0] = mas->alloc; + node->node_count = 1; + } + else + { + node->node_count = 0; + } + + mas->alloc = node; + node->total = ++allocated; + requested--; + } + + node = mas->alloc; + node->request_count = 0; + while (requested) + { + max_req = MAPLE_ALLOC_SLOTS - node->node_count; + slots = (void **) &node->slot[node->node_count]; + max_req = min(requested, max_req); + count = mt_alloc_bulk(gfp, max_req, slots); + if (!count) + goto nomem_bulk; + + if (node->node_count == 0) + { + node->slot[0]->node_count = 0; + node->slot[0]->request_count = 0; + } + + node->node_count += count; + allocated += count; + node = node->slot[0]; + requested -= count; + } + mas->alloc->total = allocated; + return; + +nomem_bulk: + /* Clean up potential freed allocations on bulk failure */ + memset(slots, 0, max_req * sizeof(unsigned long)); +nomem_one: + mas_set_alloc_req(mas, requested); + if (mas->alloc && !(((unsigned long) mas->alloc & 0x1))) + mas->alloc->total = allocated; + mas_set_err(mas, -ENOMEM); +} + +/* + * mas_free() - Free an encoded maple node + * @mas: The maple state + * @used: The encoded maple node to free. + * + * Uses rcu free if necessary, pushes @used back on the maple state allocations + * otherwise. + */ +static inline void mas_free(struct ma_state *mas, struct maple_enode *used) +{ + struct maple_node *tmp = mte_to_node(used); + + if (mt_in_rcu(mas->tree)) + ma_free_rcu(tmp); + else + mas_push_node(mas, tmp); +} + +/* + * mas_node_count_gfp() - Check if enough nodes are allocated and request more + * if there is not enough nodes. + * @mas: The maple state + * @count: The number of nodes needed + * @gfp: the gfp flags + */ +static void mas_node_count_gfp(struct ma_state *mas, int count, gfp_t gfp) +{ + unsigned long allocated = mas_allocated(mas); + + if (allocated < (unsigned long) count) + { + mas_set_alloc_req(mas, count - allocated); + mas_alloc_nodes(mas, gfp); + } +} + +/* + * mas_node_count() - Check if enough nodes are allocated and request more if + * there is not enough nodes. + * @mas: The maple state + * @count: The number of nodes needed + * + * Note: Uses GFP_NOWAIT | __GFP_NOWARN for gfp flags. + */ +static void mas_node_count(struct ma_state *mas, int count) +{ + return mas_node_count_gfp(mas, count, GFP_NOWAIT | __GFP_NOWARN); +} + +/* + * mas_start() - Sets up maple state for operations. + * @mas: The maple state. + * + * If mas->status == mas_start, then set the min, max and depth to + * defaults. + * + * Return: + * - If mas->node is an error or not mas_start, return NULL. + * - If it's an empty tree: NULL & mas->status == ma_none + * - If it's a single entry: The entry & mas->status == mas_root + * - If it's a tree: NULL & mas->status == safe root node. + */ +static inline struct maple_enode *mas_start(struct ma_state *mas) +{ + if (likely(mas_is_start(mas))) + { + struct maple_enode *root; + + mas->min = 0; + mas->max = ULONG_MAX; + + retry: + mas->depth = 0; + root = mas_root(mas); + /* Tree with nodes */ + if (likely(xa_is_node(root))) + { + mas->depth = 1; + mas->status = ma_active; + mas->node = mte_safe_root(root); + mas->offset = 0; + if (mte_dead_node(mas->node)) + goto retry; + + return NULL; + } + + /* empty tree */ + if (unlikely(!root)) + { + mas->node = NULL; + mas->status = ma_none; + mas->offset = MAPLE_NODE_SLOTS; + return NULL; + } + + /* Single entry tree */ + mas->status = ma_root; + mas->offset = MAPLE_NODE_SLOTS; + + /* Single entry tree. */ + if (mas->index > 0) + return NULL; + + return root; + } + + return NULL; +} + +/* + * ma_data_end() - Find the end of the data in a node. + * @node: The maple node + * @type: The maple node type + * @pivots: The array of pivots in the node + * @max: The maximum value in the node + * + * Uses metadata to find the end of the data when possible. + * Return: The zero indexed last slot with data (may be null). + */ +__always_inline unsigned char ma_data_end(struct maple_node *node, enum maple_type type, + unsigned long *pivots, unsigned long max) +{ + unsigned char offset; + + if (!pivots) + return 0; + + if (type == maple_arange_64) + return ma_meta_end(node, type); + + offset = mt_pivots[type] - 1; + if (likely(!pivots[offset])) + return ma_meta_end(node, type); + + if (likely(pivots[offset] == max)) + return offset; + + return mt_pivots[type]; +} + +/* + * mas_data_end() - Find the end of the data (slot). + * @mas: the maple state + * + * This method is optimized to check the metadata of a node if the node type + * supports data end metadata. + * + * Return: The zero indexed last slot with data (may be null). + */ +static inline unsigned char mas_data_end(struct ma_state *mas) +{ + enum maple_type type; + struct maple_node *node; + unsigned char offset; + unsigned long *pivots; + + type = mte_node_type(mas->node); + node = mas_mn(mas); + if (type == maple_arange_64) + return ma_meta_end(node, type); + + pivots = ma_pivots(node, type); + if (unlikely(ma_dead_node(node))) + return 0; + + offset = mt_pivots[type] - 1; + if (likely(!pivots[offset])) + return ma_meta_end(node, type); + + if (likely(pivots[offset] == mas->max)) + return offset; + + return mt_pivots[type]; +} + +/* + * mas_leaf_max_gap() - Returns the largest gap in a leaf node + * @mas - the maple state + * + * Return: The maximum gap in the leaf. + */ +static unsigned long mas_leaf_max_gap(struct ma_state *mas) +{ + enum maple_type mt; + unsigned long pstart, gap, max_gap; + struct maple_node *mn; + unsigned long *pivots; + void __rcu **slots; + unsigned char i; + unsigned char max_piv; + + mt = mte_node_type(mas->node); + mn = mas_mn(mas); + slots = ma_slots(mn, mt); + max_gap = 0; + if (unlikely(ma_is_dense(mt))) + { + gap = 0; + for (i = 0; i < mt_slots[mt]; i++) + { + if (slots[i]) + { + if (gap > max_gap) + max_gap = gap; + gap = 0; + } + else + { + gap++; + } + } + if (gap > max_gap) + max_gap = gap; + return max_gap; + } + + /* + * Check the first implied pivot optimizes the loop below and slot 1 may + * be skipped if there is a gap in slot 0. + */ + pivots = ma_pivots(mn, mt); + if (likely(!slots[0])) + { + max_gap = pivots[0] - mas->min + 1; + i = 2; + } + else + { + i = 1; + } + + /* reduce max_piv as the special case is checked before the loop */ + max_piv = ma_data_end(mn, mt, pivots, mas->max) - 1; + /* + * Check end implied pivot which can only be a gap on the right most + * node. + */ + if (unlikely(mas->max == ULONG_MAX) && !slots[max_piv + 1]) + { + gap = ULONG_MAX - pivots[max_piv]; + if (gap > max_gap) + max_gap = gap; + + if (max_gap > pivots[max_piv] - mas->min) + return max_gap; + } + + for (; i <= max_piv; i++) + { + /* data == no gap. */ + if (likely(slots[i])) + continue; + + pstart = pivots[i - 1]; + gap = pivots[i] - pstart; + if (gap > max_gap) + max_gap = gap; + + /* There cannot be two gaps in a row. */ + i++; + } + return max_gap; +} + +/* + * ma_max_gap() - Get the maximum gap in a maple node (non-leaf) + * @node: The maple node + * @gaps: The pointer to the gaps + * @mt: The maple node type + * @*off: Pointer to store the offset location of the gap. + * + * Uses the metadata data end to scan backwards across set gaps. + * + * Return: The maximum gap value + */ +static inline unsigned long ma_max_gap(struct maple_node *node, unsigned long *gaps, + enum maple_type mt, unsigned char *off) +{ + unsigned char offset, i; + unsigned long max_gap = 0; + + i = offset = ma_meta_end(node, mt); + do + { + if (gaps[i] > max_gap) + { + max_gap = gaps[i]; + offset = i; + } + } while (i--); + + *off = offset; + return max_gap; +} + +/* + * mas_max_gap() - find the largest gap in a non-leaf node and set the slot. + * @mas: The maple state. + * + * Return: The gap value. + */ +static inline unsigned long mas_max_gap(struct ma_state *mas) +{ + unsigned long *gaps; + unsigned char offset; + enum maple_type mt; + struct maple_node *node; + + mt = mte_node_type(mas->node); + if (ma_is_leaf(mt)) + return mas_leaf_max_gap(mas); + + node = mas_mn(mas); + MAS_BUG_ON(mas, mt != maple_arange_64); + offset = ma_meta_gap(node); + gaps = ma_gaps(node, mt); + return gaps[offset]; +} + +/* + * mas_parent_gap() - Set the parent gap and any gaps above, as needed + * @mas: The maple state + * @offset: The gap offset in the parent to set + * @new: The new gap value. + * + * Set the parent gap then continue to set the gap upwards, using the metadata + * of the parent to see if it is necessary to check the node above. + */ +static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset, unsigned long new) +{ + unsigned long meta_gap = 0; + struct maple_node *pnode; + struct maple_enode *penode; + unsigned long *pgaps; + unsigned char meta_offset; + enum maple_type pmt; + + pnode = mte_parent(mas->node); + pmt = mas_parent_type(mas, mas->node); + penode = mt_mk_node(pnode, pmt); + pgaps = ma_gaps(pnode, pmt); + +ascend: + MAS_BUG_ON(mas, pmt != maple_arange_64); + meta_offset = ma_meta_gap(pnode); + meta_gap = pgaps[meta_offset]; + + pgaps[offset] = new; + + if (meta_gap == new) + return; + + if (offset != meta_offset) + { + if (meta_gap > new) + return; + + ma_set_meta_gap(pnode, pmt, offset); + } + else if (new < meta_gap) + { + new = ma_max_gap(pnode, pgaps, pmt, &meta_offset); + ma_set_meta_gap(pnode, pmt, meta_offset); + } + + if (ma_is_root(pnode)) + return; + + /* Go to the parent node. */ + pnode = mte_parent(penode); + pmt = mas_parent_type(mas, penode); + pgaps = ma_gaps(pnode, pmt); + offset = mte_parent_slot(penode); + penode = mt_mk_node(pnode, pmt); + goto ascend; +} + +/* + * mas_update_gap() - Update a nodes gaps and propagate up if necessary. + * @mas - the maple state. + */ +static inline void mas_update_gap(struct ma_state *mas) +{ + unsigned char pslot; + unsigned long p_gap; + unsigned long max_gap; + + if (!mt_is_alloc(mas->tree)) + return; + + if (mte_is_root(mas->node)) + return; + + max_gap = mas_max_gap(mas); + + pslot = mte_parent_slot(mas->node); + p_gap = ma_gaps(mte_parent(mas->node), mas_parent_type(mas, mas->node))[pslot]; + + if (p_gap != max_gap) + mas_parent_gap(mas, pslot, max_gap); +} + +/* + * mas_adopt_children() - Set the parent pointer of all nodes in @parent to + * @parent with the slot encoded. + * @mas - the maple state (for the tree) + * @parent - the maple encoded node containing the children. + */ +static inline void mas_adopt_children(struct ma_state *mas, struct maple_enode *parent) +{ + enum maple_type type = mte_node_type(parent); + struct maple_node *node = mte_to_node(parent); + void __rcu **slots = ma_slots(node, type); + unsigned long *pivots = ma_pivots(node, type); + struct maple_enode *child; + unsigned char offset; + + offset = ma_data_end(node, type, pivots, mas->max); + do + { + child = mas_slot_locked(mas, slots, offset); + mas_set_parent(mas, child, parent, offset); + } while (offset--); +} + +#define __must_hold(x) + +/* + * mas_put_in_tree() - Put a new node in the tree, smp_wmb(), and mark the old + * node as dead. + * @mas - the maple state with the new node + * @old_enode - The old maple encoded node to replace. + */ +static inline void mas_put_in_tree(struct ma_state *mas, struct maple_enode *old_enode) + __must_hold(mas->tree->ma_lock) +{ + unsigned char offset; + void __rcu **slots; + + if (mte_is_root(mas->node)) + { + mas_mn(mas)->parent = ma_parent_ptr(mas_tree_parent(mas)); + rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); + mas_set_height(mas); + } + else + { + + offset = mte_parent_slot(mas->node); + slots = ma_slots(mte_parent(mas->node), mas_parent_type(mas, mas->node)); + rcu_assign_pointer(slots[offset], mas->node); + } + + mte_set_node_dead(old_enode); +} + +/* + * mas_replace_node() - Replace a node by putting it in the tree, marking it + * dead, and freeing it. + * the parent encoding to locate the maple node in the tree. + * @mas - the ma_state with @mas->node pointing to the new node. + * @old_enode - The old maple encoded node. + */ +static inline void mas_replace_node(struct ma_state *mas, struct maple_enode *old_enode) + __must_hold(mas->tree->ma_lock) +{ + mas_put_in_tree(mas, old_enode); + mas_free(mas, old_enode); +} + +/* + * mas_find_child() - Find a child who has the parent @mas->node. + * @mas: the maple state with the parent. + * @child: the maple state to store the child. + */ +static inline bool mas_find_child(struct ma_state *mas, struct ma_state *child) + __must_hold(mas->tree->ma_lock) +{ + enum maple_type mt; + unsigned char offset; + unsigned char end; + unsigned long *pivots; + struct maple_enode *entry; + struct maple_node *node; + void __rcu **slots; + + mt = mte_node_type(mas->node); + node = mas_mn(mas); + slots = ma_slots(node, mt); + pivots = ma_pivots(node, mt); + end = ma_data_end(node, mt, pivots, mas->max); + for (offset = mas->offset; offset <= end; offset++) + { + entry = mas_slot_locked(mas, slots, offset); + if (mte_parent(entry) == node) + { + *child = *mas; + mas->offset = offset + 1; + child->offset = offset; + mas_descend(child); + child->offset = 0; + return true; + } + } + return false; +} + +/* + * mab_shift_right() - Shift the data in mab right. Note, does not clean out the + * old data or set b_node->b_end. + * @b_node: the maple_big_node + * @shift: the shift count + */ +static inline void mab_shift_right(struct maple_big_node *b_node, unsigned char shift) +{ + unsigned long size = b_node->b_end * sizeof(unsigned long); + + memmove(b_node->pivot + shift, b_node->pivot, size); + memmove(b_node->slot + shift, b_node->slot, size); + if (b_node->type == maple_arange_64) + memmove(b_node->gap + shift, b_node->gap, size); +} + +/* + * mab_middle_node() - Check if a middle node is needed (unlikely) + * @b_node: the maple_big_node that contains the data. + * @size: the amount of data in the b_node + * @split: the potential split location + * @slot_count: the size that can be stored in a single node being considered. + * + * Return: true if a middle node is required. + */ +static inline bool mab_middle_node(struct maple_big_node *b_node, int split, + unsigned char slot_count) +{ + unsigned char size = b_node->b_end; + + if (size >= 2 * slot_count) + return true; + + if (!b_node->slot[split] && (size >= 2 * slot_count - 1)) + return true; + + return false; +} + +/* + * mab_no_null_split() - ensure the split doesn't fall on a NULL + * @b_node: the maple_big_node with the data + * @split: the suggested split location + * @slot_count: the number of slots in the node being considered. + * + * Return: the split location. + */ +static inline int mab_no_null_split(struct maple_big_node *b_node, unsigned char split, + unsigned char slot_count) +{ + if (!b_node->slot[split]) + { + /* + * If the split is less than the max slot && the right side will + * still be sufficient, then increment the split on NULL. + */ + if ((split < slot_count - 1) && (b_node->b_end - split) > (mt_min_slots[b_node->type])) + split++; + else + split--; + } + return split; +} + +// Onyx patch +#ifndef __clang__ +#pragma GCC diagnostic ignored "-Wsign-compare" +#endif + +/* + * mab_calc_split() - Calculate the split location and if there needs to be two + * splits. + * @bn: The maple_big_node with the data + * @mid_split: The second split, if required. 0 otherwise. + * + * Return: The first split location. The middle split is set in @mid_split. + */ +static inline int mab_calc_split(struct ma_state *mas, struct maple_big_node *bn, + unsigned char *mid_split, unsigned long min) +{ + unsigned char b_end = bn->b_end; + int split = b_end / 2; /* Assume equal split. */ + unsigned char slot_min, slot_count = mt_slots[bn->type]; + + /* + * To support gap tracking, all NULL entries are kept together and a node cannot + * end on a NULL entry, with the exception of the left-most leaf. The + * limitation means that the split of a node must be checked for this condition + * and be able to put more data in one direction or the other. + */ + if (unlikely((mas->mas_flags & MA_STATE_BULK))) + { + *mid_split = 0; + split = b_end - mt_min_slots[bn->type]; + + if (!ma_is_leaf(bn->type)) + return split; + + mas->mas_flags |= MA_STATE_REBALANCE; + if (!bn->slot[split]) + split--; + return split; + } + + /* + * Although extremely rare, it is possible to enter what is known as the 3-way + * split scenario. The 3-way split comes about by means of a store of a range + * that overwrites the end and beginning of two full nodes. The result is a set + * of entries that cannot be stored in 2 nodes. Sometimes, these two nodes can + * also be located in different parent nodes which are also full. This can + * carry upwards all the way to the root in the worst case. + */ + if (unlikely(mab_middle_node(bn, split, slot_count))) + { + split = b_end / 3; + *mid_split = split * 2; + } + else + { + slot_min = mt_min_slots[bn->type]; + + *mid_split = 0; + /* + * Avoid having a range less than the slot count unless it + * causes one node to be deficient. + * NOTE: mt_min_slots is 1 based, b_end and split are zero. + */ + while ((split < slot_count - 1) && ((bn->pivot[split] - min) < slot_count - 1) && + (b_end - split > slot_min)) + split++; + } + + /* Avoid ending a node on a NULL entry */ + split = mab_no_null_split(bn, split, slot_count); + + if (unlikely(*mid_split)) + *mid_split = mab_no_null_split(bn, *mid_split, slot_count); + + return split; +} + +/* + * mas_mab_cp() - Copy data from a maple state inclusively to a maple_big_node + * and set @b_node->b_end to the next free slot. + * @mas: The maple state + * @mas_start: The starting slot to copy + * @mas_end: The end slot to copy (inclusively) + * @b_node: The maple_big_node to place the data + * @mab_start: The starting location in maple_big_node to store the data. + */ +static inline void mas_mab_cp(struct ma_state *mas, unsigned char mas_start, unsigned char mas_end, + struct maple_big_node *b_node, unsigned char mab_start) +{ + enum maple_type mt; + struct maple_node *node; + void __rcu **slots; + unsigned long *pivots, *gaps; + int i = mas_start, j = mab_start; + unsigned char piv_end; + + node = mas_mn(mas); + mt = mte_node_type(mas->node); + pivots = ma_pivots(node, mt); + if (!i) + { + b_node->pivot[j] = pivots[i++]; + if (unlikely(i > mas_end)) + goto complete; + j++; + } + + piv_end = min(mas_end, mt_pivots[mt]); + for (; i < piv_end; i++, j++) + { + b_node->pivot[j] = pivots[i]; + if (unlikely(!b_node->pivot[j])) + break; + + if (unlikely(mas->max == b_node->pivot[j])) + goto complete; + } + + if (likely(i <= mas_end)) + b_node->pivot[j] = mas_safe_pivot(mas, pivots, i, mt); + +complete: + b_node->b_end = ++j; + j -= mab_start; + slots = ma_slots(node, mt); + memcpy(b_node->slot + mab_start, slots + mas_start, sizeof(void *) * j); + if (!ma_is_leaf(mt) && mt_is_alloc(mas->tree)) + { + gaps = ma_gaps(node, mt); + memcpy(b_node->gap + mab_start, gaps + mas_start, sizeof(unsigned long) * j); + } +} + +/* + * mas_leaf_set_meta() - Set the metadata of a leaf if possible. + * @node: The maple node + * @mt: The maple type + * @end: The node end + */ +static inline void mas_leaf_set_meta(struct maple_node *node, enum maple_type mt, unsigned char end) +{ + if (end < mt_slots[mt] - 1) + ma_set_meta(node, mt, 0, end); +} + +/* + * mab_mas_cp() - Copy data from maple_big_node to a maple encoded node. + * @b_node: the maple_big_node that has the data + * @mab_start: the start location in @b_node. + * @mab_end: The end location in @b_node (inclusively) + * @mas: The maple state with the maple encoded node. + */ +static inline void mab_mas_cp(struct maple_big_node *b_node, unsigned char mab_start, + unsigned char mab_end, struct ma_state *mas, bool new_max) +{ + int i, j = 0; + enum maple_type mt = mte_node_type(mas->node); + struct maple_node *node = mte_to_node(mas->node); + void __rcu **slots = ma_slots(node, mt); + unsigned long *pivots = ma_pivots(node, mt); + unsigned long *gaps = NULL; + unsigned char end; + + if (mab_end - mab_start > mt_pivots[mt]) + mab_end--; + + if (!pivots[mt_pivots[mt] - 1]) + slots[mt_pivots[mt]] = NULL; + + i = mab_start; + do + { + pivots[j++] = b_node->pivot[i++]; + } while (i <= mab_end && likely(b_node->pivot[i])); + + memcpy(slots, b_node->slot + mab_start, sizeof(void *) * (i - mab_start)); + + if (new_max) + mas->max = b_node->pivot[i - 1]; + + end = j - 1; + if (likely(!ma_is_leaf(mt) && mt_is_alloc(mas->tree))) + { + unsigned long max_gap = 0; + unsigned char offset = 0; + + gaps = ma_gaps(node, mt); + do + { + gaps[--j] = b_node->gap[--i]; + if (gaps[j] > max_gap) + { + offset = j; + max_gap = gaps[j]; + } + } while (j); + + ma_set_meta(node, mt, offset, end); + } + else + { + mas_leaf_set_meta(node, mt, end); + } +} + +/* + * mas_bulk_rebalance() - Rebalance the end of a tree after a bulk insert. + * @mas: The maple state + * @end: The maple node end + * @mt: The maple node type + */ +static inline void mas_bulk_rebalance(struct ma_state *mas, unsigned char end, enum maple_type mt) +{ + if (!(mas->mas_flags & MA_STATE_BULK)) + return; + + if (mte_is_root(mas->node)) + return; + + if (end > mt_min_slots[mt]) + { + mas->mas_flags &= ~MA_STATE_REBALANCE; + return; + } +} + +/* + * mas_store_b_node() - Store an @entry into the b_node while also copying the + * data from a maple encoded node. + * @wr_mas: the maple write state + * @b_node: the maple_big_node to fill with data + * @offset_end: the offset to end copying + * + * Return: The actual end of the data stored in @b_node + */ +static noinline_for_kasan void mas_store_b_node(struct ma_wr_state *wr_mas, + struct maple_big_node *b_node, + unsigned char offset_end) +{ + unsigned char slot; + unsigned char b_end; + /* Possible underflow of piv will wrap back to 0 before use. */ + unsigned long piv; + struct ma_state *mas = wr_mas->mas; + + b_node->type = wr_mas->type; + b_end = 0; + slot = mas->offset; + if (slot) + { + /* Copy start data up to insert. */ + mas_mab_cp(mas, 0, slot - 1, b_node, 0); + b_end = b_node->b_end; + piv = b_node->pivot[b_end - 1]; + } + else + piv = mas->min - 1; + + if (piv + 1 < mas->index) + { + /* Handle range starting after old range */ + b_node->slot[b_end] = wr_mas->content; + if (!wr_mas->content) + b_node->gap[b_end] = mas->index - 1 - piv; + b_node->pivot[b_end++] = mas->index - 1; + } + + /* Store the new entry. */ + mas->offset = b_end; + b_node->slot[b_end] = wr_mas->entry; + b_node->pivot[b_end] = mas->last; + + /* Appended. */ + if (mas->last >= mas->max) + goto b_end; + + /* Handle new range ending before old range ends */ + piv = mas_safe_pivot(mas, wr_mas->pivots, offset_end, wr_mas->type); + if (piv > mas->last) + { + if (piv == ULONG_MAX) + mas_bulk_rebalance(mas, b_node->b_end, wr_mas->type); + + if (offset_end != slot) + wr_mas->content = mas_slot_locked(mas, wr_mas->slots, offset_end); + + b_node->slot[++b_end] = wr_mas->content; + if (!wr_mas->content) + b_node->gap[b_end] = piv - mas->last + 1; + b_node->pivot[b_end] = piv; + } + + slot = offset_end + 1; + if (slot > mas->end) + goto b_end; + + /* Copy end data to the end of the node. */ + mas_mab_cp(mas, slot, mas->end + 1, b_node, ++b_end); + b_node->b_end--; + return; + +b_end: + b_node->b_end = b_end; +} + +/* + * mas_prev_sibling() - Find the previous node with the same parent. + * @mas: the maple state + * + * Return: True if there is a previous sibling, false otherwise. + */ +static inline bool mas_prev_sibling(struct ma_state *mas) +{ + unsigned int p_slot = mte_parent_slot(mas->node); + + if (mte_is_root(mas->node)) + return false; + + if (!p_slot) + return false; + + mas_ascend(mas); + mas->offset = p_slot - 1; + mas_descend(mas); + return true; +} + +/* + * mas_next_sibling() - Find the next node with the same parent. + * @mas: the maple state + * + * Return: true if there is a next sibling, false otherwise. + */ +static inline bool mas_next_sibling(struct ma_state *mas) +{ + MA_STATE(parent, mas->tree, mas->index, mas->last); + + if (mte_is_root(mas->node)) + return false; + + parent = *mas; + mas_ascend(&parent); + parent.offset = mte_parent_slot(mas->node) + 1; + if (parent.offset > mas_data_end(&parent)) + return false; + + *mas = parent; + mas_descend(mas); + return true; +} + +/* + * mte_node_or_none() - Set the enode and state. + * @enode: The encoded maple node. + * + * Set the node to the enode and the status. + */ +static inline void mas_node_or_none(struct ma_state *mas, struct maple_enode *enode) +{ + if (enode) + { + mas->node = enode; + mas->status = ma_active; + } + else + { + mas->node = NULL; + mas->status = ma_none; + } +} + +/* + * mas_wr_node_walk() - Find the correct offset for the index in the @mas. + * @wr_mas: The maple write state + * + * Uses mas_slot_locked() and does not need to worry about dead nodes. + */ +static inline void mas_wr_node_walk(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + unsigned char count, offset; + + if (unlikely(ma_is_dense(wr_mas->type))) + { + wr_mas->r_max = wr_mas->r_min = mas->index; + mas->offset = mas->index = mas->min; + return; + } + + wr_mas->node = mas_mn(wr_mas->mas); + wr_mas->pivots = ma_pivots(wr_mas->node, wr_mas->type); + count = mas->end = ma_data_end(wr_mas->node, wr_mas->type, wr_mas->pivots, mas->max); + offset = mas->offset; + + while (offset < count && mas->index > wr_mas->pivots[offset]) + offset++; + + wr_mas->r_max = offset < count ? wr_mas->pivots[offset] : mas->max; + wr_mas->r_min = mas_safe_min(mas, wr_mas->pivots, offset); + wr_mas->offset_end = mas->offset = offset; +} + +/* + * mast_rebalance_next() - Rebalance against the next node + * @mast: The maple subtree state + * @old_r: The encoded maple node to the right (next node). + */ +static inline void mast_rebalance_next(struct maple_subtree_state *mast) +{ + unsigned char b_end = mast->bn->b_end; + + mas_mab_cp(mast->orig_r, 0, mt_slot_count(mast->orig_r->node), mast->bn, b_end); + mast->orig_r->last = mast->orig_r->max; +} + +/* + * mast_rebalance_prev() - Rebalance against the previous node + * @mast: The maple subtree state + * @old_l: The encoded maple node to the left (previous node) + */ +static inline void mast_rebalance_prev(struct maple_subtree_state *mast) +{ + unsigned char end = mas_data_end(mast->orig_l) + 1; + unsigned char b_end = mast->bn->b_end; + + mab_shift_right(mast->bn, end); + mas_mab_cp(mast->orig_l, 0, end - 1, mast->bn, 0); + mast->l->min = mast->orig_l->min; + mast->orig_l->index = mast->orig_l->min; + mast->bn->b_end = end + b_end; + mast->l->offset += end; +} + +/* + * mast_spanning_rebalance() - Rebalance nodes with nearest neighbour favouring + * the node to the right. Checking the nodes to the right then the left at each + * level upwards until root is reached. + * Data is copied into the @mast->bn. + * @mast: The maple_subtree_state. + */ +static inline bool mast_spanning_rebalance(struct maple_subtree_state *mast) +{ + struct ma_state r_tmp = *mast->orig_r; + struct ma_state l_tmp = *mast->orig_l; + unsigned char depth = 0; + + do + { + mas_ascend(mast->orig_r); + mas_ascend(mast->orig_l); + depth++; + if (mast->orig_r->offset < mas_data_end(mast->orig_r)) + { + mast->orig_r->offset++; + do + { + mas_descend(mast->orig_r); + mast->orig_r->offset = 0; + } while (--depth); + + mast_rebalance_next(mast); + *mast->orig_l = l_tmp; + return true; + } + else if (mast->orig_l->offset != 0) + { + mast->orig_l->offset--; + do + { + mas_descend(mast->orig_l); + mast->orig_l->offset = mas_data_end(mast->orig_l); + } while (--depth); + + mast_rebalance_prev(mast); + *mast->orig_r = r_tmp; + return true; + } + } while (!mte_is_root(mast->orig_r->node)); + + *mast->orig_r = r_tmp; + *mast->orig_l = l_tmp; + return false; +} + +/* + * mast_ascend() - Ascend the original left and right maple states. + * @mast: the maple subtree state. + * + * Ascend the original left and right sides. Set the offsets to point to the + * data already in the new tree (@mast->l and @mast->r). + */ +static inline void mast_ascend(struct maple_subtree_state *mast) +{ + MA_WR_STATE(wr_mas, mast->orig_r, NULL); + mas_ascend(mast->orig_l); + mas_ascend(mast->orig_r); + + mast->orig_r->offset = 0; + mast->orig_r->index = mast->r->max; + /* last should be larger than or equal to index */ + if (mast->orig_r->last < mast->orig_r->index) + mast->orig_r->last = mast->orig_r->index; + + wr_mas.type = mte_node_type(mast->orig_r->node); + mas_wr_node_walk(&wr_mas); + /* Set up the left side of things */ + mast->orig_l->offset = 0; + mast->orig_l->index = mast->l->min; + wr_mas.mas = mast->orig_l; + wr_mas.type = mte_node_type(mast->orig_l->node); + mas_wr_node_walk(&wr_mas); + + mast->bn->type = wr_mas.type; +} + +/* + * mas_new_ma_node() - Create and return a new maple node. Helper function. + * @mas: the maple state with the allocations. + * @b_node: the maple_big_node with the type encoding. + * + * Use the node type from the maple_big_node to allocate a new node from the + * ma_state. This function exists mainly for code readability. + * + * Return: A new maple encoded node + */ +static inline struct maple_enode *mas_new_ma_node(struct ma_state *mas, + struct maple_big_node *b_node) +{ + return mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), b_node->type); +} + +/* + * mas_mab_to_node() - Set up right and middle nodes + * + * @mas: the maple state that contains the allocations. + * @b_node: the node which contains the data. + * @left: The pointer which will have the left node + * @right: The pointer which may have the right node + * @middle: the pointer which may have the middle node (rare) + * @mid_split: the split location for the middle node + * + * Return: the split of left. + */ +static inline unsigned char mas_mab_to_node(struct ma_state *mas, struct maple_big_node *b_node, + struct maple_enode **left, struct maple_enode **right, + struct maple_enode **middle, unsigned char *mid_split, + unsigned long min) +{ + unsigned char split = 0; + unsigned char slot_count = mt_slots[b_node->type]; + + *left = mas_new_ma_node(mas, b_node); + *right = NULL; + *middle = NULL; + *mid_split = 0; + + if (b_node->b_end < slot_count) + { + split = b_node->b_end; + } + else + { + split = mab_calc_split(mas, b_node, mid_split, min); + *right = mas_new_ma_node(mas, b_node); + } + + if (*mid_split) + *middle = mas_new_ma_node(mas, b_node); + + return split; +} + +/* + * mab_set_b_end() - Add entry to b_node at b_node->b_end and increment the end + * pointer. + * @b_node - the big node to add the entry + * @mas - the maple state to get the pivot (mas->max) + * @entry - the entry to add, if NULL nothing happens. + */ +static inline void mab_set_b_end(struct maple_big_node *b_node, struct ma_state *mas, void *entry) +{ + if (!entry) + return; + + b_node->slot[b_node->b_end] = entry; + if (mt_is_alloc(mas->tree)) + b_node->gap[b_node->b_end] = mas_max_gap(mas); + b_node->pivot[b_node->b_end++] = mas->max; +} + +/* + * mas_set_split_parent() - combine_then_separate helper function. Sets the parent + * of @mas->node to either @left or @right, depending on @slot and @split + * + * @mas - the maple state with the node that needs a parent + * @left - possible parent 1 + * @right - possible parent 2 + * @slot - the slot the mas->node was placed + * @split - the split location between @left and @right + */ +static inline void mas_set_split_parent(struct ma_state *mas, struct maple_enode *left, + struct maple_enode *right, unsigned char *slot, + unsigned char split) +{ + if (mas_is_none(mas)) + return; + + if ((*slot) <= split) + mas_set_parent(mas, mas->node, left, *slot); + else if (right) + mas_set_parent(mas, mas->node, right, (*slot) - split - 1); + + (*slot)++; +} + +/* + * mte_mid_split_check() - Check if the next node passes the mid-split + * @**l: Pointer to left encoded maple node. + * @**m: Pointer to middle encoded maple node. + * @**r: Pointer to right encoded maple node. + * @slot: The offset + * @*split: The split location. + * @mid_split: The middle split. + */ +static inline void mte_mid_split_check(struct maple_enode **l, struct maple_enode **r, + struct maple_enode *right, unsigned char slot, + unsigned char *split, unsigned char mid_split) +{ + if (*r == right) + return; + + if (slot < mid_split) + return; + + *l = *r; + *r = right; + *split = mid_split; +} + +/* + * mast_set_split_parents() - Helper function to set three nodes parents. Slot + * is taken from @mast->l. + * @mast - the maple subtree state + * @left - the left node + * @right - the right node + * @split - the split location. + */ +static inline void mast_set_split_parents(struct maple_subtree_state *mast, + struct maple_enode *left, struct maple_enode *middle, + struct maple_enode *right, unsigned char split, + unsigned char mid_split) +{ + unsigned char slot; + struct maple_enode *l = left; + struct maple_enode *r = right; + + if (mas_is_none(mast->l)) + return; + + if (middle) + r = middle; + + slot = mast->l->offset; + + mte_mid_split_check(&l, &r, right, slot, &split, mid_split); + mas_set_split_parent(mast->l, l, r, &slot, split); + + mte_mid_split_check(&l, &r, right, slot, &split, mid_split); + mas_set_split_parent(mast->m, l, r, &slot, split); + + mte_mid_split_check(&l, &r, right, slot, &split, mid_split); + mas_set_split_parent(mast->r, l, r, &slot, split); +} + +/* + * mas_topiary_node() - Dispose of a single node + * @mas: The maple state for pushing nodes + * @enode: The encoded maple node + * @in_rcu: If the tree is in rcu mode + * + * The node will either be RCU freed or pushed back on the maple state. + */ +static inline void mas_topiary_node(struct ma_state *mas, struct ma_state *tmp_mas, bool in_rcu) +{ + struct maple_node *tmp; + struct maple_enode *enode; + + if (mas_is_none(tmp_mas)) + return; + + enode = tmp_mas->node; + tmp = mte_to_node(enode); + mte_set_node_dead(enode); + if (in_rcu) + ma_free_rcu(tmp); + else + mas_push_node(mas, tmp); +} + +/* + * mas_topiary_replace() - Replace the data with new data, then repair the + * parent links within the new tree. Iterate over the dead sub-tree and collect + * the dead subtrees and topiary the nodes that are no longer of use. + * + * The new tree will have up to three children with the correct parent. Keep + * track of the new entries as they need to be followed to find the next level + * of new entries. + * + * The old tree will have up to three children with the old parent. Keep track + * of the old entries as they may have more nodes below replaced. Nodes within + * [index, last] are dead subtrees, others need to be freed and followed. + * + * @mas: The maple state pointing at the new data + * @old_enode: The maple encoded node being replaced + * + */ +static inline void mas_topiary_replace(struct ma_state *mas, struct maple_enode *old_enode) +{ + struct ma_state tmp[3], tmp_next[3]; + MA_TOPIARY(subtrees, mas->tree); + bool in_rcu; + int i, n; + + /* Place data in tree & then mark node as old */ + mas_put_in_tree(mas, old_enode); + + /* Update the parent pointers in the tree */ + tmp[0] = *mas; + tmp[0].offset = 0; + tmp[1].status = ma_none; + tmp[2].status = ma_none; + while (!mte_is_leaf(tmp[0].node)) + { + n = 0; + for (i = 0; i < 3; i++) + { + if (mas_is_none(&tmp[i])) + continue; + + while (n < 3) + { + if (!mas_find_child(&tmp[i], &tmp_next[n])) + break; + n++; + } + + mas_adopt_children(&tmp[i], tmp[i].node); + } + + if (MAS_WARN_ON(mas, n == 0)) + break; + + while (n < 3) + tmp_next[n++].status = ma_none; + + for (i = 0; i < 3; i++) + tmp[i] = tmp_next[i]; + } + + /* Collect the old nodes that need to be discarded */ + if (mte_is_leaf(old_enode)) + return mas_free(mas, old_enode); + + tmp[0] = *mas; + tmp[0].offset = 0; + tmp[0].node = old_enode; + tmp[1].status = ma_none; + tmp[2].status = ma_none; + in_rcu = mt_in_rcu(mas->tree); + do + { + n = 0; + for (i = 0; i < 3; i++) + { + if (mas_is_none(&tmp[i])) + continue; + + while (n < 3) + { + if (!mas_find_child(&tmp[i], &tmp_next[n])) + break; + + if ((tmp_next[n].min >= tmp_next->index) && (tmp_next[n].max <= tmp_next->last)) + { + mat_add(&subtrees, tmp_next[n].node); + tmp_next[n].status = ma_none; + } + else + { + n++; + } + } + } + + if (MAS_WARN_ON(mas, n == 0)) + break; + + while (n < 3) + tmp_next[n++].status = ma_none; + + for (i = 0; i < 3; i++) + { + mas_topiary_node(mas, &tmp[i], in_rcu); + tmp[i] = tmp_next[i]; + } + } while (!mte_is_leaf(tmp[0].node)); + + for (i = 0; i < 3; i++) + mas_topiary_node(mas, &tmp[i], in_rcu); + + mas_mat_destroy(mas, &subtrees); +} + +/* + * mas_wmb_replace() - Write memory barrier and replace + * @mas: The maple state + * @old: The old maple encoded node that is being replaced. + * + * Updates gap as necessary. + */ +static inline void mas_wmb_replace(struct ma_state *mas, struct maple_enode *old_enode) +{ + /* Insert the new data in the tree */ + mas_topiary_replace(mas, old_enode); + + if (mte_is_leaf(mas->node)) + return; + + mas_update_gap(mas); +} + +/* + * mast_cp_to_nodes() - Copy data out to nodes. + * @mast: The maple subtree state + * @left: The left encoded maple node + * @middle: The middle encoded maple node + * @right: The right encoded maple node + * @split: The location to split between left and (middle ? middle : right) + * @mid_split: The location to split between middle and right. + */ +static inline void mast_cp_to_nodes(struct maple_subtree_state *mast, struct maple_enode *left, + struct maple_enode *middle, struct maple_enode *right, + unsigned char split, unsigned char mid_split) +{ + bool new_lmax = true; + + mas_node_or_none(mast->l, left); + mas_node_or_none(mast->m, middle); + mas_node_or_none(mast->r, right); + + mast->l->min = mast->orig_l->min; + if (split == mast->bn->b_end) + { + mast->l->max = mast->orig_r->max; + new_lmax = false; + } + + mab_mas_cp(mast->bn, 0, split, mast->l, new_lmax); + + if (middle) + { + mab_mas_cp(mast->bn, 1 + split, mid_split, mast->m, true); + mast->m->min = mast->bn->pivot[split] + 1; + split = mid_split; + } + + mast->r->max = mast->orig_r->max; + if (right) + { + mab_mas_cp(mast->bn, 1 + split, mast->bn->b_end, mast->r, false); + mast->r->min = mast->bn->pivot[split] + 1; + } +} + +/* + * mast_combine_cp_left - Copy in the original left side of the tree into the + * combined data set in the maple subtree state big node. + * @mast: The maple subtree state + */ +static inline void mast_combine_cp_left(struct maple_subtree_state *mast) +{ + unsigned char l_slot = mast->orig_l->offset; + + if (!l_slot) + return; + + mas_mab_cp(mast->orig_l, 0, l_slot - 1, mast->bn, 0); +} + +/* + * mast_combine_cp_right: Copy in the original right side of the tree into the + * combined data set in the maple subtree state big node. + * @mast: The maple subtree state + */ +static inline void mast_combine_cp_right(struct maple_subtree_state *mast) +{ + if (mast->bn->pivot[mast->bn->b_end - 1] >= mast->orig_r->max) + return; + + mas_mab_cp(mast->orig_r, mast->orig_r->offset + 1, mt_slot_count(mast->orig_r->node), mast->bn, + mast->bn->b_end); + mast->orig_r->last = mast->orig_r->max; +} + +/* + * mast_sufficient: Check if the maple subtree state has enough data in the big + * node to create at least one sufficient node + * @mast: the maple subtree state + */ +static inline bool mast_sufficient(struct maple_subtree_state *mast) +{ + if (mast->bn->b_end > mt_min_slot_count(mast->orig_l->node)) + return true; + + return false; +} + +/* + * mast_overflow: Check if there is too much data in the subtree state for a + * single node. + * @mast: The maple subtree state + */ +static inline bool mast_overflow(struct maple_subtree_state *mast) +{ + if (mast->bn->b_end >= mt_slot_count(mast->orig_l->node)) + return true; + + return false; +} + +static inline void *mtree_range_walk(struct ma_state *mas) +{ + unsigned long *pivots; + unsigned char offset; + struct maple_node *node; + struct maple_enode *next, *last; + enum maple_type type; + void __rcu **slots; + unsigned char end; + unsigned long max, min; + unsigned long prev_max, prev_min; + + next = mas->node; + min = mas->min; + max = mas->max; + do + { + last = next; + node = mte_to_node(next); + type = mte_node_type(next); + pivots = ma_pivots(node, type); + end = ma_data_end(node, type, pivots, max); + prev_min = min; + prev_max = max; + if (pivots[0] >= mas->index) + { + offset = 0; + max = pivots[0]; + goto next; + } + + offset = 1; + while (offset < end) + { + if (pivots[offset] >= mas->index) + { + max = pivots[offset]; + break; + } + offset++; + } + + min = pivots[offset - 1] + 1; + next: + slots = ma_slots(node, type); + next = mt_slot(mas->tree, slots, offset); + if (unlikely(ma_dead_node(node))) + goto dead_node; + } while (!ma_is_leaf(type)); + + mas->end = end; + mas->offset = offset; + mas->index = min; + mas->last = max; + mas->min = prev_min; + mas->max = prev_max; + mas->node = last; + return (void *) next; + +dead_node: + mas_reset(mas); + return NULL; +} + +/* + * mas_spanning_rebalance() - Rebalance across two nodes which may not be peers. + * @mas: The starting maple state + * @mast: The maple_subtree_state, keeps track of 4 maple states. + * @count: The estimated count of iterations needed. + * + * Follow the tree upwards from @l_mas and @r_mas for @count, or until the root + * is hit. First @b_node is split into two entries which are inserted into the + * next iteration of the loop. @b_node is returned populated with the final + * iteration. @mas is used to obtain allocations. orig_l_mas keeps track of the + * nodes that will remain active by using orig_l_mas->index and orig_l_mas->last + * to account of what has been copied into the new sub-tree. The update of + * orig_l_mas->last is used in mas_consume to find the slots that will need to + * be either freed or destroyed. orig_l_mas->depth keeps track of the height of + * the new sub-tree in case the sub-tree becomes the full tree. + * + * Return: the number of elements in b_node during the last loop. + */ +static int mas_spanning_rebalance(struct ma_state *mas, struct maple_subtree_state *mast, + unsigned char count) +{ + unsigned char split, mid_split; + unsigned char slot = 0; + struct maple_enode *left = NULL, *middle = NULL, *right = NULL; + struct maple_enode *old_enode; + + MA_STATE(l_mas, mas->tree, mas->index, mas->index); + MA_STATE(r_mas, mas->tree, mas->index, mas->last); + MA_STATE(m_mas, mas->tree, mas->index, mas->index); + + /* + * The tree needs to be rebalanced and leaves need to be kept at the same level. + * Rebalancing is done by use of the ``struct maple_topiary``. + */ + mast->l = &l_mas; + mast->m = &m_mas; + mast->r = &r_mas; + l_mas.status = r_mas.status = m_mas.status = ma_none; + + /* Check if this is not root and has sufficient data. */ + if (((mast->orig_l->min != 0) || (mast->orig_r->max != ULONG_MAX)) && + unlikely(mast->bn->b_end <= mt_min_slots[mast->bn->type])) + mast_spanning_rebalance(mast); + + l_mas.depth = 0; + + /* + * Each level of the tree is examined and balanced, pushing data to the left or + * right, or rebalancing against left or right nodes is employed to avoid + * rippling up the tree to limit the amount of churn. Once a new sub-section of + * the tree is created, there may be a mix of new and old nodes. The old nodes + * will have the incorrect parent pointers and currently be in two trees: the + * original tree and the partially new tree. To remedy the parent pointers in + * the old tree, the new data is swapped into the active tree and a walk down + * the tree is performed and the parent pointers are updated. + * See mas_topiary_replace() for more information. + */ + while (count--) + { + mast->bn->b_end--; + mast->bn->type = mte_node_type(mast->orig_l->node); + split = + mas_mab_to_node(mas, mast->bn, &left, &right, &middle, &mid_split, mast->orig_l->min); + mast_set_split_parents(mast, left, middle, right, split, mid_split); + mast_cp_to_nodes(mast, left, middle, right, split, mid_split); + + /* + * Copy data from next level in the tree to mast->bn from next + * iteration + */ + memset(mast->bn, 0, sizeof(struct maple_big_node)); + mast->bn->type = mte_node_type(left); + l_mas.depth++; + + /* Root already stored in l->node. */ + if (mas_is_root_limits(mast->l)) + goto new_root; + + mast_ascend(mast); + mast_combine_cp_left(mast); + l_mas.offset = mast->bn->b_end; + mab_set_b_end(mast->bn, &l_mas, left); + mab_set_b_end(mast->bn, &m_mas, middle); + mab_set_b_end(mast->bn, &r_mas, right); + + /* Copy anything necessary out of the right node. */ + mast_combine_cp_right(mast); + mast->orig_l->last = mast->orig_l->max; + + if (mast_sufficient(mast)) + continue; + + if (mast_overflow(mast)) + continue; + + /* May be a new root stored in mast->bn */ + if (mas_is_root_limits(mast->orig_l)) + break; + + mast_spanning_rebalance(mast); + + /* rebalancing from other nodes may require another loop. */ + if (!count) + count++; + } + + l_mas.node = mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), mte_node_type(mast->orig_l->node)); + l_mas.depth++; + mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, &l_mas, true); + mas_set_parent(mas, left, l_mas.node, slot); + if (middle) + mas_set_parent(mas, middle, l_mas.node, ++slot); + + if (right) + mas_set_parent(mas, right, l_mas.node, ++slot); + + if (mas_is_root_limits(mast->l)) + { + new_root: + mas_mn(mast->l)->parent = ma_parent_ptr(mas_tree_parent(mas)); + while (!mte_is_root(mast->orig_l->node)) + mast_ascend(mast); + } + else + { + mas_mn(&l_mas)->parent = mas_mn(mast->orig_l)->parent; + } + + old_enode = mast->orig_l->node; + mas->depth = l_mas.depth; + mas->node = l_mas.node; + mas->min = l_mas.min; + mas->max = l_mas.max; + mas->offset = l_mas.offset; + mas_wmb_replace(mas, old_enode); + mtree_range_walk(mas); + return mast->bn->b_end; +} + +/* + * mas_rebalance() - Rebalance a given node. + * @mas: The maple state + * @b_node: The big maple node. + * + * Rebalance two nodes into a single node or two new nodes that are sufficient. + * Continue upwards until tree is sufficient. + * + * Return: the number of elements in b_node during the last loop. + */ +static inline int mas_rebalance(struct ma_state *mas, struct maple_big_node *b_node) +{ + char empty_count = mas_mt_height(mas); + struct maple_subtree_state mast; + unsigned char shift, b_end = ++b_node->b_end; + + MA_STATE(l_mas, mas->tree, mas->index, mas->last); + MA_STATE(r_mas, mas->tree, mas->index, mas->last); + + trace_ma_op(__func__, mas); + + /* + * Rebalancing occurs if a node is insufficient. Data is rebalanced + * against the node to the right if it exists, otherwise the node to the + * left of this node is rebalanced against this node. If rebalancing + * causes just one node to be produced instead of two, then the parent + * is also examined and rebalanced if it is insufficient. Every level + * tries to combine the data in the same way. If one node contains the + * entire range of the tree, then that node is used as a new root node. + */ + mas_node_count(mas, empty_count * 2 - 1); + if (mas_is_err(mas)) + return 0; + + mast.orig_l = &l_mas; + mast.orig_r = &r_mas; + mast.bn = b_node; + mast.bn->type = mte_node_type(mas->node); + + l_mas = r_mas = *mas; + + if (mas_next_sibling(&r_mas)) + { + mas_mab_cp(&r_mas, 0, mt_slot_count(r_mas.node), b_node, b_end); + r_mas.last = r_mas.index = r_mas.max; + } + else + { + mas_prev_sibling(&l_mas); + shift = mas_data_end(&l_mas) + 1; + mab_shift_right(b_node, shift); + mas->offset += shift; + mas_mab_cp(&l_mas, 0, shift - 1, b_node, 0); + b_node->b_end = shift + b_end; + l_mas.index = l_mas.last = l_mas.min; + } + + return mas_spanning_rebalance(mas, &mast, empty_count); +} + +/* + * mas_destroy_rebalance() - Rebalance left-most node while destroying the maple + * state. + * @mas: The maple state + * @end: The end of the left-most node. + * + * During a mass-insert event (such as forking), it may be necessary to + * rebalance the left-most node when it is not sufficient. + */ +static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end) +{ + enum maple_type mt = mte_node_type(mas->node); + struct maple_node reuse, *newnode, *parent, *new_left, *left, *node; + struct maple_enode *eparent, *old_eparent; + unsigned char offset, tmp, split = mt_slots[mt] / 2; + void __rcu **l_slots, **slots; + unsigned long *l_pivs, *pivs, gap; + bool in_rcu = mt_in_rcu(mas->tree); + + MA_STATE(l_mas, mas->tree, mas->index, mas->last); + + l_mas = *mas; + mas_prev_sibling(&l_mas); + + /* set up node. */ + if (in_rcu) + { + /* Allocate for both left and right as well as parent. */ + mas_node_count(mas, 3); + if (mas_is_err(mas)) + return; + + newnode = mas_pop_node(mas); + } + else + { + newnode = &reuse; + } + + node = mas_mn(mas); + newnode->parent = node->parent; + slots = ma_slots(newnode, mt); + pivs = ma_pivots(newnode, mt); + left = mas_mn(&l_mas); + l_slots = ma_slots(left, mt); + l_pivs = ma_pivots(left, mt); + if (!l_slots[split]) + split++; + tmp = mas_data_end(&l_mas) - split; + + memcpy(slots, l_slots + split + 1, sizeof(void *) * tmp); + memcpy(pivs, l_pivs + split + 1, sizeof(unsigned long) * tmp); + pivs[tmp] = l_mas.max; + memcpy(slots + tmp, ma_slots(node, mt), sizeof(void *) * end); + memcpy(pivs + tmp, ma_pivots(node, mt), sizeof(unsigned long) * end); + + l_mas.max = l_pivs[split]; + mas->min = l_mas.max + 1; + old_eparent = mt_mk_node(mte_parent(l_mas.node), mas_parent_type(&l_mas, l_mas.node)); + tmp += end; + if (!in_rcu) + { + unsigned char max_p = mt_pivots[mt]; + unsigned char max_s = mt_slots[mt]; + + if (tmp < max_p) + memset(pivs + tmp, 0, sizeof(unsigned long) * (max_p - tmp)); + + if (tmp < mt_slots[mt]) + memset(slots + tmp, 0, sizeof(void *) * (max_s - tmp)); + + memcpy(node, newnode, sizeof(struct maple_node)); + ma_set_meta(node, mt, 0, tmp - 1); + mte_set_pivot(old_eparent, mte_parent_slot(l_mas.node), l_pivs[split]); + + /* Remove data from l_pivs. */ + tmp = split + 1; + memset(l_pivs + tmp, 0, sizeof(unsigned long) * (max_p - tmp)); + memset(l_slots + tmp, 0, sizeof(void *) * (max_s - tmp)); + ma_set_meta(left, mt, 0, split); + eparent = old_eparent; + + goto done; + } + + /* RCU requires replacing both l_mas, mas, and parent. */ + mas->node = mt_mk_node(newnode, mt); + ma_set_meta(newnode, mt, 0, tmp); + + new_left = mas_pop_node(mas); + new_left->parent = left->parent; + mt = mte_node_type(l_mas.node); + slots = ma_slots(new_left, mt); + pivs = ma_pivots(new_left, mt); + memcpy(slots, l_slots, sizeof(void *) * split); + memcpy(pivs, l_pivs, sizeof(unsigned long) * split); + ma_set_meta(new_left, mt, 0, split); + l_mas.node = mt_mk_node(new_left, mt); + + /* replace parent. */ + offset = mte_parent_slot(mas->node); + mt = mas_parent_type(&l_mas, l_mas.node); + parent = mas_pop_node(mas); + slots = ma_slots(parent, mt); + pivs = ma_pivots(parent, mt); + memcpy(parent, mte_to_node(old_eparent), sizeof(struct maple_node)); + rcu_assign_pointer(slots[offset], mas->node); + rcu_assign_pointer(slots[offset - 1], l_mas.node); + pivs[offset - 1] = l_mas.max; + eparent = mt_mk_node(parent, mt); +done: + gap = mas_leaf_max_gap(mas); + mte_set_gap(eparent, mte_parent_slot(mas->node), gap); + gap = mas_leaf_max_gap(&l_mas); + mte_set_gap(eparent, mte_parent_slot(l_mas.node), gap); + mas_ascend(mas); + + if (in_rcu) + { + mas_replace_node(mas, old_eparent); + mas_adopt_children(mas, mas->node); + } + + mas_update_gap(mas); +} + +/* + * mas_split_final_node() - Split the final node in a subtree operation. + * @mast: the maple subtree state + * @mas: The maple state + * @height: The height of the tree in case it's a new root. + */ +static inline void mas_split_final_node(struct maple_subtree_state *mast, struct ma_state *mas, + int height) +{ + struct maple_enode *ancestor; + + if (mte_is_root(mas->node)) + { + if (mt_is_alloc(mas->tree)) + mast->bn->type = maple_arange_64; + else + mast->bn->type = maple_range_64; + mas->depth = height; + } + /* + * Only a single node is used here, could be root. + * The Big_node data should just fit in a single node. + */ + ancestor = mas_new_ma_node(mas, mast->bn); + mas_set_parent(mas, mast->l->node, ancestor, mast->l->offset); + mas_set_parent(mas, mast->r->node, ancestor, mast->r->offset); + mte_to_node(ancestor)->parent = mas_mn(mas)->parent; + + mast->l->node = ancestor; + mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true); + mas->offset = mast->bn->b_end - 1; +} + +/* + * mast_fill_bnode() - Copy data into the big node in the subtree state + * @mast: The maple subtree state + * @mas: the maple state + * @skip: The number of entries to skip for new nodes insertion. + */ +static inline void mast_fill_bnode(struct maple_subtree_state *mast, struct ma_state *mas, + unsigned char skip) +{ + bool cp = true; + unsigned char split; + + memset(mast->bn->gap, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->gap)); + memset(mast->bn->slot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->slot)); + memset(mast->bn->pivot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->pivot)); + mast->bn->b_end = 0; + + if (mte_is_root(mas->node)) + { + cp = false; + } + else + { + mas_ascend(mas); + mas->offset = mte_parent_slot(mas->node); + } + + if (cp && mast->l->offset) + mas_mab_cp(mas, 0, mast->l->offset - 1, mast->bn, 0); + + split = mast->bn->b_end; + mab_set_b_end(mast->bn, mast->l, mast->l->node); + mast->r->offset = mast->bn->b_end; + mab_set_b_end(mast->bn, mast->r, mast->r->node); + if (mast->bn->pivot[mast->bn->b_end - 1] == mas->max) + cp = false; + + if (cp) + mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1, mast->bn, mast->bn->b_end); + + mast->bn->b_end--; + mast->bn->type = mte_node_type(mas->node); +} + +/* + * mast_split_data() - Split the data in the subtree state big node into regular + * nodes. + * @mast: The maple subtree state + * @mas: The maple state + * @split: The location to split the big node + */ +static inline void mast_split_data(struct maple_subtree_state *mast, struct ma_state *mas, + unsigned char split) +{ + unsigned char p_slot; + + mab_mas_cp(mast->bn, 0, split, mast->l, true); + mte_set_pivot(mast->r->node, 0, mast->r->max); + mab_mas_cp(mast->bn, split + 1, mast->bn->b_end, mast->r, false); + mast->l->offset = mte_parent_slot(mas->node); + mast->l->max = mast->bn->pivot[split]; + mast->r->min = mast->l->max + 1; + if (mte_is_leaf(mas->node)) + return; + + p_slot = mast->orig_l->offset; + mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node, &p_slot, split); + mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node, &p_slot, split); +} + +/* + * mas_push_data() - Instead of splitting a node, it is beneficial to push the + * data to the right or left node if there is room. + * @mas: The maple state + * @height: The current height of the maple state + * @mast: The maple subtree state + * @left: Push left or not. + * + * Keeping the height of the tree low means faster lookups. + * + * Return: True if pushed, false otherwise. + */ +static inline bool mas_push_data(struct ma_state *mas, int height, struct maple_subtree_state *mast, + bool left) +{ + unsigned char slot_total = mast->bn->b_end; + unsigned char end, space, split; + + MA_STATE(tmp_mas, mas->tree, mas->index, mas->last); + tmp_mas = *mas; + tmp_mas.depth = mast->l->depth; + + if (left && !mas_prev_sibling(&tmp_mas)) + return false; + else if (!left && !mas_next_sibling(&tmp_mas)) + return false; + + end = mas_data_end(&tmp_mas); + slot_total += end; + space = 2 * mt_slot_count(mas->node) - 2; + /* -2 instead of -1 to ensure there isn't a triple split */ + if (ma_is_leaf(mast->bn->type)) + space--; + + if (mas->max == ULONG_MAX) + space--; + + if (slot_total >= space) + return false; + + /* Get the data; Fill mast->bn */ + mast->bn->b_end++; + if (left) + { + mab_shift_right(mast->bn, end + 1); + mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0); + mast->bn->b_end = slot_total + 1; + } + else + { + mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end); + } + + /* Configure mast for splitting of mast->bn */ + split = mt_slots[mast->bn->type] - 2; + if (left) + { + /* Switch mas to prev node */ + *mas = tmp_mas; + /* Start using mast->l for the left side. */ + tmp_mas.node = mast->l->node; + *mast->l = tmp_mas; + } + else + { + tmp_mas.node = mast->r->node; + *mast->r = tmp_mas; + split = slot_total - split; + } + split = mab_no_null_split(mast->bn, split, mt_slots[mast->bn->type]); + /* Update parent slot for split calculation. */ + if (left) + mast->orig_l->offset += end + 1; + + mast_split_data(mast, mas, split); + mast_fill_bnode(mast, mas, 2); + mas_split_final_node(mast, mas, height + 1); + return true; +} + +/* + * mas_split() - Split data that is too big for one node into two. + * @mas: The maple state + * @b_node: The maple big node + * Return: 1 on success, 0 on failure. + */ +static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) +{ + struct maple_subtree_state mast; + int height = 0; + unsigned char mid_split, split = 0; + struct maple_enode *old; + + /* + * Splitting is handled differently from any other B-tree; the Maple + * Tree splits upwards. Splitting up means that the split operation + * occurs when the walk of the tree hits the leaves and not on the way + * down. The reason for splitting up is that it is impossible to know + * how much space will be needed until the leaf is (or leaves are) + * reached. Since overwriting data is allowed and a range could + * overwrite more than one range or result in changing one entry into 3 + * entries, it is impossible to know if a split is required until the + * data is examined. + * + * Splitting is a balancing act between keeping allocations to a minimum + * and avoiding a 'jitter' event where a tree is expanded to make room + * for an entry followed by a contraction when the entry is removed. To + * accomplish the balance, there are empty slots remaining in both left + * and right nodes after a split. + */ + MA_STATE(l_mas, mas->tree, mas->index, mas->last); + MA_STATE(r_mas, mas->tree, mas->index, mas->last); + MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last); + MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last); + + trace_ma_op(__func__, mas); + mas->depth = mas_mt_height(mas); + /* Allocation failures will happen early. */ + mas_node_count(mas, 1 + mas->depth * 2); + if (mas_is_err(mas)) + return 0; + + mast.l = &l_mas; + mast.r = &r_mas; + mast.orig_l = &prev_l_mas; + mast.orig_r = &prev_r_mas; + mast.bn = b_node; + + while (height++ <= mas->depth) + { + if (mt_slots[b_node->type] > b_node->b_end) + { + mas_split_final_node(&mast, mas, height); + break; + } + + l_mas = r_mas = *mas; + l_mas.node = mas_new_ma_node(mas, b_node); + r_mas.node = mas_new_ma_node(mas, b_node); + /* + * Another way that 'jitter' is avoided is to terminate a split up early if the + * left or right node has space to spare. This is referred to as "pushing left" + * or "pushing right" and is similar to the B* tree, except the nodes left or + * right can rarely be reused due to RCU, but the ripple upwards is halted which + * is a significant savings. + */ + /* Try to push left. */ + if (mas_push_data(mas, height, &mast, true)) + break; + /* Try to push right. */ + if (mas_push_data(mas, height, &mast, false)) + break; + + split = mab_calc_split(mas, b_node, &mid_split, prev_l_mas.min); + mast_split_data(&mast, mas, split); + /* + * Usually correct, mab_mas_cp in the above call overwrites + * r->max. + */ + mast.r->max = mas->max; + mast_fill_bnode(&mast, mas, 1); + prev_l_mas = *mast.l; + prev_r_mas = *mast.r; + } + + /* Set the original node as dead */ + old = mas->node; + mas->node = l_mas.node; + mas_wmb_replace(mas, old); + mtree_range_walk(mas); + return 1; +} + +/* + * mas_reuse_node() - Reuse the node to store the data. + * @wr_mas: The maple write state + * @bn: The maple big node + * @end: The end of the data. + * + * Will always return false in RCU mode. + * + * Return: True if node was reused, false otherwise. + */ +static inline bool mas_reuse_node(struct ma_wr_state *wr_mas, struct maple_big_node *bn, + unsigned char end) +{ + /* Need to be rcu safe. */ + if (mt_in_rcu(wr_mas->mas->tree)) + return false; + + if (end > bn->b_end) + { + int clear = mt_slots[wr_mas->type] - bn->b_end; + + memset(wr_mas->slots + bn->b_end, 0, sizeof(void *) * clear--); + memset(wr_mas->pivots + bn->b_end, 0, sizeof(void *) * clear); + } + mab_mas_cp(bn, 0, bn->b_end, wr_mas->mas, false); + return true; +} + +/* + * mas_commit_b_node() - Commit the big node into the tree. + * @wr_mas: The maple write state + * @b_node: The maple big node + * @end: The end of the data. + */ +static noinline_for_kasan int mas_commit_b_node(struct ma_wr_state *wr_mas, + struct maple_big_node *b_node, unsigned char end) +{ + struct maple_node *node; + struct maple_enode *old_enode; + unsigned char b_end = b_node->b_end; + enum maple_type b_type = b_node->type; + + old_enode = wr_mas->mas->node; + if ((b_end < mt_min_slots[b_type]) && (!mte_is_root(old_enode)) && + (mas_mt_height(wr_mas->mas) > 1)) + return mas_rebalance(wr_mas->mas, b_node); + + if (b_end >= mt_slots[b_type]) + return mas_split(wr_mas->mas, b_node); + + if (mas_reuse_node(wr_mas, b_node, end)) + goto reuse_node; + + mas_node_count(wr_mas->mas, 1); + if (mas_is_err(wr_mas->mas)) + return 0; + + node = mas_pop_node(wr_mas->mas); + node->parent = mas_mn(wr_mas->mas)->parent; + wr_mas->mas->node = mt_mk_node(node, b_type); + mab_mas_cp(b_node, 0, b_end, wr_mas->mas, false); + mas_replace_node(wr_mas->mas, old_enode); +reuse_node: + mas_update_gap(wr_mas->mas); + wr_mas->mas->end = b_end; + return 1; +} + +/* + * mas_root_expand() - Expand a root to a node + * @mas: The maple state + * @entry: The entry to store into the tree + */ +static inline int mas_root_expand(struct ma_state *mas, void *entry) +{ + void *contents = mas_root_locked(mas); + enum maple_type type = maple_leaf_64; + struct maple_node *node; + void __rcu **slots; + unsigned long *pivots; + int slot = 0; + + mas_node_count(mas, 1); + if (unlikely(mas_is_err(mas))) + return 0; + + node = mas_pop_node(mas); + pivots = ma_pivots(node, type); + slots = ma_slots(node, type); + node->parent = ma_parent_ptr(mas_tree_parent(mas)); + mas->node = mt_mk_node(node, type); + mas->status = ma_active; + + if (mas->index) + { + if (contents) + { + rcu_assign_pointer(slots[slot], contents); + if (likely(mas->index > 1)) + slot++; + } + pivots[slot++] = mas->index - 1; + } + + rcu_assign_pointer(slots[slot], entry); + mas->offset = slot; + pivots[slot] = mas->last; + if (mas->last != ULONG_MAX) + pivots[++slot] = ULONG_MAX; + + mas->depth = 1; + mas_set_height(mas); + ma_set_meta(node, maple_leaf_64, 0, slot); + /* swap the new root into the tree */ + rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); + return slot; +} + +static inline void mas_store_root(struct ma_state *mas, void *entry) +{ + if (likely((mas->last != 0) || (mas->index != 0))) + mas_root_expand(mas, entry); + else if (((unsigned long) (entry) &3) == 2) + mas_root_expand(mas, entry); + else + { + rcu_assign_pointer(mas->tree->ma_root, entry); + mas->status = ma_start; + } +} + +/* + * mas_is_span_wr() - Check if the write needs to be treated as a write that + * spans the node. + * @mas: The maple state + * @piv: The pivot value being written + * @type: The maple node type + * @entry: The data to write + * + * Spanning writes are writes that start in one node and end in another OR if + * the write of a %NULL will cause the node to end with a %NULL. + * + * Return: True if this is a spanning write, false otherwise. + */ +static bool mas_is_span_wr(struct ma_wr_state *wr_mas) +{ + unsigned long max = wr_mas->r_max; + unsigned long last = wr_mas->mas->last; + enum maple_type type = wr_mas->type; + void *entry = wr_mas->entry; + + /* Contained in this pivot, fast path */ + if (last < max) + return false; + + if (ma_is_leaf(type)) + { + max = wr_mas->mas->max; + if (last < max) + return false; + } + + if (last == max) + { + /* + * The last entry of leaf node cannot be NULL unless it is the + * rightmost node (writing ULONG_MAX), otherwise it spans slots. + */ + if (entry || last == ULONG_MAX) + return false; + } + + trace_ma_write(__func__, wr_mas->mas, wr_mas->r_max, entry); + return true; +} + +static inline void mas_wr_walk_descend(struct ma_wr_state *wr_mas) +{ + wr_mas->type = mte_node_type(wr_mas->mas->node); + mas_wr_node_walk(wr_mas); + wr_mas->slots = ma_slots(wr_mas->node, wr_mas->type); +} + +static inline void mas_wr_walk_traverse(struct ma_wr_state *wr_mas) +{ + wr_mas->mas->max = wr_mas->r_max; + wr_mas->mas->min = wr_mas->r_min; + wr_mas->mas->node = wr_mas->content; + wr_mas->mas->offset = 0; + wr_mas->mas->depth++; +} +/* + * mas_wr_walk() - Walk the tree for a write. + * @wr_mas: The maple write state + * + * Uses mas_slot_locked() and does not need to worry about dead nodes. + * + * Return: True if it's contained in a node, false on spanning write. + */ +static bool mas_wr_walk(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + + while (true) + { + mas_wr_walk_descend(wr_mas); + if (unlikely(mas_is_span_wr(wr_mas))) + return false; + + wr_mas->content = mas_slot_locked(mas, wr_mas->slots, mas->offset); + if (ma_is_leaf(wr_mas->type)) + return true; + + mas_wr_walk_traverse(wr_mas); + } + + return true; +} + +static bool mas_wr_walk_index(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + + while (true) + { + mas_wr_walk_descend(wr_mas); + wr_mas->content = mas_slot_locked(mas, wr_mas->slots, mas->offset); + if (ma_is_leaf(wr_mas->type)) + return true; + mas_wr_walk_traverse(wr_mas); + } + return true; +} +/* + * mas_extend_spanning_null() - Extend a store of a %NULL to include surrounding %NULLs. + * @l_wr_mas: The left maple write state + * @r_wr_mas: The right maple write state + */ +static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas, + struct ma_wr_state *r_wr_mas) +{ + struct ma_state *r_mas = r_wr_mas->mas; + struct ma_state *l_mas = l_wr_mas->mas; + unsigned char l_slot; + + l_slot = l_mas->offset; + if (!l_wr_mas->content) + l_mas->index = l_wr_mas->r_min; + + if ((l_mas->index == l_wr_mas->r_min) && + (l_slot && !mas_slot_locked(l_mas, l_wr_mas->slots, l_slot - 1))) + { + if (l_slot > 1) + l_mas->index = l_wr_mas->pivots[l_slot - 2] + 1; + else + l_mas->index = l_mas->min; + + l_mas->offset = l_slot - 1; + } + + if (!r_wr_mas->content) + { + if (r_mas->last < r_wr_mas->r_max) + r_mas->last = r_wr_mas->r_max; + r_mas->offset++; + } + else if ((r_mas->last == r_wr_mas->r_max) && (r_mas->last < r_mas->max) && + !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) + { + r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots, r_wr_mas->type, r_mas->offset + 1); + r_mas->offset++; + } +} + +static inline void *mas_state_walk(struct ma_state *mas) +{ + void *entry; + + entry = mas_start(mas); + if (mas_is_none(mas)) + return NULL; + + if (mas_is_ptr(mas)) + return entry; + + return mtree_range_walk(mas); +} + +/* + * mtree_lookup_walk() - Internal quick lookup that does not keep maple state up + * to date. + * + * @mas: The maple state. + * + * Note: Leaves mas in undesirable state. + * Return: The entry for @mas->index or %NULL on dead node. + */ +static inline void *mtree_lookup_walk(struct ma_state *mas) +{ + unsigned long *pivots; + unsigned char offset; + struct maple_node *node; + struct maple_enode *next; + enum maple_type type; + void __rcu **slots; + unsigned char end; + + next = mas->node; + do + { + node = mte_to_node(next); + type = mte_node_type(next); + pivots = ma_pivots(node, type); + end = mt_pivots[type]; + offset = 0; + do + { + if (pivots[offset] >= mas->index) + break; + } while (++offset < end); + + slots = ma_slots(node, type); + next = mt_slot(mas->tree, slots, offset); + if (unlikely(ma_dead_node(node))) + goto dead_node; + } while (!ma_is_leaf(type)); + + return (void *) next; + +dead_node: + mas_reset(mas); + return NULL; +} + +static void mte_destroy_walk(struct maple_enode *, struct maple_tree *); +/* + * mas_new_root() - Create a new root node that only contains the entry passed + * in. + * @mas: The maple state + * @entry: The entry to store. + * + * Only valid when the index == 0 and the last == ULONG_MAX + * + * Return 0 on error, 1 on success. + */ +static inline int mas_new_root(struct ma_state *mas, void *entry) +{ + struct maple_enode *root = mas_root_locked(mas); + enum maple_type type = maple_leaf_64; + struct maple_node *node; + void __rcu **slots; + unsigned long *pivots; + + if (!entry && !mas->index && mas->last == ULONG_MAX) + { + mas->depth = 0; + mas_set_height(mas); + rcu_assign_pointer(mas->tree->ma_root, entry); + mas->status = ma_start; + goto done; + } + + mas_node_count(mas, 1); + if (mas_is_err(mas)) + return 0; + + node = mas_pop_node(mas); + pivots = ma_pivots(node, type); + slots = ma_slots(node, type); + node->parent = ma_parent_ptr(mas_tree_parent(mas)); + mas->node = mt_mk_node(node, type); + mas->status = ma_active; + rcu_assign_pointer(slots[0], entry); + pivots[0] = mas->last; + mas->depth = 1; + mas_set_height(mas); + rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); + +done: + if (xa_is_node(root)) + mte_destroy_walk(root, mas->tree); + + return 1; +} +/* + * mas_wr_spanning_store() - Create a subtree with the store operation completed + * and new nodes where necessary, then place the sub-tree in the actual tree. + * Note that mas is expected to point to the node which caused the store to + * span. + * @wr_mas: The maple write state + * + * Return: 0 on error, positive on success. + */ +static inline int mas_wr_spanning_store(struct ma_wr_state *wr_mas) +{ + struct maple_subtree_state mast; + struct maple_big_node b_node; + struct ma_state *mas; + unsigned char height; + + /* Left and Right side of spanning store */ + MA_STATE(l_mas, NULL, 0, 0); + MA_STATE(r_mas, NULL, 0, 0); + MA_WR_STATE(r_wr_mas, &r_mas, wr_mas->entry); + MA_WR_STATE(l_wr_mas, &l_mas, wr_mas->entry); + + /* + * A store operation that spans multiple nodes is called a spanning + * store and is handled early in the store call stack by the function + * mas_is_span_wr(). When a spanning store is identified, the maple + * state is duplicated. The first maple state walks the left tree path + * to ``index``, the duplicate walks the right tree path to ``last``. + * The data in the two nodes are combined into a single node, two nodes, + * or possibly three nodes (see the 3-way split above). A ``NULL`` + * written to the last entry of a node is considered a spanning store as + * a rebalance is required for the operation to complete and an overflow + * of data may happen. + */ + mas = wr_mas->mas; + trace_ma_op(__func__, mas); + + if (unlikely(!mas->index && mas->last == ULONG_MAX)) + return mas_new_root(mas, wr_mas->entry); + /* + * Node rebalancing may occur due to this store, so there may be three new + * entries per level plus a new root. + */ + height = mas_mt_height(mas); + mas_node_count(mas, 1 + height * 3); + if (mas_is_err(mas)) + return 0; + + /* + * Set up right side. Need to get to the next offset after the spanning + * store to ensure it's not NULL and to combine both the next node and + * the node with the start together. + */ + r_mas = *mas; + /* Avoid overflow, walk to next slot in the tree. */ + if (r_mas.last + 1) + r_mas.last++; + + r_mas.index = r_mas.last; + mas_wr_walk_index(&r_wr_mas); + r_mas.last = r_mas.index = mas->last; + + /* Set up left side. */ + l_mas = *mas; + mas_wr_walk_index(&l_wr_mas); + + if (!wr_mas->entry) + { + mas_extend_spanning_null(&l_wr_mas, &r_wr_mas); + mas->offset = l_mas.offset; + mas->index = l_mas.index; + mas->last = l_mas.last = r_mas.last; + } + + /* expanding NULLs may make this cover the entire range */ + if (!l_mas.index && r_mas.last == ULONG_MAX) + { + mas_set_range(mas, 0, ULONG_MAX); + return mas_new_root(mas, wr_mas->entry); + } + + memset(&b_node, 0, sizeof(struct maple_big_node)); + /* Copy l_mas and store the value in b_node. */ + mas_store_b_node(&l_wr_mas, &b_node, l_mas.end); + /* Copy r_mas into b_node. */ + if (r_mas.offset <= r_mas.end) + mas_mab_cp(&r_mas, r_mas.offset, r_mas.end, &b_node, b_node.b_end + 1); + else + b_node.b_end++; + + /* Stop spanning searches by searching for just index. */ + l_mas.index = l_mas.last = mas->index; + + mast.bn = &b_node; + mast.orig_l = &l_mas; + mast.orig_r = &r_mas; + /* Combine l_mas and r_mas and split them up evenly again. */ + return mas_spanning_rebalance(mas, &mast, height + 1); +} + +/* + * mas_wr_node_store() - Attempt to store the value in a node + * @wr_mas: The maple write state + * + * Attempts to reuse the node, but may allocate. + * + * Return: True if stored, false otherwise + */ +static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas, unsigned char new_end) +{ + struct ma_state *mas = wr_mas->mas; + void __rcu **dst_slots; + unsigned long *dst_pivots; + unsigned char dst_offset, offset_end = wr_mas->offset_end; + struct maple_node reuse, *newnode; + unsigned char copy_size, node_pivots = mt_pivots[wr_mas->type]; + bool in_rcu = mt_in_rcu(mas->tree); + + /* Check if there is enough data. The room is enough. */ + if (!mte_is_root(mas->node) && (new_end <= mt_min_slots[wr_mas->type]) && + !(mas->mas_flags & MA_STATE_BULK)) + return false; + + if (mas->last == wr_mas->end_piv) + offset_end++; /* don't copy this offset */ + else if (unlikely(wr_mas->r_max == ULONG_MAX)) + mas_bulk_rebalance(mas, mas->end, wr_mas->type); + + /* set up node. */ + if (in_rcu) + { + mas_node_count(mas, 1); + if (mas_is_err(mas)) + return false; + + newnode = mas_pop_node(mas); + } + else + { + memset(&reuse, 0, sizeof(struct maple_node)); + newnode = &reuse; + } + + newnode->parent = mas_mn(mas)->parent; + dst_pivots = ma_pivots(newnode, wr_mas->type); + dst_slots = ma_slots(newnode, wr_mas->type); + /* Copy from start to insert point */ + memcpy(dst_pivots, wr_mas->pivots, sizeof(unsigned long) * mas->offset); + memcpy(dst_slots, wr_mas->slots, sizeof(void *) * mas->offset); + + /* Handle insert of new range starting after old range */ + if (wr_mas->r_min < mas->index) + { + rcu_assign_pointer(dst_slots[mas->offset], wr_mas->content); + dst_pivots[mas->offset++] = mas->index - 1; + } + + /* Store the new entry and range end. */ + if (mas->offset < node_pivots) + dst_pivots[mas->offset] = mas->last; + rcu_assign_pointer(dst_slots[mas->offset], wr_mas->entry); + + /* + * this range wrote to the end of the node or it overwrote the rest of + * the data + */ + if (offset_end > mas->end) + goto done; + + dst_offset = mas->offset + 1; + /* Copy to the end of node if necessary. */ + copy_size = mas->end - offset_end + 1; + memcpy(dst_slots + dst_offset, wr_mas->slots + offset_end, sizeof(void *) * copy_size); + memcpy(dst_pivots + dst_offset, wr_mas->pivots + offset_end, + sizeof(unsigned long) * (copy_size - 1)); + + if (new_end < node_pivots) + dst_pivots[new_end] = mas->max; + +done: + mas_leaf_set_meta(newnode, maple_leaf_64, new_end); + if (in_rcu) + { + struct maple_enode *old_enode = mas->node; + + mas->node = mt_mk_node(newnode, wr_mas->type); + mas_replace_node(mas, old_enode); + } + else + { + memcpy(wr_mas->node, newnode, sizeof(struct maple_node)); + } + trace_ma_write(__func__, mas, 0, wr_mas->entry); + mas_update_gap(mas); + mas->end = new_end; + return true; +} + +/* + * mas_wr_slot_store: Attempt to store a value in a slot. + * @wr_mas: the maple write state + * + * Return: True if stored, false otherwise + */ +static inline bool mas_wr_slot_store(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + unsigned char offset = mas->offset; + void __rcu **slots = wr_mas->slots; + bool gap = false; + + gap |= !mt_slot_locked(mas->tree, slots, offset); + gap |= !mt_slot_locked(mas->tree, slots, offset + 1); + + if (wr_mas->offset_end - offset == 1) + { + if (mas->index == wr_mas->r_min) + { + /* Overwriting the range and a part of the next one */ + rcu_assign_pointer(slots[offset], wr_mas->entry); + wr_mas->pivots[offset] = mas->last; + } + else + { + /* Overwriting a part of the range and the next one */ + rcu_assign_pointer(slots[offset + 1], wr_mas->entry); + wr_mas->pivots[offset] = mas->index - 1; + mas->offset++; /* Keep mas accurate. */ + } + } + else if (!mt_in_rcu(mas->tree)) + { + /* + * Expand the range, only partially overwriting the previous and + * next ranges + */ + gap |= !mt_slot_locked(mas->tree, slots, offset + 2); + rcu_assign_pointer(slots[offset + 1], wr_mas->entry); + wr_mas->pivots[offset] = mas->index - 1; + wr_mas->pivots[offset + 1] = mas->last; + mas->offset++; /* Keep mas accurate. */ + } + else + { + return false; + } + + trace_ma_write(__func__, mas, 0, wr_mas->entry); + /* + * Only update gap when the new entry is empty or there is an empty + * entry in the original two ranges. + */ + if (!wr_mas->entry || gap) + mas_update_gap(mas); + + return true; +} + +static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + + if (!wr_mas->slots[wr_mas->offset_end]) + { + /* If this one is null, the next and prev are not */ + mas->last = wr_mas->end_piv; + } + else + { + /* Check next slot(s) if we are overwriting the end */ + if ((mas->last == wr_mas->end_piv) && (mas->end != wr_mas->offset_end) && + !wr_mas->slots[wr_mas->offset_end + 1]) + { + wr_mas->offset_end++; + if (wr_mas->offset_end == mas->end) + mas->last = mas->max; + else + mas->last = wr_mas->pivots[wr_mas->offset_end]; + wr_mas->end_piv = mas->last; + } + } + + if (!wr_mas->content) + { + /* If this one is null, the next and prev are not */ + mas->index = wr_mas->r_min; + } + else + { + /* Check prev slot if we are overwriting the start */ + if (mas->index == wr_mas->r_min && mas->offset && !wr_mas->slots[mas->offset - 1]) + { + mas->offset--; + wr_mas->r_min = mas->index = mas_safe_min(mas, wr_mas->pivots, mas->offset); + wr_mas->r_max = wr_mas->pivots[mas->offset]; + } + } +} + +static inline void mas_wr_end_piv(struct ma_wr_state *wr_mas) +{ + while ((wr_mas->offset_end < wr_mas->mas->end) && + (wr_mas->mas->last > wr_mas->pivots[wr_mas->offset_end])) + wr_mas->offset_end++; + + if (wr_mas->offset_end < wr_mas->mas->end) + wr_mas->end_piv = wr_mas->pivots[wr_mas->offset_end]; + else + wr_mas->end_piv = wr_mas->mas->max; + + if (!wr_mas->entry) + mas_wr_extend_null(wr_mas); +} + +static inline unsigned char mas_wr_new_end(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + unsigned char new_end = mas->end + 2; + + new_end -= wr_mas->offset_end - mas->offset; + if (wr_mas->r_min == mas->index) + new_end--; + + if (wr_mas->end_piv == mas->last) + new_end--; + + return new_end; +} + +/* + * mas_wr_append: Attempt to append + * @wr_mas: the maple write state + * @new_end: The end of the node after the modification + * + * This is currently unsafe in rcu mode since the end of the node may be cached + * by readers while the node contents may be updated which could result in + * inaccurate information. + * + * Return: True if appended, false otherwise + */ +static inline bool mas_wr_append(struct ma_wr_state *wr_mas, unsigned char new_end) +{ + struct ma_state *mas; + void __rcu **slots; + unsigned char end; + + mas = wr_mas->mas; + if (mt_in_rcu(mas->tree)) + return false; + + end = mas->end; + if (mas->offset != end) + return false; + + if (new_end < mt_pivots[wr_mas->type]) + { + wr_mas->pivots[new_end] = wr_mas->pivots[end]; + ma_set_meta(wr_mas->node, wr_mas->type, 0, new_end); + } + + slots = wr_mas->slots; + if (new_end == end + 1) + { + if (mas->last == wr_mas->r_max) + { + /* Append to end of range */ + rcu_assign_pointer(slots[new_end], wr_mas->entry); + wr_mas->pivots[end] = mas->index - 1; + mas->offset = new_end; + } + else + { + /* Append to start of range */ + rcu_assign_pointer(slots[new_end], wr_mas->content); + wr_mas->pivots[end] = mas->last; + rcu_assign_pointer(slots[end], wr_mas->entry); + } + } + else + { + /* Append to the range without touching any boundaries. */ + rcu_assign_pointer(slots[new_end], wr_mas->content); + wr_mas->pivots[end + 1] = mas->last; + rcu_assign_pointer(slots[end + 1], wr_mas->entry); + wr_mas->pivots[end] = mas->index - 1; + mas->offset = end + 1; + } + + if (!wr_mas->content || !wr_mas->entry) + mas_update_gap(mas); + + mas->end = new_end; + trace_ma_write(__func__, mas, new_end, wr_mas->entry); + return true; +} + +/* + * mas_wr_bnode() - Slow path for a modification. + * @wr_mas: The write maple state + * + * This is where split, rebalance end up. + */ +static void mas_wr_bnode(struct ma_wr_state *wr_mas) +{ + struct maple_big_node b_node; + + trace_ma_write(__func__, wr_mas->mas, 0, wr_mas->entry); + memset(&b_node, 0, sizeof(struct maple_big_node)); + mas_store_b_node(wr_mas, &b_node, wr_mas->offset_end); + mas_commit_b_node(wr_mas, &b_node, wr_mas->mas->end); +} + +static inline void mas_wr_modify(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + unsigned char new_end; + + /* Direct replacement */ + if (wr_mas->r_min == mas->index && wr_mas->r_max == mas->last) + { + rcu_assign_pointer(wr_mas->slots[mas->offset], wr_mas->entry); + if (!!wr_mas->entry ^ !!wr_mas->content) + mas_update_gap(mas); + return; + } + + /* + * new_end exceeds the size of the maple node and cannot enter the fast + * path. + */ + new_end = mas_wr_new_end(wr_mas); + if (new_end >= mt_slots[wr_mas->type]) + goto slow_path; + + /* Attempt to append */ + if (mas_wr_append(wr_mas, new_end)) + return; + + if (new_end == mas->end && mas_wr_slot_store(wr_mas)) + return; + + if (mas_wr_node_store(wr_mas, new_end)) + return; + + if (mas_is_err(mas)) + return; + +slow_path: + mas_wr_bnode(wr_mas); +} + +/* + * mas_wr_store_entry() - Internal call to store a value + * @mas: The maple state + * @entry: The entry to store. + * + * Return: The contents that was stored at the index. + */ +static inline void *mas_wr_store_entry(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + + wr_mas->content = mas_start(mas); + if (mas_is_none(mas) || mas_is_ptr(mas)) + { + mas_store_root(mas, wr_mas->entry); + return wr_mas->content; + } + + if (unlikely(!mas_wr_walk(wr_mas))) + { + mas_wr_spanning_store(wr_mas); + return wr_mas->content; + } + + /* At this point, we are at the leaf node that needs to be altered. */ + mas_wr_end_piv(wr_mas); + /* New root for a single pointer */ + if (unlikely(!mas->index && mas->last == ULONG_MAX)) + { + mas_new_root(mas, wr_mas->entry); + return wr_mas->content; + } + + mas_wr_modify(wr_mas); + return wr_mas->content; +} + +/** + * mas_insert() - Internal call to insert a value + * @mas: The maple state + * @entry: The entry to store + * + * Return: %NULL or the contents that already exists at the requested index + * otherwise. The maple state needs to be checked for error conditions. + */ +static inline void *mas_insert(struct ma_state *mas, void *entry) +{ + MA_WR_STATE(wr_mas, mas, entry); + + /* + * Inserting a new range inserts either 0, 1, or 2 pivots within the + * tree. If the insert fits exactly into an existing gap with a value + * of NULL, then the slot only needs to be written with the new value. + * If the range being inserted is adjacent to another range, then only a + * single pivot needs to be inserted (as well as writing the entry). If + * the new range is within a gap but does not touch any other ranges, + * then two pivots need to be inserted: the start - 1, and the end. As + * usual, the entry must be written. Most operations require a new node + * to be allocated and replace an existing node to ensure RCU safety, + * when in RCU mode. The exception to requiring a newly allocated node + * is when inserting at the end of a node (appending). When done + * carefully, appending can reuse the node in place. + */ + wr_mas.content = mas_start(mas); + if (wr_mas.content) + goto exists; + + if (mas_is_none(mas) || mas_is_ptr(mas)) + { + mas_store_root(mas, entry); + return NULL; + } + + /* spanning writes always overwrite something */ + if (!mas_wr_walk(&wr_mas)) + goto exists; + + /* At this point, we are at the leaf node that needs to be altered. */ + wr_mas.offset_end = mas->offset; + wr_mas.end_piv = wr_mas.r_max; + + if (wr_mas.content || (mas->last > wr_mas.r_max)) + goto exists; + + if (!entry) + return NULL; + + mas_wr_modify(&wr_mas); + return wr_mas.content; + +exists: + mas_set_err(mas, -EEXIST); + return wr_mas.content; +} + +/** + * mas_alloc_cyclic() - Internal call to find somewhere to store an entry + * @mas: The maple state. + * @startp: Pointer to ID. + * @range_lo: Lower bound of range to search. + * @range_hi: Upper bound of range to search. + * @entry: The entry to store. + * @next: Pointer to next ID to allocate. + * @gfp: The GFP_FLAGS to use for allocations. + * + * Return: 0 if the allocation succeeded without wrapping, 1 if the + * allocation succeeded after wrapping, or -EBUSY if there are no + * free entries. + */ +int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp, void *entry, + unsigned long range_lo, unsigned long range_hi, unsigned long *next, gfp_t gfp) +{ + unsigned long min = range_lo; + int ret = 0; + + range_lo = max(min, *next); + ret = mas_empty_area(mas, range_lo, range_hi, 1); + if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) + { + mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; + ret = 1; + } + if (ret < 0 && range_lo > min) + { + ret = mas_empty_area(mas, min, range_hi, 1); + if (ret == 0) + ret = 1; + } + if (ret < 0) + return ret; + + do + { + mas_insert(mas, entry); + } while (mas_nomem(mas, gfp)); + if (mas_is_err(mas)) + return xa_err(mas->node); + + *startp = mas->index; + *next = *startp + 1; + if (*next == 0) + mas->tree->ma_flags |= MT_FLAGS_ALLOC_WRAPPED; + + return ret; +} +EXPORT_SYMBOL(mas_alloc_cyclic); + +__always_inline void mas_rewalk(struct ma_state *mas, unsigned long index) +{ +retry: + mas_set(mas, index); + mas_state_walk(mas); + if (mas_is_start(mas)) + goto retry; +} + +__always_inline bool mas_rewalk_if_dead(struct ma_state *mas, struct maple_node *node, + const unsigned long index) +{ + if (unlikely(ma_dead_node(node))) + { + mas_rewalk(mas, index); + return true; + } + return false; +} + +/* + * mas_prev_node() - Find the prev non-null entry at the same level in the + * tree. The prev value will be mas->node[mas->offset] or the status will be + * ma_none. + * @mas: The maple state + * @min: The lower limit to search + * + * The prev node value will be mas->node[mas->offset] or the status will be + * ma_none. + * Return: 1 if the node is dead, 0 otherwise. + */ +static int mas_prev_node(struct ma_state *mas, unsigned long min) +{ + enum maple_type mt; + int offset, level; + void __rcu **slots; + struct maple_node *node; + unsigned long *pivots; + unsigned long max; + + node = mas_mn(mas); + if (!mas->min) + goto no_entry; + + max = mas->min - 1; + if (max < min) + goto no_entry; + + level = 0; + do + { + if (ma_is_root(node)) + goto no_entry; + + /* Walk up. */ + if (unlikely(mas_ascend(mas))) + return 1; + offset = mas->offset; + level++; + node = mas_mn(mas); + } while (!offset); + + offset--; + mt = mte_node_type(mas->node); + while (level > 1) + { + level--; + slots = ma_slots(node, mt); + mas->node = mas_slot(mas, slots, offset); + if (unlikely(ma_dead_node(node))) + return 1; + + mt = mte_node_type(mas->node); + node = mas_mn(mas); + pivots = ma_pivots(node, mt); + offset = ma_data_end(node, mt, pivots, max); + if (unlikely(ma_dead_node(node))) + return 1; + } + + slots = ma_slots(node, mt); + mas->node = mas_slot(mas, slots, offset); + pivots = ma_pivots(node, mt); + if (unlikely(ma_dead_node(node))) + return 1; + + if (likely(offset)) + mas->min = pivots[offset - 1] + 1; + mas->max = max; + mas->offset = mas_data_end(mas); + if (unlikely(mte_dead_node(mas->node))) + return 1; + + mas->end = mas->offset; + return 0; + +no_entry: + if (unlikely(ma_dead_node(node))) + return 1; + + mas->status = ma_underflow; + return 0; +} + +/* + * mas_prev_slot() - Get the entry in the previous slot + * + * @mas: The maple state + * @max: The minimum starting range + * @empty: Can be empty + * @set_underflow: Set the @mas->node to underflow state on limit. + * + * Return: The entry in the previous slot which is possibly NULL + */ +static void *mas_prev_slot(struct ma_state *mas, unsigned long min, bool empty) +{ + void *entry; + void __rcu **slots; + unsigned long pivot; + enum maple_type type; + unsigned long *pivots; + struct maple_node *node; + unsigned long save_point = mas->index; + +retry: + node = mas_mn(mas); + type = mte_node_type(mas->node); + pivots = ma_pivots(node, type); + if (unlikely(mas_rewalk_if_dead(mas, node, save_point))) + goto retry; + + if (mas->min <= min) + { + pivot = mas_safe_min(mas, pivots, mas->offset); + + if (unlikely(mas_rewalk_if_dead(mas, node, save_point))) + goto retry; + + if (pivot <= min) + goto underflow; + } + +again: + if (likely(mas->offset)) + { + mas->offset--; + mas->last = mas->index - 1; + mas->index = mas_safe_min(mas, pivots, mas->offset); + } + else + { + if (mas->index <= min) + goto underflow; + + if (mas_prev_node(mas, min)) + { + mas_rewalk(mas, save_point); + goto retry; + } + + if (WARN_ON_ONCE(mas_is_underflow(mas))) + return NULL; + + mas->last = mas->max; + node = mas_mn(mas); + type = mte_node_type(mas->node); + pivots = ma_pivots(node, type); + mas->index = pivots[mas->offset - 1] + 1; + } + + slots = ma_slots(node, type); + entry = mas_slot(mas, slots, mas->offset); + if (unlikely(mas_rewalk_if_dead(mas, node, save_point))) + goto retry; + + if (likely(entry)) + return entry; + + if (!empty) + { + if (mas->index <= min) + { + mas->status = ma_underflow; + return NULL; + } + + goto again; + } + + return entry; + +underflow: + mas->status = ma_underflow; + return NULL; +} + +/* + * mas_next_node() - Get the next node at the same level in the tree. + * @mas: The maple state + * @max: The maximum pivot value to check. + * + * The next value will be mas->node[mas->offset] or the status will have + * overflowed. + * Return: 1 on dead node, 0 otherwise. + */ +static int mas_next_node(struct ma_state *mas, struct maple_node *node, unsigned long max) +{ + unsigned long min; + unsigned long *pivots; + struct maple_enode *enode; + struct maple_node *tmp; + int level = 0; + unsigned char node_end; + enum maple_type mt; + void __rcu **slots; + + if (mas->max >= max) + goto overflow; + + min = mas->max + 1; + level = 0; + do + { + if (ma_is_root(node)) + goto overflow; + + /* Walk up. */ + if (unlikely(mas_ascend(mas))) + return 1; + + level++; + node = mas_mn(mas); + mt = mte_node_type(mas->node); + pivots = ma_pivots(node, mt); + node_end = ma_data_end(node, mt, pivots, mas->max); + if (unlikely(ma_dead_node(node))) + return 1; + + } while (unlikely(mas->offset == node_end)); + + slots = ma_slots(node, mt); + mas->offset++; + enode = mas_slot(mas, slots, mas->offset); + if (unlikely(ma_dead_node(node))) + return 1; + + if (level > 1) + mas->offset = 0; + + while (unlikely(level > 1)) + { + level--; + mas->node = enode; + node = mas_mn(mas); + mt = mte_node_type(mas->node); + slots = ma_slots(node, mt); + enode = mas_slot(mas, slots, 0); + if (unlikely(ma_dead_node(node))) + return 1; + } + + if (!mas->offset) + pivots = ma_pivots(node, mt); + + mas->max = mas_safe_pivot(mas, pivots, mas->offset, mt); + tmp = mte_to_node(enode); + mt = mte_node_type(enode); + pivots = ma_pivots(tmp, mt); + mas->end = ma_data_end(tmp, mt, pivots, mas->max); + if (unlikely(ma_dead_node(node))) + return 1; + + mas->node = enode; + mas->min = min; + return 0; + +overflow: + if (unlikely(ma_dead_node(node))) + return 1; + + mas->status = ma_overflow; + return 0; +} + +/* + * mas_next_slot() - Get the entry in the next slot + * + * @mas: The maple state + * @max: The maximum starting range + * @empty: Can be empty + * @set_overflow: Should @mas->node be set to overflow when the limit is + * reached. + * + * Return: The entry in the next slot which is possibly NULL + */ +static void *mas_next_slot(struct ma_state *mas, unsigned long max, bool empty) +{ + void __rcu **slots; + unsigned long *pivots; + unsigned long pivot; + enum maple_type type; + struct maple_node *node; + unsigned long save_point = mas->last; + void *entry; + +retry: + node = mas_mn(mas); + type = mte_node_type(mas->node); + pivots = ma_pivots(node, type); + if (unlikely(mas_rewalk_if_dead(mas, node, save_point))) + goto retry; + + if (mas->max >= max) + { + if (likely(mas->offset < mas->end)) + pivot = pivots[mas->offset]; + else + pivot = mas->max; + + if (unlikely(mas_rewalk_if_dead(mas, node, save_point))) + goto retry; + + if (pivot >= max) + { /* Was at the limit, next will extend beyond */ + mas->status = ma_overflow; + return NULL; + } + } + + if (likely(mas->offset < mas->end)) + { + mas->index = pivots[mas->offset] + 1; + again: + mas->offset++; + if (likely(mas->offset < mas->end)) + mas->last = pivots[mas->offset]; + else + mas->last = mas->max; + } + else + { + if (mas->last >= max) + { + mas->status = ma_overflow; + return NULL; + } + + if (mas_next_node(mas, node, max)) + { + mas_rewalk(mas, save_point); + goto retry; + } + + if (WARN_ON_ONCE(mas_is_overflow(mas))) + return NULL; + + mas->offset = 0; + mas->index = mas->min; + node = mas_mn(mas); + type = mte_node_type(mas->node); + pivots = ma_pivots(node, type); + mas->last = pivots[0]; + } + + slots = ma_slots(node, type); + entry = mt_slot(mas->tree, slots, mas->offset); + if (unlikely(mas_rewalk_if_dead(mas, node, save_point))) + goto retry; + + if (entry) + return entry; + + if (!empty) + { + if (mas->last >= max) + { + mas->status = ma_overflow; + return NULL; + } + + mas->index = mas->last + 1; + goto again; + } + + return entry; +} + +/* + * mas_next_entry() - Internal function to get the next entry. + * @mas: The maple state + * @limit: The maximum range start. + * + * Set the @mas->node to the next entry and the range_start to + * the beginning value for the entry. Does not check beyond @limit. + * Sets @mas->index and @mas->last to the range, Does not update @mas->index and + * @mas->last on overflow. + * Restarts on dead nodes. + * + * Return: the next entry or %NULL. + */ +static inline void *mas_next_entry(struct ma_state *mas, unsigned long limit) +{ + if (mas->last >= limit) + { + mas->status = ma_overflow; + return NULL; + } + + return mas_next_slot(mas, limit, false); +} + +/* + * mas_rev_awalk() - Internal function. Reverse allocation walk. Find the + * highest gap address of a given size in a given node and descend. + * @mas: The maple state + * @size: The needed size. + * + * Return: True if found in a leaf, false otherwise. + * + */ +static bool mas_rev_awalk(struct ma_state *mas, unsigned long size, unsigned long *gap_min, + unsigned long *gap_max) +{ + enum maple_type type = mte_node_type(mas->node); + struct maple_node *node = mas_mn(mas); + unsigned long *pivots, *gaps; + void __rcu **slots; + unsigned long gap = 0; + unsigned long max, min; + unsigned char offset; + + if (unlikely(mas_is_err(mas))) + return true; + + if (ma_is_dense(type)) + { + /* dense nodes. */ + mas->offset = (unsigned char) (mas->index - mas->min); + return true; + } + + pivots = ma_pivots(node, type); + slots = ma_slots(node, type); + gaps = ma_gaps(node, type); + offset = mas->offset; + min = mas_safe_min(mas, pivots, offset); + /* Skip out of bounds. */ + while (mas->last < min) + min = mas_safe_min(mas, pivots, --offset); + + max = mas_safe_pivot(mas, pivots, offset, type); + while (mas->index <= max) + { + gap = 0; + if (gaps) + gap = gaps[offset]; + else if (!mas_slot(mas, slots, offset)) + gap = max - min + 1; + + if (gap) + { + if ((size <= gap) && (size <= mas->last - min + 1)) + break; + + if (!gaps) + { + /* Skip the next slot, it cannot be a gap. */ + if (offset < 2) + goto ascend; + + offset -= 2; + max = pivots[offset]; + min = mas_safe_min(mas, pivots, offset); + continue; + } + } + + if (!offset) + goto ascend; + + offset--; + max = min - 1; + min = mas_safe_min(mas, pivots, offset); + } + + if (unlikely((mas->index > max) || (size - 1 > max - mas->index))) + goto no_space; + + if (unlikely(ma_is_leaf(type))) + { + mas->offset = offset; + *gap_min = min; + *gap_max = min + gap - 1; + return true; + } + + /* descend, only happens under lock. */ + mas->node = mas_slot(mas, slots, offset); + mas->min = min; + mas->max = max; + mas->offset = mas_data_end(mas); + return false; + +ascend: + if (!mte_is_root(mas->node)) + return false; + +no_space: + mas_set_err(mas, -EBUSY); + return false; +} + +static inline bool mas_anode_descend(struct ma_state *mas, unsigned long size) +{ + enum maple_type type = mte_node_type(mas->node); + unsigned long pivot, min, gap = 0; + unsigned char offset, data_end; + unsigned long *gaps, *pivots; + void __rcu **slots; + struct maple_node *node; + bool found = false; + + if (ma_is_dense(type)) + { + mas->offset = (unsigned char) (mas->index - mas->min); + return true; + } + + node = mas_mn(mas); + pivots = ma_pivots(node, type); + slots = ma_slots(node, type); + gaps = ma_gaps(node, type); + offset = mas->offset; + min = mas_safe_min(mas, pivots, offset); + data_end = ma_data_end(node, type, pivots, mas->max); + for (; offset <= data_end; offset++) + { + pivot = mas_safe_pivot(mas, pivots, offset, type); + + /* Not within lower bounds */ + if (mas->index > pivot) + goto next_slot; + + if (gaps) + gap = gaps[offset]; + else if (!mas_slot(mas, slots, offset)) + gap = min(pivot, mas->last) - max(mas->index, min) + 1; + else + goto next_slot; + + if (gap >= size) + { + if (ma_is_leaf(type)) + { + found = true; + goto done; + } + if (mas->index <= pivot) + { + mas->node = mas_slot(mas, slots, offset); + mas->min = min; + mas->max = pivot; + offset = 0; + break; + } + } + next_slot: + min = pivot + 1; + if (mas->last <= pivot) + { + mas_set_err(mas, -EBUSY); + return true; + } + } + + if (mte_is_root(mas->node)) + found = true; +done: + mas->offset = offset; + return found; +} + +/** + * mas_walk() - Search for @mas->index in the tree. + * @mas: The maple state. + * + * mas->index and mas->last will be set to the range if there is a value. If + * mas->status is ma_none, reset to ma_start + * + * Return: the entry at the location or %NULL. + */ +void *mas_walk(struct ma_state *mas) +{ + void *entry; + + if (!mas_is_active(mas) || !mas_is_start(mas)) + mas->status = ma_start; +retry: + entry = mas_state_walk(mas); + if (mas_is_start(mas)) + { + goto retry; + } + else if (mas_is_none(mas)) + { + mas->index = 0; + mas->last = ULONG_MAX; + } + else if (mas_is_ptr(mas)) + { + if (!mas->index) + { + mas->last = 0; + return entry; + } + + mas->index = 1; + mas->last = ULONG_MAX; + mas->status = ma_none; + return NULL; + } + + return entry; +} +EXPORT_SYMBOL_GPL(mas_walk); + +static inline bool mas_rewind_node(struct ma_state *mas) +{ + unsigned char slot; + + do + { + if (mte_is_root(mas->node)) + { + slot = mas->offset; + if (!slot) + return false; + } + else + { + mas_ascend(mas); + slot = mas->offset; + } + } while (!slot); + + mas->offset = --slot; + return true; +} + +/* + * mas_skip_node() - Internal function. Skip over a node. + * @mas: The maple state. + * + * Return: true if there is another node, false otherwise. + */ +static inline bool mas_skip_node(struct ma_state *mas) +{ + if (mas_is_err(mas)) + return false; + + do + { + if (mte_is_root(mas->node)) + { + if (mas->offset >= mas_data_end(mas)) + { + mas_set_err(mas, -EBUSY); + return false; + } + } + else + { + mas_ascend(mas); + } + } while (mas->offset >= mas_data_end(mas)); + + mas->offset++; + return true; +} + +/* + * mas_awalk() - Allocation walk. Search from low address to high, for a gap of + * @size + * @mas: The maple state + * @size: The size of the gap required + * + * Search between @mas->index and @mas->last for a gap of @size. + */ +static inline void mas_awalk(struct ma_state *mas, unsigned long size) +{ + struct maple_enode *last = NULL; + + /* + * There are 4 options: + * go to child (descend) + * go back to parent (ascend) + * no gap found. (return, slot == MAPLE_NODE_SLOTS) + * found the gap. (return, slot != MAPLE_NODE_SLOTS) + */ + while (!mas_is_err(mas) && !mas_anode_descend(mas, size)) + { + if (last == mas->node) + mas_skip_node(mas); + else + last = mas->node; + } +} + +/* + * mas_sparse_area() - Internal function. Return upper or lower limit when + * searching for a gap in an empty tree. + * @mas: The maple state + * @min: the minimum range + * @max: The maximum range + * @size: The size of the gap + * @fwd: Searching forward or back + */ +static inline int mas_sparse_area(struct ma_state *mas, unsigned long min, unsigned long max, + unsigned long size, bool fwd) +{ + if (!unlikely(mas_is_none(mas)) && min == 0) + { + min++; + /* + * At this time, min is increased, we need to recheck whether + * the size is satisfied. + */ + if (min > max || max - min + 1 < size) + return -EBUSY; + } + /* mas_is_ptr */ + + if (fwd) + { + mas->index = min; + mas->last = min + size - 1; + } + else + { + mas->last = max; + mas->index = max - size + 1; + } + return 0; +} + +/* + * mas_empty_area() - Get the lowest address within the range that is + * sufficient for the size requested. + * @mas: The maple state + * @min: The lowest value of the range + * @max: The highest value of the range + * @size: The size needed + */ +int mas_empty_area(struct ma_state *mas, unsigned long min, unsigned long max, unsigned long size) +{ + unsigned char offset; + unsigned long *pivots; + enum maple_type mt; + struct maple_node *node; + + if (min > max) + return -EINVAL; + + if (size == 0 || max - min < size - 1) + return -EINVAL; + + if (mas_is_start(mas)) + mas_start(mas); + else if (mas->offset >= 2) + mas->offset -= 2; + else if (!mas_skip_node(mas)) + return -EBUSY; + + /* Empty set */ + if (mas_is_none(mas) || mas_is_ptr(mas)) + return mas_sparse_area(mas, min, max, size, true); + + /* The start of the window can only be within these values */ + mas->index = min; + mas->last = max; + mas_awalk(mas, size); + + if (unlikely(mas_is_err(mas))) + return xa_err(mas->node); + + offset = mas->offset; + if (unlikely(offset == MAPLE_NODE_SLOTS)) + return -EBUSY; + + node = mas_mn(mas); + mt = mte_node_type(mas->node); + pivots = ma_pivots(node, mt); + min = mas_safe_min(mas, pivots, offset); + if (mas->index < min) + mas->index = min; + mas->last = mas->index + size - 1; + mas->end = ma_data_end(node, mt, pivots, mas->max); + return 0; +} +EXPORT_SYMBOL_GPL(mas_empty_area); + +/* + * mas_empty_area_rev() - Get the highest address within the range that is + * sufficient for the size requested. + * @mas: The maple state + * @min: The lowest value of the range + * @max: The highest value of the range + * @size: The size needed + */ +int mas_empty_area_rev(struct ma_state *mas, unsigned long min, unsigned long max, + unsigned long size) +{ + struct maple_enode *last = mas->node; + + if (min > max) + return -EINVAL; + + if (size == 0 || max - min < size - 1) + return -EINVAL; + + if (mas_is_start(mas)) + mas_start(mas); + else if ((mas->offset < 2) && (!mas_rewind_node(mas))) + return -EBUSY; + + if (unlikely(mas_is_none(mas) || mas_is_ptr(mas))) + return mas_sparse_area(mas, min, max, size, false); + else if (mas->offset >= 2) + mas->offset -= 2; + else + mas->offset = mas_data_end(mas); + + /* The start of the window can only be within these values. */ + mas->index = min; + mas->last = max; + + while (!mas_rev_awalk(mas, size, &min, &max)) + { + if (last == mas->node) + { + if (!mas_rewind_node(mas)) + return -EBUSY; + } + else + { + last = mas->node; + } + } + + if (mas_is_err(mas)) + return xa_err(mas->node); + + if (unlikely(mas->offset == MAPLE_NODE_SLOTS)) + return -EBUSY; + + /* Trim the upper limit to the max. */ + if (max < mas->last) + mas->last = max; + + mas->index = mas->last - size + 1; + mas->end = mas_data_end(mas); + return 0; +} +EXPORT_SYMBOL_GPL(mas_empty_area_rev); + +/* + * mte_dead_leaves() - Mark all leaves of a node as dead. + * @mas: The maple state + * @slots: Pointer to the slot array + * @type: The maple node type + * + * Must hold the write lock. + * + * Return: The number of leaves marked as dead. + */ +static inline unsigned char mte_dead_leaves(struct maple_enode *enode, struct maple_tree *mt, + void __rcu **slots) +{ + struct maple_node *node; + enum maple_type type; + void *entry; + int offset; + + for (offset = 0; offset < mt_slot_count(enode); offset++) + { + entry = mt_slot(mt, slots, offset); + type = mte_node_type(entry); + node = mte_to_node(entry); + /* Use both node and type to catch LE & BE metadata */ + if (!node || !type) + break; + + mte_set_node_dead(entry); + node->type = type; + rcu_assign_pointer(slots[offset], node); + } + + return offset; +} + +/** + * mte_dead_walk() - Walk down a dead tree to just before the leaves + * @enode: The maple encoded node + * @offset: The starting offset + * + * Note: This can only be used from the RCU callback context. + */ +static void __rcu **mte_dead_walk(struct maple_enode **enode, unsigned char offset) +{ + struct maple_node *node, *next; + void __rcu **slots = NULL; + + next = mte_to_node(*enode); + do + { + *enode = ma_enode_ptr(next); + node = mte_to_node(*enode); + slots = ma_slots(node, node->type); + next = rcu_dereference_protected(slots[offset], lock_is_held(&rcu_callback_map)); + offset = 0; + } while (!ma_is_leaf(next->type)); + + return slots; +} + +/** + * mt_free_walk() - Walk & free a tree in the RCU callback context + * @head: The RCU head that's within the node. + * + * Note: This can only be used from the RCU callback context. + */ +static void mt_free_walk(struct rcu_head *head) +{ + void __rcu **slots; + struct maple_node *node, *start; + struct maple_enode *enode; + unsigned char offset; + enum maple_type type; + + node = container_of(head, struct maple_node, rcu); + + if (ma_is_leaf(node->type)) + goto free_leaf; + + start = node; + enode = mt_mk_node(node, node->type); + slots = mte_dead_walk(&enode, 0); + node = mte_to_node(enode); + do + { + mt_free_bulk(node->slot_len, slots); + offset = node->parent_slot + 1; + enode = node->piv_parent; + if (mte_to_node(enode) == node) + goto free_leaf; + + type = mte_node_type(enode); + slots = ma_slots(mte_to_node(enode), type); + if ((offset < mt_slots[type]) && + rcu_dereference_protected(slots[offset], lock_is_held(&rcu_callback_map))) + slots = mte_dead_walk(&enode, offset); + node = mte_to_node(enode); + } while ((node != start) || (node->slot_len < offset)); + + slots = ma_slots(node, node->type); + mt_free_bulk(node->slot_len, slots); + +free_leaf: + mt_free_rcu(&node->rcu); +} + +static inline void __rcu **mte_destroy_descend(struct maple_enode **enode, struct maple_tree *mt, + struct maple_enode *prev, unsigned char offset) +{ + struct maple_node *node; + struct maple_enode *next = *enode; + void __rcu **slots = NULL; + enum maple_type type; + unsigned char next_offset = 0; + + do + { + *enode = next; + node = mte_to_node(*enode); + type = mte_node_type(*enode); + slots = ma_slots(node, type); + next = mt_slot_locked(mt, slots, next_offset); + if ((mte_dead_node(next))) + next = mt_slot_locked(mt, slots, ++next_offset); + + mte_set_node_dead(*enode); + node->type = type; + node->piv_parent = prev; + node->parent_slot = offset; + offset = next_offset; + next_offset = 0; + prev = *enode; + } while (!mte_is_leaf(next)); + + return slots; +} + +static void mt_destroy_walk(struct maple_enode *enode, struct maple_tree *mt, bool free) +{ + void __rcu **slots; + struct maple_node *node = mte_to_node(enode); + struct maple_enode *start; + + if (mte_is_leaf(enode)) + { + node->type = mte_node_type(enode); + goto free_leaf; + } + + start = enode; + slots = mte_destroy_descend(&enode, mt, start, 0); + node = mte_to_node(enode); // Updated in the above call. + do + { + enum maple_type type; + unsigned char offset; + struct maple_enode *parent, *tmp; + + node->slot_len = mte_dead_leaves(enode, mt, slots); + if (free) + mt_free_bulk(node->slot_len, slots); + offset = node->parent_slot + 1; + enode = node->piv_parent; + if (mte_to_node(enode) == node) + goto free_leaf; + + type = mte_node_type(enode); + slots = ma_slots(mte_to_node(enode), type); + if (offset >= mt_slots[type]) + goto next; + + tmp = mt_slot_locked(mt, slots, offset); + if (mte_node_type(tmp) && mte_to_node(tmp)) + { + parent = enode; + enode = tmp; + slots = mte_destroy_descend(&enode, mt, parent, offset); + } + next: + node = mte_to_node(enode); + } while (start != enode); + + node = mte_to_node(enode); + node->slot_len = mte_dead_leaves(enode, mt, slots); + if (free) + mt_free_bulk(node->slot_len, slots); + +free_leaf: + if (free) + mt_free_rcu(&node->rcu); + else + mt_clear_meta(mt, node, node->type); +} + +/* + * mte_destroy_walk() - Free a tree or sub-tree. + * @enode: the encoded maple node (maple_enode) to start + * @mt: the tree to free - needed for node types. + * + * Must hold the write lock. + */ +static inline void mte_destroy_walk(struct maple_enode *enode, struct maple_tree *mt) +{ + struct maple_node *node = mte_to_node(enode); + + if (mt_in_rcu(mt)) + { + mt_destroy_walk(enode, mt, false); + call_rcu(&node->rcu, mt_free_walk); + } + else + { + mt_destroy_walk(enode, mt, true); + } +} + +static void mas_wr_store_setup(struct ma_wr_state *wr_mas) +{ + if (!mas_is_active(wr_mas->mas)) + { + if (mas_is_start(wr_mas->mas)) + return; + + if (unlikely(mas_is_paused(wr_mas->mas))) + goto reset; + + if (unlikely(mas_is_none(wr_mas->mas))) + goto reset; + + if (unlikely(mas_is_overflow(wr_mas->mas))) + goto reset; + + if (unlikely(mas_is_underflow(wr_mas->mas))) + goto reset; + } + + /* + * A less strict version of mas_is_span_wr() where we allow spanning + * writes within this node. This is to stop partial walks in + * mas_prealloc() from being reset. + */ + if (wr_mas->mas->last > wr_mas->mas->max) + goto reset; + + if (wr_mas->entry) + return; + + if (mte_is_leaf(wr_mas->mas->node) && wr_mas->mas->last == wr_mas->mas->max) + goto reset; + + return; + +reset: + mas_reset(wr_mas->mas); +} + +/* Interface */ + +/** + * mas_store() - Store an @entry. + * @mas: The maple state. + * @entry: The entry to store. + * + * The @mas->index and @mas->last is used to set the range for the @entry. + * Note: The @mas should have pre-allocated entries to ensure there is memory to + * store the entry. Please see mas_expected_entries()/mas_destroy() for more details. + * + * Return: the first entry between mas->index and mas->last or %NULL. + */ +void *mas_store(struct ma_state *mas, void *entry) +{ + MA_WR_STATE(wr_mas, mas, entry); + + trace_ma_write(__func__, mas, 0, entry); +#ifdef CONFIG_DEBUG_MAPLE_TREE + if (MAS_WARN_ON(mas, mas->index > mas->last)) + pr_err("Error %lX > %lX %p\n", mas->index, mas->last, entry); + + if (mas->index > mas->last) + { + mas_set_err(mas, -EINVAL); + return NULL; + } + +#endif + + /* + * Storing is the same operation as insert with the added caveat that it + * can overwrite entries. Although this seems simple enough, one may + * want to examine what happens if a single store operation was to + * overwrite multiple entries within a self-balancing B-Tree. + */ + mas_wr_store_setup(&wr_mas); + mas_wr_store_entry(&wr_mas); + return wr_mas.content; +} +EXPORT_SYMBOL_GPL(mas_store); + +/** + * mas_store_gfp() - Store a value into the tree. + * @mas: The maple state + * @entry: The entry to store + * @gfp: The GFP_FLAGS to use for allocations if necessary. + * + * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not + * be allocated. + */ +int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp) +{ + MA_WR_STATE(wr_mas, mas, entry); + + mas_wr_store_setup(&wr_mas); + trace_ma_write(__func__, mas, 0, entry); +retry: + mas_wr_store_entry(&wr_mas); + if (unlikely(mas_nomem(mas, gfp))) + goto retry; + + if (unlikely(mas_is_err(mas))) + return xa_err(mas->node); + + return 0; +} +EXPORT_SYMBOL_GPL(mas_store_gfp); + +/** + * mas_store_prealloc() - Store a value into the tree using memory + * preallocated in the maple state. + * @mas: The maple state + * @entry: The entry to store. + */ +void mas_store_prealloc(struct ma_state *mas, void *entry) +{ + MA_WR_STATE(wr_mas, mas, entry); + + mas_wr_store_setup(&wr_mas); + trace_ma_write(__func__, mas, 0, entry); + mas_wr_store_entry(&wr_mas); + MAS_WR_BUG_ON(&wr_mas, mas_is_err(mas)); + mas_destroy(mas); +} +EXPORT_SYMBOL_GPL(mas_store_prealloc); + +/** + * mas_preallocate() - Preallocate enough nodes for a store operation + * @mas: The maple state + * @entry: The entry that will be stored + * @gfp: The GFP_FLAGS to use for allocations. + * + * Return: 0 on success, -ENOMEM if memory could not be allocated. + */ +int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) +{ + MA_WR_STATE(wr_mas, mas, entry); + unsigned char node_size; + int request = 1; + int ret; + + if (unlikely(!mas->index && mas->last == ULONG_MAX)) + goto ask_now; + + mas_wr_store_setup(&wr_mas); + wr_mas.content = mas_start(mas); + /* Root expand */ + if (unlikely(mas_is_none(mas) || mas_is_ptr(mas))) + goto ask_now; + + if (unlikely(!mas_wr_walk(&wr_mas))) + { + /* Spanning store, use worst case for now */ + request = 1 + mas_mt_height(mas) * 3; + goto ask_now; + } + + /* At this point, we are at the leaf node that needs to be altered. */ + /* Exact fit, no nodes needed. */ + if (wr_mas.r_min == mas->index && wr_mas.r_max == mas->last) + return 0; + + mas_wr_end_piv(&wr_mas); + node_size = mas_wr_new_end(&wr_mas); + + /* Slot store, does not require additional nodes */ + if (node_size == mas->end) + { + /* reuse node */ + if (!mt_in_rcu(mas->tree)) + return 0; + /* shifting boundary */ + if (wr_mas.offset_end - mas->offset == 1) + return 0; + } + + if (node_size >= mt_slots[wr_mas.type]) + { + /* Split, worst case for now. */ + request = 1 + mas_mt_height(mas) * 2; + goto ask_now; + } + + /* New root needs a single node */ + if (unlikely(mte_is_root(mas->node))) + goto ask_now; + + /* Potential spanning rebalance collapsing a node, use worst-case */ + if (node_size - 1 <= mt_min_slots[wr_mas.type]) + request = mas_mt_height(mas) * 2 - 1; + + /* node store, slot store needs one node */ +ask_now: + mas_node_count_gfp(mas, request, gfp); + mas->mas_flags |= MA_STATE_PREALLOC; + if (likely(!mas_is_err(mas))) + return 0; + + mas_set_alloc_req(mas, 0); + ret = xa_err(mas->node); + mas_reset(mas); + mas_destroy(mas); + mas_reset(mas); + return ret; +} +EXPORT_SYMBOL_GPL(mas_preallocate); + +/* + * mas_destroy() - destroy a maple state. + * @mas: The maple state + * + * Upon completion, check the left-most node and rebalance against the node to + * the right if necessary. Frees any allocated nodes associated with this maple + * state. + */ +void mas_destroy(struct ma_state *mas) +{ + struct maple_alloc *node; + unsigned long total; + + /* + * When using mas_for_each() to insert an expected number of elements, + * it is possible that the number inserted is less than the expected + * number. To fix an invalid final node, a check is performed here to + * rebalance the previous node with the final node. + */ + if (mas->mas_flags & MA_STATE_REBALANCE) + { + unsigned char end; + + mas_start(mas); + mtree_range_walk(mas); + end = mas->end + 1; + if (end < mt_min_slot_count(mas->node) - 1) + mas_destroy_rebalance(mas, end); + + mas->mas_flags &= ~MA_STATE_REBALANCE; + } + mas->mas_flags &= ~(MA_STATE_BULK | MA_STATE_PREALLOC); + + total = mas_allocated(mas); + while (total) + { + node = mas->alloc; + mas->alloc = node->slot[0]; + if (node->node_count > 1) + { + size_t count = node->node_count - 1; + + mt_free_bulk(count, (void __rcu **) &node->slot[1]); + total -= count; + } + mt_free_one(ma_mnode_ptr(node)); + total--; + } + + mas->alloc = NULL; +} +EXPORT_SYMBOL_GPL(mas_destroy); + +#define DIV_ROUND_UP(n, d) (((n) + (d) -1) / (d)) + +/* + * mas_expected_entries() - Set the expected number of entries that will be inserted. + * @mas: The maple state + * @nr_entries: The number of expected entries. + * + * This will attempt to pre-allocate enough nodes to store the expected number + * of entries. The allocations will occur using the bulk allocator interface + * for speed. Please call mas_destroy() on the @mas after inserting the entries + * to ensure any unused nodes are freed. + * + * Return: 0 on success, -ENOMEM if memory could not be allocated. + */ +int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries) +{ + int nonleaf_cap = MAPLE_ARANGE64_SLOTS - 2; + struct maple_enode *enode = mas->node; + int nr_nodes; + int ret; + + /* + * Sometimes it is necessary to duplicate a tree to a new tree, such as + * forking a process and duplicating the VMAs from one tree to a new + * tree. When such a situation arises, it is known that the new tree is + * not going to be used until the entire tree is populated. For + * performance reasons, it is best to use a bulk load with RCU disabled. + * This allows for optimistic splitting that favours the left and reuse + * of nodes during the operation. + */ + + /* Optimize splitting for bulk insert in-order */ + mas->mas_flags |= MA_STATE_BULK; + + /* + * Avoid overflow, assume a gap between each entry and a trailing null. + * If this is wrong, it just means allocation can happen during + * insertion of entries. + */ + nr_nodes = max(nr_entries, nr_entries * 2 + 1); + if (!mt_is_alloc(mas->tree)) + nonleaf_cap = MAPLE_RANGE64_SLOTS - 2; + + /* Leaves; reduce slots to keep space for expansion */ + nr_nodes = DIV_ROUND_UP(nr_nodes, MAPLE_RANGE64_SLOTS - 2); + /* Internal nodes */ + nr_nodes += DIV_ROUND_UP(nr_nodes, nonleaf_cap); + /* Add working room for split (2 nodes) + new parents */ + mas_node_count_gfp(mas, nr_nodes + 3, GFP_KERNEL); + + /* Detect if allocations run out */ + mas->mas_flags |= MA_STATE_PREALLOC; + + if (!mas_is_err(mas)) + return 0; + + ret = xa_err(mas->node); + mas->node = enode; + mas_destroy(mas); + return ret; +} +EXPORT_SYMBOL_GPL(mas_expected_entries); + +#define fallthrough __attribute__((__fallthrough__)) + +static bool mas_next_setup(struct ma_state *mas, unsigned long max, void **entry) +{ + bool was_none = mas_is_none(mas); + + if (unlikely(mas->last >= max)) + { + mas->status = ma_overflow; + return true; + } + + switch (mas->status) + { + case ma_active: + return false; + case ma_none: + fallthrough; + case ma_pause: + mas->status = ma_start; + fallthrough; + case ma_start: + mas_walk(mas); /* Retries on dead nodes handled by mas_walk */ + break; + case ma_overflow: + /* Overflowed before, but the max changed */ + mas->status = ma_active; + break; + case ma_underflow: + /* The user expects the mas to be one before where it is */ + mas->status = ma_active; + *entry = mas_walk(mas); + if (*entry) + return true; + break; + case ma_root: + break; + case ma_error: + return true; + } + + if (likely(mas_is_active(mas))) /* Fast path */ + return false; + + if (mas_is_ptr(mas)) + { + *entry = NULL; + if (was_none && mas->index == 0) + { + mas->index = mas->last = 0; + return true; + } + mas->index = 1; + mas->last = ULONG_MAX; + mas->status = ma_none; + return true; + } + + if (mas_is_none(mas)) + return true; + + return false; +} + +/** + * mas_next() - Get the next entry. + * @mas: The maple state + * @max: The maximum index to check. + * + * Returns the next entry after @mas->index. + * Must hold rcu_read_lock or the write lock. + * Can return the zero entry. + * + * Return: The next entry or %NULL + */ +void *mas_next(struct ma_state *mas, unsigned long max) +{ + void *entry = NULL; + + if (mas_next_setup(mas, max, &entry)) + return entry; + + /* Retries on dead nodes handled by mas_next_slot */ + return mas_next_slot(mas, max, false); +} +EXPORT_SYMBOL_GPL(mas_next); + +/** + * mas_next_range() - Advance the maple state to the next range + * @mas: The maple state + * @max: The maximum index to check. + * + * Sets @mas->index and @mas->last to the range. + * Must hold rcu_read_lock or the write lock. + * Can return the zero entry. + * + * Return: The next entry or %NULL + */ +void *mas_next_range(struct ma_state *mas, unsigned long max) +{ + void *entry = NULL; + + if (mas_next_setup(mas, max, &entry)) + return entry; + + /* Retries on dead nodes handled by mas_next_slot */ + return mas_next_slot(mas, max, true); +} +EXPORT_SYMBOL_GPL(mas_next_range); + +/** + * mt_next() - get the next value in the maple tree + * @mt: The maple tree + * @index: The start index + * @max: The maximum index to check + * + * Takes RCU read lock internally to protect the search, which does not + * protect the returned pointer after dropping RCU read lock. + * See also: Documentation/core-api/maple_tree.rst + * + * Return: The entry higher than @index or %NULL if nothing is found. + */ +void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max) +{ + void *entry = NULL; + MA_STATE(mas, mt, index, index); + + rcu_read_lock(); + entry = mas_next(&mas, max); + rcu_read_unlock(); + return entry; +} +EXPORT_SYMBOL_GPL(mt_next); + +static bool mas_prev_setup(struct ma_state *mas, unsigned long min, void **entry) +{ + if (unlikely(mas->index <= min)) + { + mas->status = ma_underflow; + return true; + } + + switch (mas->status) + { + case ma_active: + return false; + case ma_start: + break; + case ma_none: + fallthrough; + case ma_pause: + mas->status = ma_start; + break; + case ma_underflow: + /* underflowed before but the min changed */ + mas->status = ma_active; + break; + case ma_overflow: + /* User expects mas to be one after where it is */ + mas->status = ma_active; + *entry = mas_walk(mas); + if (*entry) + return true; + break; + case ma_root: + break; + case ma_error: + return true; + } + + if (mas_is_start(mas)) + mas_walk(mas); + + if (unlikely(mas_is_ptr(mas))) + { + if (!mas->index) + { + mas->status = ma_none; + return true; + } + mas->index = mas->last = 0; + *entry = mas_root(mas); + return true; + } + + if (mas_is_none(mas)) + { + if (mas->index) + { + /* Walked to out-of-range pointer? */ + mas->index = mas->last = 0; + mas->status = ma_root; + *entry = mas_root(mas); + return true; + } + return true; + } + + return false; +} + +/** + * mas_prev() - Get the previous entry + * @mas: The maple state + * @min: The minimum value to check. + * + * Must hold rcu_read_lock or the write lock. + * Will reset mas to ma_start if the status is ma_none. Will stop on not + * searchable nodes. + * + * Return: the previous value or %NULL. + */ +void *mas_prev(struct ma_state *mas, unsigned long min) +{ + void *entry = NULL; + + if (mas_prev_setup(mas, min, &entry)) + return entry; + + return mas_prev_slot(mas, min, false); +} +EXPORT_SYMBOL_GPL(mas_prev); + +/** + * mas_prev_range() - Advance to the previous range + * @mas: The maple state + * @min: The minimum value to check. + * + * Sets @mas->index and @mas->last to the range. + * Must hold rcu_read_lock or the write lock. + * Will reset mas to ma_start if the node is ma_none. Will stop on not + * searchable nodes. + * + * Return: the previous value or %NULL. + */ +void *mas_prev_range(struct ma_state *mas, unsigned long min) +{ + void *entry = NULL; + + if (mas_prev_setup(mas, min, &entry)) + return entry; + + return mas_prev_slot(mas, min, true); +} +EXPORT_SYMBOL_GPL(mas_prev_range); + +/** + * mt_prev() - get the previous value in the maple tree + * @mt: The maple tree + * @index: The start index + * @min: The minimum index to check + * + * Takes RCU read lock internally to protect the search, which does not + * protect the returned pointer after dropping RCU read lock. + * See also: Documentation/core-api/maple_tree.rst + * + * Return: The entry before @index or %NULL if nothing is found. + */ +void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min) +{ + void *entry = NULL; + MA_STATE(mas, mt, index, index); + + rcu_read_lock(); + entry = mas_prev(&mas, min); + rcu_read_unlock(); + return entry; +} +EXPORT_SYMBOL_GPL(mt_prev); + +/** + * mas_pause() - Pause a mas_find/mas_for_each to drop the lock. + * @mas: The maple state to pause + * + * Some users need to pause a walk and drop the lock they're holding in + * order to yield to a higher priority thread or carry out an operation + * on an entry. Those users should call this function before they drop + * the lock. It resets the @mas to be suitable for the next iteration + * of the loop after the user has reacquired the lock. If most entries + * found during a walk require you to call mas_pause(), the mt_for_each() + * iterator may be more appropriate. + * + */ +void mas_pause(struct ma_state *mas) +{ + mas->status = ma_pause; + mas->node = NULL; +} +EXPORT_SYMBOL_GPL(mas_pause); + +/** + * mas_find_setup() - Internal function to set up mas_find*(). + * @mas: The maple state + * @max: The maximum index + * @entry: Pointer to the entry + * + * Returns: True if entry is the answer, false otherwise. + */ +__always_inline bool mas_find_setup(struct ma_state *mas, unsigned long max, void **entry) +{ + switch (mas->status) + { + case ma_active: + if (mas->last < max) + return false; + return true; + case ma_start: + break; + case ma_pause: + if (unlikely(mas->last >= max)) + return true; + + mas->index = ++mas->last; + mas->status = ma_start; + break; + case ma_none: + if (unlikely(mas->last >= max)) + return true; + + mas->index = mas->last; + mas->status = ma_start; + break; + case ma_underflow: + /* mas is pointing at entry before unable to go lower */ + if (unlikely(mas->index >= max)) + { + mas->status = ma_overflow; + return true; + } + + mas->status = ma_active; + *entry = mas_walk(mas); + if (*entry) + return true; + break; + case ma_overflow: + if (unlikely(mas->last >= max)) + return true; + + mas->status = ma_active; + *entry = mas_walk(mas); + if (*entry) + return true; + break; + case ma_root: + break; + case ma_error: + return true; + } + + if (mas_is_start(mas)) + { + /* First run or continue */ + if (mas->index > max) + return true; + + *entry = mas_walk(mas); + if (*entry) + return true; + } + + if (unlikely(mas_is_ptr(mas))) + goto ptr_out_of_range; + + if (unlikely(mas_is_none(mas))) + return true; + + if (mas->index == max) + return true; + + return false; + +ptr_out_of_range: + mas->status = ma_none; + mas->index = 1; + mas->last = ULONG_MAX; + return true; +} + +/** + * mas_find() - On the first call, find the entry at or after mas->index up to + * %max. Otherwise, find the entry after mas->index. + * @mas: The maple state + * @max: The maximum value to check. + * + * Must hold rcu_read_lock or the write lock. + * If an entry exists, last and index are updated accordingly. + * May set @mas->status to ma_overflow. + * + * Return: The entry or %NULL. + */ +void *mas_find(struct ma_state *mas, unsigned long max) +{ + void *entry = NULL; + + if (mas_find_setup(mas, max, &entry)) + return entry; + + /* Retries on dead nodes handled by mas_next_slot */ + entry = mas_next_slot(mas, max, false); + /* Ignore overflow */ + mas->status = ma_active; + return entry; +} +EXPORT_SYMBOL_GPL(mas_find); + +/** + * mas_find_range() - On the first call, find the entry at or after + * mas->index up to %max. Otherwise, advance to the next slot mas->index. + * @mas: The maple state + * @max: The maximum value to check. + * + * Must hold rcu_read_lock or the write lock. + * If an entry exists, last and index are updated accordingly. + * May set @mas->status to ma_overflow. + * + * Return: The entry or %NULL. + */ +void *mas_find_range(struct ma_state *mas, unsigned long max) +{ + void *entry = NULL; + + if (mas_find_setup(mas, max, &entry)) + return entry; + + /* Retries on dead nodes handled by mas_next_slot */ + return mas_next_slot(mas, max, true); +} +EXPORT_SYMBOL_GPL(mas_find_range); + +/** + * mas_find_rev_setup() - Internal function to set up mas_find_*_rev() + * @mas: The maple state + * @min: The minimum index + * @entry: Pointer to the entry + * + * Returns: True if entry is the answer, false otherwise. + */ +static bool mas_find_rev_setup(struct ma_state *mas, unsigned long min, void **entry) +{ + + switch (mas->status) + { + case ma_active: + goto active; + case ma_start: + break; + case ma_pause: + if (unlikely(mas->index <= min)) + { + mas->status = ma_underflow; + return true; + } + mas->last = --mas->index; + mas->status = ma_start; + break; + case ma_none: + if (mas->index <= min) + goto none; + + mas->last = mas->index; + mas->status = ma_start; + break; + case ma_overflow: /* user expects the mas to be one after where it is */ + if (unlikely(mas->index <= min)) + { + mas->status = ma_underflow; + return true; + } + + mas->status = ma_active; + break; + case ma_underflow: /* user expects the mas to be one before where it is */ + if (unlikely(mas->index <= min)) + return true; + + mas->status = ma_active; + break; + case ma_root: + break; + case ma_error: + return true; + } + + if (mas_is_start(mas)) + { + /* First run or continue */ + if (mas->index < min) + return true; + + *entry = mas_walk(mas); + if (*entry) + return true; + } + + if (unlikely(mas_is_ptr(mas))) + goto none; + + if (unlikely(mas_is_none(mas))) + { + /* + * Walked to the location, and there was nothing so the previous + * location is 0. + */ + mas->last = mas->index = 0; + mas->status = ma_root; + *entry = mas_root(mas); + return true; + } + +active: + if (mas->index < min) + return true; + + return false; + +none: + mas->status = ma_none; + return true; +} + +/** + * mas_find_rev: On the first call, find the first non-null entry at or below + * mas->index down to %min. Otherwise find the first non-null entry below + * mas->index down to %min. + * @mas: The maple state + * @min: The minimum value to check. + * + * Must hold rcu_read_lock or the write lock. + * If an entry exists, last and index are updated accordingly. + * May set @mas->status to ma_underflow. + * + * Return: The entry or %NULL. + */ +void *mas_find_rev(struct ma_state *mas, unsigned long min) +{ + void *entry = NULL; + + if (mas_find_rev_setup(mas, min, &entry)) + return entry; + + /* Retries on dead nodes handled by mas_prev_slot */ + return mas_prev_slot(mas, min, false); +} +EXPORT_SYMBOL_GPL(mas_find_rev); + +/** + * mas_find_range_rev: On the first call, find the first non-null entry at or + * below mas->index down to %min. Otherwise advance to the previous slot after + * mas->index down to %min. + * @mas: The maple state + * @min: The minimum value to check. + * + * Must hold rcu_read_lock or the write lock. + * If an entry exists, last and index are updated accordingly. + * May set @mas->status to ma_underflow. + * + * Return: The entry or %NULL. + */ +void *mas_find_range_rev(struct ma_state *mas, unsigned long min) +{ + void *entry = NULL; + + if (mas_find_rev_setup(mas, min, &entry)) + return entry; + + /* Retries on dead nodes handled by mas_prev_slot */ + return mas_prev_slot(mas, min, true); +} +EXPORT_SYMBOL_GPL(mas_find_range_rev); + +/** + * mas_erase() - Find the range in which index resides and erase the entire + * range. + * @mas: The maple state + * + * Must hold the write lock. + * Searches for @mas->index, sets @mas->index and @mas->last to the range and + * erases that range. + * + * Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated. + */ +void *mas_erase(struct ma_state *mas) +{ + void *entry; + MA_WR_STATE(wr_mas, mas, NULL); + + if (!mas_is_active(mas) || !mas_is_start(mas)) + mas->status = ma_start; + + /* Retry unnecessary when holding the write lock. */ + entry = mas_state_walk(mas); + if (!entry) + return NULL; + +write_retry: + /* Must reset to ensure spanning writes of last slot are detected */ + mas_reset(mas); + mas_wr_store_setup(&wr_mas); + mas_wr_store_entry(&wr_mas); + if (mas_nomem(mas, GFP_KERNEL)) + goto write_retry; + + return entry; +} +EXPORT_SYMBOL_GPL(mas_erase); + +/** + * mas_nomem() - Check if there was an error allocating and do the allocation + * if necessary If there are allocations, then free them. + * @mas: The maple state + * @gfp: The GFP_FLAGS to use for allocations + * Return: true on allocation, false otherwise. + */ +bool mas_nomem(struct ma_state *mas, gfp_t gfp) __must_hold(mas->tree->ma_lock) +{ + if (likely(mas->node != MA_ERROR(-ENOMEM))) + { + mas_destroy(mas); + return false; + } + + if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) + { + mtree_unlock(mas->tree); + mas_alloc_nodes(mas, gfp); + mtree_lock(mas->tree); + } + else + { + mas_alloc_nodes(mas, gfp); + } + + if (!mas_allocated(mas)) + return false; + + mas->status = ma_start; + return true; +} + +void maple_tree_init(void) +{ + maple_node_cache = kmem_cache_create("maple_node", sizeof(struct maple_node), + sizeof(struct maple_node), SLAB_PANIC, NULL); +} + +/** + * mtree_load() - Load a value stored in a maple tree + * @mt: The maple tree + * @index: The index to load + * + * Return: the entry or %NULL + */ +void *mtree_load(struct maple_tree *mt, unsigned long index) +{ + MA_STATE(mas, mt, index, index); + void *entry; + + trace_ma_read(__func__, &mas); + rcu_read_lock(); +retry: + entry = mas_start(&mas); + if (unlikely(mas_is_none(&mas))) + goto unlock; + + if (unlikely(mas_is_ptr(&mas))) + { + if (index) + entry = NULL; + + goto unlock; + } + + entry = mtree_lookup_walk(&mas); + if (!entry && unlikely(mas_is_start(&mas))) + goto retry; +unlock: + rcu_read_unlock(); + if (xa_is_zero(entry)) + return NULL; + + return entry; +} +EXPORT_SYMBOL(mtree_load); + +/** + * mtree_store_range() - Store an entry at a given range. + * @mt: The maple tree + * @index: The start of the range + * @last: The end of the range + * @entry: The entry to store + * @gfp: The GFP_FLAGS to use for allocations + * + * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not + * be allocated. + */ +int mtree_store_range(struct maple_tree *mt, unsigned long index, unsigned long last, void *entry, + gfp_t gfp) +{ + MA_STATE(mas, mt, index, last); + MA_WR_STATE(wr_mas, &mas, entry); + + trace_ma_write(__func__, &mas, 0, entry); + if (WARN_ON_ONCE(xa_is_advanced(entry))) + return -EINVAL; + + if (index > last) + return -EINVAL; + + mtree_lock(mt); +retry: + mas_wr_store_entry(&wr_mas); + if (mas_nomem(&mas, gfp)) + goto retry; + + mtree_unlock(mt); + if (mas_is_err(&mas)) + return xa_err(mas.node); + + return 0; +} +EXPORT_SYMBOL(mtree_store_range); + +/** + * mtree_store() - Store an entry at a given index. + * @mt: The maple tree + * @index: The index to store the value + * @entry: The entry to store + * @gfp: The GFP_FLAGS to use for allocations + * + * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not + * be allocated. + */ +int mtree_store(struct maple_tree *mt, unsigned long index, void *entry, gfp_t gfp) +{ + return mtree_store_range(mt, index, index, entry, gfp); +} +EXPORT_SYMBOL(mtree_store); + +/** + * mtree_insert_range() - Insert an entry at a given range if there is no value. + * @mt: The maple tree + * @first: The start of the range + * @last: The end of the range + * @entry: The entry to store + * @gfp: The GFP_FLAGS to use for allocations. + * + * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid + * request, -ENOMEM if memory could not be allocated. + */ +int mtree_insert_range(struct maple_tree *mt, unsigned long first, unsigned long last, void *entry, + gfp_t gfp) +{ + MA_STATE(ms, mt, first, last); + + if (WARN_ON_ONCE(xa_is_advanced(entry))) + return -EINVAL; + + if (first > last) + return -EINVAL; + + mtree_lock(mt); +retry: + mas_insert(&ms, entry); + if (mas_nomem(&ms, gfp)) + goto retry; + + mtree_unlock(mt); + if (mas_is_err(&ms)) + return xa_err(ms.node); + + return 0; +} +EXPORT_SYMBOL(mtree_insert_range); + +/** + * mtree_insert() - Insert an entry at a given index if there is no value. + * @mt: The maple tree + * @index : The index to store the value + * @entry: The entry to store + * @gfp: The GFP_FLAGS to use for allocations. + * + * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid + * request, -ENOMEM if memory could not be allocated. + */ +int mtree_insert(struct maple_tree *mt, unsigned long index, void *entry, gfp_t gfp) +{ + return mtree_insert_range(mt, index, index, entry, gfp); +} +EXPORT_SYMBOL(mtree_insert); + +int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp, void *entry, unsigned long size, + unsigned long min, unsigned long max, gfp_t gfp) +{ + int ret = 0; + + MA_STATE(mas, mt, 0, 0); + if (!mt_is_alloc(mt)) + return -EINVAL; + + if (WARN_ON_ONCE(mt_is_reserved(entry))) + return -EINVAL; + + mtree_lock(mt); +retry: + ret = mas_empty_area(&mas, min, max, size); + if (ret) + goto unlock; + + mas_insert(&mas, entry); + /* + * mas_nomem() may release the lock, causing the allocated area + * to be unavailable, so try to allocate a free area again. + */ + if (mas_nomem(&mas, gfp)) + goto retry; + + if (mas_is_err(&mas)) + ret = xa_err(mas.node); + else + *startp = mas.index; + +unlock: + mtree_unlock(mt); + return ret; +} +EXPORT_SYMBOL(mtree_alloc_range); + +/** + * mtree_alloc_cyclic() - Find somewhere to store this entry in the tree. + * @mt: The maple tree. + * @startp: Pointer to ID. + * @range_lo: Lower bound of range to search. + * @range_hi: Upper bound of range to search. + * @entry: The entry to store. + * @next: Pointer to next ID to allocate. + * @gfp: The GFP_FLAGS to use for allocations. + * + * Finds an empty entry in @mt after @next, stores the new index into + * the @id pointer, stores the entry at that index, then updates @next. + * + * @mt must be initialized with the MT_FLAGS_ALLOC_RANGE flag. + * + * Context: Any context. Takes and releases the mt.lock. May sleep if + * the @gfp flags permit. + * + * Return: 0 if the allocation succeeded without wrapping, 1 if the + * allocation succeeded after wrapping, -ENOMEM if memory could not be + * allocated, -EINVAL if @mt cannot be used, or -EBUSY if there are no + * free entries. + */ +int mtree_alloc_cyclic(struct maple_tree *mt, unsigned long *startp, void *entry, + unsigned long range_lo, unsigned long range_hi, unsigned long *next, + gfp_t gfp) +{ + int ret; + + MA_STATE(mas, mt, 0, 0); + + if (!mt_is_alloc(mt)) + return -EINVAL; + if (WARN_ON_ONCE(mt_is_reserved(entry))) + return -EINVAL; + mtree_lock(mt); + ret = mas_alloc_cyclic(&mas, startp, entry, range_lo, range_hi, next, gfp); + mtree_unlock(mt); + return ret; +} +EXPORT_SYMBOL(mtree_alloc_cyclic); + +int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp, void *entry, + unsigned long size, unsigned long min, unsigned long max, gfp_t gfp) +{ + int ret = 0; + + MA_STATE(mas, mt, 0, 0); + if (!mt_is_alloc(mt)) + return -EINVAL; + + if (WARN_ON_ONCE(mt_is_reserved(entry))) + return -EINVAL; + + mtree_lock(mt); +retry: + ret = mas_empty_area_rev(&mas, min, max, size); + if (ret) + goto unlock; + + mas_insert(&mas, entry); + /* + * mas_nomem() may release the lock, causing the allocated area + * to be unavailable, so try to allocate a free area again. + */ + if (mas_nomem(&mas, gfp)) + goto retry; + + if (mas_is_err(&mas)) + ret = xa_err(mas.node); + else + *startp = mas.index; + +unlock: + mtree_unlock(mt); + return ret; +} +EXPORT_SYMBOL(mtree_alloc_rrange); + +/** + * mtree_erase() - Find an index and erase the entire range. + * @mt: The maple tree + * @index: The index to erase + * + * Erasing is the same as a walk to an entry then a store of a NULL to that + * ENTIRE range. In fact, it is implemented as such using the advanced API. + * + * Return: The entry stored at the @index or %NULL + */ +void *mtree_erase(struct maple_tree *mt, unsigned long index) +{ + void *entry = NULL; + + MA_STATE(mas, mt, index, index); + trace_ma_op(__func__, &mas); + + mtree_lock(mt); + entry = mas_erase(&mas); + mtree_unlock(mt); + + return entry; +} +EXPORT_SYMBOL(mtree_erase); + +/* + * mas_dup_free() - Free an incomplete duplication of a tree. + * @mas: The maple state of a incomplete tree. + * + * The parameter @mas->node passed in indicates that the allocation failed on + * this node. This function frees all nodes starting from @mas->node in the + * reverse order of mas_dup_build(). There is no need to hold the source tree + * lock at this time. + */ +static void mas_dup_free(struct ma_state *mas) +{ + struct maple_node *node; + enum maple_type type; + void __rcu **slots; + unsigned char count, i; + + /* Maybe the first node allocation failed. */ + if (mas_is_none(mas)) + return; + + while (!mte_is_root(mas->node)) + { + mas_ascend(mas); + if (mas->offset) + { + mas->offset--; + do + { + mas_descend(mas); + mas->offset = mas_data_end(mas); + } while (!mte_is_leaf(mas->node)); + + mas_ascend(mas); + } + + node = mte_to_node(mas->node); + type = mte_node_type(mas->node); + slots = ma_slots(node, type); + count = mas_data_end(mas) + 1; + for (i = 0; i < count; i++) + ((unsigned long *) slots)[i] &= ~MAPLE_NODE_MASK; + mt_free_bulk(count, slots); + } + + node = mte_to_node(mas->node); + mt_free_one(node); +} + +/* + * mas_copy_node() - Copy a maple node and replace the parent. + * @mas: The maple state of source tree. + * @new_mas: The maple state of new tree. + * @parent: The parent of the new node. + * + * Copy @mas->node to @new_mas->node, set @parent to be the parent of + * @new_mas->node. If memory allocation fails, @mas is set to -ENOMEM. + */ +static inline void mas_copy_node(struct ma_state *mas, struct ma_state *new_mas, + struct maple_pnode *parent) +{ + struct maple_node *node = mte_to_node(mas->node); + struct maple_node *new_node = mte_to_node(new_mas->node); + unsigned long val; + + /* Copy the node completely. */ + memcpy(new_node, node, sizeof(struct maple_node)); + /* Update the parent node pointer. */ + val = (unsigned long) node->parent & MAPLE_NODE_MASK; + new_node->parent = ma_parent_ptr(val | (unsigned long) parent); +} + +/* + * mas_dup_alloc() - Allocate child nodes for a maple node. + * @mas: The maple state of source tree. + * @new_mas: The maple state of new tree. + * @gfp: The GFP_FLAGS to use for allocations. + * + * This function allocates child nodes for @new_mas->node during the duplication + * process. If memory allocation fails, @mas is set to -ENOMEM. + */ +static inline void mas_dup_alloc(struct ma_state *mas, struct ma_state *new_mas, gfp_t gfp) +{ + struct maple_node *node = mte_to_node(mas->node); + struct maple_node *new_node = mte_to_node(new_mas->node); + enum maple_type type; + unsigned char request, count, i; + void __rcu **slots; + void __rcu **new_slots; + unsigned long val; + + /* Allocate memory for child nodes. */ + type = mte_node_type(mas->node); + new_slots = ma_slots(new_node, type); + request = mas_data_end(mas) + 1; + count = mt_alloc_bulk(gfp, request, (void **) new_slots); + if (unlikely(count < request)) + { + memset(new_slots, 0, request * sizeof(void *)); + mas_set_err(mas, -ENOMEM); + return; + } + + /* Restore node type information in slots. */ + slots = ma_slots(node, type); + for (i = 0; i < count; i++) + { + val = (unsigned long) mt_slot_locked(mas->tree, slots, i); + val &= MAPLE_NODE_MASK; + ((unsigned long *) new_slots)[i] |= val; + } +} + +/* + * mas_dup_build() - Build a new maple tree from a source tree + * @mas: The maple state of source tree, need to be in MAS_START state. + * @new_mas: The maple state of new tree, need to be in MAS_START state. + * @gfp: The GFP_FLAGS to use for allocations. + * + * This function builds a new tree in DFS preorder. If the memory allocation + * fails, the error code -ENOMEM will be set in @mas, and @new_mas points to the + * last node. mas_dup_free() will free the incomplete duplication of a tree. + * + * Note that the attributes of the two trees need to be exactly the same, and the + * new tree needs to be empty, otherwise -EINVAL will be set in @mas. + */ +static inline void mas_dup_build(struct ma_state *mas, struct ma_state *new_mas, gfp_t gfp) +{ + struct maple_node *node; + struct maple_pnode *parent = NULL; + struct maple_enode *root; + enum maple_type type; + + if (unlikely(mt_attr(mas->tree) != mt_attr(new_mas->tree)) || + unlikely(!mtree_empty(new_mas->tree))) + { + mas_set_err(mas, -EINVAL); + return; + } + + root = mas_start(mas); + if (mas_is_ptr(mas) || mas_is_none(mas)) + goto set_new_tree; + + node = mt_alloc_one(gfp); + if (!node) + { + new_mas->status = ma_none; + mas_set_err(mas, -ENOMEM); + return; + } + + type = mte_node_type(mas->node); + root = mt_mk_node(node, type); + new_mas->node = root; + new_mas->min = 0; + new_mas->max = ULONG_MAX; + root = mte_mk_root(root); + while (1) + { + mas_copy_node(mas, new_mas, parent); + if (!mte_is_leaf(mas->node)) + { + /* Only allocate child nodes for non-leaf nodes. */ + mas_dup_alloc(mas, new_mas, gfp); + if (unlikely(mas_is_err(mas))) + return; + } + else + { + /* + * This is the last leaf node and duplication is + * completed. + */ + if (mas->max == ULONG_MAX) + goto done; + + /* This is not the last leaf node and needs to go up. */ + do + { + mas_ascend(mas); + mas_ascend(new_mas); + } while (mas->offset == mas_data_end(mas)); + + /* Move to the next subtree. */ + mas->offset++; + new_mas->offset++; + } + + mas_descend(mas); + parent = ma_parent_ptr(mte_to_node(new_mas->node)); + mas_descend(new_mas); + mas->offset = 0; + new_mas->offset = 0; + } +done: + /* Specially handle the parent of the root node. */ + mte_to_node(root)->parent = ma_parent_ptr(mas_tree_parent(new_mas)); +set_new_tree: + /* Make them the same height */ + new_mas->tree->ma_flags = mas->tree->ma_flags; + rcu_assign_pointer(new_mas->tree->ma_root, root); +} + +/** + * __mt_dup(): Duplicate an entire maple tree + * @mt: The source maple tree + * @new: The new maple tree + * @gfp: The GFP_FLAGS to use for allocations + * + * This function duplicates a maple tree in Depth-First Search (DFS) pre-order + * traversal. It uses memcpy() to copy nodes in the source tree and allocate + * new child nodes in non-leaf nodes. The new node is exactly the same as the + * source node except for all the addresses stored in it. It will be faster than + * traversing all elements in the source tree and inserting them one by one into + * the new tree. + * The user needs to ensure that the attributes of the source tree and the new + * tree are the same, and the new tree needs to be an empty tree, otherwise + * -EINVAL will be returned. + * Note that the user needs to manually lock the source tree and the new tree. + * + * Return: 0 on success, -ENOMEM if memory could not be allocated, -EINVAL If + * the attributes of the two trees are different or the new tree is not an empty + * tree. + */ +int __mt_dup(struct maple_tree *mt, struct maple_tree *new, gfp_t gfp) +{ + int ret = 0; + MA_STATE(mas, mt, 0, 0); + MA_STATE(new_mas, new, 0, 0); + + mas_dup_build(&mas, &new_mas, gfp); + if (unlikely(mas_is_err(&mas))) + { + ret = xa_err(mas.node); + if (ret == -ENOMEM) + mas_dup_free(&new_mas); + } + + return ret; +} +EXPORT_SYMBOL(__mt_dup); + +#define SINGLE_DEPTH_NESTING 1 +#define spin_lock_nested(s, c) spin_lock(((void) (c), s)) +/** + * mtree_dup(): Duplicate an entire maple tree + * @mt: The source maple tree + * @new: The new maple tree + * @gfp: The GFP_FLAGS to use for allocations + * + * This function duplicates a maple tree in Depth-First Search (DFS) pre-order + * traversal. It uses memcpy() to copy nodes in the source tree and allocate + * new child nodes in non-leaf nodes. The new node is exactly the same as the + * source node except for all the addresses stored in it. It will be faster than + * traversing all elements in the source tree and inserting them one by one into + * the new tree. + * The user needs to ensure that the attributes of the source tree and the new + * tree are the same, and the new tree needs to be an empty tree, otherwise + * -EINVAL will be returned. + * + * Return: 0 on success, -ENOMEM if memory could not be allocated, -EINVAL If + * the attributes of the two trees are different or the new tree is not an empty + * tree. + */ +int mtree_dup(struct maple_tree *mt, struct maple_tree *new, gfp_t gfp) +{ + int ret = 0; + MA_STATE(mas, mt, 0, 0); + MA_STATE(new_mas, new, 0, 0); + + mas_lock(&new_mas); + mas_lock_nested(&mas, SINGLE_DEPTH_NESTING); + mas_dup_build(&mas, &new_mas, gfp); + mas_unlock(&mas); + if (unlikely(mas_is_err(&mas))) + { + ret = xa_err(mas.node); + if (ret == -ENOMEM) + mas_dup_free(&new_mas); + } + + mas_unlock(&new_mas); + return ret; +} +EXPORT_SYMBOL(mtree_dup); + +/** + * __mt_destroy() - Walk and free all nodes of a locked maple tree. + * @mt: The maple tree + * + * Note: Does not handle locking. + */ +void __mt_destroy(struct maple_tree *mt) +{ + void *root = mt_root_locked(mt); + + rcu_assign_pointer(mt->ma_root, NULL); + if (xa_is_node(root)) + mte_destroy_walk(root, mt); + + mt->ma_flags = mt_attr(mt); +} +EXPORT_SYMBOL_GPL(__mt_destroy); + +/** + * mtree_destroy() - Destroy a maple tree + * @mt: The maple tree + * + * Frees all resources used by the tree. Handles locking. + */ +void mtree_destroy(struct maple_tree *mt) +{ + mtree_lock(mt); + __mt_destroy(mt); + mtree_unlock(mt); +} +EXPORT_SYMBOL(mtree_destroy); + +/** + * mt_find() - Search from the start up until an entry is found. + * @mt: The maple tree + * @index: Pointer which contains the start location of the search + * @max: The maximum value of the search range + * + * Takes RCU read lock internally to protect the search, which does not + * protect the returned pointer after dropping RCU read lock. + * See also: Documentation/core-api/maple_tree.rst + * + * In case that an entry is found @index is updated to point to the next + * possible entry independent whether the found entry is occupying a + * single index or a range if indices. + * + * Return: The entry at or after the @index or %NULL + */ +void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max) +{ + MA_STATE(mas, mt, *index, *index); + void *entry; +#ifdef CONFIG_DEBUG_MAPLE_TREE + unsigned long copy = *index; +#endif + + trace_ma_read(__func__, &mas); + + if ((*index) > max) + return NULL; + + rcu_read_lock(); +retry: + entry = mas_state_walk(&mas); + if (mas_is_start(&mas)) + goto retry; + + if (unlikely(xa_is_zero(entry))) + entry = NULL; + + if (entry) + goto unlock; + + while (mas_is_active(&mas) && (mas.last < max)) + { + entry = mas_next_entry(&mas, max); + if (likely(entry && !xa_is_zero(entry))) + break; + } + + if (unlikely(xa_is_zero(entry))) + entry = NULL; +unlock: + rcu_read_unlock(); + if (likely(entry)) + { + *index = mas.last + 1; +#ifdef CONFIG_DEBUG_MAPLE_TREE + if (MT_WARN_ON(mt, (*index) && ((*index) <= copy))) + pr_err("index not increased! %lx <= %lx\n", *index, copy); +#endif + } + + return entry; +} +EXPORT_SYMBOL(mt_find); + +/** + * mt_find_after() - Search from the start up until an entry is found. + * @mt: The maple tree + * @index: Pointer which contains the start location of the search + * @max: The maximum value to check + * + * Same as mt_find() except that it checks @index for 0 before + * searching. If @index == 0, the search is aborted. This covers a wrap + * around of @index to 0 in an iterator loop. + * + * Return: The entry at or after the @index or %NULL + */ +void *mt_find_after(struct maple_tree *mt, unsigned long *index, unsigned long max) +{ + if (!(*index)) + return NULL; + + return mt_find(mt, index, max); +} +EXPORT_SYMBOL(mt_find_after); + +#ifdef CONFIG_DEBUG_MAPLE_TREE +atomic_t maple_tree_tests_run; +EXPORT_SYMBOL_GPL(maple_tree_tests_run); +atomic_t maple_tree_tests_passed; +EXPORT_SYMBOL_GPL(maple_tree_tests_passed); + +#ifndef __KERNEL__ +extern void kmem_cache_set_non_kernel(struct kmem_cache *, unsigned int); +void mt_set_non_kernel(unsigned int val) +{ + kmem_cache_set_non_kernel(maple_node_cache, val); +} + +extern unsigned long kmem_cache_get_alloc(struct kmem_cache *); +unsigned long mt_get_alloc_size(void) +{ + return kmem_cache_get_alloc(maple_node_cache); +} + +extern void kmem_cache_zero_nr_tallocated(struct kmem_cache *); +void mt_zero_nr_tallocated(void) +{ + kmem_cache_zero_nr_tallocated(maple_node_cache); +} + +extern unsigned int kmem_cache_nr_tallocated(struct kmem_cache *); +unsigned int mt_nr_tallocated(void) +{ + return kmem_cache_nr_tallocated(maple_node_cache); +} + +extern unsigned int kmem_cache_nr_allocated(struct kmem_cache *); +unsigned int mt_nr_allocated(void) +{ + return kmem_cache_nr_allocated(maple_node_cache); +} + +void mt_cache_shrink(void) +{ +} +#else +/* + * mt_cache_shrink() - For testing, don't use this. + * + * Certain testcases can trigger an OOM when combined with other memory + * debugging configuration options. This function is used to reduce the + * possibility of an out of memory even due to kmem_cache objects remaining + * around for longer than usual. + */ +void mt_cache_shrink(void) +{ + kmem_cache_shrink(maple_node_cache); +} +EXPORT_SYMBOL_GPL(mt_cache_shrink); + +#endif /* not defined __KERNEL__ */ +/* + * mas_get_slot() - Get the entry in the maple state node stored at @offset. + * @mas: The maple state + * @offset: The offset into the slot array to fetch. + * + * Return: The entry stored at @offset. + */ +static inline struct maple_enode *mas_get_slot(struct ma_state *mas, unsigned char offset) +{ + return mas_slot(mas, ma_slots(mas_mn(mas), mte_node_type(mas->node)), offset); +} + +/* Depth first search, post-order */ +static void mas_dfs_postorder(struct ma_state *mas, unsigned long max) +{ + + struct maple_enode *p, *mn = mas->node; + unsigned long p_min, p_max; + + mas_next_node(mas, mas_mn(mas), max); + if (!mas_is_overflow(mas)) + return; + + if (mte_is_root(mn)) + return; + + mas->node = mn; + mas_ascend(mas); + do + { + p = mas->node; + p_min = mas->min; + p_max = mas->max; + mas_prev_node(mas, 0); + } while (!mas_is_underflow(mas)); + + mas->node = p; + mas->max = p_max; + mas->min = p_min; +} + +/* Tree validations */ +static void mt_dump_node(const struct maple_tree *mt, void *entry, unsigned long min, + unsigned long max, unsigned int depth, enum mt_dump_format format); +static void mt_dump_range(unsigned long min, unsigned long max, unsigned int depth, + enum mt_dump_format format) +{ + static const char spaces[] = " "; + + switch (format) + { + case mt_dump_hex: + if (min == max) + pr_info("%.*s%lx: ", depth * 2, spaces, min); + else + pr_info("%.*s%lx-%lx: ", depth * 2, spaces, min, max); + break; + case mt_dump_dec: + if (min == max) + pr_info("%.*s%lu: ", depth * 2, spaces, min); + else + pr_info("%.*s%lu-%lu: ", depth * 2, spaces, min, max); + } +} + +static void mt_dump_entry(void *entry, unsigned long min, unsigned long max, unsigned int depth, + enum mt_dump_format format) +{ + mt_dump_range(min, max, depth, format); + + if (xa_is_value(entry)) + pr_cont("value %ld (0x%lx) [%p]\n", xa_to_value(entry), xa_to_value(entry), entry); + else if (xa_is_zero(entry)) + pr_cont("zero (%ld)\n", xa_to_internal(entry)); + else if (mt_is_reserved(entry)) + pr_cont("UNKNOWN ENTRY (%p)\n", entry); + else + pr_cont("%p\n", entry); +} + +static void mt_dump_range64(const struct maple_tree *mt, void *entry, unsigned long min, + unsigned long max, unsigned int depth, enum mt_dump_format format) +{ + struct maple_range_64 *node = &mte_to_node(entry)->mr64; + bool leaf = mte_is_leaf(entry); + unsigned long first = min; + int i; + + pr_cont(" contents: "); + for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) + { + switch (format) + { + case mt_dump_hex: + pr_cont("%p %lX ", node->slot[i], node->pivot[i]); + break; + case mt_dump_dec: + pr_cont("%p %lu ", node->slot[i], node->pivot[i]); + } + } + pr_cont("%p\n", node->slot[i]); + for (i = 0; i < MAPLE_RANGE64_SLOTS; i++) + { + unsigned long last = max; + + if (i < (MAPLE_RANGE64_SLOTS - 1)) + last = node->pivot[i]; + else if (!node->slot[i] && max != mt_node_max(entry)) + break; + if (last == 0 && i > 0) + break; + if (leaf) + mt_dump_entry(mt_slot(mt, node->slot, i), first, last, depth + 1, format); + else if (node->slot[i]) + mt_dump_node(mt, mt_slot(mt, node->slot, i), first, last, depth + 1, format); + + if (last == max) + break; + if (last > max) + { + switch (format) + { + case mt_dump_hex: + pr_err("node %p last (%lx) > max (%lx) at pivot %d!\n", node, last, max, i); + break; + case mt_dump_dec: + pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n", node, last, max, i); + } + } + first = last + 1; + } +} + +static void mt_dump_arange64(const struct maple_tree *mt, void *entry, unsigned long min, + unsigned long max, unsigned int depth, enum mt_dump_format format) +{ + struct maple_arange_64 *node = &mte_to_node(entry)->ma64; + bool leaf = mte_is_leaf(entry); + unsigned long first = min; + int i; + + pr_cont(" contents: "); + for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) + { + switch (format) + { + case mt_dump_hex: + pr_cont("%lx ", node->gap[i]); + break; + case mt_dump_dec: + pr_cont("%lu ", node->gap[i]); + } + } + pr_cont("| %02X %02X| ", node->meta.end, node->meta.gap); + for (i = 0; i < MAPLE_ARANGE64_SLOTS - 1; i++) + { + switch (format) + { + case mt_dump_hex: + pr_cont("%p %lX ", node->slot[i], node->pivot[i]); + break; + case mt_dump_dec: + pr_cont("%p %lu ", node->slot[i], node->pivot[i]); + } + } + pr_cont("%p\n", node->slot[i]); + for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) + { + unsigned long last = max; + + if (i < (MAPLE_ARANGE64_SLOTS - 1)) + last = node->pivot[i]; + else if (!node->slot[i]) + break; + if (last == 0 && i > 0) + break; + if (leaf) + mt_dump_entry(mt_slot(mt, node->slot, i), first, last, depth + 1, format); + else if (node->slot[i]) + mt_dump_node(mt, mt_slot(mt, node->slot, i), first, last, depth + 1, format); + + if (last == max) + break; + if (last > max) + { + pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n", node, last, max, i); + break; + } + first = last + 1; + } +} + +static void mt_dump_node(const struct maple_tree *mt, void *entry, unsigned long min, + unsigned long max, unsigned int depth, enum mt_dump_format format) +{ + struct maple_node *node = mte_to_node(entry); + unsigned int type = mte_node_type(entry); + unsigned int i; + + mt_dump_range(min, max, depth, format); + + pr_cont("node %p depth %d type %d parent %p", node, depth, type, node ? node->parent : NULL); + switch (type) + { + case maple_dense: + pr_cont("\n"); + for (i = 0; i < MAPLE_NODE_SLOTS; i++) + { + if (min + i > max) + pr_cont("OUT OF RANGE: "); + mt_dump_entry(mt_slot(mt, node->slot, i), min + i, min + i, depth, format); + } + break; + case maple_leaf_64: + case maple_range_64: + mt_dump_range64(mt, entry, min, max, depth, format); + break; + case maple_arange_64: + mt_dump_arange64(mt, entry, min, max, depth, format); + break; + + default: + pr_cont(" UNKNOWN TYPE\n"); + } +} + +void mt_dump(const struct maple_tree *mt, enum mt_dump_format format) +{ + void *entry = rcu_dereference_check(mt->ma_root, mt_locked(mt)); + + pr_info("maple_tree(%p) flags %X, height %u root %p\n", mt, mt->ma_flags, mt_height(mt), entry); + if (!xa_is_node(entry)) + mt_dump_entry(entry, 0, 0, 0, format); + else if (entry) + mt_dump_node(mt, entry, 0, mt_node_max(entry), 0, format); +} +EXPORT_SYMBOL_GPL(mt_dump); + +/* + * Calculate the maximum gap in a node and check if that's what is reported in + * the parent (unless root). + */ +static void mas_validate_gaps(struct ma_state *mas) +{ + struct maple_enode *mte = mas->node; + struct maple_node *p_mn, *node = mte_to_node(mte); + enum maple_type mt = mte_node_type(mas->node); + unsigned long gap = 0, max_gap = 0; + unsigned long p_end, p_start = mas->min; + unsigned char p_slot, offset; + unsigned long *gaps = NULL; + unsigned long *pivots = ma_pivots(node, mt); + unsigned int i; + + if (ma_is_dense(mt)) + { + for (i = 0; i < mt_slot_count(mte); i++) + { + if (mas_get_slot(mas, i)) + { + if (gap > max_gap) + max_gap = gap; + gap = 0; + continue; + } + gap++; + } + goto counted; + } + + gaps = ma_gaps(node, mt); + for (i = 0; i < mt_slot_count(mte); i++) + { + p_end = mas_safe_pivot(mas, pivots, i, mt); + + if (!gaps) + { + if (!mas_get_slot(mas, i)) + gap = p_end - p_start + 1; + } + else + { + void *entry = mas_get_slot(mas, i); + + gap = gaps[i]; + MT_BUG_ON(mas->tree, !entry); + + if (gap > p_end - p_start + 1) + { + pr_err("%p[%u] %lu >= %lu - %lu + 1 (%lu)\n", mas_mn(mas), i, gap, p_end, p_start, + p_end - p_start + 1); + MT_BUG_ON(mas->tree, gap > p_end - p_start + 1); + } + } + + if (gap > max_gap) + max_gap = gap; + + p_start = p_end + 1; + if (p_end >= mas->max) + break; + } + +counted: + if (mt == maple_arange_64) + { + MT_BUG_ON(mas->tree, !gaps); + offset = ma_meta_gap(node); + if (offset > i) + { + pr_err("gap offset %p[%u] is invalid\n", node, offset); + MT_BUG_ON(mas->tree, 1); + } + + if (gaps[offset] != max_gap) + { + pr_err("gap %p[%u] is not the largest gap %lu\n", node, offset, max_gap); + MT_BUG_ON(mas->tree, 1); + } + + for (i++; i < mt_slot_count(mte); i++) + { + if (gaps[i] != 0) + { + pr_err("gap %p[%u] beyond node limit != 0\n", node, i); + MT_BUG_ON(mas->tree, 1); + } + } + } + + if (mte_is_root(mte)) + return; + + p_slot = mte_parent_slot(mas->node); + p_mn = mte_parent(mte); + MT_BUG_ON(mas->tree, max_gap > mas->max); + if (ma_gaps(p_mn, mas_parent_type(mas, mte))[p_slot] != max_gap) + { + pr_err("gap %p[%u] != %lu\n", p_mn, p_slot, max_gap); + mt_dump(mas->tree, mt_dump_hex); + MT_BUG_ON(mas->tree, 1); + } +} + +static void mas_validate_parent_slot(struct ma_state *mas) +{ + struct maple_node *parent; + struct maple_enode *node; + enum maple_type p_type; + unsigned char p_slot; + void __rcu **slots; + int i; + + if (mte_is_root(mas->node)) + return; + + p_slot = mte_parent_slot(mas->node); + p_type = mas_parent_type(mas, mas->node); + parent = mte_parent(mas->node); + slots = ma_slots(parent, p_type); + MT_BUG_ON(mas->tree, mas_mn(mas) == parent); + + /* Check prev/next parent slot for duplicate node entry */ + + for (i = 0; i < mt_slots[p_type]; i++) + { + node = mas_slot(mas, slots, i); + if (i == p_slot) + { + if (node != mas->node) + pr_err("parent %p[%u] does not have %p\n", parent, i, mas_mn(mas)); + MT_BUG_ON(mas->tree, node != mas->node); + } + else if (node == mas->node) + { + pr_err("Invalid child %p at parent %p[%u] p_slot %u\n", mas_mn(mas), parent, i, p_slot); + MT_BUG_ON(mas->tree, node == mas->node); + } + } +} + +static void mas_validate_child_slot(struct ma_state *mas) +{ + enum maple_type type = mte_node_type(mas->node); + void __rcu **slots = ma_slots(mte_to_node(mas->node), type); + unsigned long *pivots = ma_pivots(mte_to_node(mas->node), type); + struct maple_enode *child; + unsigned char i; + + if (mte_is_leaf(mas->node)) + return; + + for (i = 0; i < mt_slots[type]; i++) + { + child = mas_slot(mas, slots, i); + + if (!child) + { + pr_err("Non-leaf node lacks child at %p[%u]\n", mas_mn(mas), i); + MT_BUG_ON(mas->tree, 1); + } + + if (mte_parent_slot(child) != i) + { + pr_err("Slot error at %p[%u]: child %p has pslot %u\n", mas_mn(mas), i, + mte_to_node(child), mte_parent_slot(child)); + MT_BUG_ON(mas->tree, 1); + } + + if (mte_parent(child) != mte_to_node(mas->node)) + { + pr_err("child %p has parent %p not %p\n", mte_to_node(child), mte_parent(child), + mte_to_node(mas->node)); + MT_BUG_ON(mas->tree, 1); + } + + if (i < mt_pivots[type] && pivots[i] == mas->max) + break; + } +} + +/* + * Validate all pivots are within mas->min and mas->max, check metadata ends + * where the maximum ends and ensure there is no slots or pivots set outside of + * the end of the data. + */ +static void mas_validate_limits(struct ma_state *mas) +{ + int i; + unsigned long prev_piv = 0; + enum maple_type type = mte_node_type(mas->node); + void __rcu **slots = ma_slots(mte_to_node(mas->node), type); + unsigned long *pivots = ma_pivots(mas_mn(mas), type); + + for (i = 0; i < mt_slots[type]; i++) + { + unsigned long piv; + + piv = mas_safe_pivot(mas, pivots, i, type); + + if (!piv && (i != 0)) + { + pr_err("Missing node limit pivot at %p[%u]", mas_mn(mas), i); + MAS_WARN_ON(mas, 1); + } + + if (prev_piv > piv) + { + pr_err("%p[%u] piv %lu < prev_piv %lu\n", mas_mn(mas), i, piv, prev_piv); + MAS_WARN_ON(mas, piv < prev_piv); + } + + if (piv < mas->min) + { + pr_err("%p[%u] %lu < %lu\n", mas_mn(mas), i, piv, mas->min); + MAS_WARN_ON(mas, piv < mas->min); + } + if (piv > mas->max) + { + pr_err("%p[%u] %lu > %lu\n", mas_mn(mas), i, piv, mas->max); + MAS_WARN_ON(mas, piv > mas->max); + } + prev_piv = piv; + if (piv == mas->max) + break; + } + + if (mas_data_end(mas) != i) + { + pr_err("node%p: data_end %u != the last slot offset %u\n", mas_mn(mas), mas_data_end(mas), + i); + MT_BUG_ON(mas->tree, 1); + } + + for (i += 1; i < mt_slots[type]; i++) + { + void *entry = mas_slot(mas, slots, i); + + if (entry && (i != mt_slots[type] - 1)) + { + pr_err("%p[%u] should not have entry %p\n", mas_mn(mas), i, entry); + MT_BUG_ON(mas->tree, entry != NULL); + } + + if (i < mt_pivots[type]) + { + unsigned long piv = pivots[i]; + + if (!piv) + continue; + + pr_err("%p[%u] should not have piv %lu\n", mas_mn(mas), i, piv); + MAS_WARN_ON(mas, i < mt_pivots[type] - 1); + } + } +} + +static void mt_validate_nulls(struct maple_tree *mt) +{ + void *entry, *last = (void *) 1; + unsigned char offset = 0; + void __rcu **slots; + MA_STATE(mas, mt, 0, 0); + + mas_start(&mas); + if (mas_is_none(&mas) || (mas_is_ptr(&mas))) + return; + + while (!mte_is_leaf(mas.node)) + mas_descend(&mas); + + slots = ma_slots(mte_to_node(mas.node), mte_node_type(mas.node)); + do + { + entry = mas_slot(&mas, slots, offset); + if (!last && !entry) + { + pr_err("Sequential nulls end at %p[%u]\n", mas_mn(&mas), offset); + } + MT_BUG_ON(mt, !last && !entry); + last = entry; + if (offset == mas_data_end(&mas)) + { + mas_next_node(&mas, mas_mn(&mas), ULONG_MAX); + if (mas_is_overflow(&mas)) + return; + offset = 0; + slots = ma_slots(mte_to_node(mas.node), mte_node_type(mas.node)); + } + else + { + offset++; + } + + } while (!mas_is_overflow(&mas)); +} + +/* + * validate a maple tree by checking: + * 1. The limits (pivots are within mas->min to mas->max) + * 2. The gap is correctly set in the parents + */ +void mt_validate(struct maple_tree *mt) +{ + unsigned char end; + + MA_STATE(mas, mt, 0, 0); + rcu_read_lock(); + mas_start(&mas); + if (!mas_is_active(&mas)) + goto done; + + while (!mte_is_leaf(mas.node)) + mas_descend(&mas); + + while (!mas_is_overflow(&mas)) + { + MAS_WARN_ON(&mas, mte_dead_node(mas.node)); + end = mas_data_end(&mas); + if (MAS_WARN_ON(&mas, (end < mt_min_slot_count(mas.node)) && (mas.max != ULONG_MAX))) + { + pr_err("Invalid size %u of %p\n", end, mas_mn(&mas)); + } + + mas_validate_parent_slot(&mas); + mas_validate_limits(&mas); + mas_validate_child_slot(&mas); + if (mt_is_alloc(mt)) + mas_validate_gaps(&mas); + mas_dfs_postorder(&mas, ULONG_MAX); + } + mt_validate_nulls(mt); +done: + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(mt_validate); + +void mas_dump(const struct ma_state *mas) +{ + pr_err("MAS: tree=%p enode=%p ", mas->tree, mas->node); + switch (mas->status) + { + case ma_active: + pr_err("(ma_active)"); + break; + case ma_none: + pr_err("(ma_none)"); + break; + case ma_root: + pr_err("(ma_root)"); + break; + case ma_start: + pr_err("(ma_start) "); + break; + case ma_pause: + pr_err("(ma_pause) "); + break; + case ma_overflow: + pr_err("(ma_overflow) "); + break; + case ma_underflow: + pr_err("(ma_underflow) "); + break; + case ma_error: + pr_err("(ma_error) "); + break; + } + + pr_err("[%u/%u] index=%lx last=%lx\n", mas->offset, mas->end, mas->index, mas->last); + pr_err(" min=%lx max=%lx alloc=%p, depth=%u, flags=%x\n", mas->min, mas->max, mas->alloc, + mas->depth, mas->mas_flags); + if (mas->index > mas->last) + pr_err("Check index & last\n"); +} +EXPORT_SYMBOL_GPL(mas_dump); + +void mas_wr_dump(const struct ma_wr_state *wr_mas) +{ + pr_err("WR_MAS: node=%p r_min=%lx r_max=%lx\n", wr_mas->node, wr_mas->r_min, wr_mas->r_max); + pr_err(" type=%u off_end=%u, node_end=%u, end_piv=%lx\n", wr_mas->type, + wr_mas->offset_end, wr_mas->mas->end, wr_mas->end_piv); +} +EXPORT_SYMBOL_GPL(mas_wr_dump); + +#endif /* CONFIG_DEBUG_MAPLE_TREE */ diff --git a/kernel/kernel/mm/Makefile b/kernel/kernel/mm/Makefile index 29820cc41..a1ce49608 100644 --- a/kernel/kernel/mm/Makefile +++ b/kernel/kernel/mm/Makefile @@ -1,5 +1,7 @@ -mm-y:= bootmem.o page.o pagealloc.o vm_object.o vm.o vmalloc.o reclaim.o amap.o anon.o mincore.o page_lru.o +mm-y:= bootmem.o page.o pagealloc.o vm_object.o vm.o vmalloc.o reclaim.o anon.o mincore.o page_lru.o mm-$(CONFIG_KUNIT)+= vm_tests.o +mm-$(CONFIG_X86)+= memory.o +mm-$(CONFIG_RISCV)+= memory.o ifeq ($(CONFIG_KASAN), y) obj-y_NOKASAN+= kernel/mm/asan/asan.o kernel/mm/asan/quarantine.o diff --git a/kernel/kernel/mm/amap.cpp b/kernel/kernel/mm/amap.cpp deleted file mode 100644 index 717da2307..000000000 --- a/kernel/kernel/mm/amap.cpp +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Copyright (c) 2023 Pedro Falcato - * This file is part of Onyx, and is released under the terms of the GPLv2 License - * check LICENSE at the root directory for more information - * - * SPDX-License-Identifier: GPL-2.0-only - */ -#include -#include -#include -#include - -#include - -__always_inline void amap_init(struct amap *amap) -{ - new (amap) struct amap; - spinlock_init(&amap->am_lock); - amap->am_refc = 1; - amap->am_size = 0; -} - -/** - * @brief Allocate a new anonymous memory map - * - * @param size Size of the amap - * @return struct amap* - */ -struct amap *amap_alloc(size_t size) -{ - struct amap *amap = (struct amap *) kmalloc(sizeof(struct amap), GFP_KERNEL); - if (!amap) - return nullptr; - amap_init(amap); - amap->am_size = size; - - return amap; -} - -void amap_free(struct amap *amap) -{ - DCHECK(amap->am_refc == 0); - auto cursor = radix_tree::cursor::from_index(&amap->am_map); - - while (!cursor.is_end()) - { - auto page = (struct page *) cursor.get(); - DCHECK_PAGE(page_flag_set(page, PAGE_FLAG_ANON), page); - if (page_mapcount(page) == 0) - { - DCHECK_PAGE(page->ref == 1, page); - } - - free_page(page); - cursor.advance(); - } - - amap->~amap(); - kfree(amap); -} - -/** - * @brief Create a copy of an amap - * - * @param amap amap to copy - * @return New amap, or NULL in case of OOM - */ -static struct amap *amap_copy(struct amap *amap) -{ - struct amap *namap = amap_alloc(amap->am_size); - if (!namap) - return nullptr; - auto ex = amap->am_map.copy( - [](unsigned long entry, void *ctx) -> unsigned long { - struct page *page = (struct page *) entry; - page_ref(page); - return entry; - }, - nullptr); - if (ex.has_error()) - { - amap_free(namap); - return nullptr; - } - - namap->am_map = cul::move(ex.value()); - return namap; -} - -/** - * @brief Add a page to an amap - * - * @param amap Amap to add to - * @param page Page to add - * @param region Region to which the amap belongs - * @param pgoff Page offset (in pfn, shifted right by PAGE_SHIFT) - * @param nocopy Don't copy if we find an old page - * @return 0 on success, negative error codes - */ -int amap_add(struct amap *amap, struct page *page, struct vm_area_struct *region, size_t pgoff, - bool nocopy) -{ - if (amap->am_refc > 1) [[unlikely]] - { - /* Note: We do not need a lock here, no one can touch this amap while am_refc > 1 */ - struct amap *namap = amap_copy(amap); - if (!namap) - return -ENOMEM; - - amap_unref(amap); - amap = namap; - region->vm_amap = amap; - } - - DCHECK_PAGE(page_flag_set(page, PAGE_FLAG_ANON), page); - - scoped_lock g{amap->am_lock}; - auto old = amap->am_map.xchg(pgoff, (unsigned long) page); - if (radix_err(old)) - return old; - - if (old != 0) - { - struct page *oldp = (struct page *) old; - if (!nocopy) - copy_page_to_page(page_to_phys(page), page_to_phys(oldp)); - page_unref(oldp); - } - - return 0; -} - -/** - * @brief Get a page from the amap - * - * @param amap Amap to lookup from - * @param pgoff Page offset (in pfn, shifted right by PAGE_SHIFT) - * @return struct page in the amap, or NULL - */ -struct page *amap_get(struct amap *amap, size_t pgoff) -{ - scoped_lock g{amap->am_lock}; - auto ex = amap->am_map.get(pgoff); - if (ex.has_error()) - return nullptr; - struct page *page = (struct page *) ex.value(); - DCHECK_PAGE(page_flag_set(page, PAGE_FLAG_ANON), page); - page_ref(page); - return page; -} - -/** - * @brief Split an amap into two - * - * @param amap Original amap - * @param region Region to which the amap belongs - * @param pgoff Page offset for the new amap - * @return New amap, or NULL - */ -struct amap *amap_split(struct amap *amap, struct vm_area_struct *region, size_t pgoff) -{ - if (amap->am_refc > 1) [[unlikely]] - { - /* Note: We do not need a lock here, no one can touch this amap while am_refc > 1 */ - struct amap *namap = amap_copy(amap); - if (!namap) - return nullptr; - - amap_unref(amap); - amap = namap; - region->vm_amap = amap; - } - - struct amap *namap = amap_alloc(0); - if (!namap) - return nullptr; - - /* Since we are the exclusive owners of this amap, and callers hold the mm address space lock, - * we do not need to lock. This saves us from GFP_ATOMIC. - */ - auto cursor = radix_tree::cursor::from_range(&amap->am_map, pgoff); - while (!cursor.is_end()) - { - /* Move pages from one amap to the other by storing to new and store(0). store(0) is done - * later in case of OOM. - */ - unsigned long curr_idx = cursor.current_idx() - pgoff; - if (namap->am_map.store(curr_idx, cursor.get()) < 0) - goto err; - cursor.advance(); - } - - cursor = radix_tree::cursor::from_range(&amap->am_map, pgoff); - while (!cursor.is_end()) - { - /* store(0) is done now.. - */ - cursor.store(0); - cursor.advance(); - } - - return namap; -err: - /* Open-coded amap_free, to avoid page_ref/page_unref shenanigans. */ - amap->~amap(); - kfree(amap); - return nullptr; -} - -/** - * @brief Truncate an amap - * - * @param amap Amap - * @param region Region to which the amap belongs - * @param new_pgsize New size, in pages - * @return 0 on success, negative error codes - */ -int amap_truncate(struct amap *amap, struct vm_area_struct *region, size_t new_pgsize) -{ - if (amap->am_refc > 1) [[unlikely]] - { - /* Note: We do not need a lock here, no one can touch this amap while am_refc > 1 */ - struct amap *namap = amap_copy(amap); - if (!namap) - return -ENOMEM; - - amap_unref(amap); - amap = namap; - region->vm_amap = amap; - } - - auto cursor = radix_tree::cursor::from_range(&amap->am_map, new_pgsize); - - while (!cursor.is_end()) - { - struct page *page = (struct page *) cursor.get(); - DCHECK_PAGE(page_flag_set(page, PAGE_FLAG_ANON), page); - page_unref(page); - cursor.store(0); - cursor.advance(); - } - - return 0; -} - -/** - * @brief Punch a hole through an amap - * - * @param amap Amap - * @param region Region to which the amap belongs - * @param first_pg First pfn of the hole - * @param end_pg End of the hole - * @return 0 on success, negative error codes - */ -int amap_punch_hole(struct amap *amap, struct vm_area_struct *region, size_t first_pg, - size_t end_pg) -{ - if (amap->am_refc > 1) [[unlikely]] - { - /* Note: We do not need a lock here, no one can touch this amap while am_refc > 1 */ - struct amap *namap = amap_copy(amap); - if (!namap) - return -ENOMEM; - - amap_unref(amap); - amap = namap; - region->vm_amap = amap; - } - - auto cursor = radix_tree::cursor::from_range(&amap->am_map, first_pg, end_pg); - - while (!cursor.is_end()) - { - struct page *page = (struct page *) cursor.get(); - DCHECK_PAGE(page_flag_set(page, PAGE_FLAG_ANON), page); - page_unref(page); - cursor.store(0); - cursor.advance(); - } - - return 0; -} diff --git a/kernel/kernel/mm/anon.cpp b/kernel/kernel/mm/anon.cpp index e4cab21e6..473968df4 100644 --- a/kernel/kernel/mm/anon.cpp +++ b/kernel/kernel/mm/anon.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Pedro Falcato + * Copyright (c) 2023 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -7,7 +7,6 @@ */ #include -#include #include #include #include @@ -20,8 +19,7 @@ int vm_anon_fault(struct vm_pf_context *ctx) { struct vm_area_struct *region = ctx->entry; struct fault_info *info = ctx->info; - struct page *page = nullptr; - unsigned long pgoff = (ctx->vpage - region->vm_start) >> PAGE_SHIFT; + struct page *page = nullptr, *oldp = nullptr; bool needs_invd = false; /* Permission checks have already been handled before .fault() */ @@ -36,27 +34,39 @@ int vm_anon_fault(struct vm_pf_context *ctx) } else { - /* Lazily allocate the vm_amap struct */ - if (!region->vm_amap) + bool copy_old = false; + if (ctx->mapping_info & PAGE_PRESENT) { - region->vm_amap = amap_alloc(vma_pages(region) << PAGE_SHIFT); - if (!region->vm_amap) - goto enomem; + oldp = phys_to_page(MAPPING_INFO_PADDR(ctx->mapping_info)); + DCHECK(info->write && !(ctx->mapping_info & PAGE_WRITABLE)); + if (oldp != vm_get_zero_page()) + copy_old = true; + needs_invd = true; + + if (copy_old && page_flag_set(oldp, PAGE_FLAG_ANON) && page_mapcount(oldp) == 1) + { + /* If this is an anon page *and* mapcount = 1, avoid allocating a new page. Since + * mapcount = 1 (AND *ANON*), no one else can grab a ref. */ + /* TODO: We might be able to explore this - we may avoid the TLB shootdown and just + * change prots, but it would require significant code refactoring as-is. */ + /* TODO: checking mapcount = 1 probably isn't this easy once we get swapping, + * because refs may come and go. Will we need the page lock? */ + page = oldp; + page_ref(page); + goto map; + } + + /* oldp's mapcount will be decremented in vm_map_page */ } - /* Allocate a brand-new zero-filled page */ - page = alloc_page(GFP_KERNEL); + /* Allocate a brand-new (possibly zero-filled) page */ + page = alloc_page((copy_old ? PAGE_ALLOC_NO_ZERO : 0) | GFP_KERNEL); if (!page) goto enomem; page_set_anon(page); - if (amap_add(region->vm_amap, page, region, pgoff, false) < 0) - { - free_page(page); - goto enomem; - } - - needs_invd = ctx->mapping_info & PAGE_PRESENT; + if (copy_old) + copy_page_to_page(page_to_phys(page), page_to_phys(oldp)); goto map; } @@ -67,6 +77,9 @@ int vm_anon_fault(struct vm_pf_context *ctx) if (needs_invd) vm_invalidate_range(ctx->vpage, 1); + /* The mapcount holds the only reference we need for anon pages... */ + if (info->write) + page_unref(page); return 0; enomem: info->error_info = VM_SIGSEGV; diff --git a/kernel/kernel/mm/memory.c b/kernel/kernel/mm/memory.c new file mode 100644 index 000000000..12a9c86ab --- /dev/null +++ b/kernel/kernel/mm/memory.c @@ -0,0 +1,1026 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the GPLv2 License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only + */ +#include +#include + +#include "pgtable.h" + +static p4d_t *__p4d_alloc(struct mm_address_space *mm) +{ + /* TODO: Deal with locking properly... */ + struct page *page = alloc_page(GFP_ATOMIC); + if (!page) + return NULL; + increment_vm_stat(mm, page_tables_size, PAGE_SIZE); + return page_to_phys(page); +} + +p4d_t *p4d_alloc(pgd_t *pgd, unsigned long addr, struct mm_address_space *mm) +{ + DCHECK(pgd_none(*pgd)); + pgprotval_t perms = addr < VM_USER_ADDR_LIMIT ? USER_PGTBL : KERNEL_PGTBL; + p4d_t *p4d = __p4d_alloc(mm); + if (!p4d) + return NULL; + set_pgd(pgd, pgd_mkpgd((unsigned long) p4d, __pgprot(perms))); + return (p4d_t *) __tovirt(p4d) + p4d_index(addr); +} + +static p4d_t *p4d_get_or_alloc(pgd_t *pgd, unsigned long addr, struct mm_address_space *mm) +{ + if (likely(!pgd_none(*pgd))) + return p4d_offset(pgd, addr); + return p4d_alloc(pgd, addr, mm); +} + +static pud_t *__pud_alloc(struct mm_address_space *mm) +{ + /* TODO: Deal with locking properly... */ + struct page *page = alloc_page(GFP_ATOMIC); + if (!page) + return NULL; + increment_vm_stat(mm, page_tables_size, PAGE_SIZE); + return page_to_phys(page); +} + +pud_t *pud_alloc(p4d_t *p4d, unsigned long addr, struct mm_address_space *mm) +{ + DCHECK(p4d_none(*p4d)); + pgprotval_t perms = addr < VM_USER_ADDR_LIMIT ? USER_PGTBL : KERNEL_PGTBL; + pud_t *pud = __pud_alloc(mm); + if (!pud) + return NULL; + set_p4d(p4d, p4d_mkp4d((unsigned long) pud, __pgprot(perms))); + return (pud_t *) __tovirt(pud) + pud_index(addr); +} + +static pud_t *pud_get_or_alloc(p4d_t *p4d, unsigned long addr, struct mm_address_space *mm) +{ + if (likely(!p4d_none(*p4d))) + return pud_offset(p4d, addr); + return pud_alloc(p4d, addr, mm); +} + +static pmd_t *__pmd_alloc(struct mm_address_space *mm) +{ + /* TODO: Deal with locking properly... */ + struct page *page = alloc_page(GFP_ATOMIC); + if (!page) + return NULL; + increment_vm_stat(mm, page_tables_size, PAGE_SIZE); + return page_to_phys(page); +} + +pmd_t *pmd_alloc(pud_t *pud, unsigned long addr, struct mm_address_space *mm) +{ + DCHECK(pud_none(*pud)); + pgprotval_t perms = addr < VM_USER_ADDR_LIMIT ? USER_PGTBL : KERNEL_PGTBL; + pmd_t *pmd = __pmd_alloc(mm); + if (!pmd) + return NULL; + set_pud(pud, pud_mkpud((unsigned long) pmd, __pgprot(perms))); + return (pmd_t *) __tovirt(pmd) + pmd_index(addr); +} + +static pmd_t *pmd_get_or_alloc(pud_t *pud, unsigned long addr, struct mm_address_space *mm) +{ + if (likely(!pud_none(*pud))) + return pmd_offset(pud, addr); + return pmd_alloc(pud, addr, mm); +} + +static pte_t *__pte_alloc(struct mm_address_space *mm) +{ + /* TODO: Deal with locking properly... */ + struct page *page = alloc_page(GFP_ATOMIC); + if (!page) + return NULL; + increment_vm_stat(mm, page_tables_size, PAGE_SIZE); + return page_to_phys(page); +} + +pte_t *pte_alloc(pmd_t *pmd, unsigned long addr, struct mm_address_space *mm) +{ + DCHECK(pmd_none(*pmd)); + pgprotval_t perms = addr < VM_USER_ADDR_LIMIT ? USER_PGTBL : KERNEL_PGTBL; + pte_t *pte = __pte_alloc(mm); + if (!pte) + return NULL; + set_pmd(pmd, pmd_mkpmd((unsigned long) pte, __pgprot(perms))); + return (pte_t *) __tovirt(pte) + pte_index(addr); +} + +static pte_t *pte_get_or_alloc(pmd_t *pmd, unsigned long addr, struct mm_address_space *mm) +{ + if (likely(!pmd_none(*pmd))) + return pte_offset(pmd, addr); + return pte_alloc(pmd, addr, mm); +} + +/** + * @brief Directly maps a page into the paging tables. + * + * @param as The target address space. + * @param virt The virtual address. + * @param phys The physical address of the page. + * @param prot Desired protection flags. + * @param vma VMA for this mapping (optional) + * @return NULL if out of memory, else virt. + */ +void *vm_map_page(struct mm_address_space *as, uint64_t virt, uint64_t phys, uint64_t prot, + struct vm_area_struct *vma) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + bool ispfnmap = vma_is_pfnmap(vma); + bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()); + + spin_lock(&as->page_table_lock); + + pgd = pgd_offset(as, virt); + + p4d = p4d_get_or_alloc(pgd, virt, as); + if (unlikely(!p4d)) + goto oom; + + pud = pud_get_or_alloc(p4d, virt, as); + if (unlikely(!pud)) + goto oom; + + pmd = pmd_get_or_alloc(pud, virt, as); + if (unlikely(!pmd)) + goto oom; + + pte = pte_get_or_alloc(pmd, virt, as); + if (unlikely(!pte)) + goto oom; + + pte_t oldpte = *pte; + pgprot_t pgprot = calc_pgprot(phys, prot); + set_pte(pte, pte_mkpte(phys, pgprot)); + + if (pte_none(oldpte)) + increment_vm_stat(as, resident_set_size, PAGE_SIZE); + + if (likely(!ispfnmap)) + { + if (unlikely(!pte_none(oldpte) && !pte_special(oldpte))) + { + /* If old was a thing, decrement the mapcount */ + struct page *oldp = phys_to_page(pte_addr(oldpte)); + page_sub_mapcount(oldp); + } + + struct page *newp = phys_to_page(phys); + if (likely(!special_mapping)) + page_add_mapcount(newp); + } + + spin_unlock(&as->page_table_lock); + return (void *) virt; +oom: + spin_unlock(&as->page_table_lock); + return NULL; +} + +static pte_t *pte_get_from_addr(struct mm_address_space *mm, unsigned long addr) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pgd = pgd_offset(mm, addr); + if (pgd_none(*pgd)) + return NULL; + + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) + return NULL; + + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) + return NULL; + DCHECK(!pud_huge(*pud)); + + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) + return NULL; + DCHECK(!pmd_huge(*pmd)); + + return pte_offset(pmd, addr); +} + +unsigned int mmu_get_clear_referenced(struct mm_address_space *mm, void *addr, struct page *page) +{ + int ret = 0; + pte_t *ptep; + spin_lock(&mm->page_table_lock); + + ptep = pte_get_from_addr(mm, (unsigned long) addr); + if (!ptep) + goto out; + + pte_t old = *ptep; + pte_t new_pte; + do + { + if (!pte_present(old) || !pte_accessed(old)) + goto out; + if (pte_addr(old) != (unsigned long) page_to_phys(page)) + goto out; + new_pte = pte_mkyoung(*ptep); + } while (!pte_cmpxchg(ptep, &old, new_pte)); + + ret = 1; + /* Architectural note: We don't need to flush the TLB. Flushing the TLB is required by x86 if we + * want the A bit to be set again, but we can just wait for an unrelated TLB flush (e.g context + * switch) to do the job for us. A TLB shootdown is too much overhead for this purpose. */ +out: + spin_unlock(&mm->page_table_lock); + return ret; +} + +static unsigned long p4d_to_mapping_info(p4d_t p4d) +{ + unsigned long ret = p4d_addr(p4d) | PAGE_PRESENT | PAGE_HUGE; + if (p4d_write(p4d)) + ret |= PAGE_WRITABLE; + if (p4d_exec(p4d)) + ret |= PAGE_EXECUTABLE; + if (p4d_global(p4d)) + ret |= PAGE_GLOBAL; + if (p4d_dirty(p4d)) + ret |= PAGE_DIRTY; + if (p4d_accessed(p4d)) + ret |= PAGE_ACCESSED; + if (p4d_user(p4d)) + ret |= PAGE_USER; + return ret; +} + +static unsigned long pud_to_mapping_info(pud_t pud) +{ + unsigned long ret = pud_addr(pud) | PAGE_PRESENT | PAGE_HUGE; + if (pud_write(pud)) + ret |= PAGE_WRITABLE; + if (pud_exec(pud)) + ret |= PAGE_EXECUTABLE; + if (pud_global(pud)) + ret |= PAGE_GLOBAL; + if (pud_dirty(pud)) + ret |= PAGE_DIRTY; + if (pud_accessed(pud)) + ret |= PAGE_ACCESSED; + if (pud_user(pud)) + ret |= PAGE_USER; + return ret; +} + +static unsigned long pmd_to_mapping_info(pmd_t pmd) +{ + unsigned long ret = pmd_addr(pmd) | PAGE_PRESENT | PAGE_HUGE; + if (pmd_write(pmd)) + ret |= PAGE_WRITABLE; + if (pmd_exec(pmd)) + ret |= PAGE_EXECUTABLE; + if (pmd_global(pmd)) + ret |= PAGE_GLOBAL; + if (pmd_dirty(pmd)) + ret |= PAGE_DIRTY; + if (pmd_accessed(pmd)) + ret |= PAGE_ACCESSED; + if (pmd_user(pmd)) + ret |= PAGE_USER; + return ret; +} + +static unsigned long pte_to_mapping_info(pte_t pte) +{ + unsigned long ret = pte_addr(pte) | PAGE_PRESENT; + if (pte_write(pte)) + ret |= PAGE_WRITABLE; + if (pte_exec(pte)) + ret |= PAGE_EXECUTABLE; + if (pte_global(pte)) + ret |= PAGE_GLOBAL; + if (pte_dirty(pte)) + ret |= PAGE_DIRTY; + if (pte_accessed(pte)) + ret |= PAGE_ACCESSED; + if (pte_user(pte)) + ret |= PAGE_USER; + return ret; +} + +unsigned long __get_mapping_info(void *addr, struct mm_address_space *mm) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + unsigned long virt = (unsigned long) addr; + pgd = pgd_offset(mm, virt); + if (!pgd_present(*pgd)) + return PAGE_NOT_PRESENT; + + p4d = p4d_offset(pgd, virt); + if (!p4d_present(*p4d)) + return PAGE_NOT_PRESENT; + if (p4d_huge(*p4d)) + return p4d_to_mapping_info(*p4d); + + pud = pud_offset(p4d, virt); + if (!pud_present(*pud)) + return PAGE_NOT_PRESENT; + if (pud_huge(*pud)) + return pud_to_mapping_info(*pud); + + pmd = pmd_offset(pud, virt); + if (!pmd_present(*pmd)) + return PAGE_NOT_PRESENT; + if (pmd_huge(*pmd)) + return pmd_to_mapping_info(*pmd); + + pte = pte_offset(pmd, virt); + if (!pte_present(*pte)) + return PAGE_NOT_PRESENT; + + return pte_to_mapping_info(*pte); +} + +#define MAX_PENDING_PAGEN 32 + +struct tlbi_tracker +{ + /* Somewhat primitive, but will do for the time being... */ + unsigned long start, end; + struct page *pending_pages[MAX_PENDING_PAGEN]; + unsigned int used_pending_pages; + bool active; +}; + +static void tlbi_tracker_init(struct tlbi_tracker *tlbi) +{ + tlbi->start = tlbi->end = 0; + tlbi->active = false; + tlbi->used_pending_pages = 0; +} + +struct unmap_info +{ + struct tlbi_tracker tlbi; + struct mm_address_space *mm; + struct vm_area_struct *vma; + int kernel : 1, full : 1, freepgtables : 1; +}; + +enum unmap_result +{ + UNMAP_OK = 0, + /* We *know* this page table is clear. Note that a page table can be clear even if this isn't + * set. + */ + UNMAP_FREE_PGTABLE = 1, + /* We *know* this page is not clear. Don't bother checking. */ + UNMAP_DONT_FREE = (1 << 1) +}; + +static void tlbi_end_batch(struct tlbi_tracker *tlbi) +{ + vm_invalidate_range(tlbi->start, (tlbi->end - tlbi->start) >> PAGE_SHIFT); + for (unsigned int i = 0; i < tlbi->used_pending_pages; i++) + page_unref(tlbi->pending_pages[i]); + tlbi->active = false; + tlbi->used_pending_pages = 0; +} + +static void tlbi_add_defer_free(struct tlbi_tracker *tlbi, struct page *page) +{ + DCHECK(tlbi->used_pending_pages < MAX_PENDING_PAGEN); + tlbi->pending_pages[tlbi->used_pending_pages++] = page; +} + +static void tlbi_remove_page(struct tlbi_tracker *tlbi, unsigned long addr, struct page *page) +{ +retry: + if (!tlbi->active) + { + tlbi->start = addr; + tlbi->end = addr + PAGE_SIZE; + tlbi->active = true; + return; + } + + /* TODO: Measure this heuristic. We need a solid, realistic benchmark that allows us to measure + * the cost of flushing too much TLB */ + /* If the new page is too far away (say, a PMD of distance), flush this batch and start anew. If + * we have a page to queue, and the defer queue is empty, flush the batch and start anew. */ + if ((long) (tlbi->start - addr) >= (long) PMD_SIZE || + (long) (addr - tlbi->end) >= (long) PMD_SIZE || + (page && tlbi->used_pending_pages == MAX_PENDING_PAGEN)) + { + tlbi_end_batch(tlbi); + goto retry; + } + + if (addr < tlbi->start) + tlbi->start = addr; + else if (addr >= tlbi->end) + tlbi->end = addr + PAGE_SIZE; + if (page) + tlbi_add_defer_free(tlbi, page); +} + +static bool tlbi_defer_page_queue_full(struct tlbi_tracker *tlbi) +{ + return tlbi->used_pending_pages == MAX_PENDING_PAGEN; +} + +static bool tlbi_covers(struct tlbi_tracker *tlbi, unsigned long start, unsigned long end) +{ + return start <= tlbi->end && tlbi->start <= end; +} + +static bool tlbi_active(struct tlbi_tracker *tlbi) +{ + return tlbi->active; +} + +/* x86 implementation of PTE removal. Intel SDM says (4.10.4.2 Recommended Invalidation) we must + * shootdown all mappings with translations under this paging structure. However, if no paging + * structure existed, we must call invlpg at least once. */ +static void x86_tlbi_remove_entry(struct mm_address_space *mm, struct tlbi_tracker *tlbi, + unsigned long pgtbl_phys, unsigned long addr, unsigned long end, + unsigned long entry_size) +{ + struct page *page = phys_to_page(pgtbl_phys); + + /* We'll pick the last address in the page table, because we're likely to go forward ("upwards") + * when doing TLB operations. But only if we don't yet cover this PMD in the existing + * invalidation. */ + if (!tlbi_covers(tlbi, addr & -entry_size, end) || tlbi_defer_page_queue_full(tlbi)) + tlbi_remove_page(tlbi, end - PAGE_SIZE, page); + else + tlbi_add_defer_free(tlbi, page); + decrement_vm_stat(mm, page_tables_size, PAGE_SIZE); +} + +static void tlbi_remove_pte(struct mm_address_space *mm, struct tlbi_tracker *tlbi, pte_t *pte, + unsigned long addr) +{ + x86_tlbi_remove_entry(mm, tlbi, (unsigned long) pte - PHYS_BASE, addr, pmd_addr_end(addr), + PMD_SIZE); +} + +static void tlbi_remove_pmd(struct mm_address_space *mm, struct tlbi_tracker *tlbi, pmd_t *pmd, + unsigned long addr) +{ + x86_tlbi_remove_entry(mm, tlbi, (unsigned long) pmd - PHYS_BASE, addr, pud_addr_end(addr), + PUD_SIZE); +} + +static void tlbi_remove_pud(struct mm_address_space *mm, struct tlbi_tracker *tlbi, pud_t *pud, + unsigned long addr) +{ + x86_tlbi_remove_entry(mm, tlbi, (unsigned long) pud - PHYS_BASE, addr, p4d_addr_end(addr), + P4D_SIZE); +} + +static void tlbi_remove_p4d(struct mm_address_space *mm, struct tlbi_tracker *tlbi, p4d_t *p4d, + unsigned long addr) +{ + x86_tlbi_remove_entry(mm, tlbi, (unsigned long) p4d - PHYS_BASE, addr, pgd_addr_end(addr), + PGD_SIZE); +} + +static void tlbi_update_page_prots(struct tlbi_tracker *tlbi, unsigned long addr, pte_t old, + pte_t new) +{ + /* TODO: We can take the spurious faults on permission upgrade, *if* the PFN is the same (if + * not, we risk exposing stale data to userspace). */ + tlbi_remove_page(tlbi, addr, NULL); +} + +static enum unmap_result pte_unmap_range(struct unmap_info *uinfo, pte_t *pte, unsigned long start, + unsigned long end) +{ + unsigned long next_start; + int clear = 0; + for (; start < end; pte++, start = next_start, clear++) + { + next_start = min(pte_addr_end(start), end); + pte_t old = *pte; + if (pte_none(old)) + continue; + + if (!uinfo->kernel && !pte_special(old)) + { + struct page *page = phys_to_page(pte_addr(old)); + page_sub_mapcount(page); + } + + decrement_vm_stat(uinfo->mm, resident_set_size, PAGE_SIZE); + set_pte(pte, __pte(0)); + tlbi_remove_page(&uinfo->tlbi, start, NULL); + } + + /* If we *know* the page table is clear, tell it to the caller so we skip expensive checks */ + if (clear == PTRS_PER_PTE) + return UNMAP_FREE_PGTABLE; + + return UNMAP_OK; +} + +static int pmd_free_pte(struct unmap_info *uinfo, pmd_t *pmd, unsigned long addr, int flags) +{ + pte_t *pte = (pte_t *) __tovirt(pmd_addr(*pmd)); + if (!(flags & UNMAP_FREE_PGTABLE)) + { + /* Check if the page table is clear */ + for (int i = 0; i < PTRS_PER_PTE; i++) + { + if (!pte_none(*(pte + i))) + return 0; + } + } + + set_pmd(pmd, __pmd(0)); + tlbi_remove_pte(uinfo->mm, &uinfo->tlbi, pte, addr); + return 1; +} + +static int pud_free_pmd(struct unmap_info *uinfo, pud_t *pud, unsigned long addr, int flags) +{ + pmd_t *pmd = (pmd_t *) __tovirt(pud_addr(*pud)); + if (!(flags & UNMAP_FREE_PGTABLE)) + { + /* Check if the page table is clear */ + for (int i = 0; i < PTRS_PER_PMD; i++) + { + if (!pmd_none(*(pmd + i))) + return 0; + } + } + + set_pud(pud, __pud(0)); + tlbi_remove_pmd(uinfo->mm, &uinfo->tlbi, pmd, addr); + return 1; +} + +static int p4d_free_pud(struct unmap_info *uinfo, p4d_t *p4d, unsigned long addr, int flags) +{ + pud_t *pud = (pud_t *) __tovirt(p4d_addr(*p4d)); + if (!(flags & UNMAP_FREE_PGTABLE)) + { + /* Check if the page table is clear */ + for (int i = 0; i < PTRS_PER_PUD; i++) + { + if (!pud_none(*(pud + i))) + return 0; + } + } + + set_p4d(p4d, __p4d(0)); + tlbi_remove_pud(uinfo->mm, &uinfo->tlbi, pud, addr); + return 1; +} + +static int pgd_free_p4d(struct unmap_info *uinfo, pgd_t *pgd, unsigned long addr, int flags) +{ + p4d_t *p4d = (p4d_t *) __tovirt(pgd_addr(*pgd)); + if (!(flags & UNMAP_FREE_PGTABLE)) + { + /* Check if the page table is clear */ + for (int i = 0; i < PTRS_PER_P4D; i++) + { + if (!p4d_none(*(p4d + i))) + return 0; + } + } + + set_pgd(pgd, __pgd(0)); + tlbi_remove_p4d(uinfo->mm, &uinfo->tlbi, p4d, addr); + return 1; +} + +static enum unmap_result pmd_unmap_range(struct unmap_info *uinfo, pmd_t *pmd, unsigned long start, + unsigned long end) +{ + unsigned long next_start; + int clear = 0; + enum unmap_result ret = UNMAP_OK; + for (; start < end; pmd++, start = next_start) + { + next_start = min(pmd_addr_end(start), end); + if (pmd_none(*pmd)) + { + clear++; + continue; + } + /* TODO: Huge page unmapping and splitting not supported yet... */ + DCHECK(!pmd_huge(*pmd)); + enum unmap_result res = pte_unmap_range(uinfo, pte_offset(pmd, start), start, next_start); + if (uinfo->freepgtables) + { + if (pmd_free_pte(uinfo, pmd, start & -PMD_SIZE, res)) + clear++; + else + ret |= UNMAP_DONT_FREE; + } + } + + if (clear == PTRS_PER_PMD) + return UNMAP_FREE_PGTABLE; + return ret; +} + +static enum unmap_result pud_unmap_range(struct unmap_info *uinfo, pud_t *pud, unsigned long start, + unsigned long end) +{ + unsigned long next_start; + int clear = 0; + enum unmap_result ret = UNMAP_OK; + for (; start < end; pud++, start = next_start) + { + next_start = min(pud_addr_end(start), end); + if (pud_none(*pud)) + { + clear++; + continue; + } + /* TODO: Huge page unmapping and splitting not supported yet... */ + DCHECK(!pud_huge(*pud)); + enum unmap_result res = pmd_unmap_range(uinfo, pmd_offset(pud, start), start, next_start); + if (!pmd_folded() && uinfo->freepgtables && !(res & UNMAP_DONT_FREE)) + { + if (pud_free_pmd(uinfo, pud, start & -PUD_SIZE, res)) + clear++; + else + ret |= UNMAP_DONT_FREE; + } + } + + if (clear == PTRS_PER_PUD) + return UNMAP_FREE_PGTABLE; + return ret; +} + +static enum unmap_result p4d_unmap_range(struct unmap_info *uinfo, p4d_t *p4d, unsigned long start, + unsigned long end) +{ + unsigned long next_start; + int clear = 0; + enum unmap_result ret = UNMAP_OK; + for (; start < end; p4d++, start = next_start) + { + next_start = min(p4d_addr_end(start), end); + if (p4d_none(*p4d)) + { + clear++; + continue; + } + /* TODO: Huge page unmapping and splitting not supported yet... */ + DCHECK(!p4d_huge(*p4d)); + enum unmap_result res = pud_unmap_range(uinfo, pud_offset(p4d, start), start, next_start); + if (!pud_folded() && uinfo->freepgtables && !(res & UNMAP_DONT_FREE)) + { + if (p4d_free_pud(uinfo, p4d, start & -P4D_SIZE, res)) + clear++; + else + ret |= UNMAP_DONT_FREE; + } + } + + if (clear == PTRS_PER_P4D) + return UNMAP_FREE_PGTABLE; + return ret; +} + +static void pgd_unmap_range(struct unmap_info *uinfo, pgd_t *pgd, unsigned long start, + unsigned long end) +{ + unsigned long next_start; + for (; start < end; pgd++, start = next_start) + { + next_start = min(pgd_addr_end(start), end); + if (pgd_none(*pgd)) + continue; + enum unmap_result res = p4d_unmap_range(uinfo, p4d_offset(pgd, start), start, next_start); + if (!p4d_folded() && uinfo->freepgtables && !(res & UNMAP_DONT_FREE)) + pgd_free_p4d(uinfo, pgd, start & -PGD_SIZE, res); + } +} + +int vm_mmu_unmap(struct mm_address_space *mm, void *addr, size_t pages, struct vm_area_struct *vma) +{ + unsigned long virt = (unsigned long) addr; + unsigned long end = virt + (pages << PAGE_SHIFT); + struct unmap_info unmap_info; + unmap_info.vma = vma; + unmap_info.mm = mm; + unmap_info.kernel = mm == &kernel_address_space; + unmap_info.full = 0; + unmap_info.freepgtables = 1; + tlbi_tracker_init(&unmap_info.tlbi); + + spin_lock(&mm->page_table_lock); + pgd_unmap_range(&unmap_info, pgd_offset(mm, virt), virt, end); + spin_unlock(&mm->page_table_lock); + + if (tlbi_active(&unmap_info.tlbi)) + tlbi_end_batch(&unmap_info.tlbi); + return 0; +} + +bool paging_write_protect(void *addr, struct mm_address_space *mm) +{ + spin_lock(&mm->page_table_lock); + pte_t *pte = pte_get_from_addr(mm, (unsigned long) addr); + if (pte) + set_pte(pte, pte_wrprotect(*pte)); + spin_unlock(&mm->page_table_lock); + return pte != NULL; +} + +static void pte_change_prot(pte_t *ptep, int vmflags) +{ + /* Note: Preserve the A bits */ + pte_t pte = *ptep; + pte_t newpte = pte_mkpte(pte_addr(pte), calc_pgprot(pte_addr(pte), vmflags)); + if (pte_accessed(pte)) + pte_val(newpte) |= _PAGE_ACCESSED; + set_pte(ptep, newpte); +} + +/* TODO: This is on the deprecated chopping block... */ +bool __paging_change_perms(struct mm_address_space *mm, void *addr, int prot) +{ + spin_lock(&mm->page_table_lock); + pte_t *pte = pte_get_from_addr(mm, (unsigned long) addr); + if (pte) + pte_change_prot(pte, prot); + spin_unlock(&mm->page_table_lock); + return pte != NULL; +} + +static void pte_protect_range(struct tlbi_tracker *tlbi, pte_t *pte, unsigned long start, + unsigned long end, int new_prots) +{ + unsigned long next_start; + for (; start < end; pte++, start = next_start) + { + next_start = min(pte_addr_end(start), end); + pte_t old = *pte; + if (pte_none(old)) + continue; + + pte_change_prot(pte, new_prots); + tlbi_update_page_prots(tlbi, start, old, *pte); + } +} + +static void pmd_protect_range(struct tlbi_tracker *tlbi, pmd_t *pmd, unsigned long start, + unsigned long end, int new_prots) +{ + unsigned long next_start; + for (; start < end; pmd++, start = next_start) + { + next_start = min(pmd_addr_end(start), end); + if (pmd_none(*pmd)) + continue; + + /* TODO: Huge page splitting not supported yet... */ + DCHECK(!pmd_huge(*pmd)); + pte_protect_range(tlbi, pte_offset(pmd, start), start, next_start, new_prots); + } +} + +static void pud_protect_range(struct tlbi_tracker *tlbi, pud_t *pud, unsigned long start, + unsigned long end, int new_prots) +{ + unsigned long next_start; + for (; start < end; pud++, start = next_start) + { + next_start = min(pud_addr_end(start), end); + if (pud_none(*pud)) + continue; + /* TODO: Huge page splitting not supported yet... */ + DCHECK(!pud_huge(*pud)); + pmd_protect_range(tlbi, pmd_offset(pud, start), start, next_start, new_prots); + } +} + +static void p4d_protect_range(struct tlbi_tracker *tlbi, p4d_t *p4d, unsigned long start, + unsigned long end, int new_prots) +{ + unsigned long next_start; + for (; start < end; p4d++, start = next_start) + { + next_start = min(p4d_addr_end(start), end); + if (p4d_none(*p4d)) + continue; + + /* TODO: Huge page splitting not supported yet... */ + DCHECK(!p4d_huge(*p4d)); + pud_protect_range(tlbi, pud_offset(p4d, start), start, next_start, new_prots); + } +} + +static void pgd_protect_range(struct tlbi_tracker *tlbi, pgd_t *pgd, unsigned long start, + unsigned long end, int new_prots) +{ + unsigned long next_start; + for (; start < end; pgd++, start = next_start) + { + next_start = min(pgd_addr_end(start), end); + if (pgd_none(*pgd)) + continue; + p4d_protect_range(tlbi, p4d_offset(pgd, start), start, next_start, new_prots); + } +} + +void vm_do_mmu_mprotect(struct mm_address_space *mm, void *address, size_t nr_pgs, int old_prots, + int new_prots) +{ + unsigned long start = (unsigned long) address; + unsigned long end = start + (nr_pgs << PAGE_SHIFT); + struct tlbi_tracker tlbi; + tlbi_tracker_init(&tlbi); + + spin_lock(&mm->page_table_lock); + pgd_protect_range(&tlbi, pgd_offset(mm, start), start, end, new_prots); + spin_unlock(&mm->page_table_lock); + + if (tlbi_active(&tlbi)) + tlbi_end_batch(&tlbi); +} + +static int pte_fork_range(struct tlbi_tracker *tlbi, pte_t *pte, pte_t *old_pte, + unsigned long start, unsigned long end, struct mm_address_space *mm, + struct vm_area_struct *old_vma) +{ + /* Let's lock the page tables. It actually *does* matter in our case, because we must have + * stable ptes (one can imagine a situation where a shared pte getting wp'd races and we get a + * writable mapping without the page being dirty). */ + spin_lock(&mm->page_table_lock); + unsigned long next_start; + for (; start < end; pte++, old_pte++, start = next_start) + { + next_start = min(pte_addr_end(start), end); + pte_t old = *old_pte; + if (pte_none(old)) + continue; + + if (!vma_is_pfnmap(old_vma) && !pte_special(old)) + page_add_mapcount(phys_to_page(pte_addr(old))); + + if (vma_private(old_vma)) + { + /* We must CoW MAP_PRIVATE */ + set_pte(old_pte, pte_wrprotect(old)); + set_pte(pte, *old_pte); + tlbi_update_page_prots(tlbi, start, old, *pte); + } + else + { + set_pte(pte, old); + } + + increment_vm_stat(mm, resident_set_size, PAGE_SIZE); + } + + spin_unlock(&mm->page_table_lock); + return 0; +} + +static int pmd_fork_range(struct tlbi_tracker *tlbi, pmd_t *pmd, pmd_t *old_pmd, + unsigned long start, unsigned long end, struct mm_address_space *mm, + struct vm_area_struct *old_vma) +{ + unsigned long next_start; + for (; start < end; pmd++, old_pmd++, start = next_start) + { + next_start = min(pmd_addr_end(start), end); + if (pmd_none(*old_pmd)) + continue; + pte_t *pte = pte_get_or_alloc(pmd, start, mm); + if (!pte) + return -ENOMEM; + + /* TODO: Huge page splitting not supported yet... */ + DCHECK(!pmd_huge(*pmd)); + int err = + pte_fork_range(tlbi, pte, pte_offset(old_pmd, start), start, next_start, mm, old_vma); + if (err < 0) + return err; + } + + return 0; +} + +static int pud_fork_range(struct tlbi_tracker *tlbi, pud_t *pud, pud_t *old_pud, + unsigned long start, unsigned long end, struct mm_address_space *mm, + struct vm_area_struct *old_vma) +{ + unsigned long next_start; + for (; start < end; pud++, old_pud++, start = next_start) + { + next_start = min(pud_addr_end(start), end); + if (pud_none(*old_pud)) + continue; + pmd_t *pmd = pmd_get_or_alloc(pud, start, mm); + if (!pmd) + return -ENOMEM; + /* TODO: Huge page splitting not supported yet... */ + DCHECK(!pud_huge(*pud)); + int err = + pmd_fork_range(tlbi, pmd, pmd_offset(old_pud, start), start, next_start, mm, old_vma); + if (err < 0) + return err; + } + + return 0; +} + +static int p4d_fork_range(struct tlbi_tracker *tlbi, p4d_t *p4d, p4d_t *old_p4d, + unsigned long start, unsigned long end, struct mm_address_space *mm, + struct vm_area_struct *old_vma) +{ + unsigned long next_start; + for (; start < end; p4d++, old_p4d++, start = next_start) + { + next_start = min(p4d_addr_end(start), end); + if (p4d_none(*old_p4d)) + continue; + pud_t *pud = pud_get_or_alloc(p4d, start, mm); + if (!pud) + return -ENOMEM; + + /* TODO: Huge page splitting not supported yet... */ + DCHECK(!p4d_huge(*p4d)); + int err = + pud_fork_range(tlbi, pud, pud_offset(old_p4d, start), start, next_start, mm, old_vma); + if (err < 0) + return -ENOMEM; + } + + return 0; +} + +static int pgd_fork_range(struct tlbi_tracker *tlbi, pgd_t *pgd, pgd_t *old_pgd, + unsigned long start, unsigned long end, struct mm_address_space *mm, + struct vm_area_struct *old_vma) +{ + unsigned long next_start; + for (; start < end; pgd++, old_pgd++, start = next_start) + { + next_start = min(pgd_addr_end(start), end); + if (pgd_none(*old_pgd)) + continue; + p4d_t *p4d = p4d_get_or_alloc(pgd, start, mm); + if (!p4d) + return -ENOMEM; + + int err = + p4d_fork_range(tlbi, p4d, p4d_offset(old_pgd, start), start, next_start, mm, old_vma); + if (err < 0) + return err; + } + + return 0; +} + +/** + * @brief Fork MMU page tables + * + * @param old_vma Old vm_area_struct + * @param mm Current address space + * @return 0 on success, negative error codes + */ +int mmu_fork_tables(struct vm_area_struct *old_vma, struct mm_address_space *mm) +{ + unsigned long start = old_vma->vm_start; + unsigned long end = old_vma->vm_end; + int err; + struct tlbi_tracker tlbi; + tlbi_tracker_init(&tlbi); + + /* Note: We can't take the page table spinlock here (hold time is too long, too many memory + * allocations may happen). We'll rely on holding the mm lock exclusively. Page table lifetime + * atm is a bit iffy, needs some solid rethinking. */ + err = pgd_fork_range(&tlbi, pgd_offset(mm, start), pgd_offset(old_vma->vm_mm, start), start, + end, mm, old_vma); + + if (tlbi_active(&tlbi)) + tlbi_end_batch(&tlbi); + return err; +} diff --git a/kernel/kernel/mm/mincore.cpp b/kernel/kernel/mm/mincore.cpp index f943e66ac..8e2b81392 100644 --- a/kernel/kernel/mm/mincore.cpp +++ b/kernel/kernel/mm/mincore.cpp @@ -19,12 +19,13 @@ static long do_pagemap(struct mm_address_space *as, unsigned long start, unsigne scoped_mutex g{as->vm_lock}; long pfns_processed = 0; - struct vm_area_struct *vma = vm_search(as, (void *) start, 1); - if (!vma) - return -ENOMEM; - while (vma) + vm_area_struct *vma; + unsigned long index = start; + void *entry_; + mt_for_each(&as->region_tree, entry_, index, end) { + vma = (vm_area_struct *) entry_; if (vma->vm_start > end) break; @@ -38,9 +39,6 @@ static long do_pagemap(struct mm_address_space *as, unsigned long start, unsigne if (start == end) break; - - vma = containerof_null_safe(bst_next(&as->region_tree, &vma->vm_tree_node), - struct vm_area_struct, vm_tree_node); } return pfns_processed; @@ -55,7 +53,7 @@ int sys_mpagemap(void *addr, size_t length, u64 *pagemap) struct page *buffer; int ret = 0; - if (start < as->start || end > as->end) + if (start < as->start || end > as->end + 1) return -EINVAL; buffer = alloc_page(GFP_KERNEL); diff --git a/kernel/kernel/mm/pagealloc.cpp b/kernel/kernel/mm/pagealloc.cpp index 36541f3d9..1b96b0afc 100644 --- a/kernel/kernel/mm/pagealloc.cpp +++ b/kernel/kernel/mm/pagealloc.cpp @@ -832,7 +832,7 @@ struct page *page_node::alloc_order(unsigned int order, unsigned long flags) else if (flags & __GFP_WAKE_PAGEDAEMON) { unsigned long cur_seq = wake_up_pagedaemon(order, attempt); - if (!(flags & __GFP_ATOMIC)) + if (!(flags & (__GFP_ATOMIC | __GFP_NOWAIT))) { wait_for_event(&paged_data.paged_waiters_queue, cur_seq < paged_data.reclaim_seq); } @@ -853,8 +853,12 @@ struct page *page_node::alloc_order(unsigned int order, unsigned long flags) return page; failure: - pr_warn("pagealloc: Failed allocation of order %u, gfp_flags %lx, on:\n", order, flags); - stack_trace(); + if (!(flags & __GFP_NOWARN)) + { + pr_warn("pagealloc: Failed allocation of order %u, gfp_flags %lx, on:\n", order, flags); + stack_trace(); + } + return nullptr; } diff --git a/kernel/kernel/mm/pgtable.h b/kernel/kernel/mm/pgtable.h new file mode 100644 index 000000000..4d005b599 --- /dev/null +++ b/kernel/kernel/mm/pgtable.h @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the GPLv2 License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only + */ +#ifndef _ONYX_PGTABLE_H +#define _ONYX_PGTABLE_H + +#include + +#ifndef set_pgd +static void set_pgd(pgd_t *pgd, pgd_t val) +{ + WRITE_ONCE(pgd_val(*pgd), pgd_val(val)); +} +#define set_pgd set_pgd +#endif + +#ifndef set_p4d +static void set_p4d(p4d_t *p4d, p4d_t val) +{ + WRITE_ONCE(p4d_val(*p4d), p4d_val(val)); +} +#define set_p4d set_p4d +#endif + +#ifndef set_pud +static void set_pud(pud_t *pud, pud_t val) +{ + WRITE_ONCE(pud_val(*pud), pud_val(val)); +} +#define set_pud set_pud +#endif + +#ifndef set_pmd +static void set_pmd(pmd_t *pmd, pmd_t val) +{ + WRITE_ONCE(pmd_val(*pmd), pmd_val(val)); +} +#define set_pmd set_pmd +#endif + +#ifndef set_pte +static void set_pte(pte_t *pte, pte_t val) +{ + WRITE_ONCE(pte_val(*pte), pte_val(val)); +} +#define set_pgd set_pgd +#endif + +static inline bool pte_cmpxchg(pte_t *pte, pte_t *expected, pte_t desired) +{ + return __atomic_compare_exchange_n(&pte->pte, &expected->pte, desired.pte, false, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); +} + +/* Dummy fallbacks for architectures that don't support certain huge page levels */ + +#ifndef ARCH_HUGE_P4D_SUPPORT + +static inline bool p4d_huge(p4d_t p4d) +{ + return false; +} + +static inline bool p4d_user(p4d_t p4d) +{ + return false; +} + +static inline bool p4d_write(p4d_t p4d) +{ + return false; +} + +static inline bool p4d_exec(p4d_t p4d) +{ + return false; +} + +static inline bool p4d_dirty(p4d_t p4d) +{ + return false; +} + +static inline bool p4d_accessed(p4d_t p4d) +{ + return false; +} + +static inline bool p4d_global(p4d_t p4d) +{ + return false; +} + +#endif + +#ifndef ARCH_HUGE_PUD_SUPPORT + +static inline bool pud_huge(pud_t pud) +{ + return false; +} + +static inline bool pud_user(pud_t pud) +{ + return false; +} + +static inline bool pud_write(pud_t pud) +{ + return false; +} + +static inline bool pud_exec(pud_t pud) +{ + return false; +} + +static inline bool pud_dirty(pud_t pud) +{ + return false; +} + +static inline bool pud_accessed(pud_t pud) +{ + return false; +} + +static inline bool pud_global(pud_t pud) +{ + return false; +} + +#endif + +#ifndef ARCH_HUGE_PMD_SUPPORT + +static inline bool pmd_huge(pmd_t pmd) +{ + return false; +} + +static inline bool pmd_user(pmd_t pmd) +{ + return false; +} + +static inline bool pmd_write(pmd_t pmd) +{ + return false; +} + +static inline bool pmd_exec(pmd_t pmd) +{ + return false; +} + +static inline bool pmd_dirty(pmd_t pmd) +{ + return false; +} + +static inline bool pmd_accessed(pmd_t pmd) +{ + return false; +} + +static inline bool pmd_global(pmd_t pmd) +{ + return false; +} + +#endif + +#define PGD_SIZE (1UL << PGD_SHIFT) +#define P4D_SIZE (1UL << P4D_SHIFT) +#define PUD_SIZE (1UL << PUD_SHIFT) +#define PMD_SIZE (1UL << PMD_SHIFT) +#define PTE_SIZE (1UL << PTE_SHIFT) + +static inline unsigned long pgd_addr_end(unsigned long addr) +{ + /* We need to be careful with overflows... */ + unsigned long end = (addr & -PGD_SIZE) + PGD_SIZE; + return end < addr ? -1UL : end; +} + +static inline unsigned long p4d_addr_end(unsigned long addr) +{ + /* We need to be careful with overflows... */ + unsigned long end = (addr & -P4D_SIZE) + P4D_SIZE; + return end < addr ? -1UL : end; +} + +static inline unsigned long pud_addr_end(unsigned long addr) +{ + /* We need to be careful with overflows... */ + unsigned long end = (addr & -PUD_SIZE) + PUD_SIZE; + return end < addr ? -1UL : end; +} + +static inline unsigned long pmd_addr_end(unsigned long addr) +{ + /* We need to be careful with overflows... */ + unsigned long end = (addr & -PMD_SIZE) + PMD_SIZE; + return end < addr ? -1UL : end; +} + +static inline unsigned long pte_addr_end(unsigned long addr) +{ + /* We need to be careful with overflows... */ + unsigned long end = (addr & -PTE_SIZE) + PTE_SIZE; + return end < addr ? -1UL : end; +} + +#endif diff --git a/kernel/kernel/mm/slab.cpp b/kernel/kernel/mm/slab.cpp index 2f1a62581..b290812e3 100644 --- a/kernel/kernel/mm/slab.cpp +++ b/kernel/kernel/mm/slab.cpp @@ -106,6 +106,8 @@ struct slab_cache *kmem_cache_create(const char *name, size_t size, size_t align auto c = slab_cache_pool.allocate(); if (!c) { + if (flags & KMEM_CACHE_PANIC) + panic("kmem_cache_create of %s failed!", name); return nullptr; } @@ -135,7 +137,7 @@ struct slab_cache *kmem_cache_create(const char *name, size_t size, size_t align } c->objsize = ALIGN_TO(c->objsize, c->alignment); - c->redzone = ALIGN_TO(c->redzone, c->alignment); + c->redzone = ALIGN_TO(c->redzone / 2, c->alignment) * 2; if (c->objsize > PAGE_SIZE) { @@ -860,6 +862,13 @@ __always_inline void kmem_cache_post_alloc(struct slab_cache *cache, unsigned in kmem_cache_post_alloc_kasan(cache, flags, object); } +__always_inline void kmem_cache_post_alloc_bulk(struct slab_cache *cache, unsigned int flags, + void **objects, size_t nr) +{ + for (size_t i = 0; i < nr; i++) + kmem_cache_post_alloc(cache, flags, objects[i]); +} + /** * @brief Allocate an object from the slab * This function call be called in nopreempt/softirq context. @@ -912,6 +921,87 @@ void *kmem_cache_alloc(struct slab_cache *cache, unsigned int flags) return ret; } +int kmem_cache_alloc_bulk_nopcpu(struct slab_cache *cache, unsigned int gfp_flags, size_t nr, + void **res) +{ + size_t i; + + for (i = 0; i < nr; i++) + { + void *ptr = kmem_cache_alloc_nopcpu(cache, gfp_flags); + if (!ptr) + goto out_nomem; + res[i] = ptr; + } + + kmem_cache_post_alloc_bulk(cache, gfp_flags, res, nr); + return nr; +out_nomem: + kmem_cache_free_bulk(cache, i, res); + return 0; +} + +/** + * @brief Allocate objects in bulk + * Allocate slab objects in bulk, while avoiding relocking as much as we can. + * + * @param cache Slab cache + * @param gfp_flags GFP flags + * @param nr Number of objects desired + * @param res Array of results (output parameter) + * @return 0 on error (ENOMEM), or the number of objects allocated + */ +size_t kmem_cache_alloc_bulk(struct slab_cache *cache, unsigned int gfp_flags, size_t nr, + void **res) +{ + size_t i = 0; + size_t ret = nr; + + if (unlikely(cache->flags & KMEM_CACHE_NOPCPU)) + return kmem_cache_alloc_bulk_nopcpu(cache, gfp_flags, nr, res); + + while (nr) + { + // Disable preemption so we can safely touch the percpu data + sched_disable_preempt(); + auto pcpu = &cache->pcpu[get_cpu_nr()]; + pcpu->touched.store(1, mem_order::release); + if (unlikely(!pcpu->size)) + { + /* Refill and try again */ + int st = kmem_cache_alloc_refill_mag(cache, pcpu, gfp_flags); + if (unlikely(st < 0)) + { + sched_enable_preempt(); + goto enomem; + } + + pcpu = &cache->pcpu[get_cpu_nr()]; + } + + DCHECK(pcpu->size > 0); + /* Attempt to fill up our res array with whatever we can find in the pcpu data. */ + unsigned long to_take = min(nr, (unsigned long) pcpu->size); + nr -= to_take; + while (to_take--) + { + void *ptr = pcpu->magazine[--pcpu->size]; + ((bufctl *) ptr)->flags = 0; + res[i++] = ptr; + pcpu->active_objs++; + } + + pcpu->touched.store(0, mem_order::release); + sched_enable_preempt(); + } + + kmem_cache_post_alloc_bulk(cache, gfp_flags, res, ret); + return ret; +enomem: + kmem_cache_free_bulk(cache, i, res); + return 0; +} + /** * @brief Free an object to its slab * This function panics on bad pointers. @@ -1007,8 +1097,10 @@ void kmem_cache_return_pcpu_batch(struct slab_cache *cache, struct slab_cache_pe memmove(pcpu->magazine, &pcpu->magazine[batchsize], (size - pcpu->size) * sizeof(void *)); } -static void kmem_cache_free_pcpu(struct slab_cache *cache, void *ptr) +__always_inline void kmem_cache_free_pcpu_single(struct slab_cache *cache, + struct slab_cache_percpu_context *pcpu, void *ptr) { + DCHECK(pcpu->size < cache->mag_limit); bufctl *buf = (bufctl *) ptr; if ((unsigned long) ptr % cache->alignment) [[unlikely]] @@ -1017,27 +1109,21 @@ static void kmem_cache_free_pcpu(struct slab_cache *cache, void *ptr) if (buf->flags == BUFCTL_PATTERN_FREE) [[unlikely]] panic("slab: Double free at %p\n", ptr); -#ifdef CONFIG_KASAN - asan_poison_shadow((unsigned long) ptr, cache->objsize, KASAN_FREED); -#endif + pcpu->magazine[pcpu->size++] = ptr; + buf->flags = BUFCTL_PATTERN_FREE; + pcpu->active_objs--; +} +static void kmem_cache_free_pcpu(struct slab_cache *cache, void *ptr) +{ sched_disable_preempt(); - auto pcpu = &cache->pcpu[get_cpu_nr()]; pcpu->touched.store(1, mem_order::release); - pcpu->active_objs--; if (pcpu->size == cache->mag_limit) [[unlikely]] - { kmem_cache_return_pcpu_batch(cache, pcpu); - } - else - { - pcpu->magazine[pcpu->size++] = ptr; - buf->flags = BUFCTL_PATTERN_FREE; - } + kmem_cache_free_pcpu_single(cache, pcpu, ptr); pcpu->touched.store(0, mem_order::release); - sched_enable_preempt(); } @@ -1045,6 +1131,9 @@ static void kmem_cache_free_pcpu(struct slab_cache *cache, void *ptr) void kasan_kfree(void *ptr, struct slab_cache *cache, size_t chunk_size) { + if ((unsigned long) ptr % cache->alignment) [[unlikely]] + panic("slab: Bad pointer %p", ptr); + bufctl *buf = (bufctl *) ptr; if (buf->flags == BUFCTL_PATTERN_FREE) [[unlikely]] @@ -1066,6 +1155,15 @@ void kasan_kfree(void *ptr, struct slab_cache *cache, size_t chunk_size) #endif } +static void kmem_cache_free_bulk_kasan(struct slab_cache *cache, size_t size, void **ptrs) +{ + for (size_t i = 0; i < size; i++) + { + if (ptrs[i]) + kasan_kfree(ptrs[i], cache, cache->objsize); + } +} + #endif /** * @brief Free a pointer to an object in a slab @@ -1113,6 +1211,64 @@ void kmem_cache_free(struct slab_cache *cache, void *ptr) kmem_cache_free_pcpu(cache, ptr); } +static void kmem_cache_free_bulk_nopcpu(struct slab_cache *cache, size_t size, void **ptrs) +{ + for (size_t i = 0; i < size; i++) + { + if (ptrs[i]) + kfree_nopcpu(ptrs[i]); + } +} + +/** + * @brief Free objects in bulk + * Free objects in bulk, avoiding relocking and doing as much as we can, in batches. + * @param cache Slab cache + * @param size Number of objects to free + * @param ptrs Pointers to free (NULL is tolerated) + */ +void kmem_cache_free_bulk(struct slab_cache *cache, size_t size, void **ptrs) +{ + size_t i = 0; + + if (unlikely(cache->flags & KMEM_CACHE_NOPCPU)) + { + kmem_cache_free_bulk_nopcpu(cache, size, ptrs); + return; + } + +#ifdef CONFIG_KASAN + kmem_cache_free_bulk_kasan(cache, size, ptrs); + return; +#endif + + while (size) + { + sched_disable_preempt(); + auto pcpu = &cache->pcpu[get_cpu_nr()]; + pcpu->touched.store(1, mem_order::release); + + if (pcpu->size == cache->mag_limit) [[unlikely]] + kmem_cache_return_pcpu_batch(cache, pcpu); + + int free_slots = cache->mag_limit - pcpu->size; + while (free_slots) + { + if (likely(ptrs[i])) + { + kmem_cache_free_pcpu_single(cache, pcpu, ptrs[i]); + free_slots--; + } + + if (--size == 0) + break; + } + + pcpu->touched.store(0, mem_order::release); + sched_enable_preempt(); + } +} + /** * @brief Free a given slab and give it back to the page backend * The given slab will be properly dissociated from its slab cache @@ -1155,17 +1311,17 @@ static void kmem_purge_remote(struct slab_rendezvous *rndvz) /** * @brief Start a slab allocator freeze - * When the slab allocator is frozen, no one can enter the per-cpu "area" of any cache. That is to - * say, cpus that were accessing their pcpu will be frozen, and new ones will not be able to get in. - * Requires preemption to be disabled, in order for us to not migrate CPUs mid-freeze. + * When the slab allocator is frozen, no one can enter the per-cpu "area" of any cache. That is + * to say, cpus that were accessing their pcpu will be frozen, and new ones will not be able to + * get in. Requires preemption to be disabled, in order for us to not migrate CPUs mid-freeze. * * @param rndvz Rendezvous structure */ static void kmem_slab_freeze_start(struct slab_rendezvous *rndvz) { - /* To start a freeze, we store the number of CPUs we're waiting for in a shared structure. As - * IPIs hit CPUs, they decrement the count. When the count hits 0, we know every CPU has hit the - * freeze and may start to reclaim (or whatever we need to do). See comments in + /* To start a freeze, we store the number of CPUs we're waiting for in a shared structure. + * As IPIs hit CPUs, they decrement the count. When the count hits 0, we know every CPU has + * hit the freeze and may start to reclaim (or whatever we need to do). See comments in * kmem_purge_remote for notes on the concurrency. */ unsigned int to_sync = get_nr_cpus() - 1; rndvz->ack = 0; @@ -1186,8 +1342,8 @@ static void kmem_slab_freeze_start(struct slab_rendezvous *rndvz) static void kmem_slab_freeze_end(struct slab_rendezvous *rndvz) { rndvz->waiting_for_cpus = get_nr_cpus() - 1; - /* Use release to make waiters see waiting_for_cpus before ack. It will also make prior stores - * visible when ack is acquired. */ + /* Use release to make waiters see waiting_for_cpus before ack. It will also make prior + * stores visible when ack is acquired. */ __atomic_store_n(&rndvz->ack, 1, __ATOMIC_RELEASE); } diff --git a/kernel/kernel/mm/vm.cpp b/kernel/kernel/mm/vm.cpp index 0a5c3f5fc..839cb43fe 100644 --- a/kernel/kernel/mm/vm.cpp +++ b/kernel/kernel/mm/vm.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2023 Pedro Falcato + * Copyright (c) 2016 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -19,11 +19,11 @@ #include #include #include +#include #include #include #include #include -#include #include #include #include @@ -44,7 +44,6 @@ #include #include -#include #include #include @@ -62,16 +61,15 @@ uintptr_t low_half_min = arch_low_half_min; uintptr_t vmalloc_space = arch_vmalloc_off; void kmalloc_init(); -int populate_shared_mapping(void *page, struct file *fd, struct vm_area_struct *entry, - size_t nr_pages); void vm_remove_region(struct mm_address_space *as, struct vm_area_struct *region); -int vm_add_region(struct mm_address_space *as, struct vm_area_struct *region); int __vm_munmap(struct mm_address_space *as, void *__addr, size_t size); -bool limits_are_contained(struct vm_area_struct *reg, unsigned long start, unsigned long limit); -bool vm_mapping_is_cow(struct vm_area_struct *entry); +static bool limits_are_contained(struct vm_area_struct *reg, unsigned long start, + unsigned long limit); +static bool vm_mapping_is_cow(struct vm_area_struct *entry); vm_area_struct *vm_search(struct mm_address_space *mm, void *addr, size_t length) REQUIRES_SHARED(mm->vm_lock); +static void vma_pre_adjust(struct vm_area_struct *vma); /** * @brief Finds a vm region. @@ -112,169 +110,108 @@ static inline void vma_free(vm_area_struct *region) kmem_cache_free(vm_area_struct_cache, (void *) region); } -bool vm_insert_region(struct mm_address_space *as, struct vm_area_struct *region) +struct vma_iterator { - return bst_insert( - &as->region_tree, ®ion->vm_tree_node, - [](struct bst_node *lhs_, struct bst_node *rhs_) -> int { - auto lhs = container_of(lhs_, vm_area_struct, vm_tree_node); - auto rhs = container_of(rhs_, vm_area_struct, vm_tree_node); - - if (check_for_overlap(lhs->vm_start, lhs->vm_end - 1, rhs->vm_start, rhs->vm_end - 1)) - { - panic("vm_insert_region: Region [%lx, %lx] and [%lx, %lx] overlap\n", lhs->vm_start, - lhs->vm_end, rhs->vm_start, rhs->vm_end); - return 0; - } - else if (rhs->vm_start > lhs->vm_start) - return 1; - else - return -1; - }); -} - -struct vm_area_struct *vm_reserve_region(struct mm_address_space *as, unsigned long start, - size_t size) -{ - MUST_HOLD_MUTEX(&as->vm_lock); - - struct vm_area_struct *region = vma_alloc(); - if (!region) - return nullptr; - - memset(region, 0, sizeof(*region)); - - region->vm_start = start; - region->vm_end = start + size; - region->vm_flags = 0; - bst_node_initialize(®ion->vm_tree_node); - - bool success = vm_insert_region(as, region); - - if (!success) - { - panic("Could not insert vm region [%lx, %lx]", start, region->vm_end - 1); - } - - region->vm_mm = as; - - return region; -} + unsigned long index; + unsigned long end; + struct mm_address_space *mm; + struct ma_state mas; +}; -#define DEBUG_VM_1 0 -#define DEBUG_VM_2 0 -#define DEBUG_VM_3 0 +#define VMA_ITERATOR(name, mm, index, end) \ + struct vma_iterator name = {index, (end) -1, mm, \ + MA_STATE_INIT(&(mm)->region_tree, index, (end) -1)} -unsigned long vm_allocate_base(struct mm_address_space *as, unsigned long min, size_t size, - u64 flags) REQUIRES(as->vm_lock) +#define CONFIG_DEBUG_MM_MMAP +#ifdef CONFIG_DEBUG_MM_MMAP +static void validate_mm_tree(struct mm_address_space *mm) { - MUST_HOLD_MUTEX(&as->vm_lock); - - if (min < as->start) - min = as->start; - - struct a : bst_node + VMA_ITERATOR(vmi, mm, 0, -1UL); + void *entry_; + size_t counting_vss = 0; + size_t counting_sss = 0; + mas_for_each(&vmi.mas, entry_, -1UL) { - unsigned long min; - } priv; - - priv.min = min; - - auto compare = [](bst_node *node0, bst_node *fake) -> int { - struct a *priv = (a *) fake; - auto reg = container_of(node0, vm_area_struct, vm_tree_node); - auto end = reg->vm_end - 1; - if (check_for_overlap(reg->vm_start, end, priv->min, priv->min + PAGE_SIZE)) - return 0; - else if (end >= priv->min) - return -1; - else // if (end < priv->min) - return 1; - }; - - struct bst_node *node = nullptr; - unsigned long last_end = min; - struct vm_area_struct *f = nullptr; + struct vm_area_struct *vma = (struct vm_area_struct *) entry_; + if (vma->vm_start != vmi.mas.index || vma->vm_end != vmi.mas.last + 1) + { + pr_err("mm: vma bounds [%016lx, %016lx] do not match maple tree [%016lx, %016lx]\n", + vma->vm_start, vma->vm_end, vmi.mas.index, vmi.mas.last + 1); + goto print_tree; + } - if (min != as->start) - { - node = bst_search(&as->region_tree, &priv, compare); + counting_vss += vma->vm_end - vma->vm_start; + if (vma_shared(vma)) + counting_sss += vma->vm_end - vma->vm_start; } - else + + if (counting_vss != mm->virtual_memory_size) { - node = bst_min(&as->region_tree, nullptr); + pr_err("mm: mm %p has wrong vss (%lx vs %lx bytes)\n", mm, counting_vss, + mm->virtual_memory_size); + goto print_tree; } - if (!node) - goto done; - - /* Check if there's a gap between the first node - * and the start of the address space - */ - - f = container_of(node, vm_area_struct, vm_tree_node); - -#if DEBUG_VM_1 - printk("Tiniest node: %016lx\n", f->base); -#endif - if (f->vm_start - min >= size) + if (counting_sss != mm->shared_set_size) { -#if DEBUG_VM_2 - printk("gap [%016lx - %016lx]\n", min, f->base); -#endif - goto done; + pr_err("mm: mm %p has wrong shared set size (%lx vs %lx bytes)\n", mm, counting_sss, + mm->shared_set_size); + goto print_tree; } - while (node) + return; +print_tree: + pr_err("mm: dumping vmas for mm %p...\n", mm); + mas_set(&vmi.mas, 0); + mas_for_each(&vmi.mas, entry_, -1UL) { - f = container_of(node, vm_area_struct, vm_tree_node); - last_end = f->vm_end; - - node = bst_next(&as->region_tree, node); - if (!node) - break; - - struct vm_area_struct *vm = container_of(node, vm_area_struct, vm_tree_node); - - if (vm->vm_start - last_end >= size && min <= vm->vm_start) - break; + struct vm_area_struct *vma = (struct vm_area_struct *) entry_; + const char *name = "[anon]"; + if (vma->vm_file) + name = vma->vm_file->f_dentry->d_name; + pr_err(" [%016lx, %016lx] vma ([%016lx, %016lx] maple tree) flags %x %s\n", vma->vm_start, + vma->vm_end, vmi.mas.index, vmi.mas.last + 1, vma->vm_flags, name); } -done: - last_end = last_end < min ? min : last_end; + pr_err("mm: dump done.\n"); +} - // We can't map something over the edge - if (last_end > as->end || last_end + size > as->end) - return -1; +#else - // The architecture may have extra checks to do - if (!arch_vm_validate_mmap_region(last_end, size, flags)) - return -1; +#define validate_mm_tree(mm) \ + do \ + { \ + } while (0) - return last_end; -} +#endif -static struct vm_area_struct *vm_allocate_region(struct mm_address_space *as, unsigned long min, - size_t size, u64 flags) REQUIRES(as->vm_lock) +bool vm_insert_region(struct mm_address_space *as, struct vm_area_struct *region) { - if (!vm_test_vs_rlimit(as, size)) - return errno = ENOMEM, nullptr; - - unsigned long new_base = vm_allocate_base(as, min, size, flags); + return mtree_insert_range(&as->region_tree, region->vm_start, region->vm_end - 1, region, + GFP_KERNEL) == 0; +} - if (new_base == (unsigned long) -1) - return nullptr; +static unsigned long vm_get_base_address(uint64_t flags, uint32_t type); - assert((new_base & (PAGE_SIZE - 1)) == 0); +static int vm_alloc_address(struct vma_iterator *vmi, u64 flags, size_t size, int type) + REQUIRES(vmi->mm->vm_lock) +{ + struct mm_address_space *mm = vmi->mm; + unsigned long min = vm_get_base_address(flags, type); - struct vm_area_struct *reg = vm_reserve_region(as, new_base, size); + if (min < mm->start) + min = mm->start; + if (mas_empty_area(&vmi->mas, min, mm->end, size) != 0) + return -ENOMEM; - if (reg) - { - increment_vm_stat(as, virtual_memory_size, size); - } + unsigned long new_base = vmi->mas.index; + CHECK((new_base & (PAGE_SIZE - 1)) == 0); + if (!arch_vm_validate_mmap_region(new_base, size, flags)) + return -ENOMEM; - return reg; + vmi->index = vmi->mas.index; + vmi->end = vmi->mas.last; + return 0; } void vm_addr_init() @@ -307,6 +244,8 @@ void vm_init() bootmem_reserve(limits.start_phys, limits.end_phys - limits.start_phys); } +extern "C" void maple_tree_init(); + /** * @brief Initialises the architecture independent parts of the VM subsystem. * @@ -325,6 +264,7 @@ void vm_late_init() kmalloc_init(); + maple_tree_init(); vm_area_struct_cache = kmem_cache_create("vm_area_struct", sizeof(vm_area_struct), 0, 0, nullptr); @@ -333,30 +273,6 @@ void vm_late_init() vm_addr_init(); - scoped_mutex g{kernel_address_space.vm_lock}; - - /* Start populating the address space */ - struct kernel_limits l; - get_kernel_limits(&l); - size_t kernel_size = l.end_virt - l.start_virt; - - struct vm_area_struct *v = vm_reserve_region(&kernel_address_space, l.start_virt, kernel_size); - if (!v) - { - panic("vmm: early boot oom"); - } - - v->vm_flags = VM_WRITE | VM_READ | VM_EXEC; - - v = vm_reserve_region(&kernel_address_space, vmalloc_space, vmalloc_len); - - if (!v) - { - panic("vmm: early boot oom"); - } - - v->vm_flags = VM_WRITE | VM_READ; - vm_zero_page = alloc_page(0); assert(vm_zero_page != nullptr); @@ -387,7 +303,7 @@ static inline bool inode_requires_wb(struct inode *i) bool vm_mapping_requires_wb(struct vm_area_struct *reg) { - return reg->vm_maptype == MAP_SHARED && reg->vm_file && inode_requires_wb(reg->vm_file->f_ino); + return vma_shared(reg) && reg->vm_file && inode_requires_wb(reg->vm_file->f_ino); } bool vm_mapping_is_anon(struct vm_area_struct *reg) @@ -407,8 +323,6 @@ void vm_make_anon(struct vm_area_struct *reg) fd_put(reg->vm_file); reg->vm_file = nullptr; } - - reg->vm_maptype |= MAP_ANONYMOUS; } bool vm_mapping_requires_write_protect(struct vm_area_struct *reg) @@ -416,7 +330,7 @@ bool vm_mapping_requires_write_protect(struct vm_area_struct *reg) return vm_mapping_requires_wb(reg); } -void vm_area_struct_destroy(struct vm_area_struct *region) +static void vma_destroy(struct vm_area_struct *region) { MUST_HOLD_MUTEX(®ion->vm_mm->vm_lock); @@ -432,25 +346,16 @@ void vm_area_struct_destroy(struct vm_area_struct *region) vmo_unref(region->vm_obj); } - if (region->vm_amap) - { - amap_unref(region->vm_amap); - } - memset_explicit(region, 0xfd, sizeof(struct vm_area_struct)); vma_free(region); } -unsigned long vm_get_base_address(uint64_t flags, uint32_t type) +static unsigned long vm_get_base_address(uint64_t flags, uint32_t type) { bool is_kernel_map = flags & VM_KERNEL; - struct mm_address_space *mm = nullptr; - - if (!is_kernel_map) - { - mm = get_current_address_space(); - } + DCHECK(!is_kernel_map); + struct mm_address_space *mm = get_current_address_space(); switch (type) { @@ -460,116 +365,30 @@ unsigned long vm_get_base_address(uint64_t flags, uint32_t type) return (uintptr_t) mm->mmap_base; } - case VM_TYPE_MODULE: { - assert(is_kernel_map == true); - - return KERNEL_VIRTUAL_BASE; - } - default: case VM_TYPE_REGULAR: { - if (is_kernel_map) - return vmalloc_space; - else - return (uintptr_t) mm->mmap_base; + return (uintptr_t) mm->mmap_base; } } } -static struct vm_area_struct *__vm_allocate_virt_region(struct mm_address_space *as, uint64_t flags, - size_t pages, uint32_t type, uint64_t prot) - REQUIRES(as->vm_lock) -{ - MUST_HOLD_MUTEX(&as->vm_lock); - - unsigned long base_addr = vm_get_base_address(flags, type); - struct vm_area_struct *region = vm_allocate_region(as, base_addr, pages << PAGE_SHIFT, flags); - - if (region) - { - if (prot & (VM_WRITE | VM_EXEC)) - prot |= VM_READ; - region->vm_flags = prot; - } - - return region; -} - vm_area_struct *vm_search(struct mm_address_space *mm, void *addr, size_t length) REQUIRES_SHARED(mm->vm_lock) { - struct search_type - { - unsigned long base; - size_t pages; - struct bst_node node; - } search; - - search.base = (unsigned long) addr & -PAGE_SIZE; - search.pages = vm_size_to_pages(length); - - auto node = bst_search(&mm->region_tree, &search.node, - [](struct bst_node *lhs_, struct bst_node *rhs_) -> int { - auto lhs = container_of(lhs_, vm_area_struct, vm_tree_node); - auto rhs = container_of(rhs_, search_type, node); - - if (check_for_overlap(lhs->vm_start, lhs->vm_end - 1, rhs->base, - rhs->base + (rhs->pages << PAGE_SHIFT) - 1)) - { - return 0; - } - else if (rhs->base > lhs->vm_start) - return 1; - else - return -1; - }); - - return containerof_null_safe(node, vm_area_struct, vm_tree_node); + unsigned long index = (unsigned long) addr; + void *entry = mt_find(&mm->region_tree, &index, index + length - 1); + struct vm_area_struct *vma = (struct vm_area_struct *) entry; + if (vma && vma->vm_start > (unsigned long) addr) + return nullptr; + return vma; } -#define VM_CREATE_REGION_AT_DEBUG 0 - -struct vm_area_struct *__vm_create_region_at(struct mm_address_space *mm, void *addr, size_t pages, - uint32_t type, uint64_t prot) REQUIRES(mm->vm_lock) +static struct vm_area_struct *__vm_create_region_at(struct mm_address_space *mm, void *addr, + size_t pages, uint32_t type, uint64_t prot) + REQUIRES(mm->vm_lock) { - struct vm_area_struct *v = nullptr; - - if (vm_search(mm, addr, pages << PAGE_SHIFT) != nullptr) - { - /* We found a region, this is not empty! */ - -#if VM_CREATE_REGION_AT_DEBUG - sched_enable_preempt(); - printk("Failed to map %p - %lx\n", addr, (unsigned long) addr + (pages << PAGE_SHIFT)); - vm_print_umap(); - sched_disable_preempt(); -#endif - - errno = EINVAL; - return nullptr; - } - - if (!vm_test_vs_rlimit(mm, pages << PAGE_SHIFT)) - { - return errno = ENOMEM, nullptr; - } - - v = vm_reserve_region(mm, (unsigned long) addr, pages << PAGE_SHIFT); - if (!v) - { - addr = nullptr; - errno = ENOMEM; - goto return_; - } - - increment_vm_stat(mm, virtual_memory_size, pages << PAGE_SHIFT); - - v->vm_start = (unsigned long) addr; - v->vm_end = v->vm_start + (pages << PAGE_SHIFT); - v->vm_flags = prot; - -return_: - return v; + /* TODO: remove once sys_mremap gets improved and tested */ + return nullptr; } /** @@ -588,33 +407,25 @@ int vm_clone_as(mm_address_space *addr_space, mm_address_space *original) return paging_clone_as(addr_space, original); } -struct fork_iteration -{ - struct mm_address_space *target_mm; - bool success; -}; - #define DEBUG_FORK_VM 0 -static bool fork_vm_area_struct(struct vm_area_struct *region, struct fork_iteration *it) +static bool fork_vm_area_struct(struct vm_area_struct *region, struct mm_address_space *mm) { bool vmo_failure, is_private, needs_to_fork_memory; bool res; struct vm_area_struct *new_region = vma_alloc(); if (!new_region) - { - goto ohno; - } + return false; memcpy(new_region, region, sizeof(*region)); + /* TODO: mtree dup */ #if DEBUG_FORK_VM printk("Forking [%016lx, %016lx] perms %x\n", region->base, region->base + (region->pages << PAGE_SHIFT) - 1, region->rwx); #endif - bst_node_initialize(&new_region->vm_tree_node); - res = vm_insert_region(it->target_mm, new_region); + res = vm_insert_region(mm, new_region); assert(res == true); @@ -622,7 +433,7 @@ static bool fork_vm_area_struct(struct vm_area_struct *region, struct fork_itera fd_get(new_region->vm_file); vmo_failure = false; - is_private = !is_mapping_shared(new_region); + is_private = vma_private(new_region); needs_to_fork_memory = is_private; if (needs_to_fork_memory) @@ -633,11 +444,6 @@ static bool fork_vm_area_struct(struct vm_area_struct *region, struct fork_itera vmo_assign_mapping(new_region->vm_obj, new_region); vmo_ref(new_region->vm_obj); } - - if (new_region->vm_amap) - { - amap_ref(new_region->vm_amap); - } } else { @@ -647,21 +453,17 @@ static bool fork_vm_area_struct(struct vm_area_struct *region, struct fork_itera if (vmo_failure) { - vm_remove_region(it->target_mm, new_region); + vm_remove_region(mm, new_region); vma_free(new_region); - goto ohno; + return false; } - new_region->vm_mm = it->target_mm; + new_region->vm_mm = mm; - if (mmu_fork_tables(region, it->target_mm) < 0) - goto ohno; - mmu_verify_address_space_accounting(it->target_mm); + if (mmu_fork_tables(region, mm) < 0) + return false; + mmu_verify_address_space_accounting(mm); return true; - -ohno: - it->success = false; - return false; } static void addr_space_delete(vm_area_struct *region) NO_THREAD_SAFETY_ANALYSIS @@ -670,7 +472,7 @@ static void addr_space_delete(vm_area_struct *region) NO_THREAD_SAFETY_ANALYSIS // is called in fork paths. do_vm_unmap(region->vm_mm, (void *) region->vm_start, vma_pages(region)); - vm_area_struct_destroy(region); + vma_destroy(region); } static void tear_down_addr_space(struct mm_address_space *addr_space) @@ -680,8 +482,11 @@ static void tear_down_addr_space(struct mm_address_space *addr_space) * If we didn't we would leak some memory. */ vm_area_struct *entry; - bst_for_every_entry_delete(&addr_space->region_tree, entry, vm_area_struct, vm_tree_node) + void *entry_; + unsigned long index = 0; + mt_for_each(&addr_space->region_tree, entry_, index, -1UL) { + entry = (vm_area_struct *) entry_; addr_space_delete(entry); } @@ -704,22 +509,19 @@ int vm_fork_address_space(struct mm_address_space *addr_space) EXCLUDES(addr_spa mmu_verify_address_space_accounting(get_current_address_space()); #endif - struct fork_iteration it = {}; - it.target_mm = addr_space; - it.success = true; - if (paging_clone_as(addr_space, current_mm) < 0) return -ENOMEM; - bst_root_initialize(&addr_space->region_tree); - addr_space->resident_set_size = 0; addr_space->virtual_memory_size = current_mm->virtual_memory_size; vm_area_struct *entry; - bst_for_every_entry(¤t_mm->region_tree, entry, vm_area_struct, vm_tree_node) + void *entry_; + unsigned long index = 0; + mt_for_each(¤t_mm->region_tree, entry_, index, -1UL) { - if (!fork_vm_area_struct(entry, &it)) + entry = (vm_area_struct *) entry_; + if (!fork_vm_area_struct(entry, addr_space)) { tear_down_addr_space(addr_space); return -1; @@ -743,7 +545,7 @@ int vm_fork_address_space(struct mm_address_space *addr_space) EXCLUDES(addr_spa assert(addr_space->active_mask.is_empty()); mutex_init(&addr_space->vm_lock); - + validate_mm_tree(addr_space); return 0; } @@ -783,51 +585,170 @@ void vm_change_perms(void *range, size_t pages, int perms) NO_THREAD_SAFETY_ANAL mutex_unlock(&as->vm_lock); } -bool vm_may_merge_with_adj(vm_area_struct *reg) +static bool vma_can_merge_into(struct vm_area_struct *vma, size_t size, int vm_flags, + struct file *file, off_t off, bool before) { - // TODO: merging is broken right now - return false; -#if 0 - auto prev_node = bst_prev(®->mm->region_tree, ®->tree_node); + if (vma->vm_file && file) + { + off_t desired_vma_off = vma->vm_offset + (vma->vm_end - vma->vm_start); + if (!before) + desired_vma_off = off + size; + if (off != desired_vma_off) + return false; + } - if (!prev_node) - return false; + return vma->vm_flags == vm_flags && vma->vm_file == file; +} - auto prev = container_of(prev_node, vm_area_struct, tree_node); +static void vma_post_adjust(struct vm_area_struct *vma) +{ + if (vma->vm_obj) + { + vm_obj_reassign_mapping(vma->vm_obj, vma); + spin_unlock(&vma->vm_obj->mapping_lock); + } +} - if (vmo_is_shared(prev->vmo)) - return false; +static struct vm_area_struct *vma_merge_around(struct vma_iterator *vmi, unsigned int vm_flags, + struct file *file, off_t off) + REQUIRES(vmi->mm->vm_lock) +{ + struct vm_area_struct *prev, *next, *ret = nullptr; + size_t new_size = vmi->end - vmi->index + 1; - return (false && prev->mapping_type == reg->mapping_type && reg->type == prev->type && - prev->base + (prev->pages << PAGE_SHIFT) == reg->base && prev->fd == reg->fd && - reg->fd == nullptr && reg->rwx == prev->rwx && !is_mapping_shared(reg) && - !is_file_backed(reg)); -#endif + prev = (struct vm_area_struct *) mas_prev(&vmi->mas, vmi->index - 1); + next = (struct vm_area_struct *) mas_next(&vmi->mas, vmi->end + 1); + + if (prev && !vma_can_merge_into(prev, new_size, vm_flags, file, off, true)) + prev = nullptr; + if (next && !vma_can_merge_into(next, new_size, vm_flags, file, off, false)) + next = nullptr; + + if (prev && next) + { + /* We can merge with prev *and* next. The whole range (prev->vm_start to next->vm_end) will + * be covered by a single VMA */ + DCHECK(prev->vm_end == vmi->index && next->vm_start == vmi->end + 1); + mas_set_range(&vmi->mas, prev->vm_start, next->vm_end - 1); + if (mas_store_gfp(&vmi->mas, prev, GFP_KERNEL) != 0) + return nullptr; + + vma_pre_adjust(prev); + prev->vm_end = next->vm_end; + if (next->vm_obj) + { + /* Remove ourselves from the vm obj */ + vmo_remove_mapping_locked(next->vm_obj, next); + } + + vma_post_adjust(prev); + vma_free(next); + ret = prev; + } + else if (prev) + { + /* Merging with prev, quite simple, just nudge vm_end */ + DCHECK(prev->vm_end == vmi->index); + mas_set_range(&vmi->mas, prev->vm_start, vmi->end); + if (mas_store_gfp(&vmi->mas, prev, GFP_KERNEL) != 0) + return nullptr; + vma_pre_adjust(prev); + prev->vm_end = vmi->end + 1; + vma_post_adjust(prev); + ret = prev; + } + else if (next) + { + /* Merging with next, nudge vm_start and vm_offset if required */ + DCHECK(next->vm_start == vmi->end + 1); + mas_set_range(&vmi->mas, vmi->index, next->vm_end - 1); + if (mas_store_gfp(&vmi->mas, next, GFP_KERNEL) != 0) + return nullptr; + vma_pre_adjust(next); + if (file) + next->vm_offset -= (next->vm_start - vmi->index); + next->vm_start = vmi->index; + vma_post_adjust(next); + ret = next; + } + + if (ret) + { + increment_vm_stat(vmi->mm, virtual_memory_size, new_size); + if (vma_shared(ret)) + increment_vm_stat(vmi->mm, shared_set_size, new_size); + } + return ret; } -void vm_merge_with_prev(vm_area_struct *reg) +static struct vm_area_struct *vma_create(struct vma_iterator *vmi, unsigned int vm_flags, + struct file *file, off_t off) { -#if 0 - auto prev_node = bst_prev(®->vm_mm->region_tree, ®->vm_tree_node); + int err = -ENOMEM; + size_t size = vmi->end - vmi->index + 1; + struct vm_area_struct *vma = nullptr; + if (!vm_test_vs_rlimit(vmi->mm, size)) + goto out_error; - assert(prev_node); - auto prev = container_of(prev_node, vm_area_struct, vm_tree_node); + vma = vma_alloc(); + if (!vma) + goto out_error; + memset(vma, 0, sizeof(*vma)); + vma->vm_start = vmi->index; + vma->vm_end = vmi->end + 1; + vma->vm_flags = vm_flags; + vma->vm_mm = vmi->mm; -#if 0 - printk("[%lx, %lx] + [%lx, %lx] =", prev->base, prev->base + (prev->pages << PAGE_SHIFT), - reg->base, reg->base + (reg->pages << PAGE_SHIFT)); -#endif - prev->vm_pages += reg->vm_pages; - auto oldsize = prev->vm_obj->size; - prev->vm_obj->size = oldsize + (reg->vm_pages << PAGE_SHIFT); - // vmo_truncate(prev->vmo, oldsize + ((reg->pages + 200) << PAGE_SHIFT), 0); - vm_remove_region(reg->vm_mm, reg); -#if 0 - printk(" [%lx, %lx]\n", prev->base, prev->base + (prev->pages << PAGE_SHIFT)); - printk("vmo size %lx -> %lx\n", oldsize, prev->vmo->size); -#endif - vma_free(reg); -#endif + err = mas_store_gfp(&vmi->mas, vma, GFP_KERNEL); + if (err) + goto free_vma; + + if (file) + { + vma->vm_offset = off; + vma->vm_file = file; + + fd_get(file); + + struct inode *ino = file->f_ino; + + if (S_ISCHR(ino->i_mode)) + { + if (!ino->i_fops->mmap) + { + err = -ENODEV; + goto unmap_vma; + } + + void *ret = ino->i_fops->mmap(vma, file); + if (!ret) + { + err = -errno; + goto unmap_vma; + } + + inode_update_atime(ino); + goto out; + } + } + + if (vma_setup_backing(vma, size >> PAGE_SHIFT, file != nullptr) < 0) + goto unmap_vma; + +out: + increment_vm_stat(vmi->mm, virtual_memory_size, size); + if (vma_shared(vma)) + increment_vm_stat(vmi->mm, shared_set_size, size); + + return vma; +unmap_vma: + if (file) + fd_put(file); + CHECK(mas_erase(&vmi->mas) == vma); +free_vma: + vma_free(vma); +out_error: + return (struct vm_area_struct *) ERR_PTR(err); } /** @@ -841,21 +762,19 @@ void vm_merge_with_prev(vm_area_struct *reg) * @param flags The mapping flags (see MAP_* as in mmap(2)). * @param file An optional pointer to a file, if it is a file mapping. * @param off The offset into the file, if it is a file mapping. - * @return A pointer to the new memory mapping, or NULL if it failed (errno is set). + * @return A pointer to the new memory mapping, or an ERR_PTR on error. */ void *vm_mmap(void *addr, size_t length, int prot, int flags, struct file *file, off_t off) { - int st = 0; - struct vm_area_struct *area = nullptr; - bool is_file_mapping = file != nullptr; - void *base = nullptr; + struct vm_area_struct *vma = nullptr; + unsigned long virt = (unsigned long) addr; u64 extra_flags = 0; struct mm_address_space *mm = get_current_address_space(); /* We don't like this offset. */ if (off & (PAGE_SIZE - 1)) - return errno = EINVAL, nullptr; + return ERR_PTR(-EINVAL); scoped_mutex g{mm->vm_lock}; @@ -865,132 +784,66 @@ void *vm_mmap(void *addr, size_t length, int prot, int flags, struct file *file, if (prot & (PROT_WRITE | PROT_EXEC)) prot |= PROT_READ; - int vm_prot = VM_USER | ((prot & PROT_READ) ? VM_READ : 0) | + int vm_prot = VM_USER | ((prot & (PROT_READ | PROT_WRITE | PROT_EXEC)) ? VM_READ : 0) | ((prot & PROT_WRITE) ? VM_WRITE : 0) | ((prot & PROT_EXEC) ? VM_EXEC : 0); + if (flags & MAP_SHARED) + vm_prot |= VM_SHARED; + /* Sanitize the address and length */ const auto aligned_len = pages << PAGE_SHIFT; if (aligned_len > arch_low_half_max) - { - st = -ENOMEM; - goto out_error; - } + return ERR_PTR(-ENOMEM); - if (is_higher_half(addr) || (unsigned long) addr & (PAGE_SIZE - 1) || - (unsigned long) addr > arch_low_half_max - aligned_len) + if (is_higher_half(addr) || virt & (PAGE_SIZE - 1) || virt > arch_low_half_max - aligned_len || + virt + aligned_len < arch_low_half_min) { if (flags & MAP_FIXED) - { - st = -ENOMEM; - goto out_error; - } + return ERR_PTR(-ENOMEM); else + { addr = nullptr; + virt = 0; + } } + VMA_ITERATOR(vmi, mm, virt, virt + aligned_len); + extra_flags = arch_vm_interpret_mmap_hint_flags(addr, flags); - if (!addr) + if (virt) { if (flags & MAP_FIXED) - { - st = -ENOMEM; - goto out_error; - } - /* Specified by POSIX, if addr == nullptr, guess an address */ - area = __vm_allocate_virt_region(mm, VM_ADDRESS_USER | extra_flags, pages, VM_TYPE_SHARED, - vm_prot); - } - else - { - if (flags & MAP_FIXED) - { __vm_munmap(mm, addr, pages << PAGE_SHIFT); - } - - area = __vm_create_region_at(mm, addr, pages, VM_TYPE_REGULAR, vm_prot); - if (!area) - { - if (flags & MAP_FIXED) - { - st = -ENOMEM; - goto out_error; - } - - area = __vm_allocate_virt_region(mm, VM_ADDRESS_USER | extra_flags, pages, - VM_TYPE_REGULAR, vm_prot); - } } - - if (!area) - { - st = -ENOMEM; - goto out_error; - } - - if (flags & MAP_SHARED) - area->vm_maptype = MAP_SHARED; else - area->vm_maptype = MAP_PRIVATE; - - if (is_file_mapping) - { - // printk("Mapping off %lx, size %lx, prots %x\n", off, length, prot); - - /* Set additional meta-data */ - - area->vm_offset = off; - area->vm_file = file; - - fd_get(file); - - struct inode *ino = file->f_ino; - - if (S_ISCHR(ino->i_mode)) - { - if (!ino->i_fops->mmap) - { - __vm_munmap(mm, (void *) area->vm_start, pages << PAGE_SHIFT); - return errno = ENODEV, nullptr; - } - - void *ret = ino->i_fops->mmap(area, file); - - if (ret) - inode_update_atime(ino); - else - __vm_munmap(mm, (void *) area->vm_start, pages << PAGE_SHIFT); - - if (ret) - goto out; - - return ret; - } - } - else if (vm_may_merge_with_adj(area)) { - base = (void *) area->vm_start; - vm_merge_with_prev(area); - return base; + if (vm_alloc_address(&vmi, VM_ADDRESS_USER | extra_flags, aligned_len, VM_TYPE_REGULAR) < 0) + return ERR_PTR(-ENOMEM); + virt = vmi.index; } - if (vm_area_struct_setup_backing(area, pages, !(flags & MAP_ANONYMOUS)) < 0) - { - __vm_munmap(mm, (void *) area->vm_start, pages << PAGE_SHIFT); - return errno = ENOMEM, nullptr; - } + if (!vm_test_vs_rlimit(mm, vmi.end - vmi.index + 1)) + return ERR_PTR(-ENOMEM); -out: - if (flags & MAP_SHARED) - increment_vm_stat(mm, shared_set_size, pages << PAGE_SHIFT); + vma = vma_merge_around(&vmi, vm_prot, file, off); + if (vma) + goto out; - base = (void *) area->vm_start; + /* vma_merge_around may touch around the vmi, reset the state */ + mas_set_range(&vmi.mas, vmi.index, vmi.end); - return base; + if (flags & MAP_ANONYMOUS) + file = nullptr; -out_error: - return errno = -st, nullptr; + vma = vma_create(&vmi, vm_prot, file, off); + if (IS_ERR(vma)) + return (void *) vma; + +out: + validate_mm_tree(mm); + return (void *) virt; } void *sys_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t off) @@ -1039,12 +892,6 @@ void *sys_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t off } ret = vm_mmap(addr, length, prot, flags, file, off); - - if (ret == nullptr) - { - ret = (void *) (unsigned long) -errno; - } - if (file) fd_put(file); @@ -1057,8 +904,6 @@ void *sys_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t off int sys_munmap(void *addr, size_t length) { - // printk("munmap [%p, %lx]\n", addr, (unsigned long) addr + length - 1); - if (is_higher_half(addr)) return -EINVAL; @@ -1074,37 +919,23 @@ int sys_munmap(void *addr, size_t length) return ret; } -void vm_copy_region(const struct vm_area_struct *source, struct vm_area_struct *dest, - int copy_amap = 1) +static void vm_copy_region(const struct vm_area_struct *source, struct vm_area_struct *dest) { dest->vm_file = source->vm_file; if (dest->vm_file) - { - /*struct file *ino = dest->fd->f_ino; - if(source->mapping_type == MAP_SHARED && inode_requires_wb(ino)) - writeback_add_region(dest);*/ fd_get(dest->vm_file); - } dest->vm_flags = source->vm_flags; - dest->vm_maptype = source->vm_maptype; dest->vm_offset = source->vm_offset; dest->vm_mm = source->vm_mm; dest->vm_obj = source->vm_obj; if (dest->vm_obj) vmo_ref(dest->vm_obj); - if (copy_amap) - { - dest->vm_amap = source->vm_amap; - if (dest->vm_amap) - amap_ref(dest->vm_amap); - } - dest->vm_ops = source->vm_ops; } -static void vma_pre_split(struct vm_area_struct *vma) +static void vma_pre_adjust(struct vm_area_struct *vma) { /* Lock the rmap intances. This stops us from every seeing an inconsistent data structure on * rmap's side. */ @@ -1126,40 +957,27 @@ static void vma_post_split(struct vm_area_struct *vma, struct vm_area_struct *ne static struct vm_area_struct *vm_split_region(struct mm_address_space *as, struct vm_area_struct *vma, unsigned long addr, - bool below) REQUIRES(as->vm_lock) + bool below, struct vma_iterator *vmi) + REQUIRES(as->vm_lock) { DCHECK((addr & (PAGE_SIZE - 1)) == 0); size_t region_off = addr - vma->vm_start; - size_t off_pages = region_off >> PAGE_SHIFT; struct vm_area_struct *newr = vma_alloc(); if (!newr) return nullptr; - memset(newr, 0, sizeof(*newr)); - vm_copy_region(vma, newr, 0); - - if (vma->vm_amap) + if (mas_expected_entries(&vmi->mas, 1) == -ENOMEM) { - auto new_amap = amap_split(vma->vm_amap, vma, off_pages); - if (!new_amap) - { - vma_free(newr); - return nullptr; - } - - /* Below region gets the original 'vma->vm_amap', top gets the new */ - if (below) - { - newr->vm_amap = vma->vm_amap; - vma->vm_amap = new_amap; - } - else - newr->vm_amap = new_amap; + vma_free(newr); + return nullptr; } + memset(newr, 0, sizeof(*newr)); + vm_copy_region(vma, newr); + DCHECK(vma->vm_end > addr); - vma_pre_split(vma); + vma_pre_adjust(vma); if (below) { @@ -1177,8 +995,11 @@ static struct vm_area_struct *vm_split_region(struct mm_address_space *as, } vma_post_split(vma, newr); - vm_insert_region(as, newr); + /* Reset the mas range to the new region */ + mas_set_range(&vmi->mas, newr->vm_start, newr->vm_end - 1); + CHECK(mas_store(&vmi->mas, newr) == vma); + DCHECK(vmi->mas.index == newr->vm_start); return newr; } @@ -1198,6 +1019,8 @@ static void vm_mprotect_handle_prot(struct vm_area_struct *region, int *pprot) } } +#if !defined(CONFIG_X86) && !defined(CONFIG_RISCV) +/* TODO: Remove once all architectures have been moved to the new shared page table code */ void vm_do_mmu_mprotect(struct mm_address_space *as, void *address, size_t nr_pgs, int old_prots, int new_prots) { @@ -1212,6 +1035,23 @@ void vm_do_mmu_mprotect(struct mm_address_space *as, void *address, size_t nr_pg vm_invalidate_range((unsigned long) addr, nr_pgs); } +#endif + +static struct vm_area_struct *vma_prepare_modify(struct vma_iterator *vmi, + struct vm_area_struct *vma, unsigned long start, + unsigned long end) REQUIRES(vmi->mm->vm_lock) +{ + if (start > vma->vm_start) + { + vma = vm_split_region(vmi->mm, vma, start, false, vmi); + if (!vma) + return nullptr; + } + + if (end < vma->vm_end) + vma = vm_split_region(vmi->mm, vma, end, true, vmi); + return vma; +} /** * @brief Changes memory protection of a memory range. @@ -1235,62 +1075,45 @@ int vm_mprotect(struct mm_address_space *as, void *__addr, size_t size, int prot * allow for a partial unmap in case of an error. Whereas this is not the case for mprotect. */ - struct vm_area_struct *region = vm_search(as, (void *) addr, PAGE_SIZE); - if (!region) + struct vm_area_struct *vma = vm_search(as, (void *) addr, PAGE_SIZE); + if (!vma) return -ENOMEM; - /* Split the head, if needed */ - if (addr > region->vm_start) - { - region = vm_split_region(as, region, addr, false); - if (!region) - return -ENOMEM; - DCHECK(addr == region->vm_start); - } + VMA_ITERATOR(vmi, as, addr, limit); - while (region) + void *entry_; + mas_for_each(&vmi.mas, entry_, vmi.end) { - /* Split the tail, if needed */ - if (limit < region->vm_end) - { - region = vm_split_region(as, region, limit, true); - if (!region) - return -ENOMEM; - DCHECK(limit == region->vm_end); - } - - if (region->vm_maptype == MAP_SHARED && region->vm_file && prot & PROT_WRITE) + vma = (vm_area_struct *) entry_; + if (vma->vm_start >= limit) + break; + vma = vma_prepare_modify(&vmi, vma, addr, limit); + if (!vma) + return -ENOMEM; + DCHECK(vma->vm_start >= addr && vma->vm_end <= limit); + if (vma_shared(vma) && vma->vm_file && prot & PROT_WRITE) { /* Block the mapping if we're trying to mprotect a shared mapping to PROT_WRITE while * not having the necessary perms on the file. */ - struct file *file = region->vm_file; + struct file *file = vma->vm_file; bool fd_has_write = fd_may_access(file, FILE_ACCESS_WRITE); if (!fd_has_write) return -EACCES; } - int old_prots = region->vm_flags; + int old_prots = vma->vm_flags; int new_prots = prot; - unsigned long vma_end = region->vm_end; - unsigned long end = ALIGN_TO(min(addr + size, vma_end), 4096); - unsigned long mprotect_size = end - addr; - DCHECK((mprotect_size & (PAGE_SIZE - 1)) == 0); - - vm_mprotect_handle_prot(region, &new_prots); - vm_do_mmu_mprotect(as, (void *) addr, mprotect_size >> PAGE_SHIFT, old_prots, new_prots); - - if (region->vm_end == limit) + vm_mprotect_handle_prot(vma, &new_prots); + vm_do_mmu_mprotect(as, (void *) vma->vm_start, (vma->vm_end - vma->vm_start) >> PAGE_SHIFT, + old_prots, new_prots); + if (vma->vm_end == limit) break; - - addr += mprotect_size; - size -= mprotect_size; - region = containerof_null_safe(bst_next(&as->region_tree, ®ion->vm_tree_node), - struct vm_area_struct, vm_tree_node); } + validate_mm_tree(as); return 0; } @@ -1322,7 +1145,7 @@ int sys_mprotect(void *addr, size_t len, int prot) return vm_mprotect(p->address_space.get(), addr, len, vm_prot); } -int vm_expand_brk(struct mm_address_space *as, size_t nr_pages) REQUIRES(as->vm_lock); +static int vm_expand_brk(struct mm_address_space *as, size_t nr_pages) REQUIRES(as->vm_lock); __always_inline int do_inc_brk(mm_address_space *as, void *oldbrk, void *newbrk) REQUIRES(as->vm_lock) @@ -1393,15 +1216,6 @@ static bool vm_print(struct vm_area_struct *region) return true; } -/** - * @brief Traverses the kernel's memory map and prints information. - * - */ -void vm_print_map(void) -{ - // rb_tree_traverse(kernel_address_space.area_tree, vm_print, nullptr); -} - /** * @brief Traverses the current process's memory map and prints information. * @@ -1466,7 +1280,7 @@ void *__map_pages_to_vaddr(struct mm_address_space *as, void *virt, void *phys, */ void *map_pages_to_vaddr(void *virt, void *phys, size_t size, size_t flags) { - return __map_pages_to_vaddr(nullptr, virt, phys, size, flags); + return __map_pages_to_vaddr(&kernel_address_space, virt, phys, size, flags); } static int vm_pf_get_page_from_vmo(struct vm_pf_context *ctx) @@ -1500,157 +1314,12 @@ static int find_page_err_to_signal(int st) } } -static int vm_prepare_write(struct inode *inode, struct page *p) -{ - /* TODO: All of this needs a good rework. We must be careful with i_size (we can't just allocate - * on a whole page like this). We need to retry if the page was truncated. This should not be - * core vm.cpp code. */ - lock_page(p); - - /* Correctness: We set the i_size before truncating pages from the page cache, so this should - * not race... I think? */ - size_t i_size = inode->i_size; - if (p->owner != inode->i_pages) - { - pr_warn("vm: (inode %lu, dev %lu) just had a truncate race, which is not yet handled " - "correctly...\n", - inode->i_inode, inode->i_dev); - unlock_page(p); - return -ENOENT; - } - - size_t len = PAGE_SIZE; - size_t offset = p->pageoff << PAGE_SHIFT; - if (offset + PAGE_SIZE > i_size) - len = i_size - offset; - - int st = inode->i_fops->prepare_write(inode, p, offset, 0, len); - filemap_mark_dirty(inode, p, p->pageoff); - unlock_page(p); - return st; -} - -int vm_handle_non_present_wp(struct fault_info *info, struct vm_pf_context *ctx) -{ - struct vm_area_struct *entry = ctx->entry; - - assert(info->read ^ info->write); - if (!info->write) - { - /* If we'll need to wp, write-protect */ - ctx->page_rwx &= ~VM_WRITE; - if (vm_mapping_is_anon(entry)) - { - ctx->page = vm_zero_page; - } - } - else - { - if (vm_mapping_requires_wb(entry)) - { - /* else handle it differently(we'll need) */ - int st = vm_pf_get_page_from_vmo(ctx); - if (st < 0) - { - info->signal = find_page_err_to_signal(st); - return -1; - } - - st = vm_prepare_write(ctx->entry->vm_file->f_ino, ctx->page); - if (st < 0) - { - page_unref(ctx->page); - ctx->page = nullptr; - info->signal = find_page_err_to_signal(st); - return -1; - } - } - else if (vm_mapping_is_anon(entry)) - { - /* This is done in vm_pf_get_page_from_vmo */ - } - } - - return 0; -} - -bool vm_mapping_is_cow(struct vm_area_struct *entry) -{ - return entry->vm_maptype == MAP_PRIVATE; -} - -int vm_handle_non_present_pf(struct vm_pf_context *ctx) -{ - struct vm_area_struct *entry = ctx->entry; - struct fault_info *info = ctx->info; - - if (vm_mapping_requires_write_protect(entry)) - { - if (vm_handle_non_present_wp(info, ctx) < 0) - return -1; - } - - /* If page wasn't set before by other fault handling code, just fetch from the vmo */ - if (ctx->page == nullptr) - { - DCHECK(entry->vm_obj != nullptr); - int st = vm_pf_get_page_from_vmo(ctx); - if (st != VMO_STATUS_OK) - { - info->signal = find_page_err_to_signal(st); - return -1; - } - } - - if (!vm_map_page(ctx->entry->vm_mm, ctx->vpage, (u64) page_to_phys(ctx->page), - ctx->page_rwx | VM_NOFLUSH, ctx->entry)) - { - page_unpin(ctx->page); - info->signal = VM_SIGSEGV; - return -1; - } - - page_unpin(ctx->page); - - return 0; -} - -int vm_handle_write_wb(struct vm_pf_context *ctx) -{ - unsigned long paddr = MAPPING_INFO_PADDR(ctx->mapping_info); - struct page *p = phys_to_page(paddr); - int st = 0; - struct inode *inode = p->owner->ino; - st = vm_prepare_write(inode, p); - if (st == 0) - { - paging_change_perms((void *) ctx->vpage, ctx->page_rwx); - vm_invalidate_range(ctx->vpage, 1); - } - - return st; -} - -int vm_handle_present_pf(struct vm_pf_context *ctx) +static bool vm_mapping_is_cow(struct vm_area_struct *entry) { - struct vm_area_struct *entry = ctx->entry; - struct fault_info *info = ctx->info; -#if 0 - printk("Handling present PF at %lx %s%s%s\n", ctx->info->fault_address, - ctx->info->write ? "W" : "-", ctx->info->read ? "R" : "-", ctx->info->exec ? "X" : "-"); -#endif - - if (info->write & !(ctx->mapping_info & PAGE_WRITABLE)) - { - if (vm_mapping_requires_wb(entry)) - return vm_handle_write_wb(ctx); - panic("Strange case inside vm_handle_present_pf"); - } - - return 0; + return vma_private(entry); } -int __vm_handle_pf(struct vm_area_struct *entry, struct fault_info *info) +static int __vm_handle_pf(struct vm_area_struct *entry, struct fault_info *info) { const pid_t pid = get_current_process()->pid_; const u64 addr = info->fault_address; @@ -1666,30 +1335,11 @@ int __vm_handle_pf(struct vm_area_struct *entry, struct fault_info *info) context.page_rwx = entry->vm_flags; context.mapping_info = get_mapping_info((void *) context.vpage); -#if 0 - struct process *p = get_current_process(); - - printk("fault on address %lx, page %lx, " - " present %s, process %d (%s)\n", context.info->fault_address, - context.vpage, context.mapping_info & PAGE_PRESENT ? "true" : "false", - p->pid, p->cmd_line); -#endif - if (entry->vm_ops && entry->vm_ops->fault) return entry->vm_ops->fault(&context); - if (context.mapping_info & PAGE_PRESENT) - { - if (vm_handle_present_pf(&context) < 0) - return -1; - } - else - { - if (vm_handle_non_present_pf(&context) < 0) - return -1; - } - - // printk("elapsed: %lu ns\n", end - start); + /* This is unreachable */ + CHECK(0); return 0; } @@ -1772,12 +1422,10 @@ static void vm_destroy_area(vm_area_struct *region) decrement_vm_stat(region->vm_mm, virtual_memory_size, region->vm_end - region->vm_start); - if (is_mapping_shared(region)) - { + if (vma_shared(region)) decrement_vm_stat(region->vm_mm, shared_set_size, region->vm_end - region->vm_start); - } - vm_area_struct_destroy(region); + vma_destroy(region); } /** @@ -1789,16 +1437,19 @@ void vm_destroy_addr_space(struct mm_address_space *mm) { bool free_pgd = true; - /* First, iterate through the rb tree and free/unmap stuff */ + /* First, iterate through the maple tree and free/unmap stuff */ scoped_mutex g{mm->vm_lock}; vm_area_struct *entry; - - bst_for_every_entry_delete(&mm->region_tree, entry, vm_area_struct, vm_tree_node) + void *entry_; + unsigned long index = 0; + mt_for_each(&mm->region_tree, entry_, index, -1UL) { + entry = (vm_area_struct *) entry_; vm_destroy_area(entry); } + mtree_destroy(&mm->region_tree); assert(mm->resident_set_size == 0); assert(mm->shared_set_size == 0); assert(mm->virtual_memory_size == 0); @@ -2120,9 +1771,9 @@ void vm_do_fatal_page_fault(struct fault_info *info) * @param is_file_backed True if file backed. * @return 0 on success, negative for errors. */ -int vm_area_struct_setup_backing(struct vm_area_struct *region, size_t pages, bool is_file_backed) +int vma_setup_backing(struct vm_area_struct *region, size_t pages, bool is_file_backed) { - bool is_shared = is_mapping_shared(region); + bool is_shared = vma_shared(region); if (!is_file_backed && is_shared) { @@ -2160,17 +1811,6 @@ int vm_area_struct_setup_backing(struct vm_area_struct *region, size_t pages, bo return 0; } -/** - * @brief Determines if a mapping is shared. - * - * @param region A pointer to the vm_area_struct. - * @return True if shared, false if not. - */ -bool is_mapping_shared(struct vm_area_struct *region) -{ - return region->vm_maptype == MAP_SHARED; -} - /** * @brief Determines if a mapping is file backed. * @@ -2193,28 +1833,7 @@ bool is_file_backed(struct vm_area_struct *region) void *map_page_list(struct page *pl, size_t size, uint64_t prot) EXCLUDES(kernel_address_space.vm_lock) { - // TODO: Maybe also use vmalloc for this? - scoped_mutex g{kernel_address_space.vm_lock}; - struct vm_area_struct *entry = __vm_allocate_virt_region( - &kernel_address_space, VM_KERNEL, vm_size_to_pages(size), VM_TYPE_REGULAR, prot); - if (!entry) - return nullptr; - void *vaddr = (void *) entry->vm_start; - - uintptr_t u = (uintptr_t) vaddr; - while (pl != nullptr) - { - if (!map_pages_to_vaddr((void *) u, page_to_phys(pl), PAGE_SIZE, prot)) - { - __vm_munmap(&kernel_address_space, vaddr, size); - return nullptr; - } - - pl = pl->next_un.next_allocation; - u += PAGE_SIZE; - } - - return vaddr; + return nullptr; } /** @@ -2235,9 +1854,6 @@ int vm_create_address_space(struct mm_address_space *mm) assert(mm->active_mask.is_empty() == true); mutex_init(&mm->vm_lock); - - bst_root_initialize(&mm->region_tree); - return 0; } @@ -2252,10 +1868,8 @@ int vm_create_brk(struct mm_address_space *mm) mm->brk = vm_mmap(vm_gen_brk_base(), 1 << PAGE_SHIFT, PROT_WRITE, MAP_PRIVATE | MAP_FIXED | MAP_ANON, nullptr, 0); - if (!mm->brk) - { - return -ENOMEM; - } + if (IS_ERR(mm->brk)) + return PTR_ERR(mm->brk); return 0; } @@ -2277,94 +1891,73 @@ void vm_remove_region(struct mm_address_space *as, struct vm_area_struct *region { MUST_HOLD_MUTEX(&as->vm_lock); - bst_delete(&as->region_tree, ®ion->vm_tree_node); -} - -int vm_add_region(struct mm_address_space *as, struct vm_area_struct *region) -{ - MUST_HOLD_MUTEX(&as->vm_lock); - - return vm_insert_region(as, region); + void *ret = mtree_erase(&as->region_tree, region->vm_start); + CHECK(ret == region); } int __vm_munmap(struct mm_address_space *as, void *__addr, size_t size) REQUIRES(as->vm_lock) { unsigned long addr = (unsigned long) __addr & -PAGE_SIZE; unsigned long limit = ALIGN_TO(((unsigned long) __addr) + size, PAGE_SIZE); - size = limit - addr; struct list_head list = LIST_HEAD_INIT(list); MUST_HOLD_MUTEX(&as->vm_lock); - struct vm_area_struct *region = vm_search(as, (void *) addr, PAGE_SIZE); - if (!region) + struct vm_area_struct *vma = vm_search(as, (void *) addr, PAGE_SIZE); + if (!vma) return -EINVAL; - /* Split the head, if needed */ - if (addr > region->vm_start) - { - region = vm_split_region(as, region, addr, false); - if (!region) - return -ENOMEM; - DCHECK(addr == region->vm_start); - } - /* Gather munmap regions into our local list. No permanent changes are done in this loop, * while regions are live *except unlinking from the BST*. */ - while (region) - { - /* Split the tail, if needed */ - if (limit < region->vm_end) - { - region = vm_split_region(as, region, limit, true); - if (!region) - goto restore; - DCHECK(limit == region->vm_end); - } - - addr = region->vm_end; - - struct vm_area_struct *next = containerof_null_safe( - bst_next(&as->region_tree, ®ion->vm_tree_node), struct vm_area_struct, vm_tree_node); - /* Note: vm_detached_node and tree_node alias each other, so we need to remove first, add - * later. - */ - vm_remove_region(as, region); - list_add_tail(®ion->vm_detached_node, &list); + VMA_ITERATOR(vmi, as, addr, limit); - if (limit == region->vm_end) + void *entry_; + mas_for_each(&vmi.mas, entry_, vmi.end) + { + vma = (vm_area_struct *) entry_; + if (vma->vm_start >= limit) + break; + vma = vma_prepare_modify(&vmi, vma, addr, limit); + if (!vma) + goto restore; + + DCHECK(vma->vm_start >= addr && vma->vm_end <= limit); + CHECK(mas_erase(&vmi.mas) == vma); + list_add_tail(&vma->vm_detached_node, &list); + if (limit == vma->vm_end) break; - region = next; } list_for_every_safe (&list) { - struct vm_area_struct *vma = container_of(l, struct vm_area_struct, vm_tree_node); - bool is_shared = is_mapping_shared(vma); + vma = container_of(l, struct vm_area_struct, vm_detached_node); + DCHECK(vma->vm_start >= addr && vma->vm_end <= limit); + bool is_shared = vma_shared(vma); unsigned long sz = vma->vm_end - vma->vm_start; vm_mmu_unmap(as, (void *) vma->vm_start, vma_pages(vma), vma); list_remove(&vma->vm_detached_node); - vm_area_struct_destroy(vma); + vma_destroy(vma); decrement_vm_stat(as, virtual_memory_size, sz); if (is_shared) decrement_vm_stat(as, shared_set_size, sz); } + validate_mm_tree(as); return 0; restore: /* Something back there just failed, restore the old regions and -ENOMEM */ list_for_every_safe (&list) { - struct vm_area_struct *vma = container_of(l, struct vm_area_struct, vm_tree_node); + vma = container_of(l, struct vm_area_struct, vm_detached_node); list_remove(&vma->vm_detached_node); - bst_node_initialize(&vma->vm_tree_node); vm_insert_region(as, vma); } + validate_mm_tree(as); return -ENOMEM; } @@ -2407,25 +2000,21 @@ void vm_invalidate_range(unsigned long addr, size_t pages) return mmu_invalidate_range(addr, pages, get_current_address_space()); } -struct vm_area_struct *vm_next_region(mm_address_space *as, vm_area_struct *region) -{ - auto node = bst_next(&as->region_tree, ®ion->vm_tree_node); - - return node ? container_of(node, vm_area_struct, vm_tree_node) : nullptr; -} - -bool vm_can_expand(struct mm_address_space *as, struct vm_area_struct *region, size_t new_size) +static bool vm_can_expand(struct mm_address_space *as, struct vm_area_struct *region, + size_t new_size) { /* Can always shrink the mapping */ if (new_size < region->vm_end - region->vm_start) return true; - auto next = vm_next_region(as, region); + unsigned long index = region->vm_end; + void *ret = mt_find_after(&as->region_tree, &index, as->end); // If there's no region after this one, we're clear to expand // TODO: What if we overflow here? - if (!next) + if (!ret) return true; + struct vm_area_struct *next = (struct vm_area_struct *) ret; /* Calculate the hole size, and if >= new_size, we're good */ size_t hole_size = next->vm_start - region->vm_start; @@ -2433,7 +2022,7 @@ bool vm_can_expand(struct mm_address_space *as, struct vm_area_struct *region, s return hole_size >= new_size; } -int __vm_expand_mapping(struct vm_area_struct *region, size_t new_size) +static int __vm_expand_mapping(struct vm_area_struct *region, size_t new_size) { size_t diff = new_size - (region->vm_end - region->vm_start); if (!vm_test_vs_rlimit(region->vm_mm, new_size)) @@ -2441,26 +2030,28 @@ int __vm_expand_mapping(struct vm_area_struct *region, size_t new_size) region->vm_end += diff; increment_vm_stat(region->vm_mm, virtual_memory_size, diff); - if (is_mapping_shared(region)) + if (vma_shared(region)) increment_vm_stat(region->vm_mm, shared_set_size, diff); + int st = mtree_store_range(®ion->vm_mm->region_tree, region->vm_start, region->vm_end - 1, + region, GFP_KERNEL); + CHECK(st == 0); + validate_mm_tree(region->vm_mm); return 0; } -int vm_expand_mapping(struct mm_address_space *as, struct vm_area_struct *region, size_t new_size) - REQUIRES(as->vm_lock) +static int vm_expand_mapping(struct mm_address_space *as, struct vm_area_struct *region, + size_t new_size) REQUIRES(as->vm_lock) { MUST_HOLD_MUTEX(&as->vm_lock); if (!vm_can_expand(as, region, new_size)) - { return -1; - } return __vm_expand_mapping(region, new_size); } -int vm_expand_brk(struct mm_address_space *as, size_t nr_pages) REQUIRES(as->vm_lock) +static int vm_expand_brk(struct mm_address_space *as, size_t nr_pages) REQUIRES(as->vm_lock) { struct vm_area_struct *brk_region = vm_find_region(as, as->brk); assert(brk_region != nullptr); @@ -2469,8 +2060,8 @@ int vm_expand_brk(struct mm_address_space *as, size_t nr_pages) REQUIRES(as->vm_ return vm_expand_mapping(as, brk_region, new_size); } -int mremap_check_for_overlap(void *__old_address, size_t old_size, void *__new_address, - size_t new_size) +static int mremap_check_for_overlap(void *__old_address, size_t old_size, void *__new_address, + size_t new_size) { unsigned long old_address = (unsigned long) __old_address; unsigned long new_address = (unsigned long) __new_address; @@ -2489,110 +2080,23 @@ void *vm_remap_create_new_mapping_of_shared_pages(struct mm_address_space *mm, v size_t new_size, int flags, void *old_address) REQUIRES(mm->vm_lock) { - void *ret = MAP_FAILED; - bool fixed = flags & MREMAP_FIXED; - struct vm_area_struct *new_mapping = nullptr; - - struct vm_area_struct *old_region = vm_find_region(mm, old_address); - if (!old_region) - { - ret = (void *) -EFAULT; - goto out; - } - - if (old_region->vm_maptype != MAP_SHARED) - { - ret = (void *) -EINVAL; - goto out; - } - - if (fixed) - { - if (vm_sanitize_address(new_address, new_size >> PAGE_SHIFT) < 0) - { - ret = (void *) -EINVAL; - goto out; - } - - if (mremap_check_for_overlap(old_address, new_size, new_address, new_size) < 0) - { - ret = (void *) -EINVAL; - goto out; - } - - new_mapping = __vm_create_region_at(mm, new_address, new_size >> PAGE_SHIFT, - VM_TYPE_REGULAR, old_region->vm_flags); - } - else - { - new_mapping = - vm_allocate_region(mm, (unsigned long) mm->mmap_base, new_size, VM_ADDRESS_USER); - if (new_mapping) - { - new_mapping->vm_flags = old_region->vm_flags; - } - } - - if (!new_mapping) - { - ret = (void *) -ENOMEM; - goto out; - } - - vm_copy_region(old_region, new_mapping); - ret = (void *) new_mapping->vm_start; -out: - return ret; + return (void *) -ENOMEM; } -void *vm_try_move(struct mm_address_space *mm, struct vm_area_struct *old_region, - unsigned long new_base, size_t new_size) REQUIRES(mm->vm_lock) +static void *vm_try_move(struct mm_address_space *mm, struct vm_area_struct *old_region, + unsigned long new_base, size_t new_size) REQUIRES(mm->vm_lock) { - vm_remove_region(mm, old_region); - - old_region->vm_start = new_base; - if (int st = __vm_expand_mapping(old_region, new_size); st < 0) - return (void *) (unsigned long) st; - - /* TODO: What to do in case of a failure? */ - vm_add_region(mm, old_region); - - /* TODO: Maybe unmapping isn't the best option on a move and we should copy mappings */ - __vm_unmap_range(mm, (void *) old_region->vm_start, vma_pages(old_region)); - - vm_print_umap(); - return (void *) old_region->vm_start; + return (void *) -ENOMEM; } -void *vm_remap_try(struct mm_address_space *as, void *old_address, size_t old_size, - void *new_address, size_t new_size, int flags) REQUIRES(as->vm_lock) +static void *vm_remap_try(struct mm_address_space *as, void *old_address, size_t old_size, + void *new_address, size_t new_size, int flags) REQUIRES(as->vm_lock) { - struct vm_area_struct *reg = vm_find_region(as, old_address); - if (!reg) - return (void *) -EFAULT; -#if 0 - struct vm_area_struct *old_reg = vm_split_region(as, reg, (unsigned long) old_address, old_size, &n); - if (!old_reg) - return (void *) -ENOMEM; - - if (vm_expand_mapping(as, old_reg, new_size) < 0) - { - if (flags & MREMAP_MAYMOVE) - { - unsigned long new_base = - vm_allocate_base(as, (unsigned long) as->mmap_base, new_size, VM_ADDRESS_USER); - return vm_try_move(as, old_reg, new_base, new_size); - } - - return (void *) -ENOMEM; - } - - return (void *) old_reg->base; -#endif return (void *) -ENOMEM; } -bool limits_are_contained(struct vm_area_struct *reg, unsigned long start, unsigned long limit) +static bool limits_are_contained(struct vm_area_struct *reg, unsigned long start, + unsigned long limit) { unsigned long reg_limit = reg->vm_end; @@ -2693,8 +2197,8 @@ void vm_wp_page(struct mm_address_space *mm, void *vaddr) mmu_invalidate_range((unsigned long) vaddr, 1, mm); } -int get_phys_pages_direct(unsigned long addr, unsigned int flags, struct page **pages, - size_t nr_pgs) +static int get_phys_pages_direct(unsigned long addr, unsigned int flags, struct page **pages, + size_t nr_pgs) { if (flags & GPP_USER) return GPP_ACCESS_FAULT; @@ -2715,7 +2219,7 @@ int get_phys_pages_direct(unsigned long addr, unsigned int flags, struct page ** return GPP_ACCESS_OK | GPP_ACCESS_PFNMAP; } -int gpp_try_to_fault_in(unsigned long addr, struct vm_area_struct *entry, unsigned int flags) +static int gpp_try_to_fault_in(unsigned long addr, struct vm_area_struct *entry, unsigned int flags) { struct fault_info finfo; finfo.signal = 0; @@ -2734,8 +2238,8 @@ int gpp_try_to_fault_in(unsigned long addr, struct vm_area_struct *entry, unsign return GPP_ACCESS_OK; } -int __get_phys_pages(struct vm_area_struct *region, unsigned long addr, unsigned int flags, - struct page **pages, size_t nr_pgs) +static int __get_phys_pages(struct vm_area_struct *region, unsigned long addr, unsigned int flags, + struct page **pages, size_t nr_pgs) { unsigned long page_rwx_mask = (flags & GPP_READ ? PAGE_PRESENT : 0) | (flags & GPP_WRITE ? PAGE_WRITABLE : 0) | @@ -2807,7 +2311,7 @@ int get_phys_pages(void *_addr, unsigned int flags, struct page **pages, size_t goto out; } - if (reg->vm_maptype == MAP_SHARED) + if (vma_shared(reg)) had_shared_pages = true; /* Do a permission check. */ @@ -2888,23 +2392,34 @@ int sys_msync(void *ptr, size_t length, int flags) /* Hogging the vm_lock is bad mkay, todo... */ scoped_mutex g{as->vm_lock}; - struct vm_area_struct *vma = vm_search(as, (void *) addr, PAGE_SIZE); + struct vm_area_struct *vma = vm_search(as, (void *) addr, length); if (vma) { /* Check if start <= addr */ if (vma->vm_start > addr) return -ENOMEM; + /* The first vma may have a gap wrt the addr, so readjust it */ + addr = vma->vm_start; } + else + return -ENOMEM; - while (vma) + VMA_ITERATOR(vmi, as, addr, limit); + void *entry_; + mas_for_each(&vmi.mas, entry_, vmi.end) { + vma = (vm_area_struct *) entry_; + + /* We must watch out for gaps in the address space and -ENOMEM there */ + if (vma->vm_start != addr) + break; if (vma->vm_start > limit || vma->vm_end < addr) break; unsigned long to_sync = cul::min(length, vma->vm_end - addr); struct file *filp = vma->vm_file; - if (flags & MS_SYNC && filp && is_mapping_shared(vma)) + if (flags & MS_SYNC && filp && vma_shared(vma)) { unsigned long start = vma->vm_offset + addr - vma->vm_start; unsigned long end = start + to_sync; @@ -2918,16 +2433,11 @@ int sys_msync(void *ptr, size_t length, int flags) addr += to_sync; length -= to_sync; - vma = containerof_null_safe(bst_next(&as->region_tree, &vma->vm_tree_node), - struct vm_area_struct, vm_tree_node); if (!length) { st = 0; break; } - - if (vma && vma->vm_start != addr) - break; } return st; @@ -2945,7 +2455,6 @@ expected, int> mm_address_space::create() return unexpected{-ENOENT}; spinlock_init(&as->page_table_lock); - bst_root_initialize(&as->region_tree); int st = vm_clone_as(as.get()); if (st < 0) @@ -2966,7 +2475,6 @@ expected, int> mm_address_space::fork() return unexpected{-ENOENT}; spinlock_init(&as->page_table_lock); - bst_root_initialize(&as->region_tree); int st = vm_fork_address_space(as.get()); if (st < 0) @@ -3017,3 +2525,21 @@ mm_address_space::~mm_address_space() { vm_destroy_addr_space(this); } + +unsigned long get_mapping_info(void *addr) +{ + struct mm_address_space *as = &kernel_address_space; + if ((unsigned long) addr < VM_HIGHER_HALF) + as = get_current_address_space(); + + return __get_mapping_info(addr, as); +} + +bool paging_change_perms(void *addr, int prot) +{ + struct mm_address_space *as = &kernel_address_space; + if ((unsigned long) addr < VM_HIGHER_HALF) + as = get_current_address_space(); + + return __paging_change_perms(as, addr, prot); +} diff --git a/kernel/kernel/mm/vm_object.cpp b/kernel/kernel/mm/vm_object.cpp index c6a24974a..cd6d180da 100644 --- a/kernel/kernel/mm/vm_object.cpp +++ b/kernel/kernel/mm/vm_object.cpp @@ -360,6 +360,18 @@ void vmo_assign_mapping(vm_object *vmo, vm_area_struct *vma) vmo_assign_mapping_locked(vmo, vma); } +/** + * @brief Removes a mapping from the VMO + * Does not take the lock + * @param vmo The target vm object + * @param vma The VMA + */ +void vmo_remove_mapping_locked(struct vm_object *vmo, struct vm_area_struct *vma) +{ + DCHECK(spin_lock_held(&vmo->mapping_lock)); + interval_tree_remove(&vmo->mappings, &vma->vm_objhead); +} + /** * @brief Removes a mapping on the VMO. * @@ -369,7 +381,7 @@ void vmo_assign_mapping(vm_object *vmo, vm_area_struct *vma) void vmo_remove_mapping(vm_object *vmo, vm_area_struct *region) { scoped_lock g{vmo->mapping_lock}; - interval_tree_remove(&vmo->mappings, ®ion->vm_objhead); + vmo_remove_mapping_locked(vmo, region); } void vm_obj_reassign_mapping(struct vm_object *vm_obj, struct vm_area_struct *vma) @@ -540,8 +552,15 @@ bool vm_obj_remove_page(struct vm_object *obj, struct page *page) if (__atomic_load_n(&page->ref, __ATOMIC_RELAXED) > expected_refs) return false; obj->unmap_page(page->pageoff << PAGE_SHIFT); + if (page_mapcount(page) > 0) + { + /* It's entirely possible the page's mapcount is larger than 0. If, for instance, the page + * removal and a fork race in any way (fork does not take the page lock). In such cases, + * fail to remove. If mapcount is 0, we know it is stable (it cannot be forked, because it + * is unmapped; and, since we hold the page lock, no new mappers may arise.) */ + return false; + } /* After this, page->ref must be 1 (if the VM system is working properly) */ - CHECK_PAGE(page_mapcount(page) == 0, page); CHECK_PAGE(page->ref == 1, page); obj->vm_pages.store(page->pageoff, 0); return true; diff --git a/kernel/kernel/mm/vm_tests.cpp b/kernel/kernel/mm/vm_tests.cpp index 5d8e643bd..11eb01a1e 100644 --- a/kernel/kernel/mm/vm_tests.cpp +++ b/kernel/kernel/mm/vm_tests.cpp @@ -14,7 +14,7 @@ struct vm_area_struct *vm_reserve_region(struct mm_address_space *as, unsigned l size_t size); unsigned long vm_allocate_base(struct mm_address_space *as, unsigned long min, size_t size, u64 flags); - +#if 0 TEST(mmap, test_range_at_end) { // Test if an allocation cannot go overboard in the address space @@ -75,3 +75,4 @@ TEST(mmap, test_48_57_bit) } #endif +#endif diff --git a/kernel/kernel/net/loopback.cpp b/kernel/kernel/net/loopback.cpp index 305464f7a..2960ec14f 100644 --- a/kernel/kernel/net/loopback.cpp +++ b/kernel/kernel/net/loopback.cpp @@ -70,6 +70,7 @@ int loopback_pollrx(netif *nif) spin_unlock(&pqueue_lock); netif_process_pbuf(nif, pbuf); + pbuf->unref(); // Relock for the next run. spin_lock(&pqueue_lock); diff --git a/kernel/kernel/process.cpp b/kernel/kernel/process.cpp index 5cd5ecb26..e10ce2ce3 100644 --- a/kernel/kernel/process.cpp +++ b/kernel/kernel/process.cpp @@ -1314,7 +1314,7 @@ ssize_t process::query_vm_regions(void *ubuf, ssize_t len, unsigned long what, s vm_for_every_region(*address_space, [&](vm_area_struct *region) -> bool { onx_process_vm_region *reg = (onx_process_vm_region *) ptr; reg->size = sizeof(onx_process_vm_region); - reg->mapping_type = region->vm_maptype; + reg->mapping_type = vma_shared(region) ? MAP_SHARED : MAP_PRIVATE; reg->protection = 0; if (region->vm_flags & VM_READ) diff --git a/kernel/kernel/vdso.cpp b/kernel/kernel/vdso.cpp index 32f5061ad..e34b2d9f9 100644 --- a/kernel/kernel/vdso.cpp +++ b/kernel/kernel/vdso.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2023 Pedro Falcato + * Copyright (c) 2016 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -55,7 +56,7 @@ class vdso bool create_vmo() { - vdso_file = anon_inode_open(S_IFCHR, &dummy_fops, "[vdso]"); + vdso_file = anon_inode_open(S_IFREG, &dummy_fops, "[vdso]"); CHECK(vdso_file); auto vmo = vdso_file->f_ino->i_pages; @@ -185,7 +186,14 @@ __attribute__((no_sanitize_undefined)) bool vdso::init() void *vdso::map() { - return vm_mmap(nullptr, length, PROT_READ | PROT_EXEC, MAP_PRIVATE, vdso_file, 0); + void *addr = vm_mmap(nullptr, length, PROT_READ | PROT_EXEC, MAP_PRIVATE, vdso_file, 0); + if (IS_ERR(addr)) + { + pr_info("vdso: Failed to map vdso: %ld\n", PTR_ERR(addr)); + return nullptr; + } + + return addr; } void *vdso_map(void) diff --git a/usystem/filesystem/trunctests/main.c b/usystem/filesystem/trunctests/main.c index 7316d55fd..531e6c299 100644 --- a/usystem/filesystem/trunctests/main.c +++ b/usystem/filesystem/trunctests/main.c @@ -186,9 +186,10 @@ static void truncation_test(int fd, void *ptr, unsigned int filesize, unsigned i if (memcmp(buffer, uncow + newsize, to_trunc)) errx(1, "read() and MAP_PRIVATE mmap contents don't match (rmap is broken?)"); -#ifndef __linux__ +#if !defined(__linux__) && !defined(__onyx__) /* Okay, Linux doesn't seem to preserve CoW'd MAP_PRIVATE memory in this case. This is weird, - * but seems to be allowed by POSIX. FreeBSD does the obvious, so does Onyx. */ + * but seems to be allowed by POSIX. FreeBSD does the obvious. Onyx (due to anon memory being + * stored in page tables) also doesn't preserve it. */ if (!memcmp(buffer, cow + newsize, to_trunc)) errx(1, "read() and cow'd MAP_PRIVATE mmap contents match (rmap is broken?)"); #endif