Skip to content

Commit

Permalink
pagecache: drain user-mapped pages when low on memory
Browse files Browse the repository at this point in the history 
WIP
  • Loading branch information
@francescolavra
francescolavra committed Feb 3, 2025
1 parent 26ee8dc commit ca9344d
Show file tree
Hide file tree
Showing 15 changed files with 474 additions and 170 deletions.
31 changes: 29 additions & 2 deletions src/kernel/flush.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@ struct flush_entry {
u64 gen;
struct refcount ref;
boolean flush;
volatile boolean wait;
u32 joined;
u64 pages[FLUSH_THRESHOLD];
int npages;
thunk completion;
closure_struct(thunk, finish);
};

Expand Down Expand Up @@ -57,6 +60,11 @@ static void _flush_handler(void)
invalidate(f->pages[i]);
}
}
if (f->wait) {
fetch_and_add_32(&f->joined, 1);
while (f->wait)
kern_pause();
}
refcount_release(&f->ref);
}
}
Expand Down Expand Up @@ -97,6 +105,9 @@ static void service_list(void)
continue;
list_delete(&f->l);
entries_count--;
thunk completion = f->completion;
if (completion)
async_apply(completion);
assert(enqueue(free_flush_entries, f));
}
}
Expand All @@ -119,7 +130,7 @@ static void queue_flush_service(void)
}
}

void page_invalidate_sync(flush_entry f)
void page_invalidate_sync(flush_entry f, thunk completion, boolean wait)
{
if (initialized) {
if (f->npages == 0) {
Expand Down Expand Up @@ -151,11 +162,27 @@ void page_invalidate_sync(flush_entry f)
f->gen = fetch_and_add((word *)&inval_gen, 1) + 1;
spin_wunlock(&flush_lock);

send_ipi(TARGET_EXCLUSIVE_BROADCAST, flush_ipi);
f->wait = false;
_flush_handler();
f->wait = wait;
if (wait) {
f->joined = 1;
f->completion = 0;
} else {
f->completion = completion;
}
send_ipi(TARGET_EXCLUSIVE_BROADCAST, flush_ipi);
irq_restore(flags);
if (wait) {
while (((volatile flush_entry)f)->joined < total_processors)
kern_pause();
apply(completion);
f->wait = false;
}
} else {
flush_tlb(false);
if (completion)
async_apply(completion);
}
}

Expand Down
83 changes: 55 additions & 28 deletions src/kernel/page.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ void update_map_flags(u64 vaddr, u64 length, pageflags flags)
assert(!intersects_linear_backed(irangel(vaddr, length)));
flush_entry fe = get_page_flush_entry();
traverse_ptes(vaddr, length, stack_closure(update_pte_flags, vaddr, length, flags, fe));
page_invalidate_sync(fe);
page_invalidate_sync(fe, 0, false);
#ifdef PAGE_DUMP_ALL
early_debug("update_map_flags ");
dump_page_tables(vaddr, length);
Expand Down Expand Up @@ -347,7 +347,7 @@ void remap_pages(u64 vaddr_new, u64 vaddr_old, u64 length)
irange(vaddr_old, vaddr_old + length))));
flush_entry fe = get_page_flush_entry();
traverse_ptes(vaddr_old, length, stack_closure(remap_entry, vaddr_new, vaddr_old, length, fe));
page_invalidate_sync(fe);
page_invalidate_sync(fe, 0, false);
#ifdef PAGE_DUMP_ALL
early_debug("remap ");
dump_page_tables(vaddr_new, length);
Expand Down Expand Up @@ -376,10 +376,9 @@ void zero_mapped_pages(u64 vaddr, u64 length)

/* called with lock held */
closure_function(4, 3, boolean, unmap_page,
u64, vstart, u64, len, range_handler, rh, flush_entry, fe,
u64, vstart, u64, len, buffer, phys_ranges, flush_entry, fe,
int level, u64 vaddr, pteptr entry)
{
range_handler rh = bound(rh);
u64 old_entry = pte_from_pteptr(entry);
if (pte_is_present(old_entry) && pte_is_mapping(level, old_entry)) {
#ifdef PAGE_UPDATE_DEBUG
Expand All @@ -405,21 +404,21 @@ closure_function(4, 3, boolean, unmap_page,
return false;
}
page_invalidate(bound(fe), vaddr);
if (rh) {
apply(rh, irangel(page_from_pte(old_entry) + map_offset, unmap_len));
buffer phys_ranges = bound(phys_ranges);
if (phys_ranges) {
range r = irangel(page_from_pte(old_entry) + map_offset, unmap_len);
return buffer_write(phys_ranges, &r, sizeof(r));
}
}
return true;
}

/* Be warned: the page table lock is held when rh is called; don't try
to modify the page table while traversing it */
void unmap_pages_with_handler(u64 virtual, u64 length, range_handler rh)
void unmap(u64 virtual, u64 length)
{
assert(!((virtual & PAGEMASK) || (length & PAGEMASK)));
flush_entry fe = get_page_flush_entry();
traverse_ptes(virtual, length, stack_closure(unmap_page, virtual, length, rh, fe));
page_invalidate_sync(fe);
traverse_ptes(virtual, length, stack_closure(unmap_page, virtual, length, 0, fe));
page_invalidate_sync(fe, 0, false);
#ifdef PAGE_DUMP_ALL
early_debug("unmap ");
dump_page_tables(virtual, length);
Expand Down Expand Up @@ -564,7 +563,7 @@ void map(u64 v, physical p, u64 length, pageflags flags)
}
page_init_debug("map_level done\n");
pagetable_unlock();
page_invalidate_sync(fe);
page_invalidate_sync(fe, 0, false);
#ifdef PAGE_DUMP_ALL
early_debug("map ");
dump_page_tables(v, length);
Expand All @@ -580,36 +579,64 @@ void map_nolock(u64 v, physical p, u64 length, pageflags flags)
map_level(table_ptr, PT_FIRST_LEVEL, r, &p, flags.w, 0);
}

void unmap(u64 virtual, u64 length)
{
page_init_debug("unmap v: ");
page_init_debug_u64(virtual);
page_init_debug(", length: ");
page_init_debug_u64(length);
page_init_debug("\n");
unmap_pages(virtual, length);
#ifdef KERNEL
closure_function(2, 0, void, unmap_and_free_phys_complete,
buffer, phys_ranges, boolean, on_stack)
{
heap h = heap_physical(get_kernel_heaps());
buffer phys_ranges = bound(phys_ranges);
range *r;
while ((r = buffer_pop(phys_ranges, sizeof(*r))))
deallocate(h, r->start, range_span(*r));
if (bound(on_stack)) {
/* clear the buffer so it can be reused if there are other iterations */
buffer_clear(phys_ranges);
} else {
deallocate_buffer(phys_ranges);
closure_finish();
}
}

closure_function(1, 1, boolean, page_dealloc,
heap, pageheap,
range r)
static void unmap_and_free_phys_sync(u64 virtual, u64 length)
{
u64 virt = pagemem.pagevirt.start + r.start;
deallocate_u64(bound(pageheap), virt, range_span(r));
return true;
kernel_context kc = (kernel_context)get_current_context(current_cpu());
buffer phys_ranges = little_stack_buffer(kc->size / 2);
thunk completion = stack_closure(unmap_and_free_phys_complete, phys_ranges, true);
boolean done, progress;
do {
flush_entry fe = get_page_flush_entry();
done = traverse_ptes(virtual, length,
stack_closure(unmap_page, virtual, length, phys_ranges, fe));
progress = buffer_length(phys_ranges) != 0;
page_invalidate_sync(fe, completion, true);
} while (!done && progress);
}

void unmap_and_free_phys(u64 virtual, u64 length)
{
unmap_pages_with_handler(virtual, length,
stack_closure(page_dealloc, (heap)heap_page_backed(get_kernel_heaps())));
heap h = heap_locked(get_kernel_heaps());
buffer phys_ranges = allocate_buffer(h, 64 * sizeof(range));
if (phys_ranges == INVALID_ADDRESS)
return unmap_and_free_phys_sync(virtual, length);
thunk completion = closure(h, unmap_and_free_phys_complete, phys_ranges, false);
if (completion == INVALID_ADDRESS) {
deallocate_buffer(phys_ranges);
return unmap_and_free_phys_sync(virtual, length);
}
flush_entry fe = get_page_flush_entry();
boolean success = traverse_ptes(virtual, length,
stack_closure(unmap_page, virtual, length, phys_ranges, fe));
page_invalidate_sync(fe, completion, !success);
if (!success)
unmap_and_free_phys_sync(virtual, length);
}

void page_free_phys(u64 phys)
{
u64 virt = pagemem.pagevirt.start + phys;
deallocate_u64((heap)get_kernel_heaps()->pages, virt, PAGESIZE);
}
#endif

static boolean init_page_map(range phys, range *curr_virt, id_heap virt_heap, pageflags flags)
{
Expand Down
8 changes: 2 additions & 6 deletions src/kernel/page.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ void init_page_tables(heap pageheap);
void init_flush(heap);
flush_entry get_page_flush_entry();
void page_invalidate(flush_entry f, u64 address);
void page_invalidate_sync(flush_entry f);
void page_invalidate_sync(flush_entry f, thunk completion, boolean wait);
void page_invalidate_flush();

void invalidate(u64 page);
Expand All @@ -43,12 +43,8 @@ void update_map_flags(u64 vaddr, u64 length, pageflags flags);
void zero_mapped_pages(u64 vaddr, u64 length);
void remap_pages(u64 vaddr_new, u64 vaddr_old, u64 length);
void unmap(u64 virtual, u64 length);
void unmap_pages_with_handler(u64 virtual, u64 length, range_handler rh);

static inline void unmap_pages(u64 virtual, u64 length)
{
unmap_pages_with_handler(virtual, length, 0);
}
#define unmap_pages(virtual, length) unmap(virtual, length)

#include <page_machine.h>

Expand Down
Loading

0 comments on commit ca9344d

Please sign in to comment.