runtime: track heap/handle table churn

Author: nickwanninger

runtime/core/Localizer.cpp

@@ -51,24 +51,26 @@ namespace alaska {
       for (size_t i = 0; i < count; i++) {
         auto hid = handle_ids[i];
         if (hid == 0) continue;
-        auto handle = reinterpret_cast<void *>((1LU << 63) | ((uint64_t)hid << ALASKA_SIZE_BITS));
-        auto *m = alaska::Mapping::from_handle_safe(handle);
-        bool moved = false;
-        if (m == NULL or m->is_free() or m->is_pinned()) {
-          moved = false;
-        } else {
-          void *ptr = m->get_pointer();
-          auto *source_page = rt.heap.pt.get_unaligned(m->get_pointer());
-          moved = tc.localize(*m, rt.localization_epoch);
-        }
-        auto size = tc.get_size(handle);
-        bytes_reach += size;
-        if (moved) {
-          moved_objects++;
-          bytes_in_dump += size;
-        } else {
-          unmoved_objects++;
-        }
+        this->seen_handles.add(hid);
+        // printf("%3d: %x\n", i, hid);
+        // auto handle = reinterpret_cast<void *>((1LU << 63) | ((uint64_t)hid << ALASKA_SIZE_BITS));
+        // auto *m = alaska::Mapping::from_handle_safe(handle);
+        // bool moved = false;
+        // if (m == NULL or m->is_free() or m->is_pinned()) {
+        //   moved = false;
+        // } else {
+        //   void *ptr = m->get_pointer();
+        //   auto *source_page = rt.heap.pt.get_unaligned(m->get_pointer());
+        //   moved = tc.localize(*m, rt.localization_epoch);
+        // }
+        // auto size = tc.get_size(handle);
+        // bytes_reach += size;
+        // if (moved) {
+        //   moved_objects++;
+        //   bytes_in_dump += size;
+        // } else {
+        //   unmoved_objects++;
+        // }
       }
       // rt.heap.compact_locality_pages();
       // rt.heap.compact_sizedpages();
@@ -77,6 +79,8 @@ namespace alaska {
     // printf("moved:%5lu unmoved:%5lu bytes:%12lu reach:%12lu\n", moved_objects, unmoved_objects,
     //     bytes_in_dump, bytes_reach);
 
+    alaska::printf("seen handles: %zu\n", this->seen_handles.size());
+
     // Push the buffer back to the queue of buffers
     struct buffer *buf = reinterpret_cast<struct buffer *>(handle_ids);
     buf->next = buffers;

runtime/core/Runtime.cpp

@@ -66,12 +66,15 @@ namespace alaska {
 #undef xstr
 #undef str
 
-    fprintf(stream, "barriers: %zu, rate:%.1f/s\n", stat_barriers.read(), stat_barriers.digest());
+    fprintf(stream, "barriers: %zu (%.1f/s)\n", stat_barriers.read(), stat_barriers.digest());
 
     for (auto *tc : this->tcs) {
-      fprintf(stream, "tc%d allocations:%zu, rate:%.1f/s", tc->id, tc->allocation_rate.read(),
+      fprintf(stream, "tc%d allocations:%zu (%.1f/s)", tc->id, tc->allocation_rate.read(),
           tc->allocation_rate.digest());
-      fprintf(stream, " frees:%zu, rate:%.1f/s", tc->free_rate.read(), tc->free_rate.digest());
+      fprintf(stream, "  frees:%zu (%.1f/s)", tc->free_rate.read(), tc->free_rate.digest());
+      fprintf(stream, "  heap_churn:%zu (%.1f/s)", tc->heap_churn.read(), tc->heap_churn.digest());
+      fprintf(stream, "  ht_churn:%zu (%.1f/s)", tc->handle_table_churn.read(),
+          tc->handle_table_churn.digest());
       fprintf(stream, "\n");
     }
     // handle_table.dump(stream);

runtime/core/ThreadCache.cpp

@@ -33,6 +33,8 @@ namespace alaska {
       : id(id)
       , runtime(rt)
       , localizer(rt.config, *this) {
+
+    handle_table_churn++;
     handle_slab = runtime.handle_table.new_slab(this);
   }
 
@@ -45,7 +47,9 @@ namespace alaska {
     ptr = ::malloc(size);
 #else
     SizedPage *page = size_classes[cls];
-    if (unlikely(page == nullptr)) page = new_sized_page(cls);
+    if (unlikely(page == nullptr)) {
+      page = new_sized_page(cls);
+    }
     ptr = page->alloc(m, size);
     if (unlikely(ptr == nullptr)) {
       // OOM?
@@ -90,6 +94,7 @@ namespace alaska {
 
 
   SizedPage *ThreadCache::new_sized_page(int cls) {
+    heap_churn++;
     // Get a new heap
     auto *heap = runtime.heap.get_sizedpage(alaska::class_to_size(cls), this);
 
@@ -106,6 +111,7 @@ namespace alaska {
 
 
   LocalityPage *ThreadCache::new_locality_page(size_t required_size) {
+    heap_churn++;
     // Get a new heap
     auto *lp = runtime.heap.get_localitypage(required_size, this);
 
@@ -208,6 +214,7 @@ namespace alaska {
     auto m = handle_slab->alloc();
 
     if (unlikely(m == NULL)) {
+      handle_table_churn++; // record that we are looking for a new handle table slab.
       auto new_handle_slab = runtime.handle_table.new_slab(this);
       this->handle_slab->set_owner(NULL);
       this->handle_slab = new_handle_slab;

runtime/include/alaska/Localizer.hpp

@@ -13,6 +13,7 @@
 
 #include <alaska/alaska.hpp>
 #include <alaska/Configuration.hpp>
+#include <ck/set.h>
 
 namespace alaska {
 
@@ -32,6 +33,8 @@ namespace alaska {
 
     struct buffer *buffers = nullptr;
 
+    ck::set<handle_id_t> seen_handles;
+
    public:
     Localizer(alaska::Configuration &config, alaska::ThreadCache &tc);
 

runtime/include/alaska/ThreadCache.hpp

@@ -99,6 +99,11 @@ namespace alaska {
     alaska::RateCounter allocation_rate;
     alaska::RateCounter free_rate;
 
+
+    // How often are we getting a new heap or handle table?
+    alaska::RateCounter heap_churn;
+    alaska::RateCounter handle_table_churn;
+
     // Each thread cache has a localizer, which can be fed with
     // "localization data" to improve object locality
     alaska::Localizer localizer;

runtime/rt/sim.cpp

@@ -14,8 +14,8 @@
 #include <semaphore.h>
 
 
-#define L1_ENTS 96
-#define L1_SETS 24
+#define L1_ENTS 16
+#define L1_SETS 4
 #define L1_WAYS (L1_ENTS / L1_SETS)
 
 #define L2_WAYS 16

runtime/yukon/yukon.cpp

@@ -108,8 +108,9 @@ static void setup_signal_handlers(void) {
   assert(sigaction(SIGUSR2, &sa, NULL) == 0);
 
 
-  if (getenv("YUKON_DUMP") == NULL) {
-    int dump_interval_ms = 10;
+  if (getenv("YUKON_DUMP") != NULL) {
+    int dump_interval_ms = atoi(getenv("YUKON_DUMP"));
+
     useconds_t dump_interval = dump_interval_ms * 1000;
     signal(SIGALRM, alarm_handler);
     // now that we have sigalarm configured, setup a ualarm for
@@ -159,16 +160,22 @@ namespace yukon {
   void dump_htlb(alaska::ThreadCache *tc) {
     auto size = 576;
     auto *space = tc->localizer.get_hotness_buffer(size);
-    memset(space, 0, size * sizeof(alaska::handle_id_t));
-
+    // memset(space, 0, size * sizeof(alaska::handle_id_t));
+    // Fence after we have a valid space for the dump
     asm volatile("fence" ::: "memory");
 
     auto start = read_cycle_counter();
+
+    // Fire off the dump..
     write_csr(CSR_HTDUMP, (uint64_t)space);
+    // ..and wait for it to finish
     wait_for_csr_zero(CSR_HTDUMP);
     auto end = read_cycle_counter();
+    // fence so we have ordering
     asm volatile("fence" ::: "memory");
+    // Feed the buffer to the localizer to do it's work
     tc->localizer.feed_hotness_buffer(size, space);
+    // Fence again... for some reason
     asm volatile("fence" ::: "memory");
     // printf("Dumping htlb took %lu cycles\n", end - start);
   }
@@ -235,11 +242,9 @@ void __attribute__((constructor(102))) alaska_init(void) {
   unsetenv("LD_PRELOAD");  // make it so we don't run alaska in subprocesses!
   yukon::get_tc();
 
-  atexit([]() {
-    printf("Setting ht addr to zero!\n");
-    yukon::set_handle_table_base(NULL);
-    printf("set!\n");
-  });
+  // atexit([]() {
+  //   yukon::set_handle_table_base(NULL);
+  // });
 }
 
 void __attribute__((destructor)) alaska_deinit(void) {}