implementation of shared heap

Signed-off-by: wenlingyun1 <wenlingyun1@xiaomi.com>

build-scripts/config_common.cmake

@@ -256,6 +256,14 @@ if (WAMR_BUILD_SHARED_MEMORY EQUAL 1)
 else ()
   add_definitions (-DWASM_ENABLE_SHARED_MEMORY=0)
 endif ()
+if (WAMR_BUILD_SHARED_HEAP EQUAL 1)
+  add_definitions (-DWASM_ENABLE_SHARED_HEAP=1)
+  message ("     Shared heap enabled")
+else ()
+  add_definitions (-DWASM_ENABLE_SHARED_HEAP=0)
+  message ("     Shared heap disabled")
+endif()
+
 if (WAMR_BUILD_MEMORY64 EQUAL 1)
   # if native is 32-bit or cross-compiled to 32-bit
   if (NOT WAMR_BUILD_TARGET MATCHES ".*64.*")

build-scripts/runtime_lib.cmake

@@ -119,6 +119,10 @@ if (WAMR_BUILD_LIB_WASI_THREADS EQUAL 1)
     set (WAMR_BUILD_SHARED_MEMORY 1)
 endif ()
 
+if (WAMR_BUILD_SHARED_HEAP EQUAL 1)
+    include (${IWASM_DIR}/libraries/shared-heap/shared_heap.cmake)
+endif ()
+
 if (WAMR_BUILD_DEBUG_INTERP EQUAL 1)
     set (WAMR_BUILD_THREAD_MGR 1)
     include (${IWASM_DIR}/libraries/debug-engine/debug_engine.cmake)
@@ -193,6 +197,7 @@ set (source_all
     ${LIBC_EMCC_SOURCE}
     ${LIB_RATS_SOURCE}
     ${DEBUG_ENGINE_SOURCE}
+    ${LIB_SHARED_HEAP_SOURCE}
 )
 
 set (WAMR_RUNTIME_LIB_SOURCE ${source_all})

core/config.h

@@ -688,4 +688,8 @@
 #endif
 #endif /* WASM_ENABLE_FUZZ_TEST != 0 */
 
+#ifndef WASM_ENABLE_SHARED_HEAP
+#define WASM_ENABLE_SHARED_HEAP 0
+#endif
+
 #endif /* end of _CONFIG_H_ */

core/iwasm/aot/aot_runtime.h

@@ -111,6 +111,10 @@ typedef struct AOTModuleInstanceExtra {
     bh_list *sub_module_inst_list;
     WASMModuleInstanceCommon **import_func_module_insts;
 #endif
+
+#if WASM_ENABLE_SHARED_HEAP != 0
+    WASMSharedHeap *shared_heap;
+#endif
 } AOTModuleInstanceExtra;
 
 #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)

core/iwasm/common/wasm_memory.c

@@ -24,6 +24,10 @@ static Memory_Mode memory_mode = MEMORY_MODE_UNKNOWN;
 
 static mem_allocator_t pool_allocator = NULL;
 
+#if WASM_ENABLE_SHARED_HEAP != 0
+static WASMSharedHeap *shared_heap_list = NULL;
+#endif
+
 static enlarge_memory_error_callback_t enlarge_memory_error_cb;
 static void *enlarge_memory_error_user_data;
 
@@ -132,6 +136,136 @@ is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst)
 #endif
 }
 
+#if WASM_ENABLE_SHARED_HEAP != 0
+static void
+set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
+{
+    if (error_buf != NULL) {
+        snprintf(error_buf, error_buf_size,
+                 "WASM module instantiate failed: %s", string);
+    }
+}
+
+static void *
+runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size)
+{
+    void *mem;
+
+    if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) {
+        set_error_buf(error_buf, error_buf_size, "allocate memory failed");
+        return NULL;
+    }
+
+    memset(mem, 0, (uint32)size);
+    return mem;
+}
+
+WASMSharedHeap *
+wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf,
+                                uint32 error_buf_size)
+{
+    uint64 heap_struct_size = sizeof(WASMSharedHeap);
+    uint32 size = init_args->size;
+    WASMSharedHeap *heap =
+        runtime_malloc(heap_struct_size, error_buf, error_buf_size);
+
+    if (!(heap = runtime_malloc(heap_struct_size, error_buf, error_buf_size))) {
+        goto fail1;
+    }
+    if (!(heap->heap_handle =
+              runtime_malloc(mem_allocator_get_heap_struct_size(), error_buf,
+                             error_buf_size))) {
+        goto fail2;
+    }
+
+    size = align_uint(size, os_getpagesize());
+    if (!(heap->base_addr = runtime_malloc(size, error_buf, error_buf_size))) {
+        goto fail3;
+    }
+    if (!mem_allocator_create_with_struct_and_pool(
+            heap->heap_handle, heap_struct_size, heap->base_addr, size)) {
+        set_error_buf(error_buf, error_buf_size, "init share heap failed");
+        goto fail4;
+    }
+
+    if (shared_heap_list == NULL) {
+        shared_heap_list = heap;
+    }
+    else {
+        heap->next = shared_heap_list;
+        shared_heap_list = heap;
+    }
+    return heap;
+
+fail4:
+    if (heap->base_addr) {
+        wasm_runtime_free(heap->base_addr);
+    }
+
+fail3:
+    if (heap->heap_handle) {
+        wasm_runtime_free(heap->heap_handle);
+    }
+fail2:
+    wasm_runtime_free(heap);
+fail1:
+    return NULL;
+}
+
+bool
+wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst,
+                                WASMSharedHeap *heap)
+{
+    WASMModuleInstance *wasm_module_inst = (WASMModuleInstance *)module_inst;
+    if (wasm_module_inst->e->shared_heap
+        && wasm_module_inst->e->shared_heap != heap) {
+        wasm_runtime_detach_shared_heap(module_inst);
+    }
+    wasm_module_inst->e->shared_heap = heap;
+
+    return true;
+}
+
+bool
+wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst)
+{
+    WASMModuleInstance *wasm_module_inst = (WASMModuleInstance *)module_inst;
+
+    wasm_module_inst->e->shared_heap = NULL;
+    return true;
+}
+
+uint64
+wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64_t size,
+                                void **p_native_addr)
+{
+    WASMModuleInstance *wasm_module_inst = (WASMModuleInstance *)module_inst;
+    WASMSharedHeap *heap = wasm_module_inst->e->shared_heap;
+    if (heap) {
+        if (heap->alloc_type == Alloc_With_Pool) {
+            *p_native_addr = mem_allocator_malloc(
+                wasm_module_inst->e->shared_heap->heap_handle, size);
+            return addr_native_to_app(p_native_addr);
+        }
+    }
+    else {
+        // TODO
+    }
+    return 0;
+}
+
+void
+wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr)
+{
+    WASMModuleInstance *wasm_module_inst = (WASMModuleInstance *)module_inst;
+
+    if (wasm_module_inst->e->shared_heap) {
+        return mem_allocator_free(wasm_module_inst->e->shared_heap->base_addr,
+                                  addr_app_to_native(ptr));
+    }
+}
+#endif
+
 bool
 wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type,
                          const MemAllocOption *alloc_option)
@@ -174,6 +308,22 @@ wasm_runtime_memory_destroy()
     }
     memory_mode = MEMORY_MODE_UNKNOWN;
 }
+#if WASM_ENABLE_SHARED_HEAP != 0
+void
+wasm_runtime_shared_heap_destroy()
+{
+    WASMSharedHeap *heap = shared_heap_list;
+    WASMSharedHeap *cur;
+    while (heap) {
+        mem_allocator_destroy(heap->heap_handle);
+        cur = heap;
+        heap = heap->next;
+        wasm_runtime_free(cur->heap_handle);
+        wasm_runtime_free(cur->base_addr);
+        wasm_runtime_free(cur);
+    }
+}
+#endif
 
 unsigned
 wasm_runtime_memory_pool_size()
@@ -315,6 +465,48 @@ wasm_runtime_get_mem_alloc_info(mem_alloc_info_t *mem_alloc_info)
     return false;
 }
 
+#if WASM_ENABLE_SHARED_HEAP != 0
+bool
+is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm,
+                           bool is_memory64, uint64 app_offset, uint32 bytes)
+{
+    WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm;
+    bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode
+              || module_inst_comm->module_type == Wasm_Module_AoT);
+    WASMSharedHeap *heap = module_inst->e->shared_heap;
+
+    if (!is_memory64) {
+        if (app_offset >= UINT32_MAX - heap->size
+            && app_offset <= UINT32_MAX - bytes) {
+            return true;
+        }
+    }
+    else {
+        if (app_offset >= UINT64_MAX - heap->size
+            && app_offset <= UINT64_MAX - bytes) {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool
+is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm,
+                              uint8 *addr, uint32 bytes)
+{
+    WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm;
+    bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode
+              || module_inst_comm->module_type == Wasm_Module_AoT);
+    WASMSharedHeap *heap = module_inst->e->shared_heap;
+
+    if (heap && heap->base_addr && addr + bytes <= heap->base_addr + heap->size
+        && addr + bytes >= heap->base_addr) {
+        return true;
+    }
+    return false;
+}
+#endif
+
 bool
 wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm,
                                uint64 app_offset, uint64 size)
@@ -342,7 +534,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm,
     /* boundary overflow check */
     if (size > max_linear_memory_size
         || app_offset > max_linear_memory_size - size) {
-        goto fail;
+        goto shared_heap_bound_check;
     }
 
     SHARED_MEMORY_LOCK(memory_inst);
@@ -354,6 +546,18 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm,
 
     SHARED_MEMORY_UNLOCK(memory_inst);
 
+shared_heap_bound_check:
+#if WASM_ENABLE_SHARED_HEAP != 0
+#if WASM_ENABLE_MEMORY64 != 0
+    if (is_app_addr_in_shared_heap(module_inst_comm, true, app_offset, size)) {
+        return true;
+    }
+#else
+    if (is_app_addr_in_shared_heap(module_inst_comm, false, app_offset, size)) {
+        return true;
+    }
+#endif
+#endif
 fail:
     wasm_set_exception(module_inst, "out of bounds memory access");
     return false;
@@ -439,6 +643,13 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm,
         return true;
     }
 
+#if WASM_ENABLE_SHARED_HEAP != 0
+    else if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr,
+                                           size)) {
+        SHARED_MEMORY_UNLOCK(memory_inst);
+        return true;
+    }
+#endif
     SHARED_MEMORY_UNLOCK(memory_inst);
 
 fail:
@@ -475,6 +686,27 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm,
             SHARED_MEMORY_UNLOCK(memory_inst);
             return addr;
         }
+#if WASM_ENABLE_SHARED_HEAP != 0
+#if WASM_ENABLE_MEMORY64 == 0
+        else if (app_offset >= UINT32_MAX - module_inst->e->shared_heap->size
+                 && app_offset <= UINT32_MAX - 1) {
+            uint64 heap_start = (uint64)(UINT32_MAX)
+                                - (uint64)module_inst->e->shared_heap->size;
+            uint64 heap_offset = (uint64)app_offset - heap_start;
+            addr = module_inst->e->shared_heap->base_addr + heap_offset;
+            return addr;
+        }
+#else
+        else if (app_offset >= UINT64_MAX - module_inst->e->shared_heap->size
+                 && app_offset <= UINT64_MAX - 1) {
+            uint64 heap_start =
+                UINT64_MAX - (uint64)module_inst->e->shared_heap->size;
+            uint64 heap_offset = (uint64)app_offset - heap_start;
+            addr = module_inst->e->shared_heap->data + heap_offset;
+            return addr;
+        }
+#endif
+#endif
         SHARED_MEMORY_UNLOCK(memory_inst);
         return NULL;
     }
@@ -499,6 +731,11 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm,
 
     bounds_checks = is_bounds_checks_enabled(module_inst_comm);
 
+#if WASM_ENABLE_SHARED_HEAP != 0
+    /* If shared heap is enabled, bounds check is always needed */
+    bounds_checks = true;
+#endif
+
     memory_inst = wasm_get_default_memory(module_inst);
     if (!memory_inst) {
         return 0;
@@ -513,6 +750,23 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm,
             SHARED_MEMORY_UNLOCK(memory_inst);
             return ret;
         }
+        else {
+#if WASM_ENABLE_SHARED_HEAP != 0
+            uint64 shared_heap_start = 0;
+#if WASM_ENABLE_MEMORY64 == 0
+            shared_heap_start =
+                UINT32_MAX - (uint64)module_inst->e->shared_heap->size;
+#else
+            shared_heap_start =
+                UINT64_MAX - (uint64)module_inst->e->shared_heap->size;
+#endif
+            ret =
+                (uint64)(addr - (uint8 *)module_inst->e->shared_heap->base_addr)
+                + shared_heap_start;
+            SHARED_MEMORY_UNLOCK(memory_inst);
+            return ret;
+#endif
+        }
     }
     /* If bounds checks is disabled, return the offset directly */
     else if (addr != NULL) {