Merge pull request #3823 from bytecodealliance/dev/shared_heap

Implement the shared heap feature for interpreter, aot and llvm jit.
Add below runtime APIs:
```C
wasm_shared_heap_t
wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args);

bool
wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst,
                                wasm_shared_heap_t shared_heap);

void
wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst);

uint64_t
wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64_t size,
                                void **p_native_addr);

void
wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64_t ptr);
```

And allow wasm app to call API shared_heap_malloc and shared_heap_free:
```C
void *shared_heap_malloc(uint32_t size);
void shared_heap_free(void *ptr);
```

.github/scripts/codeql_buildscript.sh

@@ -126,6 +126,16 @@ if [[ $? != 0 ]]; then
     exit 1;
 fi
 
+# build iwasm with multi-memory enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MULTI_MEMORY=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with multi-memory enabled!"
+    exit 1;
+fi
+
 # build iwasm with hardware boundary check disabled
 cd ${WAMR_DIR}/product-mini/platforms/linux
 rm -rf build && mkdir build && cd build
@@ -280,3 +290,23 @@ if [[ $? != 0 ]]; then
     echo "Failed to build iwasm with linux perf support enabled!"
     exit 1;
 fi
+
+# build iwasm with shared heap enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_SHARED_HEAP=1
+make -j
+if [[ $? != 0 ]]; then
+    echo "Failed to build iwasm with shared heap enabled!"
+    exit 1;
+fi
+
+# build iwasm with dynamic aot debug enabled
+cd ${WAMR_DIR}/product-mini/platforms/linux
+rm -rf build && mkdir build && cd build
+cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_DYNAMIC_AOT_DEBUG=1
+make -j
+if [[ $? != 0 ]];
+    echo "Failed to build iwasm dynamic aot debug enabled!"
+    exit 1;
+fi

.github/workflows/compilation_on_android_ubuntu.yml

@@ -578,6 +578,15 @@ jobs:
           ./run.sh test1
           ./run.sh test2
 
+      - name: Build Sample [shared-heap]
+        run: |
+          cd samples/shared-heap
+          mkdir build && cd build
+          cmake ..
+          cmake --build . --config Debug --parallel 4
+          ./shared_heap_test
+          ./shared_heap_test --aot
+
   test:
     needs:
       [

.github/workflows/compilation_on_macos.yml

@@ -386,3 +386,12 @@ jobs:
           ./build.sh
           ./run.sh test1
           ./run.sh test2
+
+      - name: Build Sample [shared-heap]
+        run: |
+          cd samples/shared-heap
+          mkdir build && cd build
+          cmake ..
+          cmake --build . --config Debug --parallel 4
+          ./shared_heap_test
+          ./shared_heap_test --aot

.github/workflows/nightly_run.yml

@@ -593,6 +593,15 @@ jobs:
           exit $?
         working-directory: ./wamr-app-framework/samples/simple
 
+      - name: Build Sample [shared-heap]
+        run: |
+          cd samples/shared-heap
+          mkdir build && cd build
+          cmake ..
+          cmake --build . --config Debug --parallel 4
+          ./shared_heap_test
+          ./shared_heap_test --aot
+
   test:
     needs:
       [

build-scripts/config_common.cmake

@@ -256,6 +256,11 @@ if (WAMR_BUILD_SHARED_MEMORY EQUAL 1)
 else ()
   add_definitions (-DWASM_ENABLE_SHARED_MEMORY=0)
 endif ()
+if (WAMR_BUILD_SHARED_HEAP EQUAL 1)
+  add_definitions (-DWASM_ENABLE_SHARED_HEAP=1)
+  message ("     Shared heap enabled")
+endif()
+
 if (WAMR_BUILD_MEMORY64 EQUAL 1)
   # if native is 32-bit or cross-compiled to 32-bit
   if (NOT WAMR_BUILD_TARGET MATCHES ".*64.*")
@@ -493,7 +498,7 @@ if (WAMR_BUILD_MODULE_INST_CONTEXT EQUAL 1)
   message ("     Module instance context enabled")
 endif ()
 if (WAMR_BUILD_GC_HEAP_VERIFY EQUAL 1)
-  add_definitions (-DWASM_ENABLE_GC_VERIFY=1)
+  add_definitions (-DBH_ENABLE_GC_VERIFY=1)
   message ("     GC heap verification enabled")
 endif ()
 if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1)

build-scripts/runtime_lib.cmake

@@ -119,6 +119,10 @@ if (WAMR_BUILD_LIB_WASI_THREADS EQUAL 1)
     set (WAMR_BUILD_SHARED_MEMORY 1)
 endif ()
 
+if (WAMR_BUILD_SHARED_HEAP EQUAL 1)
+    include (${IWASM_DIR}/libraries/shared-heap/shared_heap.cmake)
+endif ()
+
 if (WAMR_BUILD_DEBUG_INTERP EQUAL 1)
     set (WAMR_BUILD_THREAD_MGR 1)
     include (${IWASM_DIR}/libraries/debug-engine/debug_engine.cmake)
@@ -193,6 +197,7 @@ set (source_all
     ${LIBC_EMCC_SOURCE}
     ${LIB_RATS_SOURCE}
     ${DEBUG_ENGINE_SOURCE}
+    ${LIB_SHARED_HEAP_SOURCE}
 )
 
 set (WAMR_RUNTIME_LIB_SOURCE ${source_all})

core/config.h

@@ -396,7 +396,9 @@
 #define APP_HEAP_SIZE_DEFAULT (8 * 1024)
 #endif
 #define APP_HEAP_SIZE_MIN (256)
-#define APP_HEAP_SIZE_MAX (512 * 1024 * 1024)
+/* The ems memory allocator supports maximal heap size 1GB,
+   see ems_gc_internal.h */
+#define APP_HEAP_SIZE_MAX (1024 * 1024 * 1024)
 
 /* Default min/max gc heap size of each app */
 #ifndef GC_HEAP_SIZE_DEFAULT
@@ -692,4 +694,8 @@
 #endif
 #endif /* WASM_ENABLE_FUZZ_TEST != 0 */
 
+#ifndef WASM_ENABLE_SHARED_HEAP
+#define WASM_ENABLE_SHARED_HEAP 0
+#endif
+
 #endif /* end of _CONFIG_H_ */

core/iwasm/aot/aot_runtime.c

@@ -57,6 +57,9 @@ bh_static_assert(sizeof(AOTMemoryInstance) == 120);
 bh_static_assert(offsetof(AOTTableInstance, elems) == 24);
 
 bh_static_assert(offsetof(AOTModuleInstanceExtra, stack_sizes) == 0);
+bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj)
+                 == 8);
+bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_start_off) == 16);
 
 bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3);
 
@@ -1895,6 +1898,24 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent,
     extra->stack_sizes =
         aot_get_data_section_addr(module, AOT_STACK_SIZES_SECTION_NAME, NULL);
 
+    /*
+     * The AOT code checks whether the n bytes to access are in shared heap
+     * by checking whether the beginning address meets:
+     *   addr >= start_off && addr <= end_off - n-bytes + 1
+     * where n is 1/2/4/8/16 and `end_off - n-bytes + 1` is constant, e.g.,
+     *   UINT32_MAX, UINT32_MAX-1, UINT32_MAX-3 for n = 1, 2 or 4 in 32-bit
+     * target. To simplify the check, when shared heap is disabled, we set
+     * the start off to UINT64_MAX in 64-bit target and UINT32_MAX in 32-bit
+     * target, so in the checking, the above formula will be false, we don't
+     * need to check whether the shared heap is enabled or not in the AOT
+     * code.
+     */
+#if UINTPTR_MAX == UINT64_MAX
+    extra->shared_heap_start_off.u64 = UINT64_MAX;
+#else
+    extra->shared_heap_start_off.u32[0] = UINT32_MAX;
+#endif
+
 #if WASM_ENABLE_PERF_PROFILING != 0
     total_size = sizeof(AOTFuncPerfProfInfo)
                  * ((uint64)module->import_func_count + module->func_count);

core/iwasm/aot/aot_runtime.h

@@ -111,6 +111,14 @@ typedef struct AOTFunctionInstance {
 
 typedef struct AOTModuleInstanceExtra {
     DefPointer(const uint32 *, stack_sizes);
+    /*
+     * Adjusted shared heap based addr to simple the calculation
+     * in the aot code. The value is:
+     *   shared_heap->base_addr - shared_heap->start_off
+     */
+    DefPointer(uint8 *, shared_heap_base_addr_adj);
+    MemBound shared_heap_start_off;
+
     WASMModuleInstanceExtraCommon common;
     AOTFunctionInstance **functions;
     uint32 function_count;
@@ -119,6 +127,10 @@ typedef struct AOTModuleInstanceExtra {
     bh_list *sub_module_inst_list;
     WASMModuleInstanceCommon **import_func_module_insts;
 #endif
+
+#if WASM_ENABLE_SHARED_HEAP != 0
+    WASMSharedHeap *shared_heap;
+#endif
 } AOTModuleInstanceExtra;
 
 #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)