From 55cb9c59c58c005d18e95711015d32ff911a6529 Mon Sep 17 00:00:00 2001 From: bianchui Date: Wed, 14 Aug 2024 08:42:01 +0800 Subject: [PATCH 01/69] Enable merged os_mmap for aot data sections (#3681) And enable merged os_mmap for aot data and text sections except on platform nuttx and esp-idf. Fix issue that aarch64 AOT module fails to load on android: https://github.com/bytecodealliance/wasm-micro-runtime/issues/2274 --- core/iwasm/aot/aot_loader.c | 158 +++++++++++++++++++++++++++++++---- core/iwasm/aot/aot_runtime.h | 7 ++ 2 files changed, 150 insertions(+), 15 deletions(-) diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 645f68b1d3..11c7495f6e 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -2378,7 +2378,6 @@ destroy_object_data_sections(AOTObjectDataSection *data_sections, } } #endif - os_munmap(data_section->data, data_section->size); } wasm_runtime_free(data_sections); } @@ -2392,6 +2391,9 @@ load_object_data_sections(const uint8 **p_buf, const uint8 *buf_end, AOTObjectDataSection *data_sections; uint64 size; uint32 i; + uint64 total_size = 0; + uint32 page_size = os_getpagesize(); + uint8 *merged_sections = NULL; /* Allocate memory */ size = sizeof(AOTObjectDataSection) * (uint64)module->data_section_count; @@ -2400,8 +2402,22 @@ load_object_data_sections(const uint8 **p_buf, const uint8 *buf_end, return false; } - /* Create each data section */ + /* First iteration: read data from buf, and calculate total memory needed */ for (i = 0; i < module->data_section_count; i++) { + read_string(buf, buf_end, data_sections[i].name); + read_uint32(buf, buf_end, data_sections[i].size); + CHECK_BUF(buf, buf_end, data_sections[i].size); + /* temporary record data ptr for merge, will be replaced after mmaped */ + if (data_sections[i].size > 0) + data_sections[i].data = (uint8 *)buf; + buf += data_sections[i].size; + total_size += align_uint64((uint64)data_sections[i].size, page_size); + } + if (total_size > UINT32_MAX) { + set_error_buf(error_buf, error_buf_size, "data sections too large"); + return false; + } + if (total_size > 0) { int map_prot = MMAP_PROT_READ | MMAP_PROT_WRITE; #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ || defined(BUILD_TARGET_RISCV64_LP64D) \ @@ -2412,29 +2428,33 @@ load_object_data_sections(const uint8 **p_buf, const uint8 *buf_end, #else int map_flags = MMAP_MAP_NONE; #endif - - read_string(buf, buf_end, data_sections[i].name); - read_uint32(buf, buf_end, data_sections[i].size); - /* Allocate memory for data */ - if (data_sections[i].size > 0 - && !(data_sections[i].data = - os_mmap(NULL, data_sections[i].size, map_prot, map_flags, - os_get_invalid_handle()))) { + merged_sections = module->merged_data_sections = + os_mmap(NULL, (uint32)total_size, map_prot, map_flags, + os_get_invalid_handle()); + if (!merged_sections) { set_error_buf(error_buf, error_buf_size, "allocate memory failed"); return false; } + module->merged_data_sections_size = (uint32)total_size; #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) #if !defined(BH_PLATFORM_LINUX_SGX) && !defined(BH_PLATFORM_WINDOWS) \ && !defined(BH_PLATFORM_DARWIN) /* address must be in the first 2 Gigabytes of the process address space */ - bh_assert((uintptr_t)data_sections[i].data < INT32_MAX); + bh_assert((uintptr_t)merged_sections < INT32_MAX); #endif #endif + } - read_byte_array(buf, buf_end, data_sections[i].data, - data_sections[i].size); + /* Second iteration: Create each data section */ + for (i = 0; i < module->data_section_count; i++) { + if (data_sections[i].size > 0) { + bh_memcpy_s(merged_sections, data_sections[i].size, + data_sections[i].data, data_sections[i].size); + data_sections[i].data = merged_sections; + merged_sections += align_uint(data_sections[i].size, page_size); + } } *p_buf = buf; @@ -2532,6 +2552,90 @@ load_init_data_section(const uint8 *buf, const uint8 *buf_end, return false; } +#if !defined(BH_PLATFORM_NUTTX) && !defined(BH_PLATFORM_ESP_IDF) +static bool +try_merge_data_and_text(const uint8 **buf, const uint8 **buf_end, + AOTModule *module, char *error_buf, + uint32 error_buf_size) +{ + uint8 *old_buf = (uint8 *)*buf; + uint8 *old_end = (uint8 *)*buf_end; + size_t code_size = (size_t)(old_end - old_buf); + uint32 page_size = os_getpagesize(); + uint64 total_size = 0; + uint32 i; + uint8 *sections; + + if (code_size == 0) { + return true; + } + + /* calc total memory needed */ + total_size += align_uint64((uint64)code_size, page_size); + for (i = 0; i < module->data_section_count; ++i) { + total_size += + align_uint64((uint64)module->data_sections[i].size, page_size); + } + /* distance between .data and .text should not greater than 4GB for some + * targets (eg. arm64 reloc need < 4G distance) */ + if (total_size > UINT32_MAX) { + return false; + } + + if (total_size != 0) { + int map_prot = MMAP_PROT_READ | MMAP_PROT_WRITE; + +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) + /* aot code and data in x86_64 must be in range 0 to 2G due + to relocation for R_X86_64_32/32S/PC32 */ + int map_flags = MMAP_MAP_32BIT; +#else + int map_flags = MMAP_MAP_NONE; +#endif + + sections = os_mmap(NULL, (uint32)total_size, map_prot, map_flags, + os_get_invalid_handle()); + if (!sections) { + /* merge failed but maybe not critical for some targes */ + return false; + } + if (os_mprotect(sections, code_size, map_prot | MMAP_PROT_EXEC) != 0) { + os_munmap(sections, (uint32)total_size); + return false; + } + + module->merged_data_text_sections = sections; + module->merged_data_text_sections_size = (uint32)total_size; + + /* order not essential just as compilers do: .text section first */ + *buf = sections; + *buf_end = sections + code_size; + bh_memcpy_s(sections, code_size, old_buf, code_size); + os_munmap(old_buf, code_size); + sections += align_uint((uint32)code_size, page_size); + + /* then .data sections */ + for (i = 0; i < module->data_section_count; ++i) { + AOTObjectDataSection *data_section = module->data_sections + i; + uint8 *old_data = data_section->data; + data_section->data = sections; + bh_memcpy_s(data_section->data, data_section->size, old_data, + data_section->size); + sections += align_uint(data_section->size, page_size); + } + if (module->merged_data_sections) { + os_munmap(module->merged_data_sections, + module->merged_data_sections_size); + module->merged_data_sections = NULL; + module->merged_data_sections_size = 0; + } + } + return true; +} +#endif //! defined(BH_PLATFORM_NUTTX) && !defined(BH_PLATFORM_ESP_IDF) + static bool load_text_section(const uint8 *buf, const uint8 *buf_end, AOTModule *module, char *error_buf, uint32 error_buf_size) @@ -3749,6 +3853,17 @@ load_from_sections(AOTModule *module, AOTSection *sections, return false; break; case AOT_SECTION_TYPE_TEXT: +#if !defined(BH_PLATFORM_NUTTX) && !defined(BH_PLATFORM_ESP_IDF) + /* try to merge .data and .text, with exceptions: + * 1. XIP mode + * 2. pre-mmapped module load from aot_load_from_sections() + * 3. nuttx & esp-idf: have separate region for MMAP_PROT_EXEC + */ + if (!module->is_indirect_mode && is_load_from_file_buf) + if (!try_merge_data_and_text(&buf, &buf_end, module, + error_buf, error_buf_size)) + LOG_WARNING("merge .data and .text sections failed"); +#endif //! defined(BH_PLATFORM_NUTTX) && !defined(BH_PLATFORM_ESP_IDF) if (!load_text_section(buf, buf_end, module, error_buf, error_buf_size)) return false; @@ -4179,7 +4294,11 @@ load(const uint8 *buf, uint32 size, AOTModule *module, if (!ret) { /* If load_from_sections() fails, then aot text is destroyed in destroy_sections() */ - destroy_sections(section_list, module->is_indirect_mode ? false : true); + destroy_sections(section_list, + module->is_indirect_mode + || module->merged_data_text_sections + ? false + : true); /* aot_unload() won't destroy aot text again */ module->code = NULL; } @@ -4329,7 +4448,8 @@ aot_unload(AOTModule *module) } #endif - if (module->code && !module->is_indirect_mode) { + if (module->code && !module->is_indirect_mode + && !module->merged_data_text_sections) { /* The layout is: literal size + literal + code (with plt table) */ uint8 *mmap_addr = module->literal - sizeof(uint32); uint32 total_size = @@ -4364,6 +4484,14 @@ aot_unload(AOTModule *module) destroy_object_data_sections(module->data_sections, module->data_section_count); + if (module->merged_data_sections) + os_munmap(module->merged_data_sections, + module->merged_data_sections_size); + + if (module->merged_data_text_sections) + os_munmap(module->merged_data_text_sections, + module->merged_data_text_sections_size); + #if WASM_ENABLE_DEBUG_AOT != 0 jit_code_entry_destroy(module->elf_hdr); #endif diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index e3704f8278..05d66386de 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -315,6 +315,13 @@ typedef struct AOTModule { /* Whether the underlying wasm binary buffer can be freed */ bool is_binary_freeable; + + /* `.data` sections merged into one mmaped to reduce the tlb cache miss */ + uint8 *merged_data_sections; + uint32 merged_data_sections_size; + /* `.data` and `.text` sections merged into one large mmaped section */ + uint8 *merged_data_text_sections; + uint32 merged_data_text_sections_size; } AOTModule; #define AOTMemoryInstance WASMMemoryInstance From 5f517e4335b169ddbb5762b1741a95a35ac4ffe0 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Fri, 16 Aug 2024 11:31:45 +0800 Subject: [PATCH 02/69] aot loader: Refine os_mmap related code (#3711) --- core/iwasm/aot/aot_loader.c | 199 +++++++++++++++--------------------- 1 file changed, 85 insertions(+), 114 deletions(-) diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 11c7495f6e..b96079d3b4 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -294,6 +294,39 @@ loader_malloc(uint64 size, char *error_buf, uint32 error_buf_size) return mem; } +static void * +loader_mmap(uint32 size, bool prot_exec, char *error_buf, uint32 error_buf_size) +{ + int map_prot = + MMAP_PROT_READ | MMAP_PROT_WRITE | (prot_exec ? MMAP_PROT_EXEC : 0); + int map_flags; + void *mem; + +#if UINTPTR_MAX == UINT64_MAX + /* The mmapped AOT data and code in 64-bit targets had better be in + range 0 to 2G, or aot loader may fail to apply some relocations, + e.g., R_X86_64_32/R_X86_64_32S/R_X86_64_PC32/R_RISCV_32. + We try to mmap with MMAP_MAP_32BIT flag first, and if fails, mmap + again without the flag. */ + map_flags = MMAP_MAP_32BIT; + if ((mem = os_mmap(NULL, size, map_prot, map_flags, + os_get_invalid_handle()))) { + /* The mmapped memory must be in the first 2 Gigabytes of the + process address space */ + bh_assert((uintptr_t)mem < INT32_MAX); + return mem; + } +#endif + + map_flags = MMAP_MAP_NONE; + if (!(mem = os_mmap(NULL, size, map_prot, map_flags, + os_get_invalid_handle()))) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + return NULL; + } + return mem; +} + static char * load_string(uint8 **p_buf, const uint8 *buf_end, AOTModule *module, bool is_load_from_file_buf, @@ -2407,7 +2440,8 @@ load_object_data_sections(const uint8 **p_buf, const uint8 *buf_end, read_string(buf, buf_end, data_sections[i].name); read_uint32(buf, buf_end, data_sections[i].size); CHECK_BUF(buf, buf_end, data_sections[i].size); - /* temporary record data ptr for merge, will be replaced after mmaped */ + /* Temporary record data ptr for merge, will be replaced after the + merged_data_sections is mmapped */ if (data_sections[i].size > 0) data_sections[i].data = (uint8 *)buf; buf += data_sections[i].size; @@ -2418,33 +2452,13 @@ load_object_data_sections(const uint8 **p_buf, const uint8 *buf_end, return false; } if (total_size > 0) { - int map_prot = MMAP_PROT_READ | MMAP_PROT_WRITE; -#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ - || defined(BUILD_TARGET_RISCV64_LP64D) \ - || defined(BUILD_TARGET_RISCV64_LP64) - /* aot code and data in x86_64 must be in range 0 to 2G due to - relocation for R_X86_64_32/32S/PC32 */ - int map_flags = MMAP_MAP_32BIT; -#else - int map_flags = MMAP_MAP_NONE; -#endif /* Allocate memory for data */ merged_sections = module->merged_data_sections = - os_mmap(NULL, (uint32)total_size, map_prot, map_flags, - os_get_invalid_handle()); + loader_mmap((uint32)total_size, false, error_buf, error_buf_size); if (!merged_sections) { - set_error_buf(error_buf, error_buf_size, "allocate memory failed"); return false; } module->merged_data_sections_size = (uint32)total_size; -#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) -#if !defined(BH_PLATFORM_LINUX_SGX) && !defined(BH_PLATFORM_WINDOWS) \ - && !defined(BH_PLATFORM_DARWIN) - /* address must be in the first 2 Gigabytes of - the process address space */ - bh_assert((uintptr_t)merged_sections < INT32_MAX); -#endif -#endif } /* Second iteration: Create each data section */ @@ -2570,71 +2584,63 @@ try_merge_data_and_text(const uint8 **buf, const uint8 **buf_end, return true; } - /* calc total memory needed */ + /* calculate the total memory needed */ total_size += align_uint64((uint64)code_size, page_size); for (i = 0; i < module->data_section_count; ++i) { total_size += align_uint64((uint64)module->data_sections[i].size, page_size); } - /* distance between .data and .text should not greater than 4GB for some - * targets (eg. arm64 reloc need < 4G distance) */ + /* distance between .data and .text should not be greater than 4GB + for some targets (e.g. arm64 reloc need < 4G distance) */ if (total_size > UINT32_MAX) { return false; } + /* code_size was checked and must be larger than 0 here */ + bh_assert(total_size > 0); - if (total_size != 0) { - int map_prot = MMAP_PROT_READ | MMAP_PROT_WRITE; + sections = loader_mmap((uint32)total_size, false, NULL, 0); + if (!sections) { + /* merge failed but may be not critical for some targets */ + return false; + } + /* change the code part to be executable */ + if (os_mprotect(sections, code_size, + MMAP_PROT_READ | MMAP_PROT_WRITE | MMAP_PROT_EXEC) + != 0) { + os_munmap(sections, (uint32)total_size); + return false; + } -#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ - || defined(BUILD_TARGET_RISCV64_LP64D) \ - || defined(BUILD_TARGET_RISCV64_LP64) - /* aot code and data in x86_64 must be in range 0 to 2G due - to relocation for R_X86_64_32/32S/PC32 */ - int map_flags = MMAP_MAP_32BIT; -#else - int map_flags = MMAP_MAP_NONE; -#endif + module->merged_data_text_sections = sections; + module->merged_data_text_sections_size = (uint32)total_size; - sections = os_mmap(NULL, (uint32)total_size, map_prot, map_flags, - os_get_invalid_handle()); - if (!sections) { - /* merge failed but maybe not critical for some targes */ - return false; - } - if (os_mprotect(sections, code_size, map_prot | MMAP_PROT_EXEC) != 0) { - os_munmap(sections, (uint32)total_size); - return false; - } + /* order not essential just as compiler does: .text section first */ + *buf = sections; + *buf_end = sections + code_size; + bh_memcpy_s(sections, code_size, old_buf, code_size); + os_munmap(old_buf, code_size); + sections += align_uint((uint32)code_size, page_size); - module->merged_data_text_sections = sections; - module->merged_data_text_sections_size = (uint32)total_size; - - /* order not essential just as compilers do: .text section first */ - *buf = sections; - *buf_end = sections + code_size; - bh_memcpy_s(sections, code_size, old_buf, code_size); - os_munmap(old_buf, code_size); - sections += align_uint((uint32)code_size, page_size); - - /* then .data sections */ - for (i = 0; i < module->data_section_count; ++i) { - AOTObjectDataSection *data_section = module->data_sections + i; - uint8 *old_data = data_section->data; - data_section->data = sections; - bh_memcpy_s(data_section->data, data_section->size, old_data, - data_section->size); - sections += align_uint(data_section->size, page_size); - } - if (module->merged_data_sections) { - os_munmap(module->merged_data_sections, - module->merged_data_sections_size); - module->merged_data_sections = NULL; - module->merged_data_sections_size = 0; - } + /* then migrate .data sections */ + for (i = 0; i < module->data_section_count; ++i) { + AOTObjectDataSection *data_section = module->data_sections + i; + uint8 *old_data = data_section->data; + data_section->data = sections; + bh_memcpy_s(data_section->data, data_section->size, old_data, + data_section->size); + sections += align_uint(data_section->size, page_size); + } + /* free the original data sections */ + if (module->merged_data_sections) { + os_munmap(module->merged_data_sections, + module->merged_data_sections_size); + module->merged_data_sections = NULL; + module->merged_data_sections_size = 0; } + return true; } -#endif //! defined(BH_PLATFORM_NUTTX) && !defined(BH_PLATFORM_ESP_IDF) +#endif /* ! defined(BH_PLATFORM_NUTTX) && !defined(BH_PLATFORM_ESP_IDF) */ static bool load_text_section(const uint8 *buf, const uint8 *buf_end, AOTModule *module, @@ -3495,16 +3501,9 @@ load_relocation_section(const uint8 *buf, const uint8 *buf_end, + sizeof(uint64) * module->real_plt_count + sizeof(uint32) * module->float_plt_count; if (size > 0) { - map_prot = MMAP_PROT_READ | MMAP_PROT_WRITE | MMAP_PROT_EXEC; - /* aot code and data in x86_64 must be in range 0 to 2G due to - relocation for R_X86_64_32/32S/PC32 */ - map_flags = MMAP_MAP_32BIT; - if (size > UINT32_MAX - || !(module->extra_plt_data = - os_mmap(NULL, (uint32)size, map_prot, map_flags, - os_get_invalid_handle()))) { - set_error_buf(error_buf, error_buf_size, "mmap memory failed"); + || !(module->extra_plt_data = loader_mmap( + (uint32)size, true, error_buf, error_buf_size))) { goto fail; } module->extra_plt_data_size = (uint32)size; @@ -3616,19 +3615,12 @@ load_relocation_section(const uint8 *buf, const uint8 *buf_end, GOTItem *got_item = module->got_item_list; uint32 got_item_idx = 0; - map_prot = MMAP_PROT_READ | MMAP_PROT_WRITE; - /* aot code and data in x86_64 must be in range 0 to 2G due to - relocation for R_X86_64_32/32S/PC32 */ - map_flags = MMAP_MAP_32BIT; - /* Create the GOT for func_ptrs, note that it is different from the .got section of a dynamic object file */ size = (uint64)sizeof(void *) * got_item_count; if (size > UINT32_MAX - || !(module->got_func_ptrs = - os_mmap(NULL, (uint32)size, map_prot, map_flags, - os_get_invalid_handle()))) { - set_error_buf(error_buf, error_buf_size, "mmap memory failed"); + || !(module->got_func_ptrs = loader_mmap( + (uint32)size, false, error_buf, error_buf_size))) { goto fail; } @@ -3863,7 +3855,7 @@ load_from_sections(AOTModule *module, AOTSection *sections, if (!try_merge_data_and_text(&buf, &buf_end, module, error_buf, error_buf_size)) LOG_WARNING("merge .data and .text sections failed"); -#endif //! defined(BH_PLATFORM_NUTTX) && !defined(BH_PLATFORM_ESP_IDF) +#endif /* ! defined(BH_PLATFORM_NUTTX) && !defined(BH_PLATFORM_ESP_IDF) */ if (!load_text_section(buf, buf_end, module, error_buf, error_buf_size)) return false; @@ -4180,37 +4172,16 @@ create_sections(AOTModule *module, const uint8 *buf, uint32 size, if (section_type == AOT_SECTION_TYPE_TEXT) { if ((section_size > 0) && !module->is_indirect_mode) { - int map_prot = - MMAP_PROT_READ | MMAP_PROT_WRITE | MMAP_PROT_EXEC; -#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ - || defined(BUILD_TARGET_RISCV64_LP64D) \ - || defined(BUILD_TARGET_RISCV64_LP64) - /* aot code and data in x86_64 must be in range 0 to 2G due - to relocation for R_X86_64_32/32S/PC32 */ - int map_flags = MMAP_MAP_32BIT; -#else - int map_flags = MMAP_MAP_NONE; -#endif total_size = (uint64)section_size + aot_get_plt_table_size(); total_size = (total_size + 3) & ~((uint64)3); if (total_size >= UINT32_MAX || !(aot_text = - os_mmap(NULL, (uint32)total_size, map_prot, - map_flags, os_get_invalid_handle()))) { + loader_mmap((uint32)total_size, true, + error_buf, error_buf_size))) { wasm_runtime_free(section); - set_error_buf(error_buf, error_buf_size, - "mmap memory failed"); goto fail; } -#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) -#if !defined(BH_PLATFORM_LINUX_SGX) && !defined(BH_PLATFORM_WINDOWS) \ - && !defined(BH_PLATFORM_DARWIN) - /* address must be in the first 2 Gigabytes of - the process address space */ - bh_assert((uintptr_t)aot_text < INT32_MAX); -#endif -#endif #if (WASM_MEM_DUAL_BUS_MIRROR != 0) mirrored_text = os_get_dbus_mirror(aot_text); From b00904b092ce478f7cfec71299db8a910a5a383a Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Thu, 22 Aug 2024 13:35:25 +0900 Subject: [PATCH 03/69] Add a comment on AOT_SECTION_TYPE_SIGNATURE (#3746) cf. https://github.com/bytecodealliance/wasm-micro-runtime/issues/3744 --- core/iwasm/aot/aot_runtime.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 0eb647987c..76c784512d 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -39,6 +39,10 @@ typedef enum AOTSectionType { AOT_SECTION_TYPE_FUNCTION = 3, AOT_SECTION_TYPE_EXPORT = 4, AOT_SECTION_TYPE_RELOCATION = 5, + /* + * Note: We haven't had anything to use AOT_SECTION_TYPE_SIGNATURE. + * It's just reserved for possible module signing features. + */ AOT_SECTION_TYPE_SIGNATURE = 6, AOT_SECTION_TYPE_CUSTOM = 100, } AOTSectionType; From e8c2952bf959f6df81972fc469a4357f04ee629a Mon Sep 17 00:00:00 2001 From: Anders Bakken Date: Thu, 22 Aug 2024 18:49:06 -0700 Subject: [PATCH 04/69] Fix arm64 issues on mac (#3688) Make wamrc normalize "arm64" to "aarch64v8". Previously the only way to make the "arm64" target was to not specify a target on 64 bit arm-based mac builds. Now arm64 and aarch64v8 are treated as the same. Make aot_loader accept "aarch64v8" on arm-based apple (as well as accepting legacy "arm64" based aot targets). This also removes __APPLE__ and __MACH__ from the block that defaults size_level to 1 since it doesn't seem to be supported for aarch64: `LLVM ERROR: Only small, tiny and large code models are allowed on AArch64` --- core/iwasm/aot/aot_loader.c | 4 ++++ core/iwasm/aot/arch/aot_reloc_aarch64.c | 11 ++--------- core/iwasm/compilation/aot_llvm.c | 9 +++++++++ wamr-compiler/main.c | 4 ++-- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index b96079d3b4..e62e8278b0 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -579,6 +579,10 @@ load_target_info_section(const uint8 *buf, const uint8 *buf_end, return false; } + /* for backwards compatibility with previous wamrc aot files */ + if (!strcmp(target_info.arch, "arm64")) + bh_strcpy_s(target_info.arch, sizeof(target_info.arch), "aarch64v8"); + /* Check machine info */ if (!check_machine_info(&target_info, error_buf, error_buf_size)) { return false; diff --git a/core/iwasm/aot/arch/aot_reloc_aarch64.c b/core/iwasm/aot/arch/aot_reloc_aarch64.c index b4bb6024a4..ec646b4e92 100644 --- a/core/iwasm/aot/arch/aot_reloc_aarch64.c +++ b/core/iwasm/aot/arch/aot_reloc_aarch64.c @@ -53,12 +53,6 @@ get_target_symbol_map(uint32 *sym_num) return target_sym_map; } -#if (defined(__APPLE__) || defined(__MACH__)) && defined(__arm64__) -#define BUILD_TARGET_AARCH64_DEFAULT "arm64" -#else -#define BUILD_TARGET_AARCH64_DEFAULT "aarch64v8" -#endif - void get_current_target(char *target_buf, uint32 target_buf_size) { @@ -68,8 +62,8 @@ get_current_target(char *target_buf, uint32 target_buf_size) /* Set to "aarch64v8" by default if sub version isn't specified */ if (strcmp(s, "AARCH64") == 0) { - s = BUILD_TARGET_AARCH64_DEFAULT; - s_size = sizeof(BUILD_TARGET_AARCH64_DEFAULT); + s = "aarch64v8"; + s_size = 9; /* strlen("aarch64v8"); */ } if (target_buf_size < s_size) { s_size = target_buf_size; @@ -83,7 +77,6 @@ get_current_target(char *target_buf, uint32 target_buf_size) /* Ensure the string is null byte ('\0') terminated */ *d = '\0'; } -#undef BUILD_TARGET_AARCH64_DEFAULT static uint32 get_plt_item_size() diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index d738cfc0e4..ab0b6ab029 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -2790,6 +2790,15 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) bh_assert(vendor_sys); bh_memcpy_s(default_arch, sizeof(default_arch), default_triple, (uint32)(vendor_sys - default_triple)); + /** + * On Mac M[1-9]+ LLVM will report arm64 as the + * architecture, for the purposes of wamr this is the + * same as aarch64v8 so we'll normalize it here. + */ + if (!strcmp(default_arch, "arm64")) { + bh_strcpy_s(default_arch, sizeof(default_arch), + "aarch64v8"); + } arch1 = default_arch; LLVMDisposeMessage(default_triple); diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index b3e731e537..8eed1c99a4 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -601,8 +601,8 @@ main(int argc, char *argv[]) LOG_VERBOSE("Set size level to 1 for Windows AOT file"); option.size_level = 1; } -#if defined(_WIN32) || defined(_WIN32_) || defined(__APPLE__) \ - || defined(__MACH__) +#if defined(_WIN32) || defined(_WIN32_) \ + || ((defined(__APPLE__) || defined(__MACH__)) && !defined(__arm64__)) if (!option.target_arch && !option.target_abi) { LOG_VERBOSE("Set size level to 1 for Windows or MacOS AOT file"); option.size_level = 1; From cb3a69f778d42444bb2c35698069b2bc1af96baa Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Wed, 28 Aug 2024 16:05:07 +0800 Subject: [PATCH 05/69] CI: Freeze version of bloaty for NuttX compilation (#3756) Fix the compilation error of this CI: https://github.com/bytecodealliance/wasm-micro-runtime/actions/runs/10575515238 ``` /__w/wasm-micro-runtime/wasm-micro-runtime/bloaty/third_party/abseil-cpp/absl/debugging/failure_signal_handler.cc:139:32: error: no matching function for call to 'max(long int, int)' 139 | size_t stack_size = (std::max(SIGSTKSZ, 65536) + page_mask) & ~page_mask; | ~~~~~~~~^~~~~~~~~~~~~~~~~ ``` --- .github/workflows/compilation_on_nuttx.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/compilation_on_nuttx.yml b/.github/workflows/compilation_on_nuttx.yml index 2f8014face..627ada8ec6 100644 --- a/.github/workflows/compilation_on_nuttx.yml +++ b/.github/workflows/compilation_on_nuttx.yml @@ -124,6 +124,7 @@ jobs: repository: google/bloaty submodules: recursive path: bloaty + ref: 34f4a66559ad4938c1e629e9b5f54630b2b4d7b0 - name: Build Bloaty run: | From d1141f6f309b2ef4857df9cf5db3ab8bf52b096a Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Thu, 29 Aug 2024 11:08:39 +0800 Subject: [PATCH 06/69] aot compiler: Allow to control stack boundary check when boundary check is enabled (#3754) In the AOT compiler, allow the user to control stack boundary check when the boundary check is enabled (e.g. `wamrc --bounds-checks=1`). Now the code logic is: 1. When `--stack-bounds-checks` is not set, it will be the same value as `--bounds-checks`. 2. When `--stack-bounds-checks` is set, it will be the option value no matter what the status of `--bounds-checks` is. --- core/iwasm/compilation/aot_llvm.c | 20 ++++++++++---------- core/iwasm/interpreter/wasm_interp_classic.c | 1 + wamr-compiler/main.c | 4 +--- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index ab0b6ab029..39f64d81d8 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -2969,12 +2969,12 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) sizeof(comp_ctx->target_arch)); if (option->bounds_checks == 1 || option->bounds_checks == 0) { - /* Set by user */ + /* Set by the user */ comp_ctx->enable_bound_check = (option->bounds_checks == 1) ? true : false; } else { - /* Unset by user, use default value */ + /* Unset by the user, use the default value */ if (strstr(comp_ctx->target_arch, "64") && !option->is_sgx_platform) { comp_ctx->enable_bound_check = false; @@ -2984,17 +2984,17 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) } } - if (comp_ctx->enable_bound_check) { - /* Always enable stack boundary check if `bounds-checks` - is enabled */ - comp_ctx->enable_stack_bound_check = true; - } - else { - /* When `bounds-checks` is disabled, we set stack boundary - check status according to the input option */ + if (option->stack_bounds_checks == 1 + || option->stack_bounds_checks == 0) { + /* Set by the user */ comp_ctx->enable_stack_bound_check = (option->stack_bounds_checks == 1) ? true : false; } + else { + /* Unset by the user, use the default value, it will be the same + * value as the bound check */ + comp_ctx->enable_stack_bound_check = comp_ctx->enable_bound_check; + } if ((comp_ctx->enable_stack_bound_check || comp_ctx->enable_stack_estimation) diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 4a8ba4e2cd..67f8c2d455 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -5739,6 +5739,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* use memmove when memory64 is enabled since len may be larger than UINT32_MAX */ memmove(mdst, msrc, len); + (void)dlen; #endif break; } diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index 8eed1c99a4..bd9e543537 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -142,9 +142,7 @@ print_help() printf(" with a runtime without the hardware bounds checks.\n"); printf(" --stack-bounds-checks=1/0 Enable or disable the bounds checks for native stack:\n"); printf(" if the option isn't set, the status is same as `--bounds-check`,\n"); - printf(" if the option is set:\n"); - printf(" (1) it is always enabled when `--bounds-checks` is enabled,\n"); - printf(" (2) else it is enabled/disabled according to the option value\n"); + printf(" if the option is set, the status is same as the option value\n"); printf(" --stack-usage= Generate a stack-usage file.\n"); printf(" Similarly to `clang -fstack-usage`.\n"); printf(" --format= Specifies the format of the output file\n"); From eab409a4df9010bce661c800ef0ddf60acae3b4c Mon Sep 17 00:00:00 2001 From: Anders Bakken Date: Thu, 29 Aug 2024 00:25:17 -0700 Subject: [PATCH 07/69] aot loader: Call os_mmap with MMAP_MAP_32BIT only when target is x86-64 or riscv64 (#3755) Mac on aarch64 uses posix_memmap.c os_mmap which doesn't do anything with the flag MMAP_MAP_32BIT for that build so this condition ends up asserting unless the mapping ends up in the first 4 gigs worth of addressable space. Thsi PR changes to call os_mmap with MMAP_MAP_32BIT flag only when the target is x86-64 or riscv64, and the macro __APPLE__ isn't enabled. The behavior is similar to what the posix os_mmap does. --- core/iwasm/aot/aot_loader.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index e62e8278b0..3a5b6fc5cc 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -302,7 +302,10 @@ loader_mmap(uint32 size, bool prot_exec, char *error_buf, uint32 error_buf_size) int map_flags; void *mem; -#if UINTPTR_MAX == UINT64_MAX +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) \ + || defined(BUILD_TARGET_RISCV64_LP64D) \ + || defined(BUILD_TARGET_RISCV64_LP64) +#ifndef __APPLE__ /* The mmapped AOT data and code in 64-bit targets had better be in range 0 to 2G, or aot loader may fail to apply some relocations, e.g., R_X86_64_32/R_X86_64_32S/R_X86_64_PC32/R_RISCV_32. @@ -316,6 +319,7 @@ loader_mmap(uint32 size, bool prot_exec, char *error_buf, uint32 error_buf_size) bh_assert((uintptr_t)mem < INT32_MAX); return mem; } +#endif #endif map_flags = MMAP_MAP_NONE; From 0b62cc89218030561189f65cc42585319e2ed38c Mon Sep 17 00:00:00 2001 From: Marcin Kolny Date: Tue, 3 Sep 2024 03:39:03 +0100 Subject: [PATCH 08/69] Update ref to the multi-memory tests (#3764) The specific commit has been deleted, I am pointing to the same commit in the main branch though. --- tests/wamr-test-suites/test_wamr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index 8254cc712c..87e1568658 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -511,7 +511,7 @@ function spec_test() pushd spec # Reset to commit: "Merge pull request #48 from backes/specify-memcpy-immediate-order" - git reset --hard 48e69f394869c55b7bbe14ac963c09f4605490b6 + git reset --hard fbc99efd7a788db300aec3dd62a14577ec404f1b git checkout 044d0d2e77bdcbe891f7e0b9dd2ac01d56435f0b -- test/core/elem.wast git apply ../../spec-test-script/multi_memory_ignore_cases.patch || exit 1 if [[ ${RUNNING_MODE} == "aot" ]]; then From 5cc94e59eca97fcc30f7484a7d1abc354f7af3cd Mon Sep 17 00:00:00 2001 From: James Ring Date: Mon, 2 Sep 2024 20:03:24 -0700 Subject: [PATCH 09/69] Improve posix mmap retry logic (#3714) - Only retry on EAGAIN, ENOMEM or EINTR. - On EINTR, don't count it against the retry budget, just keep retrying. EINTR can happen in bursts. - Log the errno on failure, and don't conditionalize that logging on BH_ENABLE_TRACE_MMAP. In other parts of the code, error logging is not conditional on that define, while turning on that tracing define makes things overly verbose. --- .../shared/platform/common/posix/posix_memmap.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/core/shared/platform/common/posix/posix_memmap.c b/core/shared/platform/common/posix/posix_memmap.c index c76abf1378..1d972f5fa3 100644 --- a/core/shared/platform/common/posix/posix_memmap.c +++ b/core/shared/platform/common/posix/posix_memmap.c @@ -138,18 +138,25 @@ os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) /* memory hasn't been mapped or was mapped failed previously */ if (addr == MAP_FAILED) { - /* try 5 times */ - for (i = 0; i < 5; i++) { + /* try 5 times on EAGAIN or ENOMEM, and keep retrying on EINTR */ + i = 0; + while (i < 5) { addr = mmap(hint, request_size, map_prot, map_flags, file, 0); if (addr != MAP_FAILED) break; + if (errno == EINTR) + continue; + if (errno != EAGAIN && errno != ENOMEM) { + break; + } + i++; } } if (addr == MAP_FAILED) { -#if BH_ENABLE_TRACE_MMAP != 0 - os_printf("mmap failed\n"); -#endif + os_printf("mmap failed with errno: %d, hint: %p, size: %" PRIu64 + ", prot: %d, flags: %d", + errno, hint, request_size, map_prot, map_flags); return NULL; } From fed0fe953ca32894f51179c97d3dcc9175456a37 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Wed, 4 Sep 2024 12:08:25 +0800 Subject: [PATCH 10/69] compilation_on_nuttx.yml: Update checkout action to suppress warnings (#3765) --- .github/workflows/compilation_on_nuttx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compilation_on_nuttx.yml b/.github/workflows/compilation_on_nuttx.yml index 627ada8ec6..e10784fe23 100644 --- a/.github/workflows/compilation_on_nuttx.yml +++ b/.github/workflows/compilation_on_nuttx.yml @@ -119,7 +119,7 @@ jobs: run: make -j$(nproc) EXTRAFLAGS=-Werror - name: Checkout Bloaty - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: google/bloaty submodules: recursive From 65521b188d9efcdfb689314707f50513c6a509b1 Mon Sep 17 00:00:00 2001 From: Marcin Kolny Date: Thu, 5 Sep 2024 04:10:18 +0100 Subject: [PATCH 11/69] Remove unnecessary code duplication in aot runtime (#3767) --- core/iwasm/aot/aot_runtime.c | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 3ca26114f2..bdb4ca9112 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -3668,33 +3668,6 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) return true; } -static inline void -aot_free_frame_internal(WASMExecEnv *exec_env) -{ - AOTFrame *cur_frame = (AOTFrame *)exec_env->cur_frame; - AOTFrame *prev_frame = cur_frame->prev_frame; - -#if WASM_ENABLE_PERF_PROFILING != 0 - uint64 time_elapsed = - (uintptr_t)os_time_thread_cputime_us() - cur_frame->time_started; - - cur_frame->func_perf_prof_info->total_exec_time += time_elapsed; - cur_frame->func_perf_prof_info->total_exec_cnt++; - - /* parent function */ - if (prev_frame) - prev_frame->func_perf_prof_info->children_exec_time += time_elapsed; -#endif - - exec_env->cur_frame = (struct WASMInterpFrame *)prev_frame; -} - -void -aot_free_frame(WASMExecEnv *exec_env) -{ - aot_free_frame_internal(exec_env); -} - #else /* else of WASM_ENABLE_GC == 0 */ bool @@ -3752,6 +3725,7 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) frame->func_index = func_index; return true; } +#endif /* end of WASM_ENABLE_GC == 0 */ static inline void aot_free_frame_internal(WASMExecEnv *exec_env) @@ -3771,7 +3745,9 @@ aot_free_frame_internal(WASMExecEnv *exec_env) prev_frame->func_perf_prof_info->children_exec_time += time_elapsed; #endif +#if WASM_ENABLE_GC != 0 wasm_exec_env_free_wasm_frame(exec_env, cur_frame); +#endif exec_env->cur_frame = (struct WASMInterpFrame *)prev_frame; } @@ -3781,7 +3757,6 @@ aot_free_frame(WASMExecEnv *exec_env) aot_free_frame_internal(exec_env); } -#endif /* end of WASM_ENABLE_GC == 0 */ void aot_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame) From b4380fb3b10e5c5b59a3f10a6df4c5e78fbc4065 Mon Sep 17 00:00:00 2001 From: Marcin Kolny Date: Thu, 5 Sep 2024 09:18:47 +0100 Subject: [PATCH 12/69] refactoring: Re-use commit IP functionality between exception handling and other cases (#3768) --- core/iwasm/compilation/aot_compiler.c | 66 +++++++++++++-------- core/iwasm/compilation/aot_compiler.h | 9 +++ core/iwasm/compilation/aot_emit_exception.c | 45 +------------- 3 files changed, 54 insertions(+), 66 deletions(-) diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index 96ed8facfe..bb6cf100a0 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -569,6 +569,46 @@ aot_gen_commit_values(AOTCompFrame *frame) return true; } +bool +aot_gen_commit_ip(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef ip_value, bool is_64bit) +{ + LLVMValueRef cur_frame = func_ctx->cur_frame; + LLVMValueRef value_offset, value_addr, value_ptr; + uint32 offset_ip; + + if (!comp_ctx->is_jit_mode) + offset_ip = comp_ctx->pointer_size * 4; + else + offset_ip = offsetof(WASMInterpFrame, ip); + + if (!(value_offset = I32_CONST(offset_ip))) { + aot_set_last_error("llvm build const failed"); + return false; + } + + if (!(value_addr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, + &value_offset, 1, "ip_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + + if (!(value_ptr = LLVMBuildBitCast( + comp_ctx->builder, value_addr, + is_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE, "ip_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!LLVMBuildStore(comp_ctx->builder, ip_value, value_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } + + return true; +} + bool aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip) { @@ -577,40 +617,19 @@ aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip) LLVMValueRef cur_frame = func_ctx->cur_frame; LLVMValueRef value_offset, value_addr, value_ptr, value; LLVMTypeRef int8_ptr_ptr_type; - uint32 offset_ip, offset_sp, n; + uint32 offset_sp, n; bool is_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; const AOTValueSlot *sp = frame->sp; const uint8 *ip = frame->frame_ip; if (!comp_ctx->is_jit_mode) { - offset_ip = frame->comp_ctx->pointer_size * 4; offset_sp = frame->comp_ctx->pointer_size * 5; } else { - offset_ip = offsetof(WASMInterpFrame, ip); offset_sp = offsetof(WASMInterpFrame, sp); } if (commit_ip) { - if (!(value_offset = I32_CONST(offset_ip))) { - aot_set_last_error("llvm build const failed"); - return false; - } - - if (!(value_addr = - LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, - &value_offset, 1, "ip_addr"))) { - aot_set_last_error("llvm build in bounds gep failed"); - return false; - } - - if (!(value_ptr = LLVMBuildBitCast( - comp_ctx->builder, value_addr, - is_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE, "ip_ptr"))) { - aot_set_last_error("llvm build bit cast failed"); - return false; - } - if (!comp_ctx->is_jit_mode) { WASMModule *module = comp_ctx->comp_data->wasm_module; if (is_64bit) @@ -630,8 +649,7 @@ aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip) return false; } - if (!LLVMBuildStore(comp_ctx->builder, value, value_ptr)) { - aot_set_last_error("llvm build store failed"); + if (!aot_gen_commit_ip(comp_ctx, func_ctx, value, is_64bit)) { return false; } } diff --git a/core/iwasm/compilation/aot_compiler.h b/core/iwasm/compilation/aot_compiler.h index ab74b7cb65..d3d55b02bf 100644 --- a/core/iwasm/compilation/aot_compiler.h +++ b/core/iwasm/compilation/aot_compiler.h @@ -195,6 +195,15 @@ aot_gen_commit_values(AOTCompFrame *frame); bool aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip); +/** + * Generate instructions to commit IP pointer to the frame. + * + * @param frame the frame information + */ +bool +aot_gen_commit_ip(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef ip_value, bool is_64bit); + bool aot_frame_store_value(AOTCompContext *comp_ctx, LLVMValueRef value, uint8 value_type, LLVMValueRef cur_frame, uint32 offset); diff --git a/core/iwasm/compilation/aot_emit_exception.c b/core/iwasm/compilation/aot_emit_exception.c index d3dcf719db..968ee78b6d 100644 --- a/core/iwasm/compilation/aot_emit_exception.c +++ b/core/iwasm/compilation/aot_emit_exception.c @@ -4,49 +4,10 @@ */ #include "aot_emit_exception.h" +#include "aot_compiler.h" #include "../interpreter/wasm_runtime.h" #include "../aot/aot_runtime.h" -static bool -commit_ip(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, - LLVMValueRef exce_ip, bool is_64bit) -{ - LLVMValueRef cur_frame = func_ctx->cur_frame; - LLVMValueRef value_offset, value_addr, value_ptr; - uint32 offset_ip; - - if (!comp_ctx->is_jit_mode) - offset_ip = comp_ctx->pointer_size * 4; - else - offset_ip = offsetof(WASMInterpFrame, ip); - - if (!(value_offset = I32_CONST(offset_ip))) { - aot_set_last_error("llvm build const failed"); - return false; - } - - if (!(value_addr = - LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, cur_frame, - &value_offset, 1, "ip_addr"))) { - aot_set_last_error("llvm build in bounds gep failed"); - return false; - } - - if (!(value_ptr = LLVMBuildBitCast( - comp_ctx->builder, value_addr, - is_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE, "ip_ptr"))) { - aot_set_last_error("llvm build bit cast failed"); - return false; - } - - if (!LLVMBuildStore(comp_ctx->builder, exce_ip, value_ptr)) { - aot_set_last_error("llvm build store failed"); - return false; - } - - return true; -} - bool aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, int32 exception_id, bool is_cond_br, LLVMValueRef cond_br_if, @@ -90,8 +51,8 @@ aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* Commit ip to current frame */ - if (!commit_ip(comp_ctx, func_ctx, func_ctx->exception_ip_phi, - is_64bit)) { + if (!aot_gen_commit_ip(comp_ctx, func_ctx, + func_ctx->exception_ip_phi, is_64bit)) { return false; } } From 6f97822c181bc9f481b31e292b7f357321ae927c Mon Sep 17 00:00:00 2001 From: Marcin Kolny Date: Thu, 5 Sep 2024 14:44:06 +0100 Subject: [PATCH 13/69] Add wamrc parameter to configure stack frame features (#3763) Those parameters can be used to reduce the size of the AOT code. There's going to be more changes related to AOT code size reduction, this is just the initial step. p.s. https://github.com/bytecodealliance/wasm-micro-runtime/issues/3758 --- core/iwasm/compilation/aot_compiler.c | 8 ++- core/iwasm/compilation/aot_emit_exception.c | 4 +- core/iwasm/compilation/aot_emit_function.c | 39 +++++++----- core/iwasm/compilation/aot_llvm.c | 2 + core/iwasm/compilation/aot_llvm.h | 3 + core/iwasm/include/aot_comp_option.h | 18 ++++++ core/iwasm/interpreter/wasm_loader.c | 1 + core/iwasm/interpreter/wasm_mini_loader.c | 1 + wamr-compiler/main.c | 70 +++++++++++++++++++++ 9 files changed, 127 insertions(+), 19 deletions(-) diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index bb6cf100a0..78b7da88d7 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -337,6 +337,10 @@ aot_gen_commit_values(AOTCompFrame *frame) LLVMValueRef value; uint32 n; + if (!frame->comp_ctx->call_stack_features.values) { + return true; + } + /* First, commit reference flags * For LLVM JIT, iterate all local and stack ref flags * For AOT, ignore local(params + locals) ref flags */ @@ -629,7 +633,7 @@ aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip) offset_sp = offsetof(WASMInterpFrame, sp); } - if (commit_ip) { + if (commit_ip && comp_ctx->call_stack_features.ip) { if (!comp_ctx->is_jit_mode) { WASMModule *module = comp_ctx->comp_data->wasm_module; if (is_64bit) @@ -654,7 +658,7 @@ aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip) } } - if (commit_sp) { + if (commit_sp && comp_ctx->call_stack_features.values) { n = (uint32)(sp - frame->lp); value = I32_CONST(offset_of_local(comp_ctx, n)); if (!value) { diff --git a/core/iwasm/compilation/aot_emit_exception.c b/core/iwasm/compilation/aot_emit_exception.c index 968ee78b6d..1527e83e51 100644 --- a/core/iwasm/compilation/aot_emit_exception.c +++ b/core/iwasm/compilation/aot_emit_exception.c @@ -41,7 +41,7 @@ aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return false; } - if (comp_ctx->aot_frame) { + if (comp_ctx->aot_frame && comp_ctx->call_stack_features.trap_ip) { /* Create exception ip phi */ if (!(func_ctx->exception_ip_phi = LLVMBuildPhi( comp_ctx->builder, is_64bit ? I64_TYPE : I32_TYPE, @@ -134,7 +134,7 @@ aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Add phi incoming value to got_exception block */ LLVMAddIncoming(func_ctx->exception_id_phi, &exce_id, &block_curr, 1); - if (comp_ctx->aot_frame) { + if (comp_ctx->aot_frame && comp_ctx->call_stack_features.trap_ip) { const uint8 *ip = comp_ctx->aot_frame->frame_ip; LLVMValueRef exce_ip = NULL; diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index 8f6e3e4563..1d565b6c09 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -682,24 +682,29 @@ alloc_frame_for_aot_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, new_frame = wasm_stack_top; - if (!(check_wasm_stack_succ = LLVMAppendBasicBlockInContext( - comp_ctx->context, func_ctx->func, "check_wasm_stack_succ"))) { - aot_set_last_error("llvm add basic block failed."); - return false; - } + if (comp_ctx->call_stack_features.bounds_checks) { + if (!(check_wasm_stack_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, + "check_wasm_stack_succ"))) { + aot_set_last_error("llvm add basic block failed."); + return false; + } - LLVMMoveBasicBlockAfter(check_wasm_stack_succ, - LLVMGetInsertBlock(comp_ctx->builder)); + LLVMMoveBasicBlockAfter(check_wasm_stack_succ, + LLVMGetInsertBlock(comp_ctx->builder)); - if (!(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGT, wasm_stack_top_max, - wasm_stack_top_bound, "cmp"))) { - aot_set_last_error("llvm build icmp failed"); - return false; - } + if (!(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGT, + wasm_stack_top_max, wasm_stack_top_bound, + "cmp"))) { + aot_set_last_error("llvm build icmp failed"); + return false; + } - if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_OPERAND_STACK_OVERFLOW, - true, cmp, check_wasm_stack_succ))) { - return false; + if (!(aot_emit_exception(comp_ctx, func_ctx, + EXCE_OPERAND_STACK_OVERFLOW, true, cmp, + check_wasm_stack_succ))) { + return false; + } } #if WASM_ENABLE_GC != 0 @@ -1285,6 +1290,10 @@ commit_params_to_frame_of_import_func(AOTCompContext *comp_ctx, { uint32 i, n; + if (!comp_ctx->call_stack_features.values) { + return true; + } + for (i = 0, n = 0; i < func_type->param_count; i++, n++) { switch (func_type->types[i]) { case VALUE_TYPE_I32: diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 39f64d81d8..3346086a95 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -2580,6 +2580,8 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) if (option->enable_aux_stack_frame) comp_ctx->enable_aux_stack_frame = true; + comp_ctx->call_stack_features = option->call_stack_features; + if (option->enable_perf_profiling) comp_ctx->enable_perf_profiling = true; diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 270e5ae451..65debbaa36 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -412,6 +412,9 @@ typedef struct AOTCompContext { /* Generate auxiliary stack frame */ bool enable_aux_stack_frame; + /* Auxiliary call stack features */ + AOTCallStackFeatures call_stack_features; + /* Function performance profiling */ bool enable_perf_profiling; diff --git a/core/iwasm/include/aot_comp_option.h b/core/iwasm/include/aot_comp_option.h index 617b68f974..4ab2e6ab60 100644 --- a/core/iwasm/include/aot_comp_option.h +++ b/core/iwasm/include/aot_comp_option.h @@ -6,6 +6,23 @@ #ifndef __AOT_COMP_OPTION_H__ #define __AOT_COMP_OPTION_H__ +typedef struct { + /* Enables or disables bounds checks for stack frames. When enabled, the AOT + * compiler generates code to check if the stack pointer is within the + * bounds of the current stack frame (and if not, traps). */ + bool bounds_checks; + + /* Enables or disables instruction pointer (IP) tracking.*/ + bool ip; + + /* Enables or disables tracking instruction pointer of a trap. Only takes + * effect when `ip` is enabled.*/ + bool trap_ip; + + /* Enables or disables parameters, locals and stack operands. */ + bool values; +} AOTCallStackFeatures; + typedef struct AOTCompOption { bool is_jit_mode; bool is_indirect_mode; @@ -22,6 +39,7 @@ typedef struct AOTCompOption { bool enable_gc; bool enable_aux_stack_check; bool enable_aux_stack_frame; + AOTCallStackFeatures call_stack_features; bool enable_perf_profiling; bool enable_memory_profiling; bool disable_llvm_intrinsics; diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 13947ac82e..092e0d152b 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -5407,6 +5407,7 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, #if WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0 \ || WASM_ENABLE_AOT_STACK_FRAME != 0 option.enable_aux_stack_frame = true; + memset(&option.call_stack_features, 1, sizeof(AOTCallStackFeatures)); #endif #if WASM_ENABLE_PERF_PROFILING != 0 option.enable_perf_profiling = true; diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 8826f98db9..a21f4490fc 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -2149,6 +2149,7 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, #if WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0 \ || WASM_ENABLE_AOT_STACK_FRAME != 0 option.enable_aux_stack_frame = true; + memset(&option.call_stack_features, 1, sizeof(AOTCallStackFeatures)); #endif #if WASM_ENABLE_PERF_PROFILING != 0 option.enable_perf_profiling = true; diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index bd9e543537..3c7ef1f4de 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -162,6 +162,12 @@ print_help() printf(" GC is enabled\n"); printf(" --disable-aux-stack-check Disable auxiliary stack overflow/underflow check\n"); printf(" --enable-dump-call-stack Enable stack trace feature\n"); + printf(" --call-stack-features=\n"); + printf(" A comma-separated list of features when generating call stacks.\n"); + printf(" By default, all features are enabled. To disable all features,\n"); + printf(" provide an empty list (i.e. --call-stack-features=). This flag\n"); + printf(" only only takes effect when --enable-dump-call-stack is set.\n"); + printf(" Available features: bounds-checks, ip, trap-ip, values.\n"); printf(" --enable-perf-profiling Enable function performance profiling\n"); printf(" --enable-memory-profiling Enable memory usage profiling\n"); printf(" --xip A shorthand of --enable-indirect-mode --disable-llvm-intrinsics\n"); @@ -259,6 +265,48 @@ split_string(char *str, int *count, const char *delimer) return res; } +static bool +parse_call_stack_features(char *features_str, + AOTCallStackFeatures *out_features) +{ + int size = 0; + char **features; + bool ret = true; + + bh_assert(features_str); + bh_assert(out_features); + + /* non-empty feature list */ + features = split_string(features_str, &size, ","); + if (!features) { + return false; + } + + while (size--) { + if (!strcmp(features[size], "bounds-checks")) { + out_features->bounds_checks = true; + } + else if (!strcmp(features[size], "ip")) { + out_features->ip = true; + } + else if (!strcmp(features[size], "trap-ip")) { + out_features->trap_ip = true; + } + else if (!strcmp(features[size], "values")) { + out_features->values = true; + } + else { + ret = false; + printf("Unsupported feature %s\n", features[size]); + goto finish; + } + } + +finish: + free(features); + return ret; +} + static uint32 resolve_segue_flags(char *str_flags) { @@ -356,6 +404,9 @@ main(int argc, char *argv[]) option.enable_ref_types = true; option.enable_gc = false; + /* Set all the features to true by default */ + memset(&option.call_stack_features, 1, sizeof(AOTCallStackFeatures)); + /* Process options */ for (argc--, argv++; argc > 0 && argv[0][0] == '-'; argc--, argv++) { if (!strcmp(argv[0], "-o")) { @@ -470,6 +521,19 @@ main(int argc, char *argv[]) else if (!strcmp(argv[0], "--enable-dump-call-stack")) { option.enable_aux_stack_frame = true; } + else if (!strncmp(argv[0], "--call-stack-features=", 22)) { + /* Reset all the features, only enable the user-defined ones */ + memset(&option.call_stack_features, 0, + sizeof(AOTCallStackFeatures)); + + if (argv[0][22] != '\0') { + if (!parse_call_stack_features(argv[0] + 22, + &option.call_stack_features)) { + printf("Failed to parse call-stack-features\n"); + PRINT_HELP_AND_EXIT(); + } + } + } else if (!strcmp(argv[0], "--enable-perf-profiling")) { option.enable_aux_stack_frame = true; option.enable_perf_profiling = true; @@ -608,6 +672,12 @@ main(int argc, char *argv[]) #endif } + if (option.enable_gc && !option.call_stack_features.values) { + LOG_WARNING("Call stack feature 'values' must be enabled for GC. The " + "feature will be enabled automatically."); + option.call_stack_features.values = true; + } + if (sgx_mode) { option.size_level = 1; option.is_sgx_platform = true; From b38a2e88a237f2abe54b73a5dacde7b552f39e54 Mon Sep 17 00:00:00 2001 From: Matt Gabrenya Date: Thu, 5 Sep 2024 21:01:54 -0600 Subject: [PATCH 14/69] Fix building iwasm_shared and iwasm_static libs on win32 (#3762) Fixes to enable building iwasm_shared and iwasm_static libraries on win32. --- CMakeLists.txt | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c7f7666897..0531ec411b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,10 +121,14 @@ set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wshadow -Wno-unused-parameter -fvisibility=hidden") -# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wconversion -Wsign-conversion") - -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wno-unused") +if (NOT WIN32) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security \ + -ffunction-sections -fdata-sections \ + -Wno-unused-parameter -Wno-pedantic") + # Remove the extra spaces for better make log + string (REGEX REPLACE " *" " " CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wno-unused") +endif() if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) @@ -145,6 +149,10 @@ include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) set (THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) +if (MSVC) + add_definitions(-DCOMPILING_WASM_RUNTIME_API=1) +endif () + # STATIC LIBRARY if (WAMR_BUILD_STATIC) add_library(iwasm_static STATIC ${WAMR_RUNTIME_LIB_SOURCE}) @@ -155,6 +163,14 @@ if (WAMR_BUILD_STATIC) target_link_libraries(iwasm_static INTERFACE boringssl_crypto) endif () + if (MINGW) + target_link_libraries (iwasm_static PRIVATE ws2_32) + endif () + + if (WIN32) + target_link_libraries(iwasm_static PRIVATE ntdll) + endif() + install (TARGETS iwasm_static ARCHIVE DESTINATION lib) endif () @@ -169,9 +185,14 @@ if (WAMR_BUILD_SHARED) endif () if (MINGW) - target_link_libraries (iwasm_shared INTERFACE -lWs2_32 -lwsock32) + target_link_libraries(iwasm_shared INTERFACE -lWs2_32 -lwsock32) + target_link_libraries(iwasm_shared PRIVATE ws2_32) endif () + if (WIN32) + target_link_libraries(iwasm_shared PRIVATE ntdll) + endif() + install (TARGETS iwasm_shared LIBRARY DESTINATION lib) endif () From cb71ca5822cbfe130920d2a2b89c0c35ca56ac3c Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Mon, 9 Sep 2024 21:58:07 +0800 Subject: [PATCH 15/69] CI: Disable parallel test in spectest for NuttX (#3780) --- .github/workflows/spec_test_on_nuttx.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/spec_test_on_nuttx.yml b/.github/workflows/spec_test_on_nuttx.yml index f2e59ba68b..2e940b0e0f 100644 --- a/.github/workflows/spec_test_on_nuttx.yml +++ b/.github/workflows/spec_test_on_nuttx.yml @@ -327,19 +327,6 @@ jobs: working-directory: apps/interpreters/wamr/wamr - name: Test - if: matrix.target_config.target != 'xtensa' - run: | - cd apps/interpreters/wamr/wamr/tests/wamr-test-suites - ./test_wamr.sh -s spec ${{ matrix.wamr_test_option.mode }} -m ${{ matrix.target_config.target }} -b -Q -P -F ${{ steps.build_firmware_path.outputs.firmware }} ${{ matrix.wamr_feature_option.mode}} - - # for xtensa, for some reasons, when running the tests - # with test_wamr.sh -P, nuttx occasionally hangs after - # "total segments stored 6" on the CI. - # i (yamamoto) couldn't reproduce it locally (macOS) even - # with the identical flash image. - # for now, run the tests without -P. - - name: Test - if: matrix.target_config.target == 'xtensa' run: | cd apps/interpreters/wamr/wamr/tests/wamr-test-suites ./test_wamr.sh -s spec ${{ matrix.wamr_test_option.mode }} -m ${{ matrix.target_config.target }} -b -Q -F ${{ steps.build_firmware_path.outputs.firmware }} ${{ matrix.wamr_feature_option.mode}} From 059935126256e317c1a8abcedf4d560f10f3bec7 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Tue, 10 Sep 2024 08:45:18 +0800 Subject: [PATCH 16/69] wasi-nn: Add a new target for llama.cpp as a wasi-nn backend (#3709) Minimum support: - [x] accept (WasmEdge) customized model parameters. metadata. - [x] Target [wasmedge-ggml examples](https://github.com/second-state/WasmEdge-WASINN-examples/tree/master/wasmedge-ggml) - [x] basic - [x] chatml - [x] gemma - [x] llama - [x] qwen --- In the future, to support if required: - [ ] Target [wasmedge-ggml examples](https://github.com/second-state/WasmEdge-WASINN-examples/tree/master/wasmedge-ggml) - [ ] command-r. (>70G memory requirement) - [ ] embedding. (embedding mode) - [ ] grammar. (use the grammar option to constrain the model to generate the JSON output) - [ ] llama-stream. (new APIS `compute_single`, `get_output_single`, `fini_single`) - [ ] llava. (image representation) - [ ] llava-base64-stream. (image representation) - [ ] multimodel. (image representation) - [ ] Target [llamaedge](https://github.com/LlamaEdge/LlamaEdge) --- build-scripts/config_common.cmake | 8 +- core/iwasm/libraries/wasi-nn/README.md | 40 +- .../libraries/wasi-nn/cmake/Findcjson.cmake | 17 + .../wasi-nn/cmake/Findllamacpp.cmake | 18 + .../wasi-nn/cmake/Findtensorflow_lite.cmake | 58 +- .../libraries/wasi-nn/cmake/wasi_nn.cmake | 80 ++- .../libraries/wasi-nn/include/wasi_nn_types.h | 9 + core/iwasm/libraries/wasi-nn/src/wasi_nn.c | 133 +++- .../libraries/wasi-nn/src/wasi_nn_llamacpp.c | 601 ++++++++++++++++++ .../wasi-nn/test/Dockerfile.wasi-nn-smoke | 43 +- .../libraries/wasi-nn/test/run_smoke_test.py | 60 ++ 11 files changed, 947 insertions(+), 120 deletions(-) create mode 100644 core/iwasm/libraries/wasi-nn/cmake/Findcjson.cmake create mode 100644 core/iwasm/libraries/wasi-nn/cmake/Findllamacpp.cmake create mode 100644 core/iwasm/libraries/wasi-nn/src/wasi_nn_llamacpp.c diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 252ba3a841..12fc06bd70 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -442,7 +442,9 @@ if (WAMR_BUILD_WASI_NN EQUAL 1) message (" WASI-NN enabled") add_definitions (-DWASM_ENABLE_WASI_NN=1) # Variant backends - if (NOT WAMR_BUILD_WASI_NN_TFLITE EQUAL 1 AND NOT WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1) + if (NOT WAMR_BUILD_WASI_NN_TFLITE EQUAL 1 AND + NOT WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1 AND + NOT WAMR_BUILD_WASI_NN_LLAMACPP EQUAL 1) message (FATAL_ERROR " Need to select a backend for WASI-NN") endif () @@ -454,6 +456,10 @@ if (WAMR_BUILD_WASI_NN EQUAL 1) message (" WASI-NN: backend openvino enabled") add_definitions (-DWASM_ENABLE_WASI_NN_OPENVINO) endif () + if (WAMR_BUILD_WASI_NN_LLAMACPP EQUAL 1) + message (" WASI-NN: backend llamacpp enabled") + add_definitions (-DWASM_ENABLE_WASI_NN_LLAMACPP) + endif () # Variant devices if (WAMR_BUILD_WASI_NN_ENABLE_GPU EQUAL 1) message (" WASI-NN: GPU enabled") diff --git a/core/iwasm/libraries/wasi-nn/README.md b/core/iwasm/libraries/wasi-nn/README.md index e0d3a25cee..5536f6d57b 100644 --- a/core/iwasm/libraries/wasi-nn/README.md +++ b/core/iwasm/libraries/wasi-nn/README.md @@ -4,7 +4,7 @@ ### Host -Enable WASI-NN in the WAMR by spefiying it in the cmake building configuration as follows, +Enable WASI-NN in the WAMR by specifying it in the cmake building configuration as follows, ```cmake set (WAMR_BUILD_WASI_NN 1) @@ -17,14 +17,15 @@ $ cmake -DWAMR_BUILD_WASI_NN=1 ... ``` > ![Caution] -> If enable `WAMR_BUID_WASI_NN`, iwasm will link a shared WAMR library instead of a static one. Wasi-nn backends will be loaded dynamically at runtime. Users shall specify the path of the backend library and register it to the iwasm runtime with `--native-lib=`. All shared libraries should be placed in the `LD_LIBRARY_PATH`. +> Enabling WAMR_BUILD_WASI_NN will cause the IWASM to link to a shared WAMR library instead of a static one. The WASI-NN backends will then be loaded dynamically when the program is run. You must ensure that all shared libraries are included in the `LD_LIBRARY_PATH`. #### Compilation options -- `WAMR_BUILD_WASI_NN`. enable wasi-nn support. can't work alone. need to identify a backend. Match legacy wasi-nn spec naming convention. use `wasi_nn` as import module names. -- `WAMR_BUILD_WASI_EPHEMERAL_NN`. Match latest wasi-nn spec naming convention. use `wasi_ephemeral_nn` as import module names. -- `WAMR_BUILD_WASI_NN_TFLITE`. identify the backend as TensorFlow Lite. -- `WAMR_BUILD_WASI_NN_OPENVINO`. identify the backend as OpenVINO. +- `WAMR_BUILD_WASI_NN`. This option enables support for WASI-NN. It cannot function independently and requires specifying a backend. It follows the original WASI-NN specification for naming conventions and uses wasi_nn for import module names. +- `WAMR_BUILD_WASI_EPHEMERAL_NN`. This option adheres to the most recent WASI-NN specification for naming conventions and uses wasi_ephemeral_nn for import module names. +- `WAMR_BUILD_WASI_NN_TFLITE`. This option designates TensorFlow Lite as the backend. +- `WAMR_BUILD_WASI_NN_OPENVINO`. This option designates OpenVINO as the backend. +- `WAMR_BUILD_WASI_NN_LLAMACPP`. This option designates Llama.cpp as the backend. ### Wasm @@ -44,7 +45,7 @@ typedef enum { fp16 = 0, fp32, up8, ip32 } tensor_type; It is required to recompile the Wasm application if you want to switch between the two sets of functions. -#### Openvino +#### Openvino installation If you're planning to use OpenVINO backends, the first step is to install OpenVINO on your computer. To do this correctly, please follow the official installation guide which you can find at this link: https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-linux.html. @@ -162,17 +163,9 @@ Supported: ### Testing with WasmEdge-WASINN Examples -To ensure everything is set up correctly, use the examples from [WasmEdge-WASINN-examples](https://github.com/second-state/WasmEdge-WASINN-examples/tree/master). These examples help verify that WASI-NN support in WAMR is functioning as expected. +To make sure everything is configured properly, refer to the examples provided at [WasmEdge-WASINN-examples](https://github.com/second-state/WasmEdge-WASINN-examples/tree/master). These examples are useful for confirming that the WASI-NN support in WAMR is working correctly. -> Note: The repository contains two types of examples. Some use the [standard wasi-nn](https://github.com/WebAssembly/wasi-nn), while others use [WasmEdge's version of wasi-nn](https://github.com/second-state/wasmedge-wasi-nn), which is enhanced to meet specific customer needs. - -The examples test the following machine learning backends: - -- OpenVINO -- PyTorch -- TensorFlow Lite - -Due to the different requirements of each backend, we'll use a Docker container for a hassle-free testing environment. +Because each backend has its own set of requirements, we recommend using a Docker container to create a straightforward testing environment without complications. #### Prepare the execution environment @@ -186,9 +179,20 @@ $ docker build -t wasi-nn-smoke:v1.0 -f ./core/iwasm/libraries/wasi-nn/test/Dock #### Execute ```bash +$ pwd +/workspaces/wasm-micro-runtime/ $ docker run --rm wasi-nn-smoke:v1.0 ``` -### Testing with bytecodealliance wasi-nn +It should be noted that the qwen example is selected as the default one about the Llama.cpp backend because it uses a small model and is easy to run. + +```bash +- openvino_mobile_image. PASS +- openvino_mobile_raw. PASS +- openvino_road_segmentation_adas. PASS +- wasmedge_ggml_qwen. PASS +``` + +### Testing with bytecodealliance WASI-NN For another example, check out [classification-example](https://github.com/bytecodealliance/wasi-nn/tree/main/rust/examples/classification-example), which focuses on OpenVINO. You can run it using the same Docker container mentioned above. diff --git a/core/iwasm/libraries/wasi-nn/cmake/Findcjson.cmake b/core/iwasm/libraries/wasi-nn/cmake/Findcjson.cmake new file mode 100644 index 0000000000..1136f41adc --- /dev/null +++ b/core/iwasm/libraries/wasi-nn/cmake/Findcjson.cmake @@ -0,0 +1,17 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FetchContent) + +set(CJSON_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/cjson") + +FetchContent_Declare( + cjson + GIT_REPOSITORY https://github.com/DaveGamble/cJSON.git + GIT_TAG v1.7.18 + SOURCE_DIR ${CJSON_SOURCE_DIR} +) + +set(ENABLE_CJSON_TEST OFF CACHE INTERNAL "Turn off tests") +set(ENABLE_CJSON_UNINSTALL OFF CACHE INTERNAL "Turn off uninstall to avoid targets conflict") +FetchContent_MakeAvailable(cjson) diff --git a/core/iwasm/libraries/wasi-nn/cmake/Findllamacpp.cmake b/core/iwasm/libraries/wasi-nn/cmake/Findllamacpp.cmake new file mode 100644 index 0000000000..431e15db59 --- /dev/null +++ b/core/iwasm/libraries/wasi-nn/cmake/Findllamacpp.cmake @@ -0,0 +1,18 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FetchContent) + +set(LLAMA_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/llama.cpp") + +FetchContent_Declare( + llamacpp + GIT_REPOSITORY https://github.com/ggerganov/llama.cpp.git + GIT_TAG b3573 + SOURCE_DIR ${LLAMA_SOURCE_DIR} +) + +set(LLAMA_BUILD_TESTS OFF) +set(LLAMA_BUILD_EXAMPLES OFF) +set(LLAMA_BUILD_SERVER OFF) +FetchContent_MakeAvailable(llamacpp) diff --git a/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake b/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake index 052dd9804f..39480741d3 100644 --- a/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake +++ b/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake @@ -1,47 +1,25 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -find_library(TENSORFLOW_LITE - NAMES tensorflow-lite - HINTS ${CMAKE_CURRENT_BINARY_DIR}/tensorflow-lite - NO_DEFAULT_PATHS +include(FetchContent) + +set(TFLITE_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/tensorflow-src") + +FetchContent_Declare( + tensorflow_lite + GIT_REPOSITORY https://github.com/tensorflow/tensorflow.git + GIT_TAG v2.12.0 + GIT_SHALLOW ON + GIT_PROGRESS ON + SOURCE_DIR ${TFLITE_SOURCE_DIR} + SOURCE_SUBDIR tensorflow/lite ) -if(NOT TENSORFLOW_LITE) - if(NOT EXISTS "${WAMR_ROOT_DIR}/core/deps/tensorflow-src") - execute_process( - COMMAND "${WAMR_ROOT_DIR}/core/deps/install_tensorflow.sh" - RESULT_VARIABLE TENSORFLOW_RESULT - ) - else() - message("Tensorflow is already downloaded.") - endif() - - set(TENSORFLOW_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/tensorflow-src") - - if(WAMR_BUILD_WASI_NN_ENABLE_GPU EQUAL 1) - # Tensorflow specific: - # * https://www.tensorflow.org/lite/guide/build_cmake#available_options_to_build_tensorflow_lite - set (TFLITE_ENABLE_GPU ON) - endif() - - if (CMAKE_SIZEOF_VOID_P EQUAL 4) - set (TFLITE_ENABLE_XNNPACK OFF) - endif() - - add_subdirectory( - "${TENSORFLOW_SOURCE_DIR}/tensorflow/lite" - "${CMAKE_CURRENT_BINARY_DIR}/tensorflow-lite" - EXCLUDE_FROM_ALL - ) -else () - message(STATUS "TensorFlow Lite library found: ${TENSORFLOW_LITE}") - set(TENSORFLOW_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/tensorflow-src") +if(WAMR_BUILD_WASI_NN_ENABLE_GPU EQUAL 1) + set(TFLITE_ENABLE_GPU ON) +endif() +if (CMAKE_SIZEOF_VOID_P EQUAL 4) + set(TFLITE_ENABLE_XNNPACK OFF) endif() -set(TENSORFLOW_LITE_INCLUDE_DIR "${TENSORFLOW_SOURCE_DIR}/tensorflow/lite") -set(FLATBUFFER_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers/include") - -include_directories(${TENSORFLOW_SOURCE_DIR}) -include_directories(${FLATBUFFER_INCLUDE_DIR}) -link_directories(${CMAKE_CURRENT_BINARY_DIR}/tensorflow-lite) +FetchContent_MakeAvailable(tensorflow_lite) diff --git a/core/iwasm/libraries/wasi-nn/cmake/wasi_nn.cmake b/core/iwasm/libraries/wasi-nn/cmake/wasi_nn.cmake index e2ad257e06..a903f0af1f 100644 --- a/core/iwasm/libraries/wasi-nn/cmake/wasi_nn.cmake +++ b/core/iwasm/libraries/wasi-nn/cmake/wasi_nn.cmake @@ -3,27 +3,6 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) -if(WAMR_BUILD_WASI_NN_TFLITE EQUAL 1) - # Find tensorflow-lite - find_package(tensorflow_lite REQUIRED) -endif() - -if(WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1) - if(NOT DEFINED ENV{OpenVINO_DIR}) - message(FATAL_ERROR - "OpenVINO_DIR is not defined. " - "Please follow https://docs.openvino.ai/2024/get-started/install-openvino.html," - "install openvino, and set environment variable OpenVINO_DIR." - "Like OpenVINO_DIR=/usr/lib/openvino-2023.2/ cmake ..." - "Or OpenVINO_DIR=/opt/intel/openvino/ cmake ..." - ) - endif() - - list(APPEND CMAKE_MODULE_PATH $ENV{OpenVINO_DIR}) - # Find OpenVINO - find_package(OpenVINO REQUIRED COMPONENTS Runtime) -endif() - # # wasi-nn general set(WASI_NN_ROOT ${CMAKE_CURRENT_LIST_DIR}/..) @@ -42,22 +21,46 @@ add_compile_definitions( # # - tflite if(WAMR_BUILD_WASI_NN_TFLITE EQUAL 1) + find_package(tensorflow_lite REQUIRED) + add_library( wasi_nn_tflite SHARED ${WASI_NN_ROOT}/src/wasi_nn_tensorflowlite.cpp ) + target_include_directories( + wasi_nn_tflite + PUBLIC + ${tensorflow_lite_SOURCE_DIR} + ) + target_link_libraries( wasi_nn_tflite PUBLIC libiwasm tensorflow-lite ) + + install(TARGETS wasi_nn_tflite DESTINATION lib) endif() # - openvino if(WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1) + if(NOT DEFINED ENV{OpenVINO_DIR}) + message(FATAL_ERROR + "OpenVINO_DIR is not defined. " + "Please follow https://docs.openvino.ai/2024/get-started/install-openvino.html," + "install openvino, and set environment variable OpenVINO_DIR." + "Like OpenVINO_DIR=/usr/lib/openvino-2023.2/ cmake ..." + "Or OpenVINO_DIR=/opt/intel/openvino/ cmake ..." + ) + endif() + + list(APPEND CMAKE_MODULE_PATH $ENV{OpenVINO_DIR}) + # Find OpenVINO + find_package(OpenVINO REQUIRED COMPONENTS Runtime) + add_library( wasi_nn_openvino SHARED @@ -71,4 +74,37 @@ if(WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1) openvino::runtime openvino::runtime::c ) -endif() \ No newline at end of file + + install(TARGETS wasi_nn_openvino DESTINATION lib) +endif() + +# - llamacpp + +if(WAMR_BUILD_WASI_NN_LLAMACPP EQUAL 1) + find_package(cjson REQUIRED) + find_package(llamacpp REQUIRED) + + add_library( + wasi_nn_llamacpp + SHARED + ${WASI_NN_ROOT}/src/wasi_nn_llamacpp.c + ) + + target_include_directories( + wasi_nn_llamacpp + PUBLIC + ${cjson_SOURCE_DIR} + ) + + target_link_libraries( + wasi_nn_llamacpp + PUBLIC + libiwasm + cjson + common + ggml + llama + ) + + install(TARGETS wasi_nn_llamacpp DESTINATION lib) +endif() diff --git a/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h b/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h index d36f5977c2..3ac694fc95 100644 --- a/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h +++ b/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h @@ -43,6 +43,11 @@ typedef enum { security, // The operation failed for an unspecified reason. unknown, + // for WasmEdge-wasi-nn + end_of_sequence = 100, // End of Sequence Found. + context_full = 101, // Context Full. + prompt_tool_long = 102, // Prompt Too Long. + model_not_found = 103, // Model Not Found. } wasi_nn_error; /** @@ -140,6 +145,9 @@ typedef uint32_t graph_execution_context; typedef wasi_nn_error (*LOAD)(void *, graph_builder_array *, graph_encoding, execution_target, graph *); typedef wasi_nn_error (*LOAD_BY_NAME)(void *, const char *, uint32_t, graph *); +typedef wasi_nn_error (*LOAD_BY_NAME_WITH_CONFIG)(void *, const char *, + uint32_t, void *, uint32_t, + graph *); typedef wasi_nn_error (*INIT_EXECUTION_CONTEXT)(void *, graph, graph_execution_context *); typedef wasi_nn_error (*SET_INPUT)(void *, graph_execution_context, uint32_t, @@ -154,6 +162,7 @@ typedef wasi_nn_error (*BACKEND_DEINITIALIZE)(void *); typedef struct { LOAD load; LOAD_BY_NAME load_by_name; + LOAD_BY_NAME_WITH_CONFIG load_by_name_with_config; INIT_EXECUTION_CONTEXT init_execution_context; SET_INPUT set_input; COMPUTE compute; diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn.c b/core/iwasm/libraries/wasi-nn/src/wasi_nn.c index 0d56981fc7..4697e931b0 100644 --- a/core/iwasm/libraries/wasi-nn/src/wasi_nn.c +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn.c @@ -29,7 +29,7 @@ struct backends_api_functions { void *backend_handle; api_function functions; -} lookup[autodetect] = { 0 }; +} lookup[autodetect + 1] = { 0 }; #define call_wasi_nn_func(backend_encoding, func, wasi_error, ...) \ do { \ @@ -168,14 +168,7 @@ wasi_nn_destroy() lookup[i].backend_handle = NULL; } - lookup[i].functions.init = NULL; - lookup[i].functions.deinit = NULL; - lookup[i].functions.load = NULL; - lookup[i].functions.load_by_name = NULL; - lookup[i].functions.init_execution_context = NULL; - lookup[i].functions.set_input = NULL; - lookup[i].functions.compute = NULL; - lookup[i].functions.get_output = NULL; + memset(&lookup[i].functions, 0, sizeof(api_function)); } } @@ -208,6 +201,10 @@ choose_a_backend() return ggml; } +#ifndef NDEBUG + NN_WARN_PRINTF("%s", dlerror()); +#endif + handle = dlopen(OPENVINO_BACKEND_LIB, RTLD_LAZY); if (handle) { NN_INFO_PRINTF("Using openvino backend"); @@ -215,6 +212,10 @@ choose_a_backend() return openvino; } +#ifndef NDEBUG + NN_WARN_PRINTF("%s", dlerror()); +#endif + handle = dlopen(TFLITE_BACKEND_LIB, RTLD_LAZY); if (handle) { NN_INFO_PRINTF("Using tflite backend"); @@ -222,6 +223,11 @@ choose_a_backend() return tensorflowlite; } +#ifndef NDEBUG + NN_WARN_PRINTF("%s", dlerror()); +#endif + + NN_WARN_PRINTF("No backend found"); return unknown_backend; } @@ -257,6 +263,14 @@ register_backend(void *handle, api_function *functions) } functions->load_by_name = load_by_name; + LOAD_BY_NAME_WITH_CONFIG load_by_name_with_config = + (LOAD_BY_NAME_WITH_CONFIG)dlsym(handle, "load_by_name_with_config"); + if (!load_by_name_with_config) { + NN_WARN_PRINTF("load_by_name_with_config() not found"); + // since only llama.cpp backend need to support this function + } + functions->load_by_name_with_config = load_by_name_with_config; + INIT_EXECUTION_CONTEXT init_execution_context = (INIT_EXECUTION_CONTEXT)dlsym(handle, "init_execution_context"); if (!init_execution_context) { @@ -329,21 +343,23 @@ graph_encoding_to_backend_lib_name(graph_encoding encoding) static bool detect_and_load_backend(graph_encoding backend_hint, struct backends_api_functions *backends, - graph_encoding *loaded_backed) + graph_encoding *loaded_backend) { - if (backend_hint >= autodetect) + if (backend_hint > autodetect) return false; if (backend_hint == autodetect) backend_hint = choose_a_backend(); + if (backend_hint == unknown_backend) + return false; + + *loaded_backend = backend_hint; + /* if already loaded */ - if (lookup[backend_hint].backend_handle) { - *loaded_backed = backend_hint; + if (lookup[backend_hint].backend_handle) return true; - } - *loaded_backed = backend_hint; const char *backend_lib_name = graph_encoding_to_backend_lib_name(backend_hint); if (!backend_lib_name) @@ -353,6 +369,7 @@ detect_and_load_backend(graph_encoding backend_hint, } /* WASI-NN implementation */ + #if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 wasi_nn_error wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_wasm *builder, @@ -392,15 +409,15 @@ wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder, goto fail; } - graph_encoding loaded_backed = autodetect; - if (!detect_and_load_backend(encoding, lookup, &loaded_backed)) { + graph_encoding loaded_backend = autodetect; + if (!detect_and_load_backend(encoding, lookup, &loaded_backend)) { res = invalid_encoding; NN_ERR_PRINTF("load backend failed"); goto fail; } WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); - wasi_nn_ctx->backend = loaded_backed; + wasi_nn_ctx->backend = loaded_backend; /* init() the backend */ call_wasi_nn_func(wasi_nn_ctx->backend, init, res, @@ -413,7 +430,6 @@ wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder, if (res != success) goto fail; - wasi_nn_ctx->backend = loaded_backed; wasi_nn_ctx->is_model_loaded = true; fail: @@ -428,8 +444,6 @@ wasi_nn_error wasi_nn_load_by_name(wasm_exec_env_t exec_env, char *name, uint32_t name_len, graph *g) { - NN_DBG_PRINTF("[WASI NN] LOAD_BY_NAME %s...", name); - wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); if (!instance) { return runtime_error; @@ -446,15 +460,23 @@ wasi_nn_load_by_name(wasm_exec_env_t exec_env, char *name, uint32_t name_len, return invalid_argument; } - graph_encoding loaded_backed = autodetect; - if (detect_and_load_backend(autodetect, lookup, &loaded_backed)) { + if (name_len == 0 || name[name_len] != '\0') { + NN_ERR_PRINTF("Invalid filename"); + return invalid_argument; + } + + NN_DBG_PRINTF("[WASI NN] LOAD_BY_NAME %s...", name); + + graph_encoding loaded_backend = autodetect; + if (!detect_and_load_backend(autodetect, lookup, &loaded_backend)) { NN_ERR_PRINTF("load backend failed"); return invalid_encoding; } WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); - wasi_nn_error res; + wasi_nn_ctx->backend = loaded_backend; + wasi_nn_error res; /* init() the backend */ call_wasi_nn_func(wasi_nn_ctx->backend, init, res, &wasi_nn_ctx->backend_ctx); @@ -466,7 +488,67 @@ wasi_nn_load_by_name(wasm_exec_env_t exec_env, char *name, uint32_t name_len, if (res != success) return res; - wasi_nn_ctx->backend = loaded_backed; + wasi_nn_ctx->backend = loaded_backend; + wasi_nn_ctx->is_model_loaded = true; + return success; +} + +wasi_nn_error +wasi_nn_load_by_name_with_config(wasm_exec_env_t exec_env, char *name, + int32_t name_len, char *config, + int32_t config_len, graph *g) +{ + wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); + if (!instance) { + return runtime_error; + } + + if (!wasm_runtime_validate_native_addr(instance, name, name_len)) { + NN_ERR_PRINTF("name is invalid"); + return invalid_argument; + } + + if (!wasm_runtime_validate_native_addr(instance, g, + (uint64)sizeof(graph))) { + NN_ERR_PRINTF("graph is invalid"); + return invalid_argument; + } + + if (name_len == 0 || name[name_len] != '\0') { + NN_ERR_PRINTF("Invalid filename"); + return invalid_argument; + } + + if (!config || config_len == 0 || config[config_len] != '\0') { + NN_ERR_PRINTF("Invalid config"); + return invalid_argument; + } + + NN_DBG_PRINTF("[WASI NN] LOAD_BY_NAME_WITH_CONFIG %s %s...", name, config); + + graph_encoding loaded_backend = autodetect; + if (!detect_and_load_backend(autodetect, lookup, &loaded_backend)) { + NN_ERR_PRINTF("load backend failed"); + return invalid_encoding; + } + + WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); + wasi_nn_ctx->backend = loaded_backend; + + wasi_nn_error res; + /* init() the backend */ + call_wasi_nn_func(wasi_nn_ctx->backend, init, res, + &wasi_nn_ctx->backend_ctx); + if (res != success) + return res; + + call_wasi_nn_func(wasi_nn_ctx->backend, load_by_name_with_config, res, + wasi_nn_ctx->backend_ctx, name, name_len, config, + config_len, g); + if (res != success) + return res; + + wasi_nn_ctx->backend = loaded_backend; wasi_nn_ctx->is_model_loaded = true; return success; } @@ -608,6 +690,7 @@ static NativeSymbol native_symbols_wasi_nn[] = { #if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 REG_NATIVE_FUNC(load, "(*iii*)i"), REG_NATIVE_FUNC(load_by_name, "(*i*)i"), + REG_NATIVE_FUNC(load_by_name_with_config, "(*i*i*)i"), REG_NATIVE_FUNC(init_execution_context, "(i*)i"), REG_NATIVE_FUNC(set_input, "(ii*)i"), REG_NATIVE_FUNC(compute, "(i)i"), diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn_llamacpp.c b/core/iwasm/libraries/wasi-nn/src/wasi_nn_llamacpp.c new file mode 100644 index 0000000000..58d29163c3 --- /dev/null +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn_llamacpp.c @@ -0,0 +1,601 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include "wasi_nn_types.h" +#include "utils/logger.h" +#include "llama.h" +#include "ggml.h" +#include "cJSON.h" + +// build info +extern int LLAMA_BUILD_NUMBER; +extern char const *LLAMA_COMMIT; +extern char const *LLAMA_COMPILER; +extern char const *LLAMA_BUILD_TARGET; + +// compatable with WasmEdge +// https://github.com/second-state/WasmEdge-WASINN-examples/blob/master/wasmedge-ggml/README.md#parameters +// https://github.com/WasmEdge/WasmEdge/blob/master/plugins/wasi_nn/ggml.cpp +struct wasi_nn_llama_config { + // Backend(plugin in WasmEdge) parameters: + bool enable_log; + bool enable_debug_log; + bool stream_stdout; + // embedding mode + bool embedding; + // TODO: can it be -1? + // can't bigger than ctx_size + int32_t n_predict; + char *reverse_prompt; + + // Used by LLaVA + // multi-model project file + char *mmproj; + char *image; + + // Model parameters (need to reload the model if updated): + // align to definition of struct llama_model_params + int32_t n_gpu_layers; + int32_t main_gpu; + // limited size: llama_max_devices() + float *tensor_split; + bool use_mmap; + + // Context parameters (used by the llama context): + uint32_t ctx_size; + uint32_t batch_size; + uint32_t ubatch_size; + uint32_t threads; + + // Sampling parameters (used by the llama sampling context). + float temp; + float topP; + float repeat_penalty; + float presence_penalty; + float frequency_penalty; +}; + +struct LlamaContext { + struct llama_context *ctx; + struct llama_model *model; + llama_token *prompt; + size_t prompt_len; + llama_token *generation; + size_t generation_len; + struct wasi_nn_llama_config config; +}; + +static void +wasm_edge_llama_default_configuration(struct wasi_nn_llama_config *output) +{ + output->enable_log = false; + output->enable_debug_log = false; + output->stream_stdout = false; + output->embedding = false; + output->n_predict = 512; + output->reverse_prompt = NULL; + + output->mmproj = NULL; + output->image = NULL; + + output->main_gpu = 0; + output->n_gpu_layers = 0; + output->tensor_split = NULL; + output->use_mmap = true; + + // 0 = from model + output->ctx_size = 0; + output->batch_size = 512; + output->ubatch_size = output->batch_size; + output->threads = 1; + + output->temp = 0.80; + output->topP = 0.95; + output->repeat_penalty = 1.10; + output->presence_penalty = 0.0; + output->frequency_penalty = 0.0; +} + +static void +wasm_edge_llama_apply_configuration(const char *config_json, + struct wasi_nn_llama_config *output) +{ + cJSON *root = cJSON_Parse(config_json); + if (root == NULL) { + const char *error_ptr = cJSON_GetErrorPtr(); + if (error_ptr != NULL) { + NN_WARN_PRINTF("Error before: %s\n", error_ptr); + } + else { + NN_WARN_PRINTF("Failed to parse JSON"); + } + return; + } + + cJSON *item = NULL; + + item = cJSON_GetObjectItem(root, "enable-log"); + if (item != NULL) { + output->enable_log = cJSON_IsTrue(item); + NN_DBG_PRINTF("apply enable-log %d", output->enable_log); + } + + item = cJSON_GetObjectItem(root, "enable-debug-log"); + if (item != NULL) { + output->enable_debug_log = cJSON_IsTrue(item); + NN_DBG_PRINTF("apply enable-debug-log %d", output->enable_debug_log); + } + + item = cJSON_GetObjectItem(root, "stream-stdout"); + if (item != NULL) { + output->stream_stdout = cJSON_IsTrue(item); + NN_DBG_PRINTF("apply stream-stdout %d", output->stream_stdout); + } + + item = cJSON_GetObjectItem(root, "embedding"); + if (item != NULL) { + output->embedding = cJSON_IsTrue(item); + NN_DBG_PRINTF("apply embedding %d", output->embedding); + } + + item = cJSON_GetObjectItem(root, "n-predict"); + if (item != NULL) { + output->n_predict = (int32_t)cJSON_GetNumberValue(item); + NN_DBG_PRINTF("apply n-predict %d", output->n_predict); + } + + item = cJSON_GetObjectItem(root, "n-gpu-layers"); + if (item != NULL) { + output->n_gpu_layers = (int32_t)cJSON_GetNumberValue(item); + NN_DBG_PRINTF("apply n_gpu_layers %d", output->n_gpu_layers); + } + + item = cJSON_GetObjectItem(root, "ctx-size"); + if (item != NULL) { + output->ctx_size = (uint32_t)cJSON_GetNumberValue(item); + NN_DBG_PRINTF("apply ctx-size %d", output->ctx_size); + } + + // more ... + + cJSON_Delete(root); +} + +static struct llama_model_params +llama_model_params_from_wasi_nn_llama_config( + struct wasi_nn_llama_config *config) +{ + struct llama_model_params result = llama_model_default_params(); + + // TODO: support more + result.main_gpu = config->main_gpu; + result.n_gpu_layers = config->n_gpu_layers; + result.use_mmap = config->use_mmap; + + return result; +} + +static struct llama_context_params +llama_context_params_from_wasi_nn_llama_config( + struct wasi_nn_llama_config *config) +{ + struct llama_context_params result = llama_context_default_params(); + + // TODO: support more + result.n_ctx = config->ctx_size; + // result.embeddings = config->embedding; + + return result; +} + +static void +llama_batch_clear(struct llama_batch *batch) +{ + batch->n_tokens = 0; +} + +static void +llama_batch_add(struct llama_batch *batch, llama_token id, llama_pos pos, + llama_seq_id *seq_ids, size_t seq_ids_len, bool logits) +{ + batch->token[batch->n_tokens] = id; + batch->pos[batch->n_tokens] = pos; + batch->n_seq_id[batch->n_tokens] = seq_ids_len; + for (size_t i = 0; i < seq_ids_len; ++i) { + batch->seq_id[batch->n_tokens][i] = seq_ids[i]; + } + batch->logits[batch->n_tokens] = logits; + + batch->n_tokens++; +} + +// always output ERROR and WARN +// INFO needs enable_log +// DEBUG needs enable_debug_log +static void +llama_log_callback_local(enum ggml_log_level level, const char *text, + void *user_data) +{ + struct LlamaContext *backend_ctx = (struct LlamaContext *)user_data; + + if (level == GGML_LOG_LEVEL_DEBUG && !backend_ctx->config.enable_debug_log) + return; + + if (level == GGML_LOG_LEVEL_INFO && !backend_ctx->config.enable_log) + return; + + printf("%s", text); +} + +static void +llama_build_output_metadata(const struct LlamaContext *backend_ctx, + char *output_buf, size_t output_buf_size) +{ + snprintf(output_buf, output_buf_size, + "{\"input_tokens\":%ld, \"output_tokens\":%ld, " + "\"llama_build_number\":%d," + "\"llama_commit\":\"%s\"}", + backend_ctx->prompt_len, backend_ctx->generation_len, + LLAMA_BUILD_NUMBER, LLAMA_COMMIT); +} + +__attribute__((visibility("default"))) wasi_nn_error +init_backend(void **ctx) +{ + struct LlamaContext *backend_ctx = calloc(1, sizeof(struct LlamaContext)); + if (!backend_ctx) { + NN_ERR_PRINTF("Allocate for OpenVINOContext failed"); + return runtime_error; + } + + llama_backend_init(); + // llama_numa_init(); + llama_log_set(llama_log_callback_local, backend_ctx); + +#ifndef NDEBUG + NN_INFO_PRINTF("llama_build_number: % d, llama_commit: %s, llama_compiler: " + "%s, llama_build_target: %s", + LLAMA_BUILD_NUMBER, LLAMA_COMMIT, LLAMA_COMPILER, + LLAMA_BUILD_TARGET); +#endif + + *ctx = (void *)backend_ctx; + return success; +} + +__attribute__((visibility("default"))) wasi_nn_error +deinit_backend(void *ctx) +{ + struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; + + if (!backend_ctx) + return invalid_argument; + + if (backend_ctx->generation) + free(backend_ctx->generation); + + if (backend_ctx->prompt) + free(backend_ctx->prompt); + + if (backend_ctx->ctx) + llama_free(backend_ctx->ctx); + + if (backend_ctx->model) + llama_free_model(backend_ctx->model); + + llama_backend_free(); + + os_free(backend_ctx); + return success; +} + +__attribute__((visibility("default"))) wasi_nn_error +load(void *ctx, graph_builder_array *builder, graph_encoding encoding, + execution_target target, graph *g) +{ + return unsupported_operation; +} + +static wasi_nn_error +__load_by_name_with_configuration(void *ctx, const char *filename, graph *g) +{ + struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; + + // make sure backend_ctx->config is initialized + + struct llama_model_params model_params = + llama_model_params_from_wasi_nn_llama_config(&backend_ctx->config); + struct llama_model *model = + llama_load_model_from_file(filename, model_params); + if (model == NULL) { + NN_ERR_PRINTF("Failed to load model from file %s", filename); + return runtime_error; + } + +#ifndef NDEBUG + char buf[128] = { 0 }; + llama_model_desc(model, buf, 127); + NN_INFO_PRINTF("Model desc %s", buf); +#endif + + backend_ctx->model = model; + + return success; +} + +__attribute__((visibility("default"))) wasi_nn_error +load_by_name(void *ctx, const char *filename, uint32_t filename_len, graph *g) +{ + struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; + + // use default params + wasm_edge_llama_default_configuration(&backend_ctx->config); + return __load_by_name_with_configuration(ctx, filename, g); +} + +__attribute__((visibility("default"))) wasi_nn_error +load_by_name_with_config(void *ctx, const char *filename, uint32_t filename_len, + const char *config, uint32_t config_len, graph *g) +{ + struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; + + wasm_edge_llama_default_configuration(&backend_ctx->config); + + if (config != NULL) { + // parse wasmedge config + wasm_edge_llama_apply_configuration(config, &backend_ctx->config); + } + else { + NN_INFO_PRINTF("No configuration provided, use default"); + } + + return __load_by_name_with_configuration(ctx, filename, g); +} + +// It is assumed that model params shouldn't be changed in Config stage. +// We only load the model once in the Load stage. +__attribute__((visibility("default"))) wasi_nn_error +init_execution_context(void *ctx, graph g, graph_execution_context *exec_ctx) +{ + struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; + + struct llama_context_params ctx_params = + llama_context_params_from_wasi_nn_llama_config(&backend_ctx->config); + struct llama_context *llama_ctx = + llama_new_context_with_model(backend_ctx->model, ctx_params); + if (llama_ctx == NULL) { + NN_ERR_PRINTF("Failed to create context for model"); + return runtime_error; + } + + backend_ctx->ctx = llama_ctx; + + NN_INFO_PRINTF("n_predict = %d, n_ctx = %d", backend_ctx->config.n_predict, + llama_n_ctx(backend_ctx->ctx)); + return success; +} + +__attribute__((visibility("default"))) wasi_nn_error +set_input(void *ctx, graph_execution_context exec_ctx, uint32_t index, + tensor *wasi_nn_tensor) +{ + struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; + // tensor->data is the prompt string. ends with \0 + char *prompt_text = (char *)wasi_nn_tensor->data; + +#ifndef NDEBUG + NN_DBG_PRINTF("--------------------------------------------------"); + NN_DBG_PRINTF("prompt_text: %s", prompt_text); + NN_DBG_PRINTF("--------------------------------------------------"); +#endif + + // tokenize the prompt + uint32_t n_token_max = llama_n_ctx(backend_ctx->ctx); + uint32_t prompt_text_len = strlen(prompt_text); + + if (backend_ctx->prompt == NULL) { + backend_ctx->prompt = calloc(n_token_max, sizeof(llama_token)); + if (backend_ctx->prompt == NULL) { + NN_ERR_PRINTF("Failed to allocate tokens_list"); + return runtime_error; + } + } + + int32_t n_tokens = + llama_tokenize(backend_ctx->model, prompt_text, prompt_text_len, + backend_ctx->prompt, n_token_max, true, false); + if (n_tokens < 0) { + NN_ERR_PRINTF("Failed to tokenize prompt text"); + return runtime_error; + } + + backend_ctx->prompt_len = n_tokens; + + // make sure the KV cache is big enough to hold all the prompt and generated + // tokens + int n_kv_req = n_tokens + (backend_ctx->config.n_predict - n_tokens); + if (n_kv_req < 0 || (uint32_t)n_kv_req > n_token_max) { + NN_ERR_PRINTF("the required KV cache size is not big enough, either " + "reduce n_predict or increase n_ctx"); + return runtime_error; + } + + return success; +} + +__attribute__((visibility("default"))) wasi_nn_error +compute(void *ctx, graph_execution_context exec_ctx) +{ + struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; + wasi_nn_error ret = runtime_error; + + // reset the generation buffer + if (backend_ctx->generation == NULL) { + backend_ctx->generation = + calloc(backend_ctx->config.n_predict, sizeof(llama_token)); + if (backend_ctx->generation == NULL) { + NN_ERR_PRINTF("Failed to allocate generation"); + return runtime_error; + } + } + + backend_ctx->generation_len = 0; + + // check KV cache + uint32_t n_ctx = llama_n_ctx(backend_ctx->ctx); + if (n_ctx <= backend_ctx->generation_len) { + NN_ERR_PRINTF( + "ctx_size(%u) is not big enough(<%ld), please increase it", n_ctx, + backend_ctx->generation_len); + return context_full; + } + + // prepare the batch + struct llama_batch batch = + llama_batch_init(backend_ctx->config.batch_size, 0, 1); + + // evaluate the initial prompt + llama_seq_id seq_ids[1] = { 0 }; + for (size_t i = 0; i < backend_ctx->prompt_len; i++) { + llama_batch_add(&batch, backend_ctx->prompt[i], i, seq_ids, + sizeof(seq_ids) / sizeof(seq_ids[0]), false); + } + + batch.logits[batch.n_tokens - 1] = true; + + if (batch.n_tokens > backend_ctx->config.n_predict) { + NN_DBG_PRINTF("n_predict(%d) is not big enough(%d), please increase it", + backend_ctx->config.n_predict, batch.n_tokens); + return prompt_tool_long; + } + + if (llama_decode(backend_ctx->ctx, batch) != 0) { + NN_ERR_PRINTF("First decode failed"); + return runtime_error; + } + + // main loop + int32_t n_cur = batch.n_tokens; + int n_decode = 0; + int32_t n_vocab = llama_n_vocab(backend_ctx->model); + llama_token_data *candidates = NULL; + + candidates = calloc(n_vocab, sizeof(llama_token_data)); + if (candidates == NULL) { + NN_ERR_PRINTF("Failed to allocate candidates"); + goto fail; + } + + while (n_cur <= backend_ctx->config.n_predict) { + // sample the next token + float *logits = + llama_get_logits_ith(backend_ctx->ctx, batch.n_tokens - 1); + + memset(candidates, 0, sizeof(llama_token_data) * n_vocab); + for (llama_token token_id = 0; token_id < n_vocab; token_id++) { + candidates[token_id].id = token_id; + candidates[token_id].logit = logits[token_id]; + candidates[token_id].p = 0.0f; + } + + llama_token_data_array candidates_p = { candidates, n_vocab, false }; + + // sample the most likely token + llama_token new_token_id = + llama_sample_token_greedy(backend_ctx->ctx, &candidates_p); + + backend_ctx->generation[backend_ctx->generation_len++] = new_token_id; + +#ifndef NDEBUG + { + char buf[128] = { 0 }; + llama_token_to_piece(backend_ctx->model, new_token_id, buf, 120, 0, + true); + printf("%d(%s),", new_token_id, buf); + } +#endif + + // is it an end of generation? + if (llama_token_is_eog(backend_ctx->model, new_token_id)) { + printf("\n"); + NN_INFO_PRINTF("reach the end of generation"); + break; + } + + // prepare the next batch + llama_batch_clear(&batch); + // push this new token for next evaluation + llama_batch_add(&batch, new_token_id, n_cur, seq_ids, + sizeof(seq_ids) / sizeof(seq_ids[0]), true); + n_decode++; + n_cur++; + + if (llama_decode(backend_ctx->ctx, batch) != 0) { + NN_ERR_PRINTF("Secondary decode failed"); + goto fail; + } + } + + printf("\n"); + ret = success; +fail: + llama_batch_free(batch); + if (candidates != NULL) { + free(candidates); + } + return ret; +} + +__attribute__((visibility("default"))) wasi_nn_error +get_output(void *ctx, graph_execution_context exec_ctx, uint32_t index, + tensor_data output_tensor, uint32_t *output_tensor_size) +{ + struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; + + // Compatibility with WasmEdge + if (index > 1) { + NN_ERR_PRINTF("Invalid output index %d", index); + return invalid_argument; + } + + // Index 1 is for the metadata of the outputs. + if (index == 1) { + char output_metadata[128] = { 0 }; + llama_build_output_metadata(backend_ctx, output_metadata, 127); + + if (backend_ctx->config.stream_stdout) { + printf("%s\n", output_metadata); + } + + memcpy(output_tensor, output_metadata, strlen(output_metadata)); + *output_tensor_size = strlen(output_metadata); + return success; + } + + // token -> piece -> output_tensor + if (backend_ctx->config.stream_stdout) { + printf("\n"); + } + + size_t end_pos = 0; + for (size_t i = 0; i < backend_ctx->generation_len; i++) { + char buf[128] = { 0 }; + llama_token_to_piece(backend_ctx->model, backend_ctx->generation[i], + buf, 120, 0, true); + + if (backend_ctx->config.stream_stdout) { + printf("%s", buf); + } + + memcpy(output_tensor + end_pos, buf, strlen(buf)); + end_pos += strlen(buf); + } + + if (backend_ctx->config.stream_stdout) { + printf("\n"); + } + + *output_tensor_size = end_pos; + return success; +} diff --git a/core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke b/core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke index 261c772613..fe3a8c5122 100644 --- a/core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke +++ b/core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke @@ -63,21 +63,35 @@ WORKDIR /workspaces/wasmedge-wasinn-examples RUN git clone --depth 1 https://github.com/second-state/WasmEdge-WASINN-examples.git . COPY core/iwasm/libraries/wasi-nn/test/bump_wasi_nn_to_0_6_0.patch . RUN git apply ./bump_wasi_nn_to_0_6_0.patch -# recompile with wasi-nn 0.6.0 -RUN cd openvino-mobilenet-image/rust && cargo build --target=wasm32-wasi -RUN cd openvino-mobilenet-raw/rust && cargo build --target=wasm32-wasi -RUN cd openvino-road-segmentation-adas/openvino-road-seg-adas && cargo build --target=wasm32-wasi -RUN cd tflite-birds_v1-image/rust && cargo build --target=wasm32-wasi -# preparation -RUN cd openvino-mobilenet-image \ +# recompile with wasi-nn 0.6.0 +WORKDIR /workspaces/wasmedge-wasinn-examples/openvino-mobilenet-image/ +RUN pushd rust \ + && cargo build --target=wasm32-wasi \ + && popd \ && ./download_mobilenet.sh . \ && ls -l mobilenet.xml mobilenet.bin -RUN cd openvino-mobilenet-raw \ +WORKDIR /workspaces/wasmedge-wasinn-examples/openvino-mobilenet-raw/ +RUN pushd rust \ + && cargo build --target=wasm32-wasi \ + && popd \ && ./download_mobilenet.sh . \ && ls -l mobilenet.xml mobilenet.bin tensor-1x224x224x3-f32.bgr +WORKDIR /workspaces/wasmedge-wasinn-examples/openvino-road-segmentation-adas/ +RUN pushd openvino-road-seg-adas \ + && cargo build --target=wasm32-wasi + +WORKDIR /workspaces/wasmedge-wasinn-examples/tflite-birds_v1-image/ +RUN pushd rust \ + && cargo build --target=wasm32-wasi + +# mount models when running +WORKDIR /workspaces/wasmedge-wasinn-examples/wasmedge-ggml/qwen +RUN wget --progress=dot:giga https://www.modelscope.cn/models/qwen/Qwen1.5-0.5B-Chat-GGUF/resolve/master/qwen1_5-0_5b-chat-q2_k.gguf +RUN cargo build --target=wasm32-wasi + # # iwasm. build from source WORKDIR /workspaces/wamr @@ -88,15 +102,16 @@ WORKDIR /workspaces/wamr/product-mini/platforms/linux RUN OpenVINO_DIR=/usr/lib/openvino-2023.2.0 \ cmake -S . -B build \ -DWAMR_BUILD_WASI_NN=1 -DWAMR_BUILD_WASI_EPHEMERAL_NN=1 \ - -DWAMR_BUILD_WASI_NN_OPENVINO=1 -DWAMR_BUILD_WASI_NN_TFLITE=1 \ - && cmake --build build - -ENV PATH=/workspaces/wamr/product-mini/platforms/linux/build:${PATH} -ENV LD_LIBRARY_PATH=/workspaces/wamr/product-mini/platforms/linux/build + -DWAMR_BUILD_WASI_NN_OPENVINO=1 \ + -DWAMR_BUILD_WASI_NN_TFLITE=1 \ + -DWAMR_BUILD_WASI_NN_LLAMACPP=1 \ + && cmake --build build \ + && cmake --install build + +ENV LD_LIBRARY_PATH=/usr/local/lib # add smoke test script COPY core/iwasm/libraries/wasi-nn/test/run_smoke_test.py / -# WORKDIR /workspaces/wasmedge-wasinn-examples CMD ["python3", "/run_smoke_test.py"] diff --git a/core/iwasm/libraries/wasi-nn/test/run_smoke_test.py b/core/iwasm/libraries/wasi-nn/test/run_smoke_test.py index a62d9cb7a8..304b0c9774 100644 --- a/core/iwasm/libraries/wasi-nn/test/run_smoke_test.py +++ b/core/iwasm/libraries/wasi-nn/test/run_smoke_test.py @@ -260,6 +260,63 @@ def filter_output(output: str) -> str: print("------------------------------------------------------------") +def execute_wasmedge_ggml_qwen(iwasm_bin: str, wasmedge_bin: str, cwd: Path): + iwasm_args = ["--dir=."] + wasm_file = ["./target/wasm32-wasi/debug/wasmedge-ggml-qwen.wasm"] + wasm_args = ["./qwen1_5-0_5b-chat-q2_k.gguf"] + + cmd = [iwasm_bin] + cmd.extend(iwasm_args) + cmd.extend(wasm_file) + cmd.extend(wasm_args) + + # print(f'Execute: {" ".join(cmd)}') + + prompt = "what is the capital of Pakistan" + + with subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=cwd, + ) as p: + # USER + p.stdout.readline() + + p.stdin.write(b"hi\n") + p.stdin.flush() + # ASSITANT + p.stdout.readline() + # xxx + p.stdout.readline() + # USER + p.stdout.readline() + + p.stdin.write(prompt.encode()) + p.stdin.write(b"\n") + p.stdin.flush() + # ASSITANT + p.stdout.readline() + # xxx + answer = p.stdout.readline().decode("utf-8") + # USER + p.stdout.readline() + + p.terminate() + + if "Karachi" in answer: + print(f"- wasmedge_ggml_qwen. PASS") + return + + print(f"- wasmedge_ggml_qwen. FAILED") + print("------------------------------------------------------------") + pprint(answer) + print("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<") + pprint("Karachi") + print("------------------------------------------------------------") + + def execute_wasmedge_wasinn_examples(iwasm_bin: str, wasmedge_bin: str): assert Path.cwd().name == "wasmedge-wasinn-examples" assert shutil.which(iwasm_bin) @@ -282,6 +339,9 @@ def execute_wasmedge_wasinn_examples(iwasm_bin: str, wasmedge_bin: str): iwasm_bin, wasmedge_bin, openvino_road_segmentation_adas_dir ) + wasmedge_ggml_qwem_dir = Path.cwd().joinpath("./wasmedge-ggml/qwen") + execute_wasmedge_ggml_qwen(iwasm_bin, wasmedge_bin, wasmedge_ggml_qwem_dir) + if __name__ == "__main__": execute_wasmedge_wasinn_examples("iwasm", "wasmedge") From cbc20788986155391eaa20a01bd947f0f0910b16 Mon Sep 17 00:00:00 2001 From: Marcin Kolny Date: Tue, 10 Sep 2024 02:05:23 +0100 Subject: [PATCH 17/69] AOT call stack optimizations (#3773) - Implement TINY / STANDARD frame modes - tiny mode is only able to keep track on the IP and func idx, STANDARD mode provides more capabilities (parameters, stack pointer etc.). - Implement FRAME_PER_FUNCTION / FRAME_PER_CALL modes - frame per function adds code at the beginning and at the end of each function for allocating / deallocating stack frame, whereas in per-call mode the frame is allocated before each call. The exception is call to the imported function, where frame-per-function mode also allocates the stack before the `call` instruction (as it can't instrument the imported function). At the moment TINY + FRAME_PER_FUNCTION is automatically enabled in case GC and perf profiling are disabled and `values` call stack feature is not requested. In all the other cases STANDARD + FRAME_PER_CALL is used. STANDARD + FRAME_PER_FUNCTION and TINY + FRAME_PER_CALL are currently not implemented but possible, and might be enabled in the future. ps. https://github.com/bytecodealliance/wasm-micro-runtime/issues/3758 --- core/iwasm/aot/aot_loader.c | 4 + core/iwasm/aot/aot_runtime.c | 200 ++++++++++++++---- core/iwasm/aot/aot_runtime.h | 9 +- core/iwasm/compilation/aot_compiler.c | 54 ++++- core/iwasm/compilation/aot_compiler.h | 9 + core/iwasm/compilation/aot_emit_aot_file.c | 6 + core/iwasm/compilation/aot_emit_control.c | 50 ++++- core/iwasm/compilation/aot_emit_function.c | 78 +++++-- core/iwasm/compilation/aot_llvm.c | 6 +- core/iwasm/compilation/aot_llvm.h | 2 +- core/iwasm/compilation/aot_stack_frame.h | 27 +++ core/iwasm/compilation/aot_stack_frame_comp.c | 148 +++++++++++++ core/iwasm/compilation/aot_stack_frame_comp.h | 33 +++ core/iwasm/include/aot_comp_option.h | 18 +- core/iwasm/interpreter/wasm_loader.c | 4 +- core/iwasm/interpreter/wasm_mini_loader.c | 4 +- wamr-compiler/main.c | 25 ++- 17 files changed, 591 insertions(+), 86 deletions(-) create mode 100644 core/iwasm/compilation/aot_stack_frame.h create mode 100644 core/iwasm/compilation/aot_stack_frame_comp.c create mode 100644 core/iwasm/compilation/aot_stack_frame_comp.h diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 3a5b6fc5cc..0abafd9ddb 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -597,6 +597,10 @@ load_target_info_section(const uint8 *buf, const uint8 *buf_end, return false; } +#if WASM_ENABLE_DUMP_CALL_STACK != 0 + module->feature_flags = target_info.feature_flags; +#endif + /* Finally, check feature flags */ return check_feature_flags(error_buf, error_buf_size, target_info.feature_flags); diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index bdb4ca9112..013c761a06 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -4,6 +4,7 @@ */ #include "aot_runtime.h" +#include "../compilation/aot_stack_frame.h" #include "bh_log.h" #include "mem_alloc.h" #include "../common/wasm_runtime_common.h" @@ -72,6 +73,10 @@ bh_static_assert(offsetof(AOTFrame, sp) == sizeof(uintptr_t) * 5); bh_static_assert(offsetof(AOTFrame, frame_ref) == sizeof(uintptr_t) * 6); bh_static_assert(offsetof(AOTFrame, lp) == sizeof(uintptr_t) * 7); +bh_static_assert(offsetof(AOTTinyFrame, func_index) == sizeof(uint32) * 0); +bh_static_assert(offsetof(AOTTinyFrame, ip_offset) == sizeof(uint32) * 1); +bh_static_assert(sizeof(AOTTinyFrame) == sizeof(uint32) * 2); + static void set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) { @@ -110,6 +115,55 @@ runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size) return mem; } +#if WASM_ENABLE_AOT_STACK_FRAME != 0 +static bool +is_tiny_frame(WASMExecEnv *exec_env) +{ + AOTModule *module = + (AOTModule *)((AOTModuleInstance *)exec_env->module_inst)->module; + + return module->feature_flags & WASM_FEATURE_TINY_STACK_FRAME; +} + +static bool +is_frame_per_function(WASMExecEnv *exec_env) +{ + AOTModule *module = + (AOTModule *)((AOTModuleInstance *)exec_env->module_inst)->module; + + return module->feature_flags & WASM_FEATURE_FRAME_PER_FUNCTION; +} + +static void * +get_top_frame(WASMExecEnv *exec_env) +{ + if (is_tiny_frame(exec_env)) { + return exec_env->wasm_stack.top > exec_env->wasm_stack.bottom + ? exec_env->wasm_stack.top - sizeof(AOTTinyFrame) + : NULL; + } + else { + return exec_env->cur_frame; + } +} + +static void * +get_prev_frame(WASMExecEnv *exec_env, void *cur_frame) +{ + bh_assert(cur_frame); + + if (is_tiny_frame(exec_env)) { + if ((uint8 *)cur_frame == exec_env->wasm_stack.bottom) { + return NULL; + } + return ((AOTTinyFrame *)cur_frame) - 1; + } + else { + return ((AOTFrame *)cur_frame)->prev_frame; + } +} +#endif + static bool check_global_init_expr(const AOTModule *module, uint32 global_index, char *error_buf, uint32 error_buf_size) @@ -2265,7 +2319,7 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function, uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count); uint64 size; #if WASM_ENABLE_AOT_STACK_FRAME != 0 - struct WASMInterpFrame *prev_frame = exec_env->cur_frame; + void *prev_frame = get_top_frame(exec_env); #endif /* Allocate memory all arguments */ @@ -2296,7 +2350,8 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function, } #if WASM_ENABLE_AOT_STACK_FRAME != 0 - if (!aot_alloc_frame(exec_env, function->func_index)) { + if (!is_frame_per_function(exec_env) + && !aot_alloc_frame(exec_env, function->func_index)) { if (argv1 != argv1_buf) wasm_runtime_free(argv1); return false; @@ -2324,7 +2379,7 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function, /* Free all frames allocated, note that some frames may be allocated in AOT code and haven't been freed if exception occurred */ - while (exec_env->cur_frame != prev_frame) + while (get_top_frame(exec_env) != prev_frame) aot_free_frame(exec_env); #endif if (!ret) { @@ -2367,9 +2422,12 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function, } else { #if WASM_ENABLE_AOT_STACK_FRAME != 0 - struct WASMInterpFrame *prev_frame = exec_env->cur_frame; - - if (!aot_alloc_frame(exec_env, function->func_index)) { + void *prev_frame = get_top_frame(exec_env); + /* Only allocate frame for frame-per-call mode; in the + frame-per-function mode the frame is allocated at the + beginning of the function. */ + if (!is_frame_per_function(exec_env) + && !aot_alloc_frame(exec_env, function->func_index)) { return false; } #endif @@ -2394,7 +2452,7 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function, /* Free all frames allocated, note that some frames may be allocated in AOT code and haven't been freed if exception occurred */ - while (exec_env->cur_frame != prev_frame) + while (get_top_frame(exec_env) != prev_frame) aot_free_frame(exec_env); #endif @@ -2880,7 +2938,7 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, goto fail; } #if WASM_ENABLE_AOT_STACK_FRAME != 0 - struct WASMInterpFrame *prev_frame = exec_env->cur_frame; + void *prev_frame = get_top_frame(exec_env); if (!aot_alloc_frame(exec_env, func_idx)) { goto fail; @@ -2894,7 +2952,7 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, /* Free all frames allocated, note that some frames may be allocated in AOT code and haven't been freed if exception occurred */ - while (exec_env->cur_frame != prev_frame) + while (get_top_frame(exec_env) != prev_frame) aot_free_frame(exec_env); #endif } @@ -3622,8 +3680,8 @@ get_func_name_from_index(const AOTModuleInstance *module_inst, WASM_ENABLE_PERF_PROFILING != 0 */ #if WASM_ENABLE_GC == 0 -bool -aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) +static bool +aot_alloc_standard_frame(WASMExecEnv *exec_env, uint32 func_index) { AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; #if WASM_ENABLE_PERF_PROFILING != 0 @@ -3670,8 +3728,8 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) #else /* else of WASM_ENABLE_GC == 0 */ -bool -aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) +static bool +aot_alloc_standard_frame(WASMExecEnv *exec_env, uint32 func_index) { AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; AOTModule *module = (AOTModule *)module_inst->module; @@ -3727,11 +3785,48 @@ aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) } #endif /* end of WASM_ENABLE_GC == 0 */ +static bool +aot_alloc_tiny_frame(WASMExecEnv *exec_env, uint32 func_index) +{ + AOTTinyFrame *new_frame = (AOTTinyFrame *)exec_env->wasm_stack.top; + + if ((uint8 *)new_frame > exec_env->wasm_stack.top_boundary) { + aot_set_exception((WASMModuleInstance *)exec_env->module_inst, + "wasm operand stack overflow"); + return false; + } + + new_frame->func_index = func_index; + exec_env->wasm_stack.top += sizeof(AOTTinyFrame); + return true; +} + +bool +aot_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) +{ + AOTModule *module = + (AOTModule *)((AOTModuleInstance *)exec_env->module_inst)->module; + + if (is_frame_per_function(exec_env) + && func_index >= module->import_func_count) { + /* in frame per function mode the frame is allocated at + the beginning of each frame, so we only need to allocate + the frame for imported functions */ + return true; + } + if (is_tiny_frame(exec_env)) { + return aot_alloc_tiny_frame(exec_env, func_index); + } + else { + return aot_alloc_standard_frame(exec_env, func_index); + } +} + static inline void -aot_free_frame_internal(WASMExecEnv *exec_env) +aot_free_standard_frame(WASMExecEnv *exec_env) { AOTFrame *cur_frame = (AOTFrame *)exec_env->cur_frame; - AOTFrame *prev_frame = cur_frame->prev_frame; + AOTFrame *prev_frame = (AOTFrame *)cur_frame->prev_frame; #if WASM_ENABLE_PERF_PROFILING != 0 uint64 time_elapsed = @@ -3751,13 +3846,24 @@ aot_free_frame_internal(WASMExecEnv *exec_env) exec_env->cur_frame = (struct WASMInterpFrame *)prev_frame; } +static inline void +aot_free_tiny_frame(WASMExecEnv *exec_env) +{ + exec_env->wasm_stack.top = + get_prev_frame(exec_env, exec_env->wasm_stack.top); +} + void aot_free_frame(WASMExecEnv *exec_env) { - aot_free_frame_internal(exec_env); + if (is_tiny_frame(exec_env)) { + aot_free_tiny_frame(exec_env); + } + else { + aot_free_standard_frame(exec_env); + } } - void aot_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame) { @@ -3806,14 +3912,13 @@ aot_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame) bool aot_create_call_stack(struct WASMExecEnv *exec_env) { - AOTFrame *cur_frame = (AOTFrame *)exec_env->cur_frame, - *first_frame = cur_frame; AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; AOTModule *module = (AOTModule *)module_inst->module; uint32 n = 0; - while (cur_frame) { - cur_frame = cur_frame->prev_frame; + void *top_frame = get_top_frame(exec_env); + while (top_frame) { + top_frame = get_prev_frame(exec_env, top_frame); n++; } @@ -3823,28 +3928,46 @@ aot_create_call_stack(struct WASMExecEnv *exec_env) return false; } - cur_frame = first_frame; - while (cur_frame) { + top_frame = get_top_frame(exec_env); + while (n-- > 0) { + uint32 func_index, ip_offset; + uint32 *lp = NULL; +#if WASM_ENABLE_GC != 0 + uint32 *sp = NULL; + uint8 *frame_ref = NULL; +#endif + if (is_tiny_frame(exec_env)) { + AOTTinyFrame *frame = (AOTTinyFrame *)top_frame; + func_index = (uint32)frame->func_index; + ip_offset = (uint32)frame->ip_offset; + } + else { + AOTFrame *frame = (AOTFrame *)top_frame; + func_index = (uint32)frame->func_index; + ip_offset = (uint32)frame->ip_offset; + lp = frame->lp; +#if WASM_ENABLE_GC != 0 + sp = frame->sp; + frame_ref = frame->frame_ref; +#endif + } WASMCApiFrame frame = { 0 }; uint32 max_local_cell_num, max_stack_cell_num; uint32 all_cell_num, lp_size; frame.instance = module_inst; frame.module_offset = 0; - frame.func_index = (uint32)cur_frame->func_index; - frame.func_offset = (uint32)cur_frame->ip_offset; - frame.func_name_wp = get_func_name_from_index( - module_inst, (uint32)cur_frame->func_index); - - if (cur_frame->func_index >= module->import_func_count) { - uint32 aot_func_idx = - (uint32)(cur_frame->func_index - module->import_func_count); + frame.func_index = func_index; + frame.func_offset = ip_offset; + frame.func_name_wp = get_func_name_from_index(module_inst, func_index); + + if (func_index >= module->import_func_count) { + uint32 aot_func_idx = func_index - module->import_func_count; max_local_cell_num = module->max_local_cell_nums[aot_func_idx]; max_stack_cell_num = module->max_stack_cell_nums[aot_func_idx]; } else { - AOTFuncType *func_type = - module->import_funcs[cur_frame->func_index].func_type; + AOTFuncType *func_type = module->import_funcs[func_index].func_type; max_local_cell_num = func_type->param_cell_num > 2 ? func_type->param_cell_num : 2; max_stack_cell_num = 0; @@ -3856,12 +3979,12 @@ aot_create_call_stack(struct WASMExecEnv *exec_env) #else lp_size = align_uint(all_cell_num * 5, 4); #endif - if (lp_size > 0) { + if (lp_size > 0 && !is_tiny_frame(exec_env)) { if (!(frame.lp = wasm_runtime_malloc(lp_size))) { destroy_c_api_frames(module_inst->frames); return false; } - bh_memcpy_s(frame.lp, lp_size, cur_frame->lp, lp_size); + bh_memcpy_s(frame.lp, lp_size, lp, lp_size); #if WASM_ENABLE_GC != 0 uint32 local_ref_flags_cell_num = @@ -3869,9 +3992,8 @@ aot_create_call_stack(struct WASMExecEnv *exec_env) .local_ref_flag_cell_num; uint8 *local_ref_flags = module->func_local_ref_flags[frame.func_index].local_ref_flags; - frame.sp = frame.lp + (cur_frame->sp - cur_frame->lp); - frame.frame_ref = (uint8 *)frame.lp - + (cur_frame->frame_ref - (uint8 *)cur_frame->lp); + frame.sp = frame.lp + (sp - lp); + frame.frame_ref = (uint8 *)frame.lp + (frame_ref - (uint8 *)lp); /* copy local ref flags from AOT module */ bh_memcpy_s(frame.frame_ref, local_ref_flags_cell_num, local_ref_flags, lp_size); @@ -3885,7 +4007,7 @@ aot_create_call_stack(struct WASMExecEnv *exec_env) return false; } - cur_frame = cur_frame->prev_frame; + top_frame = get_prev_frame(exec_env, top_frame); } return true; diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 76c784512d..f6bff00bf2 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -25,12 +25,15 @@ extern "C" { #define WASM_FEATURE_REF_TYPES (1 << 3) #define WASM_FEATURE_GARBAGE_COLLECTION (1 << 4) #define WASM_FEATURE_EXCEPTION_HANDLING (1 << 5) -#define WASM_FEATURE_MEMORY64 (1 << 6) +#define WASM_FEATURE_TINY_STACK_FRAME (1 << 6) #define WASM_FEATURE_MULTI_MEMORY (1 << 7) #define WASM_FEATURE_DYNAMIC_LINKING (1 << 8) #define WASM_FEATURE_COMPONENT_MODEL (1 << 9) #define WASM_FEATURE_RELAXED_SIMD (1 << 10) #define WASM_FEATURE_FLEXIBLE_VECTORS (1 << 11) +/* Stack frame is created at the beginning of the function, + * and not at the beginning of each function call */ +#define WASM_FEATURE_FRAME_PER_FUNCTION (1 << 12) typedef enum AOTSectionType { AOT_SECTION_TYPE_TARGET_INFO = 0, @@ -326,6 +329,10 @@ typedef struct AOTModule { /* `.data` and `.text` sections merged into one large mmaped section */ uint8 *merged_data_text_sections; uint32 merged_data_text_sections_size; + +#if WASM_ENABLE_AOT_STACK_FRAME != 0 + uint32 feature_flags; +#endif } AOTModule; #define AOTMemoryInstance WASMMemoryInstance diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index 78b7da88d7..e560049725 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -16,6 +16,7 @@ #include "aot_emit_parametric.h" #include "aot_emit_table.h" #include "aot_emit_gc.h" +#include "aot_stack_frame_comp.h" #include "simd/simd_access_lanes.h" #include "simd/simd_bitmask_extracts.h" #include "simd/simd_bit_shifts.h" @@ -253,6 +254,13 @@ store_value(AOTCompContext *comp_ctx, LLVMValueRef value, uint8 value_type, return true; } +void +aot_call_stack_features_init_default(AOTCallStackFeatures *features) +{ + memset(features, 1, sizeof(AOTCallStackFeatures)); + features->frame_per_function = false; +} + bool aot_frame_store_value(AOTCompContext *comp_ctx, LLVMValueRef value, uint8 value_type, LLVMValueRef cur_frame, uint32 offset) @@ -573,9 +581,10 @@ aot_gen_commit_values(AOTCompFrame *frame) return true; } -bool -aot_gen_commit_ip(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, - LLVMValueRef ip_value, bool is_64bit) +static bool +aot_standard_frame_gen_commit_ip(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef ip_value, bool is_64bit) { LLVMValueRef cur_frame = func_ctx->cur_frame; LLVMValueRef value_offset, value_addr, value_ptr; @@ -613,6 +622,23 @@ aot_gen_commit_ip(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } +bool +aot_gen_commit_ip(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef ip_value, bool is_64bit) +{ + switch (comp_ctx->aux_stack_frame_type) { + case AOT_STACK_FRAME_TYPE_STANDARD: + return aot_standard_frame_gen_commit_ip(comp_ctx, func_ctx, + ip_value, is_64bit); + case AOT_STACK_FRAME_TYPE_TINY: + return aot_tiny_frame_gen_commit_ip(comp_ctx, func_ctx, ip_value); + default: + aot_set_last_error( + "unsupported mode when generating commit_ip code"); + return false; + } +} + bool aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip) { @@ -962,6 +988,7 @@ static bool aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) { AOTFuncContext *func_ctx = comp_ctx->func_ctxes[func_index]; + LLVMValueRef func_index_ref; uint8 *frame_ip = func_ctx->aot_func->code, opcode, *p_f32, *p_f64; uint8 *frame_ip_end = frame_ip + func_ctx->aot_func->code_size; uint8 *param_types = NULL; @@ -984,16 +1011,27 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) LLVMMetadataRef location; #endif - if (comp_ctx->enable_aux_stack_frame) { + /* Start to translate the opcodes */ + LLVMPositionBuilderAtEnd( + comp_ctx->builder, + func_ctx->block_stack.block_list_head->llvm_entry_block); + + if (comp_ctx->aux_stack_frame_type + && comp_ctx->call_stack_features.frame_per_function) { + INT_CONST(func_index_ref, + func_index + comp_ctx->comp_data->import_func_count, I32_TYPE, + true); + if (!aot_alloc_frame_per_function_frame_for_aot_func(comp_ctx, func_ctx, + func_index_ref)) { + return false; + } + } + if (comp_ctx->aux_stack_frame_type) { if (!init_comp_frame(comp_ctx, func_ctx, func_index)) { return false; } } - /* Start to translate the opcodes */ - LLVMPositionBuilderAtEnd( - comp_ctx->builder, - func_ctx->block_stack.block_list_head->llvm_entry_block); while (frame_ip < frame_ip_end) { opcode = *frame_ip++; diff --git a/core/iwasm/compilation/aot_compiler.h b/core/iwasm/compilation/aot_compiler.h index d3d55b02bf..895d2416b4 100644 --- a/core/iwasm/compilation/aot_compiler.h +++ b/core/iwasm/compilation/aot_compiler.h @@ -661,6 +661,15 @@ set_local_gc_ref(AOTCompFrame *frame, int n, LLVMValueRef value, uint8 ref_type) #define F64_CONST(v) LLVMConstReal(F64_TYPE, v) #define I8_CONST(v) LLVMConstInt(INT8_TYPE, v, true) +#define INT_CONST(variable, value, type, is_signed) \ + do { \ + variable = LLVMConstInt(type, value, is_signed); \ + if (!variable) { \ + aot_set_last_error("llvm build const failed"); \ + return false; \ + } \ + } while (0) + #define LLVM_CONST(name) (comp_ctx->llvm_consts.name) #define I1_ZERO LLVM_CONST(i1_zero) #define I1_ONE LLVM_CONST(i1_one) diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index e05f83b09e..20f29057c6 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -4433,6 +4433,12 @@ aot_obj_data_create(AOTCompContext *comp_ctx) if (comp_ctx->enable_gc) { obj_data->target_info.feature_flags |= WASM_FEATURE_GARBAGE_COLLECTION; } + if (comp_ctx->aux_stack_frame_type == AOT_STACK_FRAME_TYPE_TINY) { + obj_data->target_info.feature_flags |= WASM_FEATURE_TINY_STACK_FRAME; + } + if (comp_ctx->call_stack_features.frame_per_function) { + obj_data->target_info.feature_flags |= WASM_FEATURE_FRAME_PER_FUNCTION; + } bh_print_time("Begin to resolve object file info"); diff --git a/core/iwasm/compilation/aot_emit_control.c b/core/iwasm/compilation/aot_emit_control.c index 7d73d8d906..945f63952b 100644 --- a/core/iwasm/compilation/aot_emit_control.c +++ b/core/iwasm/compilation/aot_emit_control.c @@ -6,6 +6,7 @@ #include "aot_emit_control.h" #include "aot_compiler.h" #include "aot_emit_exception.h" +#include "aot_stack_frame_comp.h" #if WASM_ENABLE_GC != 0 #include "aot_emit_gc.h" #endif @@ -38,13 +39,24 @@ format_block_name(char *name, uint32 name_size, uint32 block_index, snprintf(name, name_size, "%s", "func_end"); } -#define CREATE_BLOCK(new_llvm_block, name) \ - do { \ - if (!(new_llvm_block = LLVMAppendBasicBlockInContext( \ - comp_ctx->context, func_ctx->func, name))) { \ - aot_set_last_error("add LLVM basic block failed."); \ - goto fail; \ - } \ +#define CREATE_BLOCK(new_llvm_block, name) \ + do { \ + if (!(new_llvm_block = LLVMAppendBasicBlockInContext( \ + comp_ctx->context, func_ctx->func, name))) { \ + aot_set_last_error("add LLVM basic block failed."); \ + goto fail; \ + } \ + if (!strcmp(name, "func_end") && comp_ctx->aux_stack_frame_type \ + && comp_ctx->call_stack_features.frame_per_function) { \ + LLVMBasicBlockRef cur_block = \ + LLVMGetInsertBlock(comp_ctx->builder); \ + SET_BUILDER_POS(new_llvm_block); \ + if (!aot_free_frame_per_function_frame_for_aot_func(comp_ctx, \ + func_ctx)) { \ + goto fail; \ + } \ + SET_BUILDER_POS(cur_block); \ + } \ } while (0) #define CURR_BLOCK() LLVMGetInsertBlock(comp_ctx->builder) @@ -93,6 +105,11 @@ format_block_name(char *name, uint32 name_size, uint32 block_index, goto fail; \ } \ SET_BUILDER_POS(block->llvm_end_block); \ + LLVMValueRef first_instr = \ + get_first_non_phi(block->llvm_end_block); \ + if (first_instr) { \ + LLVMPositionBuilderBefore(comp_ctx->builder, first_instr); \ + } \ for (_i = 0; _i < block->result_count; _i++) { \ if (!(block->result_phis[_i] = LLVMBuildPhi( \ comp_ctx->builder, \ @@ -158,6 +175,18 @@ get_target_block(AOTFuncContext *func_ctx, uint32 br_depth) return block; } +LLVMValueRef +get_first_non_phi(LLVMBasicBlockRef block) +{ + LLVMValueRef instr = LLVMGetFirstInstruction(block); + + while (instr && LLVMIsAPHINode(instr)) { + instr = LLVMGetNextInstruction(instr); + } + + return instr; +} + static void clear_frame_locals(AOTCompFrame *aot_frame) { @@ -1361,6 +1390,13 @@ aot_compile_op_return(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, (*p_frame_ip - 1) - comp_ctx->comp_data->wasm_module->buf_code); #endif + if (comp_ctx->aux_stack_frame_type + && comp_ctx->call_stack_features.frame_per_function + && !aot_free_frame_per_function_frame_for_aot_func(comp_ctx, + func_ctx)) { + return false; + } + if (block_func->result_count) { /* Store extra result values to function parameters */ for (i = 0; i < block_func->result_count - 1; i++) { diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index 1d565b6c09..fbef02e20f 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -7,6 +7,7 @@ #include "aot_emit_exception.h" #include "aot_emit_control.h" #include "aot_emit_table.h" +#include "aot_stack_frame_comp.h" #include "../aot/aot_runtime.h" #if WASM_ENABLE_GC != 0 #include "aot_emit_gc.h" @@ -1403,6 +1404,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef *param_values = NULL, value_ret = NULL, func; LLVMValueRef import_func_idx, res; LLVMValueRef ext_ret, ext_ret_ptr, ext_ret_idx; + LLVMValueRef func_idx_ref; int32 i, j = 0, param_count, result_count, ext_ret_count; uint64 total_size; uint8 wasm_ret_type; @@ -1447,12 +1449,28 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return false; } - if (comp_ctx->enable_aux_stack_frame) { #if WASM_ENABLE_AOT_STACK_FRAME != 0 - if (!alloc_frame_for_aot_func(comp_ctx, func_ctx, func_idx)) - return false; -#endif + if (comp_ctx->aux_stack_frame_type) { + if (func_idx < import_func_count + && comp_ctx->call_stack_features.frame_per_function) { + INT_CONST(func_idx_ref, func_idx, I32_TYPE, true); + if (!aot_alloc_frame_per_function_frame_for_aot_func( + comp_ctx, func_ctx, func_idx_ref)) { + return false; + } + } + else if (!comp_ctx->call_stack_features.frame_per_function) { + if (comp_ctx->aux_stack_frame_type + != AOT_STACK_FRAME_TYPE_STANDARD) { + aot_set_last_error("unsupported mode"); + return false; + } + if (!alloc_frame_for_aot_func(comp_ctx, func_ctx, func_idx)) { + return false; + } + } } +#endif /* Get param cell number */ param_cell_num = func_type->param_cell_num; @@ -1522,7 +1540,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } if (func_idx < import_func_count) { - if (comp_ctx->enable_aux_stack_frame + if (comp_ctx->aux_stack_frame_type == AOT_STACK_FRAME_TYPE_STANDARD && !commit_params_to_frame_of_import_func( comp_ctx, func_ctx, func_type, param_values + 1)) { goto fail; @@ -1813,12 +1831,26 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } - if (comp_ctx->enable_aux_stack_frame) { #if WASM_ENABLE_AOT_STACK_FRAME != 0 - if (!free_frame_for_aot_func(comp_ctx, func_ctx)) - goto fail; -#endif + if (comp_ctx->aux_stack_frame_type) { + if (func_idx < import_func_count + && comp_ctx->call_stack_features.frame_per_function) { + if (!aot_free_frame_per_function_frame_for_aot_func(comp_ctx, + func_ctx)) { + goto fail; + } + } + else if (!comp_ctx->call_stack_features.frame_per_function) { + if (comp_ctx->aux_stack_frame_type + != AOT_STACK_FRAME_TYPE_STANDARD) { + aot_set_last_error("unsupported mode"); + } + if (!free_frame_for_aot_func(comp_ctx, func_ctx)) { + goto fail; + } + } } +#endif /* Insert suspend check point */ if (comp_ctx->enable_thread_mgr) { @@ -2439,7 +2471,8 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - if (comp_ctx->enable_aux_stack_frame) { + if (comp_ctx->aux_stack_frame_type + && !comp_ctx->call_stack_features.frame_per_function) { #if WASM_ENABLE_AOT_STACK_FRAME != 0 /* TODO: use current frame instead of allocating new frame for WASM_OP_RETURN_CALL_INDIRECT */ @@ -2508,7 +2541,13 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Translate call import block */ LLVMPositionBuilderAtEnd(comp_ctx->builder, block_call_import); - if (comp_ctx->enable_aux_stack_frame + if (comp_ctx->aot_frame && comp_ctx->call_stack_features.frame_per_function + && !aot_alloc_frame_per_function_frame_for_aot_func(comp_ctx, func_ctx, + func_idx)) { + goto fail; + } + + if (comp_ctx->aux_stack_frame_type == AOT_STACK_FRAME_TYPE_STANDARD && !commit_params_to_frame_of_import_func(comp_ctx, func_ctx, func_type, param_values + 1)) { goto fail; @@ -2545,6 +2584,12 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, && !check_call_return(comp_ctx, func_ctx, res)) goto fail; + if (comp_ctx->aot_frame && comp_ctx->call_stack_features.frame_per_function + && !aot_free_frame_per_function_frame_for_aot_func(comp_ctx, + func_ctx)) { + goto fail; + } + block_curr = LLVMGetInsertBlock(comp_ctx->builder); for (i = 0; i < func_result_count; i++) { LLVMAddIncoming(result_phis[i], &value_rets[i], &block_curr, 1); @@ -2629,7 +2674,8 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, PUSH(result_phis[i], func_type->types[func_param_count + i]); } - if (comp_ctx->enable_aux_stack_frame) { + if (comp_ctx->aux_stack_frame_type + && !comp_ctx->call_stack_features.frame_per_function) { #if WASM_ENABLE_AOT_STACK_FRAME != 0 if (!free_frame_for_aot_func(comp_ctx, func_ctx)) goto fail; @@ -2936,7 +2982,8 @@ aot_compile_op_call_ref(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - if (comp_ctx->enable_aux_stack_frame) { + if (comp_ctx->aux_stack_frame_type + && !comp_ctx->call_stack_features.frame_per_function) { #if WASM_ENABLE_AOT_STACK_FRAME != 0 /* TODO: use current frame instead of allocating new frame for WASM_OP_RETURN_CALL_REF */ @@ -3005,7 +3052,7 @@ aot_compile_op_call_ref(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Translate call import block */ LLVMPositionBuilderAtEnd(comp_ctx->builder, block_call_import); - if (comp_ctx->enable_aux_stack_frame + if (comp_ctx->aux_stack_frame_type == AOT_STACK_FRAME_TYPE_STANDARD && !commit_params_to_frame_of_import_func(comp_ctx, func_ctx, func_type, param_values + 1)) { goto fail; @@ -3133,7 +3180,8 @@ aot_compile_op_call_ref(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, PUSH(result_phis[i], func_type->types[func_param_count + i]); } - if (comp_ctx->enable_aux_stack_frame) { + if (comp_ctx->aux_stack_frame_type + && !comp_ctx->call_stack_features.frame_per_function) { #if WASM_ENABLE_AOT_STACK_FRAME != 0 if (!free_frame_for_aot_func(comp_ctx, func_ctx)) goto fail; diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 3346086a95..820a55e965 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1771,7 +1771,7 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx, goto fail; } - if (comp_ctx->enable_aux_stack_frame + if (comp_ctx->aux_stack_frame_type && !create_aux_stack_frame(comp_ctx, func_ctx)) { goto fail; } @@ -2577,9 +2577,7 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) if (option->enable_ref_types) comp_ctx->enable_ref_types = true; - if (option->enable_aux_stack_frame) - comp_ctx->enable_aux_stack_frame = true; - + comp_ctx->aux_stack_frame_type = option->aux_stack_frame_type; comp_ctx->call_stack_features = option->call_stack_features; if (option->enable_perf_profiling) diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 65debbaa36..43212e5027 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -410,7 +410,7 @@ typedef struct AOTCompContext { bool enable_aux_stack_check; /* Generate auxiliary stack frame */ - bool enable_aux_stack_frame; + AOTStackFrameType aux_stack_frame_type; /* Auxiliary call stack features */ AOTCallStackFeatures call_stack_features; diff --git a/core/iwasm/compilation/aot_stack_frame.h b/core/iwasm/compilation/aot_stack_frame.h new file mode 100644 index 0000000000..6155ee6e9d --- /dev/null +++ b/core/iwasm/compilation/aot_stack_frame.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2024 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_STACK_FRAME_H_ +#define _AOT_STACK_FRAME_H_ + +#include "platform_common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + /* The non-imported function index of current function */ + uint32 func_index; + + /* Instruction pointer: offset to the bytecode array */ + uint32 ip_offset; +} AOTTinyFrame; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/core/iwasm/compilation/aot_stack_frame_comp.c b/core/iwasm/compilation/aot_stack_frame_comp.c new file mode 100644 index 0000000000..342dfe806e --- /dev/null +++ b/core/iwasm/compilation/aot_stack_frame_comp.c @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2024 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ +#include "aot_stack_frame_comp.h" +#include "aot_emit_exception.h" + +#define ADD_IN_BOUNDS_GEP(variable, type, pointer, indices, num_indices) \ + do { \ + if (!(variable = \ + LLVMBuildInBoundsGEP2(comp_ctx->builder, type, pointer, \ + indices, num_indices, #variable))) { \ + aot_set_last_error("llvm build in bounds gep failed"); \ + return false; \ + } \ + } while (0) + +#define ADD_STORE(value, pointer) \ + do { \ + if (!LLVMBuildStore(comp_ctx->builder, value, pointer)) { \ + aot_set_last_error("llvm build store failed"); \ + return false; \ + } \ + } while (0) + +#define ADD_LOAD(value, type, pointer) \ + do { \ + if (!(value = \ + LLVMBuildLoad2(comp_ctx->builder, type, pointer, #value))) { \ + aot_set_last_error("llvm build load failed"); \ + return false; \ + } \ + } while (0) + +static bool +aot_alloc_tiny_frame_for_aot_func(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef func_index) +{ + LLVMValueRef wasm_stack_top_ptr = func_ctx->wasm_stack_top_ptr, + wasm_stack_top_bound = func_ctx->wasm_stack_top_bound, + wasm_stack_top, cmp; + LLVMBasicBlockRef check_wasm_stack_succ; + LLVMValueRef offset; + + ADD_LOAD(wasm_stack_top, INT8_PTR_TYPE, wasm_stack_top_ptr); + + if (comp_ctx->call_stack_features.bounds_checks) { + if (!(check_wasm_stack_succ = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, + "check_wasm_stack_succ"))) { + aot_set_last_error("llvm add basic block failed."); + return false; + } + + LLVMMoveBasicBlockAfter(check_wasm_stack_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGE, wasm_stack_top, + wasm_stack_top_bound, "cmp"))) { + aot_set_last_error("llvm build icmp failed"); + return false; + } + + if (!(aot_emit_exception(comp_ctx, func_ctx, + EXCE_OPERAND_STACK_OVERFLOW, true, cmp, + check_wasm_stack_succ))) { + return false; + } + } + + /* Save the func_idx on the top of the stack */ + ADD_STORE(func_index, wasm_stack_top); + + /* increment the stack pointer */ + INT_CONST(offset, sizeof(AOTTinyFrame), I32_TYPE, true); + ADD_IN_BOUNDS_GEP(wasm_stack_top, INT8_TYPE, wasm_stack_top, &offset, 1); + ADD_STORE(wasm_stack_top, wasm_stack_top_ptr); + + return true; +} + +static bool +aot_free_tiny_frame_for_aot_func(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + LLVMValueRef wasm_stack_top_ptr = func_ctx->wasm_stack_top_ptr, + wasm_stack_top; + LLVMValueRef offset; + + ADD_LOAD(wasm_stack_top, INT8_PTR_TYPE, wasm_stack_top_ptr); + + INT_CONST(offset, -sizeof(AOTTinyFrame), + comp_ctx->pointer_size == 8 ? I64_TYPE : I32_TYPE, true); + ADD_IN_BOUNDS_GEP(wasm_stack_top, INT8_TYPE, wasm_stack_top, &offset, 1); + ADD_STORE(wasm_stack_top, wasm_stack_top_ptr); + + return true; +} + +bool +aot_tiny_frame_gen_commit_ip(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef ip_value) +{ + LLVMValueRef wasm_stack_top_ptr = func_ctx->wasm_stack_top_ptr, + wasm_stack_top; + LLVMValueRef offset, ip_addr; + + bh_assert(ip_value); + + ADD_LOAD(wasm_stack_top, INT8_PTR_TYPE, wasm_stack_top_ptr); + + INT_CONST(offset, -4, comp_ctx->pointer_size == 8 ? I64_TYPE : I32_TYPE, + true); + ADD_IN_BOUNDS_GEP(ip_addr, INT8_TYPE, wasm_stack_top, &offset, 1); + + ADD_STORE(ip_value, ip_addr); + + return true; +} + +bool +aot_alloc_frame_per_function_frame_for_aot_func(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef func_index) +{ + switch (comp_ctx->aux_stack_frame_type) { + case AOT_STACK_FRAME_TYPE_TINY: + return aot_alloc_tiny_frame_for_aot_func(comp_ctx, func_ctx, + func_index); + default: + aot_set_last_error("unsupported mode"); + return false; + } +} + +bool +aot_free_frame_per_function_frame_for_aot_func(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx) +{ + switch (comp_ctx->aux_stack_frame_type) { + case AOT_STACK_FRAME_TYPE_TINY: + return aot_free_tiny_frame_for_aot_func(comp_ctx, func_ctx); + default: + aot_set_last_error("unsupported mode"); + return false; + } +} diff --git a/core/iwasm/compilation/aot_stack_frame_comp.h b/core/iwasm/compilation/aot_stack_frame_comp.h new file mode 100644 index 0000000000..7980b8c08b --- /dev/null +++ b/core/iwasm/compilation/aot_stack_frame_comp.h @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2024 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_STACK_FRAME_COMP_H_ +#define _AOT_STACK_FRAME_COMP_H_ + +#include "aot_stack_frame.h" +#include "aot_compiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +bool +aot_alloc_frame_per_function_frame_for_aot_func(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef func_index); + +bool +aot_free_frame_per_function_frame_for_aot_func(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx); + +bool +aot_tiny_frame_gen_commit_ip(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef ip_value); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/core/iwasm/include/aot_comp_option.h b/core/iwasm/include/aot_comp_option.h index 4ab2e6ab60..67ec81cd36 100644 --- a/core/iwasm/include/aot_comp_option.h +++ b/core/iwasm/include/aot_comp_option.h @@ -21,8 +21,24 @@ typedef struct { /* Enables or disables parameters, locals and stack operands. */ bool values; + + /* If enabled, stack frame is generated at the beginning of each + * function (frame-per-function mode). Otherwise, stack frame is + * generated before each call of a function (frame-per-call mode). */ + bool frame_per_function; } AOTCallStackFeatures; +void +aot_call_stack_features_init_default(AOTCallStackFeatures *features); + +typedef enum { + AOT_STACK_FRAME_OFF = 0, + /* Use a small stack frame data structure (AOTTinyFrame) */ + AOT_STACK_FRAME_TYPE_TINY, + /* Use a regular stack frame data structure (AOTFrame) */ + AOT_STACK_FRAME_TYPE_STANDARD, +} AOTStackFrameType; + typedef struct AOTCompOption { bool is_jit_mode; bool is_indirect_mode; @@ -38,7 +54,7 @@ typedef struct AOTCompOption { bool enable_ref_types; bool enable_gc; bool enable_aux_stack_check; - bool enable_aux_stack_frame; + AOTStackFrameType aux_stack_frame_type; AOTCallStackFeatures call_stack_features; bool enable_perf_profiling; bool enable_memory_profiling; diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 092e0d152b..ed85bb789e 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -5406,8 +5406,8 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, option.enable_aux_stack_check = true; #if WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0 \ || WASM_ENABLE_AOT_STACK_FRAME != 0 - option.enable_aux_stack_frame = true; - memset(&option.call_stack_features, 1, sizeof(AOTCallStackFeatures)); + option.aux_stack_frame_type = AOT_STACK_FRAME_TYPE_STANDARD; + aot_call_stack_features_init_default(&option.call_stack_features); #endif #if WASM_ENABLE_PERF_PROFILING != 0 option.enable_perf_profiling = true; diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index a21f4490fc..34f4a18311 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -2148,8 +2148,8 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, option.enable_aux_stack_check = true; #if WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0 \ || WASM_ENABLE_AOT_STACK_FRAME != 0 - option.enable_aux_stack_frame = true; - memset(&option.call_stack_features, 1, sizeof(AOTCallStackFeatures)); + option.aux_stack_frame_type = AOT_STACK_FRAME_TYPE_STANDARD; + aot_call_stack_features_init_default(&option.call_stack_features); #endif #if WASM_ENABLE_PERF_PROFILING != 0 option.enable_perf_profiling = true; diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index 3c7ef1f4de..53c75c84e5 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -307,6 +307,13 @@ parse_call_stack_features(char *features_str, return ret; } +static bool +can_enable_tiny_frame(const AOTCompOption *opt) +{ + return !opt->call_stack_features.values && !opt->enable_gc + && !opt->enable_perf_profiling; +} + static uint32 resolve_segue_flags(char *str_flags) { @@ -403,9 +410,7 @@ main(int argc, char *argv[]) option.enable_bulk_memory = true; option.enable_ref_types = true; option.enable_gc = false; - - /* Set all the features to true by default */ - memset(&option.call_stack_features, 1, sizeof(AOTCallStackFeatures)); + aot_call_stack_features_init_default(&option.call_stack_features); /* Process options */ for (argc--, argv++; argc > 0 && argv[0][0] == '-'; argc--, argv++) { @@ -519,7 +524,7 @@ main(int argc, char *argv[]) option.enable_aux_stack_check = false; } else if (!strcmp(argv[0], "--enable-dump-call-stack")) { - option.enable_aux_stack_frame = true; + option.aux_stack_frame_type = AOT_STACK_FRAME_TYPE_STANDARD; } else if (!strncmp(argv[0], "--call-stack-features=", 22)) { /* Reset all the features, only enable the user-defined ones */ @@ -535,7 +540,7 @@ main(int argc, char *argv[]) } } else if (!strcmp(argv[0], "--enable-perf-profiling")) { - option.enable_aux_stack_frame = true; + option.aux_stack_frame_type = AOT_STACK_FRAME_TYPE_STANDARD; option.enable_perf_profiling = true; } else if (!strcmp(argv[0], "--enable-memory-profiling")) { @@ -550,7 +555,7 @@ main(int argc, char *argv[]) option.is_indirect_mode = true; } else if (!strcmp(argv[0], "--enable-gc")) { - option.enable_aux_stack_frame = true; + option.aux_stack_frame_type = AOT_STACK_FRAME_TYPE_STANDARD; option.enable_gc = true; } else if (!strcmp(argv[0], "--disable-llvm-intrinsics")) { @@ -652,6 +657,14 @@ main(int argc, char *argv[]) if (!use_dummy_wasm && (argc == 0 || !out_file_name)) PRINT_HELP_AND_EXIT(); + if (option.aux_stack_frame_type == AOT_STACK_FRAME_TYPE_STANDARD + && can_enable_tiny_frame(&option)) { + LOG_VERBOSE("Use tiny frame mode for stack frames"); + option.aux_stack_frame_type = AOT_STACK_FRAME_TYPE_TINY; + /* for now we only enable frame per function for a TINY frame mode */ + option.call_stack_features.frame_per_function = true; + } + if (!size_level_set) { /** * Set opt level to 1 by default for Windows and MacOS as From f453d9d5ce2cb58080154ecbbb4642409beba056 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 10 Sep 2024 10:42:23 +0900 Subject: [PATCH 18/69] Appease GCC strict prototypes warning (#3775) --- core/iwasm/aot/aot_reloc.h | 2 +- core/iwasm/aot/aot_runtime.h | 2 +- core/iwasm/aot/arch/aot_reloc_aarch64.c | 2 +- core/iwasm/aot/arch/aot_reloc_arm.c | 194 +++++++++--------- core/iwasm/aot/arch/aot_reloc_mips.c | 2 +- core/iwasm/aot/arch/aot_reloc_riscv.c | 106 +++++----- core/iwasm/aot/arch/aot_reloc_thumb.c | 194 +++++++++--------- core/iwasm/aot/arch/aot_reloc_x86_64.c | 2 +- core/iwasm/aot/arch/aot_reloc_xtensa.c | 74 +++---- core/iwasm/aot/debug/jit_debug.c | 10 +- core/iwasm/aot/debug/jit_debug.h | 4 +- core/iwasm/common/wasm_memory.c | 4 +- core/iwasm/common/wasm_memory.h | 4 +- core/iwasm/common/wasm_native.c | 4 +- core/iwasm/common/wasm_native.h | 4 +- core/iwasm/common/wasm_runtime_common.c | 18 +- core/iwasm/common/wasm_runtime_common.h | 8 +- core/iwasm/common/wasm_shared_memory.h | 4 +- core/iwasm/compilation/aot.h | 2 +- core/iwasm/interpreter/wasm_interp_fast.c | 2 +- core/iwasm/interpreter/wasm_loader.c | 2 +- core/iwasm/interpreter/wasm_mini_loader.c | 2 +- .../lib_wasi_threads_wrapper.c | 2 +- .../libraries/thread-mgr/thread_manager.h | 8 +- .../platform/include/platform_api_extension.h | 10 +- .../shared/platform/nuttx/platform_internal.h | 2 +- product-mini/platforms/common/libc_wasi.c | 2 +- product-mini/platforms/nuttx/CMakeLists.txt | 2 +- product-mini/platforms/nuttx/wamr.mk | 2 +- product-mini/platforms/posix/main.c | 2 +- 30 files changed, 338 insertions(+), 338 deletions(-) diff --git a/core/iwasm/aot/aot_reloc.h b/core/iwasm/aot/aot_reloc.h index 8ead3cd934..f7ada4d8df 100644 --- a/core/iwasm/aot/aot_reloc.h +++ b/core/iwasm/aot/aot_reloc.h @@ -226,7 +226,7 @@ SymbolMap * get_target_symbol_map(uint32 *sym_num); uint32 -get_plt_table_size(); +get_plt_table_size(void); void init_plt_table(uint8 *plt); diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index f6bff00bf2..3ff0e0e3ce 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -648,7 +648,7 @@ aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str, void **p_native_addr); uint32 -aot_get_plt_table_size(); +aot_get_plt_table_size(void); void * aot_memmove(void *dest, const void *src, size_t n); diff --git a/core/iwasm/aot/arch/aot_reloc_aarch64.c b/core/iwasm/aot/arch/aot_reloc_aarch64.c index ec646b4e92..26815334f7 100644 --- a/core/iwasm/aot/arch/aot_reloc_aarch64.c +++ b/core/iwasm/aot/arch/aot_reloc_aarch64.c @@ -79,7 +79,7 @@ get_current_target(char *target_buf, uint32 target_buf_size) } static uint32 -get_plt_item_size() +get_plt_item_size(void) { /* 6*4 bytes instructions and 8 bytes symbol address */ return 32; diff --git a/core/iwasm/aot/arch/aot_reloc_arm.c b/core/iwasm/aot/arch/aot_reloc_arm.c index bb492bfeba..0be17ef4cc 100644 --- a/core/iwasm/aot/arch/aot_reloc_arm.c +++ b/core/iwasm/aot/arch/aot_reloc_arm.c @@ -12,102 +12,102 @@ #define R_ARM_MOVT_ABS 44 /* clang-format off */ -void __adddf3(); -void __addsf3(); -void __aeabi_d2f(); -void __aeabi_d2iz(); -void __aeabi_d2lz(); -void __aeabi_d2uiz(); -void __aeabi_d2ulz(); -void __aeabi_dadd(); -void __aeabi_dcmpeq(); -void __aeabi_dcmpge(); -void __aeabi_dcmpgt(); -void __aeabi_dcmple(); -void __aeabi_dcmplt(); -void __aeabi_dcmpun(); -void __aeabi_ddiv(); -void __aeabi_dmul(); -void __aeabi_dsub(); -void __aeabi_f2d(); -void __aeabi_f2iz(); -void __aeabi_f2lz(); -void __aeabi_f2ulz(); -void __aeabi_fadd(); -void __aeabi_fcmpeq(); -void __aeabi_fcmpge(); -void __aeabi_fcmpgt(); -void __aeabi_fcmple(); -void __aeabi_fcmplt(); -void __aeabi_fcmpun(); -void __aeabi_fdiv(); -void __aeabi_fmul(); -void __aeabi_fsub(); -void __aeabi_i2d(); -void __aeabi_i2f(); -void __aeabi_idiv(); -void __aeabi_idivmod(); -void __aeabi_l2d(); -void __aeabi_l2f(); -void __aeabi_ldivmod(); -void __aeabi_memclr(); -void __aeabi_memcpy(); -void __aeabi_memmove(); -void __aeabi_memset(); -void __aeabi_ui2d(); -void __aeabi_ui2f(); -void __aeabi_uidiv(); -void __aeabi_uidivmod(); -void __aeabi_ul2d(); -void __aeabi_ul2f(); -void __aeabi_uldivmod(); -void __clzsi2(); -void __divdf3(); -void __divdi3(); -void __divsf3(); -void __divsi3(); -void __eqdf2(); -void __eqsf2(); -void __extendsfdf2(); -void __fixdfdi(); -void __fixdfsi(); -void __fixsfdi(); -void __fixsfsi(); -void __fixunsdfdi(); -void __fixunsdfsi(); -void __fixunssfdi(); -void __floatdidf(); -void __floatdisf(); -void __floatsidf(); -void __floatsisf(); -void __floatundidf(); -void __floatundisf(); -void __floatunsidf(); -void __floatunsisf(); -void __gedf2(); -void __gesf2(); -void __gtdf2(); -void __gtsf2(); -void __ledf2(); -void __lesf2(); -void __ltdf2(); -void __ltsf2(); -void __moddi3(); -void __modsi3(); -void __muldf3(); -void __mulsf3(); -void __nedf2(); -void __nesf2(); -void __subdf3(); -void __subsf3(); -void __truncdfsf2(); -void __udivdi3(); -void __udivmoddi4(); -void __udivsi3(); -void __umoddi3(); -void __umodsi3(); -void __unorddf2(); -void __unordsf2(); +void __adddf3(void); +void __addsf3(void); +void __aeabi_d2f(void); +void __aeabi_d2iz(void); +void __aeabi_d2lz(void); +void __aeabi_d2uiz(void); +void __aeabi_d2ulz(void); +void __aeabi_dadd(void); +void __aeabi_dcmpeq(void); +void __aeabi_dcmpge(void); +void __aeabi_dcmpgt(void); +void __aeabi_dcmple(void); +void __aeabi_dcmplt(void); +void __aeabi_dcmpun(void); +void __aeabi_ddiv(void); +void __aeabi_dmul(void); +void __aeabi_dsub(void); +void __aeabi_f2d(void); +void __aeabi_f2iz(void); +void __aeabi_f2lz(void); +void __aeabi_f2ulz(void); +void __aeabi_fadd(void); +void __aeabi_fcmpeq(void); +void __aeabi_fcmpge(void); +void __aeabi_fcmpgt(void); +void __aeabi_fcmple(void); +void __aeabi_fcmplt(void); +void __aeabi_fcmpun(void); +void __aeabi_fdiv(void); +void __aeabi_fmul(void); +void __aeabi_fsub(void); +void __aeabi_i2d(void); +void __aeabi_i2f(void); +void __aeabi_idiv(void); +void __aeabi_idivmod(void); +void __aeabi_l2d(void); +void __aeabi_l2f(void); +void __aeabi_ldivmod(void); +void __aeabi_memclr(void); +void __aeabi_memcpy(void); +void __aeabi_memmove(void); +void __aeabi_memset(void); +void __aeabi_ui2d(void); +void __aeabi_ui2f(void); +void __aeabi_uidiv(void); +void __aeabi_uidivmod(void); +void __aeabi_ul2d(void); +void __aeabi_ul2f(void); +void __aeabi_uldivmod(void); +void __clzsi2(void); +void __divdf3(void); +void __divdi3(void); +void __divsf3(void); +void __divsi3(void); +void __eqdf2(void); +void __eqsf2(void); +void __extendsfdf2(void); +void __fixdfdi(void); +void __fixdfsi(void); +void __fixsfdi(void); +void __fixsfsi(void); +void __fixunsdfdi(void); +void __fixunsdfsi(void); +void __fixunssfdi(void); +void __floatdidf(void); +void __floatdisf(void); +void __floatsidf(void); +void __floatsisf(void); +void __floatundidf(void); +void __floatundisf(void); +void __floatunsidf(void); +void __floatunsisf(void); +void __gedf2(void); +void __gesf2(void); +void __gtdf2(void); +void __gtsf2(void); +void __ledf2(void); +void __lesf2(void); +void __ltdf2(void); +void __ltsf2(void); +void __moddi3(void); +void __modsi3(void); +void __muldf3(void); +void __mulsf3(void); +void __nedf2(void); +void __nesf2(void); +void __subdf3(void); +void __subsf3(void); +void __truncdfsf2(void); +void __udivdi3(void); +void __udivmoddi4(void); +void __udivsi3(void); +void __umoddi3(void); +void __umodsi3(void); +void __unorddf2(void); +void __unordsf2(void); /* clang-format on */ static SymbolMap target_sym_map[] = { @@ -255,7 +255,7 @@ get_current_target(char *target_buf, uint32 target_buf_size) #undef BUILD_TARGET_ARM_DEFAULT uint32 -get_plt_item_size() +get_plt_item_size(void) { /* 8 bytes instructions and 4 bytes symbol address */ return 12; diff --git a/core/iwasm/aot/arch/aot_reloc_mips.c b/core/iwasm/aot/arch/aot_reloc_mips.c index f9f06a053e..4b856119c4 100644 --- a/core/iwasm/aot/arch/aot_reloc_mips.c +++ b/core/iwasm/aot/arch/aot_reloc_mips.c @@ -28,7 +28,7 @@ get_current_target(char *target_buf, uint32 target_buf_size) } static uint32 -get_plt_item_size() +get_plt_item_size(void) { return 0; } diff --git a/core/iwasm/aot/arch/aot_reloc_riscv.c b/core/iwasm/aot/arch/aot_reloc_riscv.c index b87bb20009..058ad0e10a 100644 --- a/core/iwasm/aot/arch/aot_reloc_riscv.c +++ b/core/iwasm/aot/arch/aot_reloc_riscv.c @@ -49,58 +49,58 @@ #endif /* clang-format off */ -void __adddf3(); -void __addsf3(); -void __divdf3(); -void __divdi3(); -void __divsf3(); -void __divsi3(); -void __eqdf2(); -void __eqsf2(); -void __extendsfdf2(); -void __fixdfdi(); -void __fixdfsi(); -void __fixsfdi(); -void __fixsfsi(); -void __fixunsdfdi(); -void __fixunsdfsi(); -void __fixunssfdi(); -void __fixunssfsi(); -void __floatdidf(); -void __floatdisf(); -void __floatsidf(); -void __floatsisf(); -void __floatundidf(); -void __floatundisf(); -void __floatunsidf(); -void __floatunsisf(); -void __gedf2(); -void __gesf2(); -void __gtdf2(); -void __gtsf2(); -void __ledf2(); -void __lesf2(); -void __ltdf2(); -void __ltsf2(); -void __moddi3(); -void __modsi3(); -void __muldf3(); -void __muldi3(); -void __mulsf3(); -void __mulsi3(); -void __nedf2(); -void __negdf2(); -void __negsf2(); -void __nesf2(); -void __subdf3(); -void __subsf3(); -void __truncdfsf2(); -void __udivdi3(); -void __udivsi3(); -void __umoddi3(); -void __umodsi3(); -void __unorddf2(); -void __unordsf2(); +void __adddf3(void); +void __addsf3(void); +void __divdf3(void); +void __divdi3(void); +void __divsf3(void); +void __divsi3(void); +void __eqdf2(void); +void __eqsf2(void); +void __extendsfdf2(void); +void __fixdfdi(void); +void __fixdfsi(void); +void __fixsfdi(void); +void __fixsfsi(void); +void __fixunsdfdi(void); +void __fixunsdfsi(void); +void __fixunssfdi(void); +void __fixunssfsi(void); +void __floatdidf(void); +void __floatdisf(void); +void __floatsidf(void); +void __floatsisf(void); +void __floatundidf(void); +void __floatundisf(void); +void __floatunsidf(void); +void __floatunsisf(void); +void __gedf2(void); +void __gesf2(void); +void __gtdf2(void); +void __gtsf2(void); +void __ledf2(void); +void __lesf2(void); +void __ltdf2(void); +void __ltsf2(void); +void __moddi3(void); +void __modsi3(void); +void __muldf3(void); +void __muldi3(void); +void __mulsf3(void); +void __mulsi3(void); +void __nedf2(void); +void __negdf2(void); +void __negsf2(void); +void __nesf2(void); +void __subdf3(void); +void __subsf3(void); +void __truncdfsf2(void); +void __udivdi3(void); +void __udivsi3(void); +void __umoddi3(void); +void __umodsi3(void); +void __unorddf2(void); +void __unordsf2(void); /* clang-format on */ static SymbolMap target_sym_map[] = { @@ -193,7 +193,7 @@ get_current_target(char *target_buf, uint32 target_buf_size) } uint32 -get_plt_item_size() +get_plt_item_size(void) { #if __riscv_xlen == 64 /* auipc + ld + jalr + nop + addr */ diff --git a/core/iwasm/aot/arch/aot_reloc_thumb.c b/core/iwasm/aot/arch/aot_reloc_thumb.c index f90507dec4..c0957a42a9 100644 --- a/core/iwasm/aot/arch/aot_reloc_thumb.c +++ b/core/iwasm/aot/arch/aot_reloc_thumb.c @@ -14,102 +14,102 @@ #define R_ARM_THM_MOVT_PREL 50 /* clang-format off */ -void __adddf3(); -void __addsf3(); -void __aeabi_d2f(); -void __aeabi_d2iz(); -void __aeabi_d2lz(); -void __aeabi_d2uiz(); -void __aeabi_d2ulz(); -void __aeabi_dadd(); -void __aeabi_dcmpeq(); -void __aeabi_dcmpge(); -void __aeabi_dcmpgt(); -void __aeabi_dcmple(); -void __aeabi_dcmplt(); -void __aeabi_dcmpun(); -void __aeabi_ddiv(); -void __aeabi_dmul(); -void __aeabi_dsub(); -void __aeabi_f2d(); -void __aeabi_f2iz(); -void __aeabi_f2lz(); -void __aeabi_f2ulz(); -void __aeabi_fadd(); -void __aeabi_fcmpeq(); -void __aeabi_fcmpge(); -void __aeabi_fcmpgt(); -void __aeabi_fcmple(); -void __aeabi_fcmplt(); -void __aeabi_fcmpun(); -void __aeabi_fdiv(); -void __aeabi_fmul(); -void __aeabi_fsub(); -void __aeabi_i2d(); -void __aeabi_i2f(); -void __aeabi_idiv(); -void __aeabi_idivmod(); -void __aeabi_l2d(); -void __aeabi_l2f(); -void __aeabi_ldivmod(); -void __aeabi_llsl(); -void __aeabi_llsr(); -void __aeabi_lmul(); -void __aeabi_ui2d(); -void __aeabi_ui2f(); -void __aeabi_uidiv(); -void __aeabi_uidivmod(); -void __aeabi_ul2d(); -void __aeabi_ul2f(); -void __aeabi_uldivmod(); -void __ashldi3(); -void __clzsi2(); -void __divdf3(); -void __divdi3(); -void __divsi3(); -void __eqdf2(); -void __eqsf2(); -void __extendsfdf2(); -void __fixdfdi(); -void __fixdfsi(); -void __fixsfdi(); -void __fixunsdfdi(); -void __fixunsdfsi(); -void __fixunssfdi(); -void __floatdidf(); -void __floatdisf(); -void __floatsidf(); -void __floatsisf(); -void __floatundidf(); -void __floatundisf(); -void __floatunsidf(); -void __floatunsisf(); -void __gedf2(); -void __gesf2(); -void __gtdf2(); -void __gtsf2(); -void __ledf2(); -void __lesf2(); -void __lshrdi3(); -void __ltdf2(); -void __ltsf2(); -void __moddi3(); -void __modsi3(); -void __muldf3(); -void __muldi3(); -void __mulsf3(); -void __nedf2(); -void __nesf2(); -void __subdf3(); -void __subsf3(); -void __truncdfsf2(); -void __udivdi3(); -void __udivmoddi4(); -void __udivsi3(); -void __umoddi3(); -void __umodsi3(); -void __unorddf2(); -void __unordsf2(); +void __adddf3(void); +void __addsf3(void); +void __aeabi_d2f(void); +void __aeabi_d2iz(void); +void __aeabi_d2lz(void); +void __aeabi_d2uiz(void); +void __aeabi_d2ulz(void); +void __aeabi_dadd(void); +void __aeabi_dcmpeq(void); +void __aeabi_dcmpge(void); +void __aeabi_dcmpgt(void); +void __aeabi_dcmple(void); +void __aeabi_dcmplt(void); +void __aeabi_dcmpun(void); +void __aeabi_ddiv(void); +void __aeabi_dmul(void); +void __aeabi_dsub(void); +void __aeabi_f2d(void); +void __aeabi_f2iz(void); +void __aeabi_f2lz(void); +void __aeabi_f2ulz(void); +void __aeabi_fadd(void); +void __aeabi_fcmpeq(void); +void __aeabi_fcmpge(void); +void __aeabi_fcmpgt(void); +void __aeabi_fcmple(void); +void __aeabi_fcmplt(void); +void __aeabi_fcmpun(void); +void __aeabi_fdiv(void); +void __aeabi_fmul(void); +void __aeabi_fsub(void); +void __aeabi_i2d(void); +void __aeabi_i2f(void); +void __aeabi_idiv(void); +void __aeabi_idivmod(void); +void __aeabi_l2d(void); +void __aeabi_l2f(void); +void __aeabi_ldivmod(void); +void __aeabi_llsl(void); +void __aeabi_llsr(void); +void __aeabi_lmul(void); +void __aeabi_ui2d(void); +void __aeabi_ui2f(void); +void __aeabi_uidiv(void); +void __aeabi_uidivmod(void); +void __aeabi_ul2d(void); +void __aeabi_ul2f(void); +void __aeabi_uldivmod(void); +void __ashldi3(void); +void __clzsi2(void); +void __divdf3(void); +void __divdi3(void); +void __divsi3(void); +void __eqdf2(void); +void __eqsf2(void); +void __extendsfdf2(void); +void __fixdfdi(void); +void __fixdfsi(void); +void __fixsfdi(void); +void __fixunsdfdi(void); +void __fixunsdfsi(void); +void __fixunssfdi(void); +void __floatdidf(void); +void __floatdisf(void); +void __floatsidf(void); +void __floatsisf(void); +void __floatundidf(void); +void __floatundisf(void); +void __floatunsidf(void); +void __floatunsisf(void); +void __gedf2(void); +void __gesf2(void); +void __gtdf2(void); +void __gtsf2(void); +void __ledf2(void); +void __lesf2(void); +void __lshrdi3(void); +void __ltdf2(void); +void __ltsf2(void); +void __moddi3(void); +void __modsi3(void); +void __muldf3(void); +void __muldi3(void); +void __mulsf3(void); +void __nedf2(void); +void __nesf2(void); +void __subdf3(void); +void __subsf3(void); +void __truncdfsf2(void); +void __udivdi3(void); +void __udivmoddi4(void); +void __udivsi3(void); +void __umoddi3(void); +void __umodsi3(void); +void __unorddf2(void); +void __unordsf2(void); /* clang-format on */ static SymbolMap target_sym_map[] = { @@ -259,7 +259,7 @@ get_current_target(char *target_buf, uint32 target_buf_size) #undef BUILD_TARGET_THUMB_V4T uint32 -get_plt_item_size() +get_plt_item_size(void) { /* 16 bytes instructions and 4 bytes symbol address */ return 20; diff --git a/core/iwasm/aot/arch/aot_reloc_x86_64.c b/core/iwasm/aot/arch/aot_reloc_x86_64.c index d1f5cb5acc..fe18d79c63 100644 --- a/core/iwasm/aot/arch/aot_reloc_x86_64.c +++ b/core/iwasm/aot/arch/aot_reloc_x86_64.c @@ -58,7 +58,7 @@ get_current_target(char *target_buf, uint32 target_buf_size) } static uint32 -get_plt_item_size() +get_plt_item_size(void) { /* size of mov instruction and jmp instruction */ return 12; diff --git a/core/iwasm/aot/arch/aot_reloc_xtensa.c b/core/iwasm/aot/arch/aot_reloc_xtensa.c index a29c9f2b9e..fca1b80da1 100644 --- a/core/iwasm/aot/arch/aot_reloc_xtensa.c +++ b/core/iwasm/aot/arch/aot_reloc_xtensa.c @@ -10,44 +10,44 @@ /* clang-format off */ /* for soft-float */ -void __floatsidf(); -void __divdf3(); -void __ltdf2(); +void __floatsidf(void); +void __divdf3(void); +void __ltdf2(void); /* for mul32 */ -void __mulsi3(); -void __muldi3(); - -void __modsi3(); - -void __divdi3(); - -void __udivdi3(); -void __unorddf2(); -void __adddf3(); -void __eqdf2(); -void __muldf3(); -void __gedf2(); -void __ledf2(); -void __fixunsdfsi(); -void __floatunsidf(); -void __subdf3(); -void __nedf2(); -void __fixdfsi(); -void __moddi3(); -void __extendsfdf2(); -void __truncdfsf2(); -void __gtdf2(); -void __umoddi3(); -void __floatdidf(); -void __divsf3(); -void __fixdfdi(); -void __floatundidf(); -void __fixsfdi(); -void __fixunssfdi(); -void __fixunsdfdi(); -void __floatdisf(); -void __floatundisf(); +void __mulsi3(void); +void __muldi3(void); + +void __modsi3(void); + +void __divdi3(void); + +void __udivdi3(void); +void __unorddf2(void); +void __adddf3(void); +void __eqdf2(void); +void __muldf3(void); +void __gedf2(void); +void __ledf2(void); +void __fixunsdfsi(void); +void __floatunsidf(void); +void __subdf3(void); +void __nedf2(void); +void __fixdfsi(void); +void __moddi3(void); +void __extendsfdf2(void); +void __truncdfsf2(void); +void __gtdf2(void); +void __umoddi3(void); +void __floatdidf(void); +void __divsf3(void); +void __fixdfdi(void); +void __floatundidf(void); +void __fixsfdi(void); +void __fixunssfdi(void); +void __fixunsdfdi(void); +void __floatdisf(void); +void __floatundisf(void); static SymbolMap target_sym_map[] = { @@ -119,7 +119,7 @@ get_current_target(char *target_buf, uint32 target_buf_size) } static uint32 -get_plt_item_size() +get_plt_item_size(void) { return 0; } diff --git a/core/iwasm/aot/debug/jit_debug.c b/core/iwasm/aot/debug/jit_debug.c index 261c205462..9f92dd393e 100644 --- a/core/iwasm/aot/debug/jit_debug.c +++ b/core/iwasm/aot/debug/jit_debug.c @@ -69,10 +69,10 @@ typedef struct JITDescriptor { * and inline assembler statement inside. */ void attribute_noinline -__jit_debug_register_code(); +__jit_debug_register_code(void); void attribute_noinline -__jit_debug_register_code() +__jit_debug_register_code(void) { int x; *(char *)&x = '\0'; @@ -96,7 +96,7 @@ extern JITDescriptor __jit_debug_descriptor; * This gives the debugger an easy way to inject custom code to * handle the events. */ -void (*__jit_debug_register_code_ptr)() = __jit_debug_register_code; +void (*__jit_debug_register_code_ptr)(void) = __jit_debug_register_code; #ifdef __cplusplus } @@ -171,7 +171,7 @@ DestroyJITCodeEntryInternal(JITCodeEntry *entry) } bool -jit_debug_engine_init() +jit_debug_engine_init(void) { if (jit_debug_engine) { return true; @@ -194,7 +194,7 @@ jit_debug_engine_init() } void -jit_debug_engine_destroy() +jit_debug_engine_destroy(void) { if (jit_debug_engine) { WASMJITEntryNode *node, *node_next; diff --git a/core/iwasm/aot/debug/jit_debug.h b/core/iwasm/aot/debug/jit_debug.h index 5e3e365199..813c8b782b 100644 --- a/core/iwasm/aot/debug/jit_debug.h +++ b/core/iwasm/aot/debug/jit_debug.h @@ -11,10 +11,10 @@ extern "C" { #endif bool -jit_debug_engine_init(); +jit_debug_engine_init(void); void -jit_debug_engine_destroy(); +jit_debug_engine_destroy(void); bool jit_code_entry_create(const uint8 *symfile_addr, uint64 symfile_size); diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 71d337549b..82eebbf301 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -159,7 +159,7 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, } void -wasm_runtime_memory_destroy() +wasm_runtime_memory_destroy(void) { if (memory_mode == MEMORY_MODE_POOL) { #if BH_ENABLE_GC_VERIFY == 0 @@ -176,7 +176,7 @@ wasm_runtime_memory_destroy() } unsigned -wasm_runtime_memory_pool_size() +wasm_runtime_memory_pool_size(void) { if (memory_mode == MEMORY_MODE_POOL) return global_pool_size; diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index a5dfefae99..2f20d3f684 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -46,10 +46,10 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, const MemAllocOption *alloc_option); void -wasm_runtime_memory_destroy(); +wasm_runtime_memory_destroy(void); unsigned -wasm_runtime_memory_pool_size(); +wasm_runtime_memory_pool_size(void); void wasm_runtime_set_mem_bound_check_bytes(WASMMemoryInstance *memory, diff --git a/core/iwasm/common/wasm_native.c b/core/iwasm/common/wasm_native.c index 9e8764a22a..0ff3053fa9 100644 --- a/core/iwasm/common/wasm_native.c +++ b/core/iwasm/common/wasm_native.c @@ -469,7 +469,7 @@ wasi_context_dtor(WASMModuleInstanceCommon *inst, void *ctx) #if WASM_ENABLE_QUICK_AOT_ENTRY != 0 static bool -quick_aot_entry_init(); +quick_aot_entry_init(void); #endif bool @@ -1461,7 +1461,7 @@ quick_aot_entry_cmp(const void *quick_aot_entry1, const void *quick_aot_entry2) } static bool -quick_aot_entry_init() +quick_aot_entry_init(void) { qsort(quick_aot_entries, sizeof(quick_aot_entries) / sizeof(QuickAOTEntry), sizeof(QuickAOTEntry), quick_aot_entry_cmp); diff --git a/core/iwasm/common/wasm_native.h b/core/iwasm/common/wasm_native.h index 5cb78bf91f..9a6afee195 100644 --- a/core/iwasm/common/wasm_native.h +++ b/core/iwasm/common/wasm_native.h @@ -100,10 +100,10 @@ wasm_native_inherit_contexts(struct WASMModuleInstanceCommon *child, #endif /* WASM_ENABLE_MODULE_INST_CONTEXT */ bool -wasm_native_init(); +wasm_native_init(void); void -wasm_native_destroy(); +wasm_native_destroy(void); #if WASM_ENABLE_QUICK_AOT_ENTRY != 0 void * diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 5dd2957dea..314dc7ddb1 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -86,7 +86,7 @@ static bh_list registered_module_list_head; static bh_list *const registered_module_list = ®istered_module_list_head; static korp_mutex registered_module_list_lock; static void -wasm_runtime_destroy_registered_module_list(); +wasm_runtime_destroy_registered_module_list(void); #endif /* WASM_ENABLE_MULTI_MODULE */ #define E_TYPE_XIP 4 @@ -97,11 +97,11 @@ val_type_to_val_kind(uint8 value_type); #if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 /* Initialize externref hashmap */ static bool -wasm_externref_map_init(); +wasm_externref_map_init(void); /* Destroy externref hashmap */ static void -wasm_externref_map_destroy(); +wasm_externref_map_destroy(void); #endif /* end of WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 */ static void @@ -438,7 +438,7 @@ wasm_runtime_get_exec_env_tls() #endif /* end of OS_ENABLE_HW_BOUND_CHECK */ static bool -wasm_runtime_env_init() +wasm_runtime_env_init(void) { if (bh_platform_init() != 0) return false; @@ -584,7 +584,7 @@ static korp_mutex runtime_lock = OS_THREAD_MUTEX_INITIALIZER; static int32 runtime_ref_count = 0; static bool -wasm_runtime_init_internal() +wasm_runtime_init_internal(void) { if (!wasm_runtime_memory_init(Alloc_With_System_Allocator, NULL)) return false; @@ -622,7 +622,7 @@ wasm_runtime_init() } static void -wasm_runtime_destroy_internal() +wasm_runtime_destroy_internal(void) { #if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 wasm_externref_map_destroy(); @@ -4747,7 +4747,7 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, || defined(BUILD_TARGET_RISCV32_ILP32D) \ || defined(BUILD_TARGET_RISCV32_ILP32F) \ || defined(BUILD_TARGET_RISCV32_ILP32) || defined(BUILD_TARGET_ARC) -typedef void (*GenericFunctionPointer)(); +typedef void (*GenericFunctionPointer)(void); void invokeNative(GenericFunctionPointer f, uint32 *args, uint32 n_stacks); @@ -5312,7 +5312,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, #if defined(BUILD_TARGET_X86_32) || defined(BUILD_TARGET_ARM) \ || defined(BUILD_TARGET_THUMB) || defined(BUILD_TARGET_MIPS) \ || defined(BUILD_TARGET_XTENSA) -typedef void (*GenericFunctionPointer)(); +typedef void (*GenericFunctionPointer)(void); void invokeNative(GenericFunctionPointer f, uint32 *args, uint32 sz); @@ -5597,7 +5597,7 @@ typedef uint32x4_t __m128i; #endif /* end of WASM_ENABLE_SIMD != 0 */ -typedef void (*GenericFunctionPointer)(); +typedef void (*GenericFunctionPointer)(void); void invokeNative(GenericFunctionPointer f, uint64 *args, uint64 n_stacks); diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 71264ca7e7..fb2c79408d 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -852,10 +852,10 @@ wasm_runtime_set_module_reader(const module_reader reader, const module_destroyer destroyer); module_reader -wasm_runtime_get_module_reader(); +wasm_runtime_get_module_reader(void); module_destroyer -wasm_runtime_get_module_destroyer(); +wasm_runtime_get_module_destroyer(void); bool wasm_runtime_register_module_internal(const char *module_name, @@ -881,7 +881,7 @@ bool wasm_runtime_is_loading_module(const char *module_name); void -wasm_runtime_destroy_loading_module_list(); +wasm_runtime_destroy_loading_module_list(void); WASMModuleCommon * wasm_runtime_search_sub_module(const WASMModuleCommon *parent_module, @@ -1168,7 +1168,7 @@ wasm_runtime_quick_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, uint32 result_count); void -wasm_runtime_show_app_heap_corrupted_prompt(); +wasm_runtime_show_app_heap_corrupted_prompt(void); #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 void diff --git a/core/iwasm/common/wasm_shared_memory.h b/core/iwasm/common/wasm_shared_memory.h index 8bbc4a8004..e1c5154a53 100644 --- a/core/iwasm/common/wasm_shared_memory.h +++ b/core/iwasm/common/wasm_shared_memory.h @@ -17,10 +17,10 @@ extern "C" { extern korp_mutex g_shared_memory_lock; bool -wasm_shared_memory_init(); +wasm_shared_memory_init(void); void -wasm_shared_memory_destroy(); +wasm_shared_memory_destroy(void); uint16 shared_memory_inc_reference(WASMMemoryInstance *memory); diff --git a/core/iwasm/compilation/aot.h b/core/iwasm/compilation/aot.h index dcf9bbe122..98d2cc6cc7 100644 --- a/core/iwasm/compilation/aot.h +++ b/core/iwasm/compilation/aot.h @@ -312,7 +312,7 @@ void aot_destroy_comp_data(AOTCompData *comp_data); char * -aot_get_last_error(); +aot_get_last_error(void); void aot_set_last_error(const char *error); diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 51963759fb..1d7ca8f908 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -6030,7 +6030,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_LABELS_AS_VALUES != 0 void ** -wasm_interp_get_handle_table() +wasm_interp_get_handle_table(void) { WASMModuleInstance module; memset(&module, 0, sizeof(WASMModuleInstance)); diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index ed85bb789e..7b99c60aaa 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -5751,7 +5751,7 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, #if WASM_ENABLE_FAST_INTERP != 0 && WASM_ENABLE_LABELS_AS_VALUES != 0 void ** -wasm_interp_get_handle_table(); +wasm_interp_get_handle_table(void); static void **handle_table; #endif diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 34f4a18311..968eaf0096 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -2532,7 +2532,7 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, #if WASM_ENABLE_FAST_INTERP != 0 && WASM_ENABLE_LABELS_AS_VALUES != 0 void ** -wasm_interp_get_handle_table(); +wasm_interp_get_handle_table(void); static void **handle_table; #endif diff --git a/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c b/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c index aeaafced75..c9512fb432 100644 --- a/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c +++ b/core/iwasm/libraries/lib-wasi-threads/lib_wasi_threads_wrapper.c @@ -29,7 +29,7 @@ typedef struct { } ThreadStartArg; static int32 -allocate_thread_id() +allocate_thread_id(void) { os_mutex_lock(&thread_id_lock); int32 id = tid_allocator_get_tid(&tid_allocator); diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.h b/core/iwasm/libraries/thread-mgr/thread_manager.h index f5ca1eaede..7ad6c772aa 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.h +++ b/core/iwasm/libraries/thread-mgr/thread_manager.h @@ -64,10 +64,10 @@ void wasm_cluster_set_max_thread_num(uint32 num); bool -thread_manager_init(); +thread_manager_init(void); void -thread_manager_destroy(); +thread_manager_destroy(void); /* Create cluster */ WASMCluster * @@ -109,7 +109,7 @@ bool wasm_cluster_register_destroy_callback(void (*callback)(WASMCluster *)); void -wasm_cluster_cancel_all_callbacks(); +wasm_cluster_cancel_all_callbacks(void); void wasm_cluster_suspend_all(WASMCluster *cluster); @@ -190,7 +190,7 @@ struct WASMCurrentEnvStatus { }; WASMCurrentEnvStatus * -wasm_cluster_create_exenv_status(); +wasm_cluster_create_exenv_status(void); void wasm_cluster_destroy_exenv_status(WASMCurrentEnvStatus *status); diff --git a/core/shared/platform/include/platform_api_extension.h b/core/shared/platform/include/platform_api_extension.h index b1c3b4f4aa..37b8399b85 100644 --- a/core/shared/platform/include/platform_api_extension.h +++ b/core/shared/platform/include/platform_api_extension.h @@ -379,19 +379,19 @@ os_sem_unlink(const char *name); * Initialize process-global state for os_wakeup_blocking_op. */ int -os_blocking_op_init(); +os_blocking_op_init(void); /** * Start accepting os_wakeup_blocking_op requests for the calling thread. */ void -os_begin_blocking_op(); +os_begin_blocking_op(void); /** * Stop accepting os_wakeup_blocking_op requests for the calling thread. */ void -os_end_blocking_op(); +os_end_blocking_op(void); /** * Wake up the specified thread. @@ -1586,7 +1586,7 @@ os_closedir(os_dir_stream dir_stream); * @return the invalid directory stream */ os_dir_stream -os_get_invalid_dir_stream(); +os_get_invalid_dir_stream(void); /** * Checks whether the given directory stream is valid. An invalid directory @@ -1605,7 +1605,7 @@ os_is_dir_stream_valid(os_dir_stream *dir_stream); * @return the invalid handle */ os_file_handle -os_get_invalid_handle(); +os_get_invalid_handle(void); /** * Checks whether the given file handle is valid. An invalid handle is diff --git a/core/shared/platform/nuttx/platform_internal.h b/core/shared/platform/nuttx/platform_internal.h index 0b54d85a9e..fef2122da8 100644 --- a/core/shared/platform/nuttx/platform_internal.h +++ b/core/shared/platform/nuttx/platform_internal.h @@ -137,7 +137,7 @@ typedef DIR *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } diff --git a/product-mini/platforms/common/libc_wasi.c b/product-mini/platforms/common/libc_wasi.c index 84e133bc05..2f0b351253 100644 --- a/product-mini/platforms/common/libc_wasi.c +++ b/product-mini/platforms/common/libc_wasi.c @@ -28,7 +28,7 @@ typedef enum { } libc_wasi_parse_result_t; static void -libc_wasi_print_help() +libc_wasi_print_help(void) { printf(" --env= Pass wasi environment variables with " "\"key=value\"\n"); diff --git a/product-mini/platforms/nuttx/CMakeLists.txt b/product-mini/platforms/nuttx/CMakeLists.txt index ac6c47b919..ca07a5d551 100644 --- a/product-mini/platforms/nuttx/CMakeLists.txt +++ b/product-mini/platforms/nuttx/CMakeLists.txt @@ -197,7 +197,7 @@ include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) # NuttX wamr lib complie required: `WAMR_SOURCES` `WAMR_CFLAGS` `WAMR_INCDIRS` # `WAMR_DEFINITIONS` set(WAMR_SOURCES ${WAMR_RUNTIME_LIB_SOURCE}) -set(WAMR_CFLAGS -Wno-strict-prototypes -Wno-shadow -Wno-unused-variable +set(WAMR_CFLAGS -Wno-shadow -Wno-unused-variable -Wno-int-conversion -Wno-implicit-function-declaration) get_directory_property(WAMR_INCDIRS INCLUDE_DIRECTORIES) get_directory_property(WAMR_DEFINITIONS COMPILE_DEFINITIONS) diff --git a/product-mini/platforms/nuttx/wamr.mk b/product-mini/platforms/nuttx/wamr.mk index 38553e8635..0ee76c7ddc 100644 --- a/product-mini/platforms/nuttx/wamr.mk +++ b/product-mini/platforms/nuttx/wamr.mk @@ -400,7 +400,7 @@ CFLAGS += -DWASM_ENABLE_EXCE_HANDLING=0 CFLAGS += -DWASM_ENABLE_TAGS=0 endif -CFLAGS += -Wno-strict-prototypes -Wno-shadow -Wno-unused-variable +CFLAGS += -Wno-shadow -Wno-unused-variable CFLAGS += -Wno-int-conversion -Wno-implicit-function-declaration CFLAGS += -I${CORE_ROOT} \ diff --git a/product-mini/platforms/posix/main.c b/product-mini/platforms/posix/main.c index c1ba169d57..14dc01f6bf 100644 --- a/product-mini/platforms/posix/main.c +++ b/product-mini/platforms/posix/main.c @@ -27,7 +27,7 @@ static char **app_argv; /* clang-format off */ static int -print_help() +print_help(void) { printf("Usage: iwasm [-options] wasm_file [args...]\n"); printf("options:\n"); From c276aca2023bf7d737491abbb93df5c7f634afc9 Mon Sep 17 00:00:00 2001 From: Benbuck Nason Date: Mon, 9 Sep 2024 18:55:15 -0700 Subject: [PATCH 19/69] Fix compile error when multi-module and tags are enabled (#3781) --- core/iwasm/interpreter/wasm_loader.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 7b99c60aaa..3a21b1fc6b 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -2474,7 +2474,8 @@ wasm_loader_resolve_tag(const char *module_name, const char *tag_name, } /* check function type */ - if (!wasm_type_equal(expected_tag_type, tag->tag_type)) { + if (!wasm_type_equal(expected_tag_type, tag->tag_type, module->types, + module->type_count)) { LOG_DEBUG("%s.%s failed the type check", module_name, tag_name); set_error_buf(error_buf, error_buf_size, "incompatible import type"); return NULL; From cd47438af92fc75231339197b03ca0a7b0512af3 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 10 Sep 2024 11:58:22 +0900 Subject: [PATCH 20/69] spec_test_on_nuttx.yml: Disable riscv32_ilp32f for now (#3777) It seems failing too frequently. cf. https://github.com/bytecodealliance/wasm-micro-runtime/issues/3776 --- .github/workflows/spec_test_on_nuttx.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/spec_test_on_nuttx.yml b/.github/workflows/spec_test_on_nuttx.yml index 2e940b0e0f..712bd06bcb 100644 --- a/.github/workflows/spec_test_on_nuttx.yml +++ b/.github/workflows/spec_test_on_nuttx.yml @@ -74,11 +74,11 @@ jobs: target: "riscv32", fpu_type: "none" }, - { - config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh", - target: "riscv32_ilp32f", - fpu_type: "fp" - }, + #{ + # config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh", + # target: "riscv32_ilp32f", + # fpu_type: "fp" + #}, # { # config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh", # target: "riscv32_ilp32d", From 1a61cb75e1d8ced8a2aed2b02f8793676664aa37 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 10 Sep 2024 15:53:11 +0900 Subject: [PATCH 21/69] Appease GCC -Wformat (#3783) I'm not sure we want to use C99 %tu here. While C99 %zu is more widely used in WAMR, %tu is rare (if any) and I'm not sure if it's ubiquitously implemented in platforms we support. --- core/shared/mem-alloc/ems/ems_alloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/shared/mem-alloc/ems/ems_alloc.c b/core/shared/mem-alloc/ems/ems_alloc.c index 4863527d62..e272b30147 100644 --- a/core/shared/mem-alloc/ems/ems_alloc.c +++ b/core/shared/mem-alloc/ems/ems_alloc.c @@ -992,8 +992,8 @@ gci_dump(gc_heap_t *heap) os_printf("#%d %08" PRIx32 " %" PRIx32 " %d %d" " %c %" PRId32 "\n", - i, (int32)((char *)cur - (char *)heap->base_addr), (int32)ut, - p, mark, inuse, (int32)hmu_obj_size(size)); + i, (uint32)((char *)cur - (char *)heap->base_addr), + (uint32)ut, p, mark, inuse, (int32)hmu_obj_size(size)); #if BH_ENABLE_GC_VERIFY != 0 if (inuse == 'V') { gc_object_prefix_t *prefix = (gc_object_prefix_t *)(cur + 1); From b882017674bf76ea50b7e1a1dc5af8e64d4b2e78 Mon Sep 17 00:00:00 2001 From: Marcin Kolny Date: Wed, 11 Sep 2024 01:59:16 +0100 Subject: [PATCH 22/69] Fix compiler warnings (#3784) The definition of os_get_invalid_handle in platform_internal.h did not match the declaration in platform_api_extension.h. --- core/shared/platform/alios/platform_internal.h | 2 +- core/shared/platform/android/platform_internal.h | 2 +- core/shared/platform/cosmopolitan/platform_internal.h | 2 +- core/shared/platform/darwin/platform_internal.h | 2 +- core/shared/platform/esp-idf/platform_internal.h | 2 +- core/shared/platform/freebsd/platform_internal.h | 2 +- core/shared/platform/linux-sgx/platform_internal.h | 2 +- core/shared/platform/linux/platform_internal.h | 2 +- core/shared/platform/riot/platform_internal.h | 2 +- core/shared/platform/rt-thread/platform_internal.h | 2 +- core/shared/platform/vxworks/platform_internal.h | 2 +- core/shared/platform/windows/platform_internal.h | 2 +- core/shared/platform/zephyr/platform_internal.h | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/shared/platform/alios/platform_internal.h b/core/shared/platform/alios/platform_internal.h index d2897a6b59..bdf3d073f3 100644 --- a/core/shared/platform/alios/platform_internal.h +++ b/core/shared/platform/alios/platform_internal.h @@ -77,7 +77,7 @@ typedef void *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } diff --git a/core/shared/platform/android/platform_internal.h b/core/shared/platform/android/platform_internal.h index 4449f21e87..7abf863f81 100644 --- a/core/shared/platform/android/platform_internal.h +++ b/core/shared/platform/android/platform_internal.h @@ -151,7 +151,7 @@ typedef DIR *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } diff --git a/core/shared/platform/cosmopolitan/platform_internal.h b/core/shared/platform/cosmopolitan/platform_internal.h index 7260211606..5c73ed5a65 100644 --- a/core/shared/platform/cosmopolitan/platform_internal.h +++ b/core/shared/platform/cosmopolitan/platform_internal.h @@ -69,7 +69,7 @@ typedef DIR *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } diff --git a/core/shared/platform/darwin/platform_internal.h b/core/shared/platform/darwin/platform_internal.h index 1cbecdc205..928aad7246 100644 --- a/core/shared/platform/darwin/platform_internal.h +++ b/core/shared/platform/darwin/platform_internal.h @@ -114,7 +114,7 @@ typedef DIR *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } diff --git a/core/shared/platform/esp-idf/platform_internal.h b/core/shared/platform/esp-idf/platform_internal.h index e0091bee13..580a06d907 100644 --- a/core/shared/platform/esp-idf/platform_internal.h +++ b/core/shared/platform/esp-idf/platform_internal.h @@ -145,7 +145,7 @@ typedef DIR *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } diff --git a/core/shared/platform/freebsd/platform_internal.h b/core/shared/platform/freebsd/platform_internal.h index bfdfe1493f..01a6e824fc 100644 --- a/core/shared/platform/freebsd/platform_internal.h +++ b/core/shared/platform/freebsd/platform_internal.h @@ -115,7 +115,7 @@ os_set_signal_number_for_blocking_op(int signo); typedef int os_file_handle; static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } diff --git a/core/shared/platform/linux-sgx/platform_internal.h b/core/shared/platform/linux-sgx/platform_internal.h index 2cc34dfcae..66960ad282 100644 --- a/core/shared/platform/linux-sgx/platform_internal.h +++ b/core/shared/platform/linux-sgx/platform_internal.h @@ -74,7 +74,7 @@ typedef DIR *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } diff --git a/core/shared/platform/linux/platform_internal.h b/core/shared/platform/linux/platform_internal.h index aeddc4ccf2..865180273e 100644 --- a/core/shared/platform/linux/platform_internal.h +++ b/core/shared/platform/linux/platform_internal.h @@ -127,7 +127,7 @@ typedef DIR *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } diff --git a/core/shared/platform/riot/platform_internal.h b/core/shared/platform/riot/platform_internal.h index 24a1d6c868..11f2ba0a72 100644 --- a/core/shared/platform/riot/platform_internal.h +++ b/core/shared/platform/riot/platform_internal.h @@ -89,7 +89,7 @@ int isnan(double x); #endif static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } diff --git a/core/shared/platform/rt-thread/platform_internal.h b/core/shared/platform/rt-thread/platform_internal.h index 69d6d5581f..b9b8c78c8f 100644 --- a/core/shared/platform/rt-thread/platform_internal.h +++ b/core/shared/platform/rt-thread/platform_internal.h @@ -124,7 +124,7 @@ typedef void *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } diff --git a/core/shared/platform/vxworks/platform_internal.h b/core/shared/platform/vxworks/platform_internal.h index 1b870c70eb..6a6b00f4be 100644 --- a/core/shared/platform/vxworks/platform_internal.h +++ b/core/shared/platform/vxworks/platform_internal.h @@ -101,7 +101,7 @@ os_sigreturn(); #define os_getpagesize getpagesize static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } diff --git a/core/shared/platform/windows/platform_internal.h b/core/shared/platform/windows/platform_internal.h index 8bb77e7cba..ed021a9aa1 100644 --- a/core/shared/platform/windows/platform_internal.h +++ b/core/shared/platform/windows/platform_internal.h @@ -188,7 +188,7 @@ typedef uint32_t os_raw_file_handle; #endif static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return NULL; } diff --git a/core/shared/platform/zephyr/platform_internal.h b/core/shared/platform/zephyr/platform_internal.h index fa8fbd861a..5bc9ca986c 100644 --- a/core/shared/platform/zephyr/platform_internal.h +++ b/core/shared/platform/zephyr/platform_internal.h @@ -209,7 +209,7 @@ typedef void *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle -os_get_invalid_handle() +os_get_invalid_handle(void) { return -1; } From 9c2083a27f7cba141b4957db4b9907890f0fe096 Mon Sep 17 00:00:00 2001 From: Marcin Kolny Date: Wed, 11 Sep 2024 09:08:37 +0100 Subject: [PATCH 23/69] Implement option for skipping function index in the callstack (#3785) Also add a script that converts instruction pointers to function indexes (or function names). https://github.com/bytecodealliance/wasm-micro-runtime/issues/3758 --- core/iwasm/aot/aot_runtime.c | 35 ++-- core/iwasm/aot/aot_runtime.h | 1 + core/iwasm/compilation/aot_emit_aot_file.c | 3 + core/iwasm/compilation/aot_emit_function.c | 41 ++--- core/iwasm/compilation/aot_stack_frame_comp.c | 4 +- core/iwasm/include/aot_comp_option.h | 12 +- test-tools/ip2function/ip2function.py | 156 ++++++++++++++++++ wamr-compiler/main.c | 11 +- 8 files changed, 229 insertions(+), 34 deletions(-) create mode 100644 test-tools/ip2function/ip2function.py diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 013c761a06..4b6d25b83c 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -134,6 +134,15 @@ is_frame_per_function(WASMExecEnv *exec_env) return module->feature_flags & WASM_FEATURE_FRAME_PER_FUNCTION; } +static bool +is_frame_func_idx_disabled(WASMExecEnv *exec_env) +{ + AOTModule *module = + (AOTModule *)((AOTModuleInstance *)exec_env->module_inst)->module; + + return module->feature_flags & WASM_FEATURE_FRAME_NO_FUNC_IDX; +} + static void * get_top_frame(WASMExecEnv *exec_env) { @@ -3952,7 +3961,7 @@ aot_create_call_stack(struct WASMExecEnv *exec_env) #endif } WASMCApiFrame frame = { 0 }; - uint32 max_local_cell_num, max_stack_cell_num; + uint32 max_local_cell_num = 0, max_stack_cell_num = 0; uint32 all_cell_num, lp_size; frame.instance = module_inst; @@ -3961,16 +3970,20 @@ aot_create_call_stack(struct WASMExecEnv *exec_env) frame.func_offset = ip_offset; frame.func_name_wp = get_func_name_from_index(module_inst, func_index); - if (func_index >= module->import_func_count) { - uint32 aot_func_idx = func_index - module->import_func_count; - max_local_cell_num = module->max_local_cell_nums[aot_func_idx]; - max_stack_cell_num = module->max_stack_cell_nums[aot_func_idx]; - } - else { - AOTFuncType *func_type = module->import_funcs[func_index].func_type; - max_local_cell_num = - func_type->param_cell_num > 2 ? func_type->param_cell_num : 2; - max_stack_cell_num = 0; + if (!is_frame_func_idx_disabled(exec_env)) { + if (func_index >= module->import_func_count) { + uint32 aot_func_idx = func_index - module->import_func_count; + max_local_cell_num = module->max_local_cell_nums[aot_func_idx]; + max_stack_cell_num = module->max_stack_cell_nums[aot_func_idx]; + } + else { + AOTFuncType *func_type = + module->import_funcs[func_index].func_type; + max_local_cell_num = func_type->param_cell_num > 2 + ? func_type->param_cell_num + : 2; + max_stack_cell_num = 0; + } } all_cell_num = max_local_cell_num + max_stack_cell_num; diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 3ff0e0e3ce..56d11a22d3 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -34,6 +34,7 @@ extern "C" { /* Stack frame is created at the beginning of the function, * and not at the beginning of each function call */ #define WASM_FEATURE_FRAME_PER_FUNCTION (1 << 12) +#define WASM_FEATURE_FRAME_NO_FUNC_IDX (1 << 13) typedef enum AOTSectionType { AOT_SECTION_TYPE_TARGET_INFO = 0, diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index 20f29057c6..8fa2053083 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -4439,6 +4439,9 @@ aot_obj_data_create(AOTCompContext *comp_ctx) if (comp_ctx->call_stack_features.frame_per_function) { obj_data->target_info.feature_flags |= WASM_FEATURE_FRAME_PER_FUNCTION; } + if (!comp_ctx->call_stack_features.func_idx) { + obj_data->target_info.feature_flags |= WASM_FEATURE_FRAME_NO_FUNC_IDX; + } bh_print_time("Begin to resolve object file info"); diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index fbef02e20f..11129ac9c0 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -885,25 +885,28 @@ alloc_frame_for_aot_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } if (!comp_ctx->is_jit_mode) { - /* aot mode: new_frame->func_idx = func_idx */ - func_idx_val = comp_ctx->pointer_size == sizeof(uint64) - ? I64_CONST(func_idx) - : I32_CONST(func_idx); - offset = I32_CONST(comp_ctx->pointer_size); - CHECK_LLVM_CONST(func_idx_val); - CHECK_LLVM_CONST(offset); - if (!(func_idx_ptr = - LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, new_frame, - &offset, 1, "func_idx_addr")) - || !(func_idx_ptr = - LLVMBuildBitCast(comp_ctx->builder, func_idx_ptr, - INTPTR_T_PTR_TYPE, "func_idx_ptr"))) { - aot_set_last_error("llvm get func_idx_ptr failed"); - return false; - } - if (!LLVMBuildStore(comp_ctx->builder, func_idx_val, func_idx_ptr)) { - aot_set_last_error("llvm build store failed"); - return false; + if (comp_ctx->call_stack_features.func_idx) { + /* aot mode: new_frame->func_idx = func_idx */ + func_idx_val = comp_ctx->pointer_size == sizeof(uint64) + ? I64_CONST(func_idx) + : I32_CONST(func_idx); + offset = I32_CONST(comp_ctx->pointer_size); + CHECK_LLVM_CONST(func_idx_val); + CHECK_LLVM_CONST(offset); + if (!(func_idx_ptr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, new_frame, &offset, 1, + "func_idx_addr")) + || !(func_idx_ptr = + LLVMBuildBitCast(comp_ctx->builder, func_idx_ptr, + INTPTR_T_PTR_TYPE, "func_idx_ptr"))) { + aot_set_last_error("llvm get func_idx_ptr failed"); + return false; + } + if (!LLVMBuildStore(comp_ctx->builder, func_idx_val, + func_idx_ptr)) { + aot_set_last_error("llvm build store failed"); + return false; + } } } else { diff --git a/core/iwasm/compilation/aot_stack_frame_comp.c b/core/iwasm/compilation/aot_stack_frame_comp.c index 342dfe806e..fb540e643f 100644 --- a/core/iwasm/compilation/aot_stack_frame_comp.c +++ b/core/iwasm/compilation/aot_stack_frame_comp.c @@ -70,7 +70,9 @@ aot_alloc_tiny_frame_for_aot_func(AOTCompContext *comp_ctx, } /* Save the func_idx on the top of the stack */ - ADD_STORE(func_index, wasm_stack_top); + if (comp_ctx->call_stack_features.func_idx) { + ADD_STORE(func_index, wasm_stack_top); + } /* increment the stack pointer */ INT_CONST(offset, sizeof(AOTTinyFrame), I32_TYPE, true); diff --git a/core/iwasm/include/aot_comp_option.h b/core/iwasm/include/aot_comp_option.h index 67ec81cd36..98f33a1608 100644 --- a/core/iwasm/include/aot_comp_option.h +++ b/core/iwasm/include/aot_comp_option.h @@ -12,11 +12,19 @@ typedef struct { * bounds of the current stack frame (and if not, traps). */ bool bounds_checks; - /* Enables or disables instruction pointer (IP) tracking.*/ + /* Enables or disables instruction pointer (IP) tracking. */ bool ip; + /* Enables or disables function index in the stack trace. Please note that + * function index can be recovered from the instruction pointer using + * ip2function.py script, so enabling this feature along with `ip` might + * often be redundant. + * This option will automatically be enabled for GC and Perf Profiling mode. + */ + bool func_idx; + /* Enables or disables tracking instruction pointer of a trap. Only takes - * effect when `ip` is enabled.*/ + * effect when `ip` is enabled. */ bool trap_ip; /* Enables or disables parameters, locals and stack operands. */ diff --git a/test-tools/ip2function/ip2function.py b/test-tools/ip2function/ip2function.py new file mode 100644 index 0000000000..fb8ecd17d4 --- /dev/null +++ b/test-tools/ip2function/ip2function.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 Amazon Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +""" +This tool corrects function names in call stacks based on the +instruction pointers. + +When the AOT file is generated with excluded func-idx in the +`--call-stack-features` parameter, the function indexes are +incorrect (likely they're zero). This script uses instruction +pointers and the original WASM file to generate a call stack +file with the correct function indexes (or function names, +when available). + +Example input (call_stack.txt) - note that `__imported_wasi_snapshot_preview1_fd_close` +had index 0, therefore it appears as a name in every line: +``` +#00: 0x0505 - __imported_wasi_snapshot_preview1_fd_close +#01: 0x0309 - __imported_wasi_snapshot_preview1_fd_close +#02: 0x037c - __imported_wasi_snapshot_preview1_fd_close +#03: 0x03b2 - __imported_wasi_snapshot_preview1_fd_close +#04: 0x03e4 - __imported_wasi_snapshot_preview1_fd_close +#05: 0x02e6 - __imported_wasi_snapshot_preview1_fd_close +``` + +Conversion command: +``` +python3 test-tools/ip2function/ip2function.py \ + --wasm-file opt-samp/tiny.wasm \ + call_stack.txt +``` + +Output: +``` +#0: 0x0505 - abort +#1: 0x0309 - baz +#2: 0x037c - bar +#3: 0x03b2 - foo +#4: 0x03e4 - __original_main +#5: 0x02e6 - _start +``` +""" + +import argparse +import bisect +import os +import re +import subprocess +import sys + +from typing import NamedTuple, Optional +from typing import TextIO +from pathlib import Path +import shutil + + +class FunctionInfo(NamedTuple): + start_address: int + idx: int + name: Optional[str] + + def __str__(self) -> str: + return self.name if self.name else f"$f{self.idx}" + + +def load_functions(wasm_objdump: Path, wasm_file: Path) -> list[FunctionInfo]: + objdump_function_pattern = re.compile( + r"^([0-9a-f]+)\sfunc\[(\d+)\](?:\s\<(.+)\>)?\:$" + ) + + def parse_objdump_function_line( + line: str, + ) -> Optional[FunctionInfo]: + match = objdump_function_pattern.match(line.strip()) + return ( + FunctionInfo(int(match[1], 16), int(match[2]), match[3]) if match else None + ) + + p = subprocess.run( + [wasm_objdump, "--disassemble", wasm_file], + check=True, + capture_output=True, + text=True, + universal_newlines=True, + ) + + return list( + filter( + None, + ( + parse_objdump_function_line(line.strip()) + for line in p.stdout.split(os.linesep) + ), + ) + ) + + +def parse_call_stack_file( + functions: list[FunctionInfo], call_stack_file: TextIO, output_file: TextIO +) -> None: + call_stack_line_pattern = re.compile(r"^(#\d+): (0x[0-9a-f]+) \- (\S+)$") + for line in call_stack_file: + match = call_stack_line_pattern.match(line.strip()) + if not match: + output_file.write(line) + continue + index = match[1] + address = match[2] + + func_pos = bisect.bisect_right( + functions, int(address, 16), key=lambda x: x.start_address + ) + if func_pos <= 0: + raise ValueError(f"Cannot find function for address {address}") + output_file.write(f"{index}: {address} - {functions[func_pos -1]}\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description="addr2line for wasm") + parser.add_argument( + "--wasm-objdump", type=Path, default="wasm-objdump", help="path to wasm objdump" + ) + parser.add_argument( + "--wasm-file", required=True, type=Path, help="path to wasm file" + ) + parser.add_argument( + "call_stack_file", type=argparse.FileType("r"), help="path to a call stack file" + ) + parser.add_argument( + "-o", + "--output", + type=argparse.FileType("w"), + default=sys.stdout, + help="Output file path (default is stdout)", + ) + + args = parser.parse_args() + + wasm_objdump: Path = shutil.which(args.wasm_objdump) + assert wasm_objdump is not None + + wasm_file: Path = args.wasm_file + assert wasm_file.exists() + + parse_call_stack_file( + load_functions(wasm_objdump, wasm_file), args.call_stack_file, args.output + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index 53c75c84e5..8dca712cd1 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -167,7 +167,7 @@ print_help() printf(" By default, all features are enabled. To disable all features,\n"); printf(" provide an empty list (i.e. --call-stack-features=). This flag\n"); printf(" only only takes effect when --enable-dump-call-stack is set.\n"); - printf(" Available features: bounds-checks, ip, trap-ip, values.\n"); + printf(" Available features: bounds-checks, ip, func-idx, trap-ip, values.\n"); printf(" --enable-perf-profiling Enable function performance profiling\n"); printf(" --enable-memory-profiling Enable memory usage profiling\n"); printf(" --xip A shorthand of --enable-indirect-mode --disable-llvm-intrinsics\n"); @@ -295,6 +295,9 @@ parse_call_stack_features(char *features_str, else if (!strcmp(features[size], "values")) { out_features->values = true; } + else if (!strcmp(features[size], "func-idx")) { + out_features->func_idx = true; + } else { ret = false; printf("Unsupported feature %s\n", features[size]); @@ -664,6 +667,12 @@ main(int argc, char *argv[]) /* for now we only enable frame per function for a TINY frame mode */ option.call_stack_features.frame_per_function = true; } + if (!option.call_stack_features.func_idx + && (option.enable_gc || option.enable_perf_profiling)) { + LOG_WARNING("'func-idx' call stack feature will be automatically " + "enabled for GC and perf profiling mode"); + option.call_stack_features.func_idx = true; + } if (!size_level_set) { /** From 9aadbfee292dba0a3e7ee4940e5031b49cc3d5bd Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Fri, 13 Sep 2024 09:43:44 +0800 Subject: [PATCH 24/69] Ignore temporary file from aider (#3787) Aider is AI pair programming in your terminal: https://aider.chat --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 355d391f1b..baf11c8916 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ .clangd .DS_Store *.o +.aider* core/deps/** core/shared/mem-alloc/tlsf From 926f6622312b52ce24bf55690cfba771299f36fa Mon Sep 17 00:00:00 2001 From: Benbuck Nason Date: Fri, 13 Sep 2024 19:31:13 -0700 Subject: [PATCH 25/69] Add memory instance support apis (#3786) Now that WAMR supports multiple memory instances, this PR adds some APIs to access them in a standard way. This involves moving some existing utility functions out from the `WASM_ENABLE_MULTI_MODULE` blocks they were nested in, but multi-memory and multi-module seem independent as far as I can tell so I assume that's okay. APIs added: ```C wasm_runtime_lookup_memory wasm_runtime_get_default_memory wasm_runtime_get_memory wasm_memory_get_cur_page_count wasm_memory_get_max_page_count wasm_memory_get_bytes_per_page wasm_memory_get_shared wasm_memory_get_base_address wasm_memory_enlarge ``` --- core/iwasm/aot/aot_runtime.c | 27 ++++- core/iwasm/aot/aot_runtime.h | 9 ++ core/iwasm/common/wasm_memory.c | 137 ++++++++++++++++++++++---- core/iwasm/include/wasm_export.h | 98 ++++++++++++++++++ core/iwasm/interpreter/wasm_runtime.c | 106 ++++++++++---------- core/iwasm/interpreter/wasm_runtime.h | 6 +- 6 files changed, 307 insertions(+), 76 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 4b6d25b83c..dc6cba3f67 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1058,7 +1058,24 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, return NULL; } -static AOTMemoryInstance * +AOTMemoryInstance * +aot_lookup_memory(AOTModuleInstance *module_inst, char const *name) +{ +#if WASM_ENABLE_MULTI_MEMORY != 0 + uint32 i; + for (i = 0; i < module_inst->export_memory_count; i++) + if (!strcmp(module_inst->export_memories[i].name, name)) + return module_inst->export_memories[i].memory; + return NULL; +#else + (void)module_inst->export_memories; + if (!module_inst->memories) + return NULL; + return module_inst->memories[0]; +#endif +} + +AOTMemoryInstance * aot_get_default_memory(AOTModuleInstance *module_inst) { if (module_inst->memories) @@ -1067,6 +1084,14 @@ aot_get_default_memory(AOTModuleInstance *module_inst) return NULL; } +AOTMemoryInstance * +aot_get_memory_with_index(AOTModuleInstance *module_inst, uint32 index) +{ + if ((index >= module_inst->memory_count) || !module_inst->memories) + return NULL; + return module_inst->memories[index]; +} + static bool memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, AOTModule *module, uint32 heap_size, diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 56d11a22d3..f13d7eefc0 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -532,6 +532,15 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst); AOTFunctionInstance * aot_lookup_function(const AOTModuleInstance *module_inst, const char *name); +AOTMemoryInstance * +aot_lookup_memory(AOTModuleInstance *module_inst, char const *name); + +AOTMemoryInstance * +aot_get_default_memory(AOTModuleInstance *module_inst); + +AOTMemoryInstance * +aot_get_memory_with_index(AOTModuleInstance *module_inst, uint32 index); + /** * Get a function in the AOT module instance. * diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 82eebbf301..03260589d3 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -673,11 +673,9 @@ wasm_get_default_memory(WASMModuleInstance *module_inst) WASMMemoryInstance * wasm_get_memory_with_idx(WASMModuleInstance *module_inst, uint32 index) { - bh_assert(index < module_inst->memory_count); - if (module_inst->memories) - return module_inst->memories[index]; - else + if ((index >= module_inst->memory_count) || !module_inst->memories) return NULL; + return module_inst->memories[index]; } void @@ -756,15 +754,10 @@ wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size) return wasm_mremap_linear_memory(NULL, 0, map_size, commit_size); } -bool -wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, - uint32 memidx) +static bool +wasm_enlarge_memory_internal(WASMModuleInstanceCommon *module, + WASMMemoryInstance *memory, uint32 inc_page_count) { -#if WASM_ENABLE_MULTI_MEMORY != 0 - WASMMemoryInstance *memory = wasm_get_memory_with_idx(module, memidx); -#else - WASMMemoryInstance *memory = wasm_get_default_memory(module); -#endif uint8 *memory_data_old, *memory_data_new, *heap_data_old; uint32 num_bytes_per_page, heap_size; uint32 cur_page_count, max_page_count, total_page_count; @@ -913,7 +906,7 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, wasm_runtime_set_mem_bound_check_bytes(memory, total_size_new); return_func: - if (!ret && enlarge_memory_error_cb) { + if (!ret && module && enlarge_memory_error_cb) { WASMExecEnv *exec_env = NULL; #if WASM_ENABLE_INTERP != 0 @@ -926,8 +919,7 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, #endif enlarge_memory_error_cb(inc_page_count, total_size_old, 0, - failure_reason, - (WASMModuleInstanceCommon *)module, exec_env, + failure_reason, module, exec_env, enlarge_memory_error_user_data); } @@ -971,15 +963,16 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) { bool ret = false; + if (module->memory_count > 0) { #if WASM_ENABLE_SHARED_MEMORY != 0 - if (module->memory_count > 0) shared_memory_lock(module->memories[0]); #endif - ret = wasm_enlarge_memory_internal(module, inc_page_count, 0); + ret = wasm_enlarge_memory_internal((WASMModuleInstanceCommon *)module, + module->memories[0], inc_page_count); #if WASM_ENABLE_SHARED_MEMORY != 0 - if (module->memory_count > 0) shared_memory_unlock(module->memories[0]); #endif + } return ret; } @@ -990,15 +983,117 @@ wasm_enlarge_memory_with_idx(WASMModuleInstance *module, uint32 inc_page_count, { bool ret = false; + if (memidx < module->memory_count) { #if WASM_ENABLE_SHARED_MEMORY != 0 - if (memidx < module->memory_count) shared_memory_lock(module->memories[memidx]); #endif - ret = wasm_enlarge_memory_internal(module, inc_page_count, memidx); + ret = wasm_enlarge_memory_internal((WASMModuleInstanceCommon *)module, + module->memories[memidx], + inc_page_count); #if WASM_ENABLE_SHARED_MEMORY != 0 - if (memidx < module->memory_count) shared_memory_unlock(module->memories[memidx]); #endif + } + + return ret; +} + +WASMMemoryInstance * +wasm_runtime_lookup_memory(WASMModuleInstanceCommon *module_inst, + const char *name) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_lookup_memory((WASMModuleInstance *)module_inst, name); +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_lookup_memory((WASMModuleInstance *)module_inst, name); +#endif + + return NULL; +} + +WASMMemoryInstance * +wasm_runtime_get_default_memory(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_get_default_memory((WASMModuleInstance *)module_inst); +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_get_default_memory((AOTModuleInstance *)module_inst); +#endif + + return NULL; +} + +WASMMemoryInstance * +wasm_runtime_get_memory(WASMModuleInstanceCommon *module_inst, uint32 index) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_get_memory_with_idx((WASMModuleInstance *)module_inst, + index); +#endif + +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_get_memory_with_index((AOTModuleInstance *)module_inst, + index); +#endif + + return NULL; +} + +uint64 +wasm_memory_get_cur_page_count(WASMMemoryInstance *memory) +{ + return memory->cur_page_count; +} + +uint64 +wasm_memory_get_max_page_count(WASMMemoryInstance *memory) +{ + return memory->max_page_count; +} + +uint64 +wasm_memory_get_bytes_per_page(WASMMemoryInstance *memory) +{ + return memory->num_bytes_per_page; +} + +bool +wasm_memory_get_shared(WASMMemoryInstance *memory) +{ + return memory->is_shared_memory; +} + +void * +wasm_memory_get_base_address(WASMMemoryInstance *memory) +{ + return memory->memory_data; +} + +bool +wasm_memory_enlarge(WASMMemoryInstance *memory, uint64 inc_page_count) +{ + bool ret = false; + + if (memory) { +#if WASM_ENABLE_SHARED_MEMORY != 0 + shared_memory_lock(memory); +#endif + ret = + wasm_enlarge_memory_internal(NULL, memory, (uint32)inc_page_count); +#if WASM_ENABLE_SHARED_MEMORY != 0 + shared_memory_unlock(memory); +#endif + } return ret; } diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 569c4deaa7..1a03f7280d 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -120,6 +120,10 @@ typedef struct WASMModuleInstanceCommon *wasm_module_inst_t; typedef void WASMFunctionInstanceCommon; typedef WASMFunctionInstanceCommon *wasm_function_inst_t; +/* Memory instance */ +struct WASMMemoryInstance; +typedef struct WASMMemoryInstance *wasm_memory_inst_t; + /* WASM section */ typedef struct wasm_section_t { struct wasm_section_t *next; @@ -939,6 +943,100 @@ WASM_RUNTIME_API_EXTERN void wasm_runtime_set_module_inst(wasm_exec_env_t exec_env, const wasm_module_inst_t module_inst); +/** + * @brief Lookup a memory instance by name + * + * @param module_inst The module instance + * @param name The name of the memory instance + * + * @return The memory instance if found, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_memory_inst_t +wasm_runtime_lookup_memory(const wasm_module_inst_t module_inst, + const char *name); + +/** + * @brief Get the default memory instance + * + * @param module_inst The module instance + * + * @return The memory instance if found, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_memory_inst_t +wasm_runtime_get_default_memory(const wasm_module_inst_t module_inst); + +/** + * @brief Get a memory instance by index + * + * @param module_inst The module instance + * @param index The index of the memory instance + * + * @return The memory instance if found, NULL otherwise + */ +WASM_RUNTIME_API_EXTERN wasm_memory_inst_t +wasm_runtime_get_memory(const wasm_module_inst_t module_inst, uint32_t index); + +/** + * @brief Get the current number of pages for a memory instance + * + * @param memory_inst The memory instance + * + * @return The current number of pages + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_memory_get_cur_page_count(const wasm_memory_inst_t memory_inst); + +/** + * @brief Get the maximum number of pages for a memory instance + * + * @param memory_inst The memory instance + * + * @return The maximum number of pages + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_memory_get_max_page_count(const wasm_memory_inst_t memory_inst); + +/** + * @brief Get the number of bytes per page for a memory instance + * + * @param memory_inst The memory instance + * + * @return The number of bytes per page + */ +WASM_RUNTIME_API_EXTERN uint64_t +wasm_memory_get_bytes_per_page(const wasm_memory_inst_t memory_inst); + +/** + * @brief Get the shared status for a memory instance + * + * @param memory_inst The memory instance + * + * @return True if shared, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_memory_get_shared(const wasm_memory_inst_t memory_inst); + +/** + * @brief Get the base address for a memory instance + * + * @param memory_inst The memory instance + * + * @return The base address on success, false otherwise + */ +WASM_RUNTIME_API_EXTERN void * +wasm_memory_get_base_address(const wasm_memory_inst_t memory_inst); + +/** + * @brief Enlarge a memory instance by a number of pages + * + * @param memory_inst The memory instance + * @param inc_page_count The number of pages to add + * + * @return True if successful, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_memory_enlarge(wasm_memory_inst_t memory_inst, uint64_t inc_page_count); + /** * Call the given WASM function of a WASM module instance with * arguments (bytecode and AoT). diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index f70f9cb736..e8f4c749e0 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -1323,42 +1323,6 @@ export_tags_instantiate(const WASMModule *module, } #endif /* end of WASM_ENABLE_TAGS != 0 */ -#if WASM_ENABLE_MULTI_MODULE != 0 -static void -export_globals_deinstantiate(WASMExportGlobInstance *globals) -{ - if (globals) - wasm_runtime_free(globals); -} - -static WASMExportGlobInstance * -export_globals_instantiate(const WASMModule *module, - WASMModuleInstance *module_inst, - uint32 export_glob_count, char *error_buf, - uint32 error_buf_size) -{ - WASMExportGlobInstance *export_globals, *export_global; - WASMExport *export = module->exports; - uint32 i; - uint64 total_size = - sizeof(WASMExportGlobInstance) * (uint64)export_glob_count; - - if (!(export_global = export_globals = - runtime_malloc(total_size, error_buf, error_buf_size))) { - return NULL; - } - - for (i = 0; i < module->export_count; i++, export ++) - if (export->kind == EXPORT_KIND_GLOBAL) { - export_global->name = export->name; - export_global->global = &module_inst->e->globals[export->index]; - export_global++; - } - - bh_assert((uint32)(export_global - export_globals) == export_glob_count); - return export_globals; -} - #if WASM_ENABLE_MULTI_MEMORY != 0 static void export_memories_deinstantiate(WASMExportMemInstance *memories) @@ -1396,6 +1360,42 @@ export_memories_instantiate(const WASMModule *module, } #endif /* end of if WASM_ENABLE_MULTI_MEMORY != 0 */ +#if WASM_ENABLE_MULTI_MODULE != 0 +static void +export_globals_deinstantiate(WASMExportGlobInstance *globals) +{ + if (globals) + wasm_runtime_free(globals); +} + +static WASMExportGlobInstance * +export_globals_instantiate(const WASMModule *module, + WASMModuleInstance *module_inst, + uint32 export_glob_count, char *error_buf, + uint32 error_buf_size) +{ + WASMExportGlobInstance *export_globals, *export_global; + WASMExport *export = module->exports; + uint32 i; + uint64 total_size = + sizeof(WASMExportGlobInstance) * (uint64)export_glob_count; + + if (!(export_global = export_globals = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + for (i = 0; i < module->export_count; i++, export ++) + if (export->kind == EXPORT_KIND_GLOBAL) { + export_global->name = export->name; + export_global->global = &module_inst->e->globals[export->index]; + export_global++; + } + + bh_assert((uint32)(export_global - export_globals) == export_glob_count); + return export_globals; +} + #endif /* end of if WASM_ENABLE_MULTI_MODULE != 0 */ static WASMFunctionInstance * @@ -2388,11 +2388,13 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, /* export */ module_inst->export_func_count = get_export_count(module, EXPORT_KIND_FUNC); +#if WASM_ENABLE_MULTI_MEMORY != 0 + module_inst->export_memory_count = + get_export_count(module, EXPORT_KIND_MEMORY); +#endif #if WASM_ENABLE_MULTI_MODULE != 0 module_inst->export_table_count = get_export_count(module, EXPORT_KIND_TABLE); - module_inst->export_memory_count = - get_export_count(module, EXPORT_KIND_MEMORY); #if WASM_ENABLE_TAGS != 0 module_inst->e->export_tag_count = get_export_count(module, EXPORT_KIND_TAG); @@ -2432,7 +2434,7 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, module, module_inst, module_inst->export_global_count, error_buf, error_buf_size))) #endif -#if WASM_ENABLE_MULTI_MODULE != 0 && WASM_ENABLE_MULTI_MEMORY != 0 +#if WASM_ENABLE_MULTI_MEMORY != 0 || (module_inst->export_memory_count > 0 && !(module_inst->export_memories = export_memories_instantiate( module, module_inst, module_inst->export_memory_count, @@ -3240,7 +3242,7 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst) export_globals_deinstantiate(module_inst->export_globals); #endif -#if WASM_ENABLE_MULTI_MODULE != 0 && WASM_ENABLE_MULTI_MEMORY != 0 +#if WASM_ENABLE_MULTI_MEMORY != 0 export_memories_deinstantiate(module_inst->export_memories); #endif @@ -3292,17 +3294,6 @@ wasm_lookup_function(const WASMModuleInstance *module_inst, const char *name) return NULL; } -#if WASM_ENABLE_MULTI_MODULE != 0 -WASMGlobalInstance * -wasm_lookup_global(const WASMModuleInstance *module_inst, const char *name) -{ - uint32 i; - for (i = 0; i < module_inst->export_global_count; i++) - if (!strcmp(module_inst->export_globals[i].name, name)) - return module_inst->export_globals[i].global; - return NULL; -} - WASMMemoryInstance * wasm_lookup_memory(const WASMModuleInstance *module_inst, const char *name) { @@ -3314,10 +3305,23 @@ wasm_lookup_memory(const WASMModuleInstance *module_inst, const char *name) return NULL; #else (void)module_inst->export_memories; + if (!module_inst->memories) + return NULL; return module_inst->memories[0]; #endif } +#if WASM_ENABLE_MULTI_MODULE != 0 +WASMGlobalInstance * +wasm_lookup_global(const WASMModuleInstance *module_inst, const char *name) +{ + uint32 i; + for (i = 0; i < module_inst->export_global_count; i++) + if (!strcmp(module_inst->export_globals[i].name, name)) + return module_inst->export_globals[i].global; + return NULL; +} + WASMTableInstance * wasm_lookup_table(const WASMModuleInstance *module_inst, const char *name) { diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 8666541f2b..c430186952 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -539,13 +539,13 @@ wasm_set_running_mode(WASMModuleInstance *module_inst, WASMFunctionInstance * wasm_lookup_function(const WASMModuleInstance *module_inst, const char *name); +WASMMemoryInstance * +wasm_lookup_memory(const WASMModuleInstance *module_inst, const char *name); + #if WASM_ENABLE_MULTI_MODULE != 0 WASMGlobalInstance * wasm_lookup_global(const WASMModuleInstance *module_inst, const char *name); -WASMMemoryInstance * -wasm_lookup_memory(const WASMModuleInstance *module_inst, const char *name); - WASMTableInstance * wasm_lookup_table(const WASMModuleInstance *module_inst, const char *name); From 92852f3719572b80a06de2df3de5bc662183022c Mon Sep 17 00:00:00 2001 From: WenLY1 <130950131+WenLY1@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:51:42 +0800 Subject: [PATCH 26/69] Implement a first version of shared heap feature (#3789) ps. https://github.com/bytecodealliance/wasm-micro-runtime/issues/3546 --- build-scripts/config_common.cmake | 5 + build-scripts/runtime_lib.cmake | 5 + core/config.h | 4 + core/iwasm/aot/aot_runtime.h | 4 + core/iwasm/common/wasm_memory.c | 470 +++++++++++++++++- core/iwasm/common/wasm_memory.h | 26 + core/iwasm/common/wasm_native.c | 13 + core/iwasm/include/wasm_export.h | 55 ++ core/iwasm/interpreter/wasm_interp_classic.c | 69 ++- core/iwasm/interpreter/wasm_interp_fast.c | 30 +- core/iwasm/interpreter/wasm_runtime.h | 15 + .../libraries/shared-heap/shared_heap.cmake | 8 + .../shared-heap/shared_heap_wrapper.c | 55 ++ .../libraries/thread-mgr/thread_manager.c | 77 +++ .../libraries/thread-mgr/thread_manager.h | 12 + 15 files changed, 826 insertions(+), 22 deletions(-) create mode 100644 core/iwasm/libraries/shared-heap/shared_heap.cmake create mode 100644 core/iwasm/libraries/shared-heap/shared_heap_wrapper.c diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 12fc06bd70..0a5f82106a 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -256,6 +256,11 @@ if (WAMR_BUILD_SHARED_MEMORY EQUAL 1) else () add_definitions (-DWASM_ENABLE_SHARED_MEMORY=0) endif () +if (WAMR_BUILD_SHARED_HEAP EQUAL 1) + add_definitions (-DWASM_ENABLE_SHARED_HEAP=1) + message (" Shared heap enabled") +endif() + if (WAMR_BUILD_MEMORY64 EQUAL 1) # if native is 32-bit or cross-compiled to 32-bit if (NOT WAMR_BUILD_TARGET MATCHES ".*64.*") diff --git a/build-scripts/runtime_lib.cmake b/build-scripts/runtime_lib.cmake index 3ab0cff4fb..a2aaf425bb 100644 --- a/build-scripts/runtime_lib.cmake +++ b/build-scripts/runtime_lib.cmake @@ -119,6 +119,10 @@ if (WAMR_BUILD_LIB_WASI_THREADS EQUAL 1) set (WAMR_BUILD_SHARED_MEMORY 1) endif () +if (WAMR_BUILD_SHARED_HEAP EQUAL 1) + include (${IWASM_DIR}/libraries/shared-heap/shared_heap.cmake) +endif () + if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) set (WAMR_BUILD_THREAD_MGR 1) include (${IWASM_DIR}/libraries/debug-engine/debug_engine.cmake) @@ -193,6 +197,7 @@ set (source_all ${LIBC_EMCC_SOURCE} ${LIB_RATS_SOURCE} ${DEBUG_ENGINE_SOURCE} + ${LIB_SHARED_HEAP_SOURCE} ) set (WAMR_RUNTIME_LIB_SOURCE ${source_all}) diff --git a/core/config.h b/core/config.h index a25eb543ea..a7925b0088 100644 --- a/core/config.h +++ b/core/config.h @@ -688,4 +688,8 @@ #endif #endif /* WASM_ENABLE_FUZZ_TEST != 0 */ +#ifndef WASM_ENABLE_SHARED_HEAP +#define WASM_ENABLE_SHARED_HEAP 0 +#endif + #endif /* end of _CONFIG_H_ */ diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 56d11a22d3..4c3676f884 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -119,6 +119,10 @@ typedef struct AOTModuleInstanceExtra { bh_list *sub_module_inst_list; WASMModuleInstanceCommon **import_func_module_insts; #endif + +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; +#endif } AOTModuleInstanceExtra; #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 82eebbf301..dc9c4aa1be 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -13,6 +13,10 @@ #include "../common/wasm_shared_memory.h" #endif +#if WASM_ENABLE_THREAD_MGR != 0 +#include "../libraries/thread-mgr/thread_manager.h" +#endif + typedef enum Memory_Mode { MEMORY_MODE_UNKNOWN = 0, MEMORY_MODE_POOL, @@ -24,6 +28,11 @@ static Memory_Mode memory_mode = MEMORY_MODE_UNKNOWN; static mem_allocator_t pool_allocator = NULL; +#if WASM_ENABLE_SHARED_HEAP != 0 +static WASMSharedHeap *shared_heap_list = NULL; +static korp_mutex shared_heap_list_lock; +#endif + static enlarge_memory_error_callback_t enlarge_memory_error_cb; static void *enlarge_memory_error_user_data; @@ -132,16 +141,370 @@ is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) #endif } +#if WASM_ENABLE_SHARED_HEAP != 0 +static void * +wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size); +static void +wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, + uint64 map_size); + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, + "Operation of shared heap failed: %s", string); + } +} + +static void * +runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +WASMSharedHeap * +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, + uint32 error_buf_size) +{ + uint64 heap_struct_size = sizeof(WASMSharedHeap); + uint32 size = init_args->size; + WASMSharedHeap *heap; + + if (!(heap = runtime_malloc(heap_struct_size, error_buf, error_buf_size))) { + goto fail1; + } + if (!(heap->heap_handle = + runtime_malloc(mem_allocator_get_heap_struct_size(), error_buf, + error_buf_size))) { + goto fail2; + } + heap->start_off_mem64 = UINT64_MAX - heap->size + 1; + heap->start_off_mem32 = UINT32_MAX - heap->size + 1; + + size = align_uint(size, os_getpagesize()); + if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { + set_error_buf(error_buf, error_buf_size, "invalid size of shared heap"); + goto fail3; + } + + if (!(heap->base_addr = wasm_mmap_linear_memory(size, size))) { + goto fail3; + } + if (!mem_allocator_create_with_struct_and_pool( + heap->heap_handle, heap_struct_size, heap->base_addr, size)) { + set_error_buf(error_buf, error_buf_size, "init share heap failed"); + goto fail4; + } + + os_mutex_lock(&shared_heap_list_lock); + if (shared_heap_list == NULL) { + shared_heap_list = heap; + } + else { + heap->next = shared_heap_list; + shared_heap_list = heap; + } + os_mutex_unlock(&shared_heap_list_lock); + return heap; + +fail4: + wasm_munmap_linear_memory(heap->base_addr, size, size); + +fail3: + wasm_runtime_free(heap->heap_handle); +fail2: + wasm_runtime_free(heap); +fail1: + return NULL; +} + +bool +wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + return wasm_cluster_attach_shared_heap(module_inst, shared_heap); +#else + return wasm_runtime_attach_shared_heap_internal(module_inst, shared_heap); +#endif +} + +bool +wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap) +{ + uint64 linear_mem_size = 0; + WASMMemoryInstance *memory = NULL; + WASMSharedHeap *heap = (WASMSharedHeap *)shared_heap; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + // check if linear memory and shared heap are overlapped + linear_mem_size = memory->memory_data_size; + + if ((memory->is_memory64 && linear_mem_size > heap->start_off_mem64) + || (!memory->is_memory64 && linear_mem_size > heap->start_off_mem32)) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + return false; + } + + if (module_inst->module_type == Wasm_Module_Bytecode) { + if (((WASMModuleInstance *)module_inst)->e->shared_heap) { + LOG_WARNING("A shared heap is already attached"); + return false; + } + ((WASMModuleInstance *)module_inst)->e->shared_heap = heap; + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + return true; +} + +void +wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_detach_shared_heap(module_inst); +#else + wasm_runtime_detach_shared_heap_internal(module_inst); +#endif +} + +void +wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) +{ + if (module_inst->module_type == Wasm_Module_Bytecode) { + ((WASMModuleInstance *)module_inst)->e->shared_heap = NULL; + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } +} + +static bool +is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, + bool is_memory64, uint64 app_offset, uint32 bytes) +{ + WASMSharedHeap *heap = NULL; + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; + } + else if (module_inst_comm->module_type == Wasm_Module_AoT) { + // TODO + } + + if (!heap) { + return false; + } + + if (!is_memory64) { + if (app_offset >= heap->start_off_mem32 + && app_offset <= UINT32_MAX - bytes + 1) { + return true; + } + } + else { + if (app_offset >= heap->start_off_mem64 + && app_offset <= UINT64_MAX - bytes + 1) { + return true; + } + } + return false; +} + +static bool +is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, + uint8 *addr, uint32 bytes) +{ + WASMSharedHeap *heap = NULL; + + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; + } + else if (module_inst_comm->module_type == Wasm_Module_AoT) { + // TODO + } + + if (heap && addr >= heap->base_addr + && addr + bytes <= heap->base_addr + heap->size + && addr + bytes > addr) { + return true; + } + return false; +} + +static uint64 +shared_heap_addr_native_to_app(WASMModuleInstanceCommon *module_inst, + WASMMemoryInstance *memory, void *addr) +{ + WASMSharedHeap *heap = NULL; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (!heap) { + LOG_WARNING("Wasm module doesn't attach to a shared heap"); + return 0; + } + if (!addr) { + LOG_WARNING("Invalid address"); + return 0; + } + + if (memory && memory->is_memory64) { + return heap->start_off_mem64 + ((uint8 *)addr - heap->base_addr); + } + else if (memory && !memory->is_memory64) { + return heap->start_off_mem32 + ((uint8 *)addr - heap->base_addr); + } + return 0; +} + +static void * +shared_heap_addr_app_to_native(WASMModuleInstanceCommon *module_inst, + WASMMemoryInstance *memory, uint64 ptr) +{ + void *addr = NULL; + WASMSharedHeap *heap = NULL; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (!heap) { + LOG_WARNING("Wasm module doesn't attach to a shared heap"); + return NULL; + } + + if (!memory) { + LOG_WARNING("Wasm memory is not initialized"); + return NULL; + } + + if (memory->is_memory64) { + addr = heap->base_addr + (ptr - heap->start_off_mem64); + } + else { + addr = heap->base_addr + (ptr - heap->start_off_mem32); + } + + return addr; +} + +static uint64 +shared_heap_get_addr_start(WASMSharedHeap *heap, WASMMemoryInstance *memory) +{ + uint64 shared_heap_start = 0; + + if (!heap || !memory) { + LOG_ERROR("Invalid heap or memory"); + return 0; + } + + if (memory && !memory->is_memory64) { + shared_heap_start = heap->start_off_mem32; + } + else if (memory && memory->is_memory64) { + shared_heap_start = heap->start_off_mem64; + } + + return shared_heap_start; +} + +uint64 +wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, + uint64_t size, void **p_native_addr) +{ + WASMSharedHeap *heap = NULL; + WASMMemoryInstance *memory = NULL; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (heap) { + *p_native_addr = mem_allocator_malloc(heap->heap_handle, size); + + return shared_heap_addr_native_to_app(module_inst, memory, + *p_native_addr); + } + else { + LOG_WARNING("Wasm module doesn't attach to a shared heap"); + } + return 0; +} + +void +wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr) +{ + WASMSharedHeap *heap = NULL; + WASMMemoryInstance *memory = NULL; + void *addr = NULL; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (!heap) { + LOG_WARNING("Wasm module doesn't attach to a shared heap"); + return; + } + + addr = shared_heap_addr_app_to_native(module_inst, memory, ptr); + + if (heap) { + mem_allocator_free(heap->base_addr, addr); + } +} +#endif + bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, const MemAllocOption *alloc_option) { + bool ret = false; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (os_mutex_init(&shared_heap_list_lock)) { + return false; + } +#endif if (mem_alloc_type == Alloc_With_Pool) { - return wasm_memory_init_with_pool(alloc_option->pool.heap_buf, - alloc_option->pool.heap_size); + ret = wasm_memory_init_with_pool(alloc_option->pool.heap_buf, + alloc_option->pool.heap_size); } else if (mem_alloc_type == Alloc_With_Allocator) { - return wasm_memory_init_with_allocator( + ret = wasm_memory_init_with_allocator( #if WASM_MEM_ALLOC_WITH_USER_DATA != 0 alloc_option->allocator.user_data, #endif @@ -151,16 +514,48 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, } else if (mem_alloc_type == Alloc_With_System_Allocator) { memory_mode = MEMORY_MODE_SYSTEM_ALLOCATOR; - return true; + ret = true; } - else { - return false; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (!ret) { + os_mutex_destroy(&shared_heap_list_lock); + } +#endif + + return ret; +} + +#if WASM_ENABLE_SHARED_HEAP != 0 +static void +wasm_runtime_shared_heap_destroy() +{ + WASMSharedHeap *heap = shared_heap_list; + WASMSharedHeap *cur; + int ret = 0; + + while (heap) { + cur = heap; + heap = heap->next; + ret = ret + mem_allocator_destroy(cur->heap_handle); + wasm_runtime_free(cur->heap_handle); + wasm_munmap_linear_memory(cur->base_addr, cur->size, cur->size); + wasm_runtime_free(cur); + } + + if (ret != 0) { + LOG_ERROR("Memory leak detected in shared heap"); } } +#endif void wasm_runtime_memory_destroy(void) { + +#if WASM_ENABLE_SHARED_HEAP != 0 + wasm_runtime_shared_heap_destroy(); +#endif + if (memory_mode == MEMORY_MODE_POOL) { #if BH_ENABLE_GC_VERIFY == 0 (void)mem_allocator_destroy(pool_allocator); @@ -342,7 +737,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, /* boundary overflow check */ if (size > max_linear_memory_size || app_offset > max_linear_memory_size - size) { - goto fail; + goto shared_heap_bound_check; } SHARED_MEMORY_LOCK(memory_inst); @@ -354,6 +749,13 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); +shared_heap_bound_check: +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_offset, size)) { + return true; + } +#endif fail: wasm_set_exception(module_inst, "out of bounds memory access"); return false; @@ -439,6 +841,13 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, return true; } +#if WASM_ENABLE_SHARED_HEAP != 0 + else if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, + size)) { + SHARED_MEMORY_UNLOCK(memory_inst); + return true; + } +#endif SHARED_MEMORY_UNLOCK(memory_inst); fail: @@ -475,6 +884,19 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); return addr; } +#if WASM_ENABLE_SHARED_HEAP != 0 + else if (is_app_addr_in_shared_heap(module_inst_comm, + memory_inst->is_memory64, + app_offset, 1)) { + uint64 heap_start = shared_heap_get_addr_start( + module_inst->e->shared_heap, memory_inst); + uint64 heap_offset = (uint64)app_offset - heap_start; + + addr = module_inst->e->shared_heap->base_addr + heap_offset; + SHARED_MEMORY_UNLOCK(memory_inst); + return addr; + } +#endif SHARED_MEMORY_UNLOCK(memory_inst); return NULL; } @@ -499,6 +921,11 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, bounds_checks = is_bounds_checks_enabled(module_inst_comm); +#if WASM_ENABLE_SHARED_HEAP != 0 + /* If shared heap is enabled, bounds check is always needed */ + bounds_checks = true; +#endif + memory_inst = wasm_get_default_memory(module_inst); if (!memory_inst) { return 0; @@ -513,6 +940,17 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); return ret; } + else { +#if WASM_ENABLE_SHARED_HEAP != 0 + uint64 shared_heap_start = shared_heap_get_addr_start( + module_inst->e->shared_heap, memory_inst); + ret = + (uint64)(addr - (uint8 *)module_inst->e->shared_heap->base_addr) + + shared_heap_start; + SHARED_MEMORY_UNLOCK(memory_inst); + return ret; +#endif + } } /* If bounds checks is disabled, return the offset directly */ else if (addr != NULL) { @@ -765,6 +1203,11 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, #else WASMMemoryInstance *memory = wasm_get_default_memory(module); #endif + +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *heap; +#endif + uint8 *memory_data_old, *memory_data_new, *heap_data_old; uint32 num_bytes_per_page, heap_size; uint32 cur_page_count, max_page_count, total_page_count; @@ -797,6 +1240,19 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, total_page_count = inc_page_count + cur_page_count; total_size_new = num_bytes_per_page * (uint64)total_page_count; +#if WASM_ENABLE_SHARED_HEAP != 0 + heap = module->e->shared_heap; + if (memory->is_memory64 && total_size_new > heap->start_off_mem64) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } + else if (!memory->is_memory64 && total_size_new > heap->start_off_mem32) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } +#endif if (inc_page_count <= 0) /* No need to enlarge memory */ return true; diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index 2f20d3f684..be89772e37 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -41,6 +41,32 @@ SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size) #define SET_LINEAR_MEMORY_SIZE(memory, size) memory->memory_data_size = size #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +WASMSharedHeap * +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, + uint32 error_buf_size); + +bool +wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap); +bool +wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap); + +void +wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst); +void +wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst); + +uint64 +wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, + uint64 size, void **p_native_addr); + +void +wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, + uint64 ptr); +#endif + bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, const MemAllocOption *alloc_option); diff --git a/core/iwasm/common/wasm_native.c b/core/iwasm/common/wasm_native.c index 0ff3053fa9..060bb2c3d3 100644 --- a/core/iwasm/common/wasm_native.c +++ b/core/iwasm/common/wasm_native.c @@ -33,6 +33,11 @@ uint32 get_spectest_export_apis(NativeSymbol **p_libc_builtin_apis); #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +uint32 +get_lib_shared_heap_export_apis(NativeSymbol **p_shared_heap_apis); +#endif + uint32 get_libc_wasi_export_apis(NativeSymbol **p_libc_wasi_apis); @@ -512,6 +517,14 @@ wasm_native_init() goto fail; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + n_native_symbols = get_lib_shared_heap_export_apis(&native_symbols); + if (n_native_symbols > 0 + && !wasm_native_register_natives("env", native_symbols, + n_native_symbols)) + goto fail; +#endif + #if WASM_ENABLE_BASE_LIB != 0 n_native_symbols = get_base_lib_export_apis(&native_symbols); if (n_native_symbols > 0 diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 569c4deaa7..5ce617ef16 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -135,6 +135,9 @@ typedef struct wasm_section_t { struct WASMExecEnv; typedef struct WASMExecEnv *wasm_exec_env_t; +struct WASMSharedHeap; +typedef struct WASMSharedHeap *wasm_shared_heap_t; + /* Package Type */ typedef enum { Wasm_Module_Bytecode = 0, @@ -320,6 +323,12 @@ typedef enum { WASM_LOG_LEVEL_VERBOSE = 4 } log_level_t; +#if WASM_ENABLE_SHARED_HEAP != 0 +typedef struct SharedHeapInitArgs { + uint32 size; +} SharedHeapInitArgs; +#endif + /** * Initialize the WASM runtime environment, and also initialize * the memory allocator with system allocator, which calls os_malloc @@ -2110,6 +2119,52 @@ wasm_runtime_detect_native_stack_overflow_size(wasm_exec_env_t exec_env, WASM_RUNTIME_API_EXTERN bool wasm_runtime_is_underlying_binary_freeable(const wasm_module_t module); +#if WASM_ENABLE_SHARED_HEAP != 0 +/** + * Create a shared heap + * @param init_args the initialization arguments + * @param error_buf buffer to output the error info if failed + * @param error_buf_size the size of the error buffer + */ +WASM_RUNTIME_API_EXTERN wasm_shared_heap_t +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, + uint32 error_buf_size); + +/** + * Attach a shared heap to a module instance + * @param module_inst the module instance + * @param shared_heap the shared heap + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst, + wasm_shared_heap_t shared_heap); + +/** + * Detach a shared heap from a module instance + * @param module_inst the module instance + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst); + +/** + * Allocate memory from a shared heap + * @param module_inst the module instance + * @param size required memory size + * @param p_native_addr native address of allocated memory + */ +WASM_RUNTIME_API_EXTERN uint64 +wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64 size, + void **p_native_addr); + +/** + * Free the memory allocated from shared heap + * @param module_inst the module instance + * @param ptr the offset in wasm app + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr); +#endif + #ifdef __cplusplus } #endif diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 67f8c2d455..f39ae3468b 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -48,6 +48,31 @@ typedef float64 CellType_F64; #if WASM_ENABLE_MEMORY64 == 0 +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 +#define GOTO_OUT_OF_BOUNDS goto out_of_bounds; +#else +#define GOTO_OUT_OF_BOUNDS +#endif + +#if WASM_ENABLE_SHARED_HEAP != 0 +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ + do { \ + if (offset1 + bytes >= UINT32_MAX - module->shared_heap->size \ + && offset1 + bytes <= UINT32_MAX) { \ + uint64 heap_start = UINT32_MAX - module->shared_heap->size; \ + uint64 heap_offset = (uint64)offset1 - heap_start; \ + maddr = module->shared_heap->data + heap_offset; \ + } \ + else { \ + GOTO_OUT_OF_BOUNDS; \ + } \ + } while (0) +#else +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS +#endif + #if (!defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0) #define CHECK_MEMORY_OVERFLOW(bytes) \ @@ -58,7 +83,7 @@ typedef float64 CellType_F64; be in valid range, no need to check it again. */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ @@ -69,7 +94,7 @@ typedef float64 CellType_F64; bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ } while (0) #else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ @@ -82,22 +107,40 @@ typedef float64 CellType_F64; #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ maddr = memory->memory_data + (uint32)(start); \ + uint64 offset1 = start; \ } while (0) #endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ #else /* else of WASM_ENABLE_MEMORY64 == 0 */ -#define CHECK_MEMORY_OVERFLOW(bytes) \ - do { \ - uint64 offset1 = (uint64)offset + (uint64)addr; \ - /* If memory64 is enabled, offset1, offset1 + bytes can overflow */ \ - if (disable_bounds_checks \ - || (offset1 >= offset && offset1 + bytes >= offset1 \ - && offset1 + bytes <= get_linear_mem_size())) \ - maddr = memory->memory_data + offset1; \ - else \ - goto out_of_bounds; \ +#if WASM_ENABLE_SHARED_HEAP != 0 +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ + do { \ + if (offset1 + bytes >= UINT64_MAX - module->shared_heap->size \ + && offset1 + bytes <= UINT64_MAX) { \ + uint64 heap_start = UINT64_MAX - module->shared_heap->size; \ + uint64 heap_offset = (uint64)offset1 - heap_start; \ + maddr = module->shared_heap->data + heap_offset; \ + } \ + else \ + goto out_of_bounds; \ + } while (0) +#else +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) goto out_of_bounds; +#endif + +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + /* If memory64 is enabled, offset1, offset1 + bytes can \ + * overflow */ \ + if (disable_bounds_checks \ + || (offset1 >= offset && offset1 + bytes >= offset1 \ + && offset1 + bytes <= get_linear_mem_size())) \ + maddr = memory->memory_data + offset1; \ + else \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ @@ -110,7 +153,7 @@ typedef float64 CellType_F64; bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ } while (0) #endif /* end of WASM_ENABLE_MEMORY64 == 0 */ diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 1d7ca8f908..84170f2135 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -37,6 +37,31 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 +#define GOTO_OUT_OF_BOUNDS goto out_of_bounds; +#else +#define GOTO_OUT_OF_BOUNDS +#endif + +#if WASM_ENABLE_SHARED_HEAP != 0 +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ + do { \ + if (offset1 + bytes >= UINT32_MAX - module->e->shared_heap->size \ + && offset1 + bytes <= UINT32_MAX) { \ + uint64 heap_start = UINT32_MAX - module->e->shared_heap->size; \ + uint64 heap_offset = (uint64)offset1 - heap_start; \ + maddr = module->e->shared_heap->base_addr + heap_offset; \ + } \ + else { \ + GOTO_OUT_OF_BOUNDS; \ + } \ + } while (0) +#else +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS +#endif + #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 #define CHECK_MEMORY_OVERFLOW(bytes) \ @@ -47,7 +72,7 @@ typedef float64 CellType_F64; be in valid range, no need to check it again. */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ @@ -58,7 +83,7 @@ typedef float64 CellType_F64; bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ } while (0) #else #define CHECK_MEMORY_OVERFLOW(bytes) \ @@ -70,6 +95,7 @@ typedef float64 CellType_F64; #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ maddr = memory->memory_data + (uint32)(start); \ + uint64 offset1 = start; \ } while (0) #endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 8666541f2b..3fa3682c03 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -92,6 +92,17 @@ typedef union { uint32 u32[2]; } MemBound; +#if WASM_ENABLE_SHARED_HEAP != 0 +typedef struct WASMSharedHeap { + struct WASMSharedHeap *next; + void *heap_handle; + uint8_t *base_addr; + uint32_t size; + uint64 start_off_mem64; + uint64 start_off_mem32; +} WASMSharedHeap; +#endif + struct WASMMemoryInstance { /* Module type */ uint32 module_type; @@ -353,6 +364,10 @@ typedef struct WASMModuleInstanceExtra { uint32 max_aux_stack_used; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; +#endif + #if WASM_ENABLE_DEBUG_INTERP != 0 \ || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ && WASM_ENABLE_LAZY_JIT != 0) diff --git a/core/iwasm/libraries/shared-heap/shared_heap.cmake b/core/iwasm/libraries/shared-heap/shared_heap.cmake new file mode 100644 index 0000000000..ec91dabcd0 --- /dev/null +++ b/core/iwasm/libraries/shared-heap/shared_heap.cmake @@ -0,0 +1,8 @@ +# Copyright (C) 2024 Xiaomi Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (LIB_SHARED_HEAP ${CMAKE_CURRENT_LIST_DIR}) +add_definitions (-DWASM_ENABLE_SHARED_HEAP=1) +include_directories(${LIB_SHARED_HEAP_DIR}) +file (GLOB source_all ${LIB_SHARED_HEAP}/*.c) +set (LIB_SHARED_HEAP_SOURCE ${source_all}) \ No newline at end of file diff --git a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c new file mode 100644 index 0000000000..164568ac23 --- /dev/null +++ b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2024 Xiaomi Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_common.h" +#include "bh_log.h" +#include "wasm_export.h" +#include "../interpreter/wasm.h" +#include "../common/wasm_runtime_common.h" +/* clang-format off */ +#define validate_native_addr(addr, size) \ + wasm_runtime_validate_native_addr(module_inst, addr, size) + +#define module_shared_malloc(size, p_native_addr) \ + wasm_runtime_shared_heap_malloc(module_inst, size, p_native_addr) + +#define module_shared_free(offset) \ + wasm_runtime_shared_heap_free(module_inst, offset) +/* clang-format on */ + +static uint32 +shared_malloc_wrapper(wasm_exec_env_t exec_env, uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + return (uint32)module_shared_malloc((uint64)size, NULL); +} + +static void +shared_free_wrapper(wasm_exec_env_t exec_env, void *ptr) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + if (!validate_native_addr(ptr, (uint64)sizeof(uint32))) + return; + + module_shared_free(addr_native_to_app(ptr)); +} + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } +/* clang-format on */ + +static NativeSymbol native_symbols_shared_heap[] = { + REG_NATIVE_FUNC(shared_malloc, "(i)i"), + REG_NATIVE_FUNC(shared_free, "(*)"), +}; + +uint32 +get_lib_shared_heap_export_apis(NativeSymbol **p_shared_heap_apis) +{ + *p_shared_heap_apis = native_symbols_shared_heap; + return sizeof(native_symbols_shared_heap) / sizeof(NativeSymbol); +} \ No newline at end of file diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index ebb56ba7ca..27ff1441f9 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -1402,6 +1402,83 @@ wasm_cluster_spread_custom_data(WASMModuleInstanceCommon *module_inst, } } +#if WASM_ENABLE_SHARED_HEAP != 0 +static void +attach_shared_heap_visitor(void *node, void *heap) +{ + WASMExecEnv *curr_exec_env = (WASMExecEnv *)node; + WASMModuleInstanceCommon *module_inst = get_module_inst(curr_exec_env); + + wasm_runtime_attach_shared_heap_internal(module_inst, heap); +} + +static void +detach_shared_heap_visitor(void *node, void *heap) +{ + WASMExecEnv *curr_exec_env = (WASMExecEnv *)node; + WASMModuleInstanceCommon *module_inst = get_module_inst(curr_exec_env); + + (void)heap; + wasm_runtime_detach_shared_heap_internal(module_inst); +} + +bool +wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *heap) +{ + WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst); + + if (module_inst->module_type == Wasm_Module_Bytecode) { + if (((WASMModuleInstance *)module_inst)->e->shared_heap) { + LOG_WARNING("A shared heap is already attached"); + return false; + } + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (exec_env == NULL) { + /* Maybe threads have not been started yet. */ + return wasm_runtime_attach_shared_heap_internal(module_inst, heap); + } + else { + WASMCluster *cluster; + + cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + + os_mutex_lock(&cluster->lock); + traverse_list(&cluster->exec_env_list, attach_shared_heap_visitor, + heap); + os_mutex_unlock(&cluster->lock); + } + return true; +} + +void +wasm_cluster_detach_shared_heap(WASMModuleInstanceCommon *module_inst) +{ + WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst); + + if (exec_env == NULL) { + /* Maybe threads have not been started yet. */ + wasm_runtime_detach_shared_heap_internal(module_inst); + } + else { + WASMCluster *cluster; + + cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + + os_mutex_lock(&cluster->lock); + traverse_list(&cluster->exec_env_list, detach_shared_heap_visitor, + NULL); + os_mutex_unlock(&cluster->lock); + } +} +#endif + #if WASM_ENABLE_MODULE_INST_CONTEXT != 0 struct inst_set_context_data { void *key; diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.h b/core/iwasm/libraries/thread-mgr/thread_manager.h index 7ad6c772aa..90d97b0be7 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.h +++ b/core/iwasm/libraries/thread-mgr/thread_manager.h @@ -11,6 +11,9 @@ #include "wasm_export.h" #include "../interpreter/wasm.h" #include "../common/wasm_runtime_common.h" +#if WASM_ENABLE_SHARED_HEAP != 0 +#include "../common/wasm_memory.h" +#endif #ifdef __cplusplus extern "C" { @@ -167,6 +170,15 @@ wasm_cluster_set_context(WASMModuleInstanceCommon *module_inst, void *key, bool wasm_cluster_is_thread_terminated(WASMExecEnv *exec_env); +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *heap); + +void +wasm_cluster_detach_shared_heap(WASMModuleInstanceCommon *module_inst); +#endif + #if WASM_ENABLE_DEBUG_INTERP != 0 #define WAMR_SIG_TRAP (5) #define WAMR_SIG_STOP (19) From d64a3ab6ecfbf83e69a2d31cf0b3fb4f6a6e3368 Mon Sep 17 00:00:00 2001 From: Benbuck Nason Date: Fri, 13 Sep 2024 19:53:01 -0700 Subject: [PATCH 27/69] Fix aot multi export memory support (#3791) --- core/iwasm/aot/aot_runtime.c | 48 ++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index dc6cba3f67..13664ca0ef 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1413,6 +1413,36 @@ create_export_funcs(AOTModuleInstance *module_inst, AOTModule *module, return true; } +#if WASM_ENABLE_MULTI_MEMORY != 0 +static WASMExportMemInstance * +export_memories_instantiate(const AOTModule *module, + AOTModuleInstance *module_inst, + uint32 export_mem_count, char *error_buf, + uint32 error_buf_size) +{ + WASMExportMemInstance *export_memories, *export_memory; + AOTExport *export = module->exports; + uint32 i; + uint64 total_size = + sizeof(WASMExportMemInstance) * (uint64)export_mem_count; + + if (!(export_memory = export_memories = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return NULL; + } + + for (i = 0; i < module->export_count; i++, export ++) + if (export->kind == EXPORT_KIND_MEMORY) { + export_memory->name = export->name; + export_memory->memory = module_inst->memories[export->index]; + export_memory++; + } + + bh_assert((uint32)(export_memory - export_memories) == export_mem_count); + return export_memories; +} +#endif /* end of if WASM_ENABLE_MULTI_MEMORY != 0 */ + static bool create_exports(AOTModuleInstance *module_inst, AOTModule *module, char *error_buf, uint32 error_buf_size) @@ -1439,6 +1469,19 @@ create_exports(AOTModuleInstance *module_inst, AOTModule *module, } } +#if WASM_ENABLE_MULTI_MEMORY == 0 + bh_assert(module_inst->export_memory_count <= 1); +#else + if (module_inst->export_memory_count) { + module_inst->export_memories = export_memories_instantiate( + module, module_inst, module_inst->export_memory_count, error_buf, + error_buf_size); + if (!module_inst->export_memories) { + return false; + } + } +#endif + return create_export_funcs(module_inst, module, error_buf, error_buf_size); } @@ -2082,6 +2125,11 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) if (module_inst->export_functions) wasm_runtime_free(module_inst->export_functions); +#if WASM_ENABLE_MULTI_MEMORY != 0 + if (module_inst->export_memories) + wasm_runtime_free(module_inst->export_memories); +#endif + if (extra->functions) { uint32 func_idx; for (func_idx = 0; func_idx < extra->function_count; ++func_idx) { From 79e695e1a3566caa38fe25e86bd242b8c6691455 Mon Sep 17 00:00:00 2001 From: Anders Bakken Date: Tue, 17 Sep 2024 18:37:57 -0700 Subject: [PATCH 28/69] Fix a compile warning in aot_emit_function.c (#3793) This just fixes an unused variable warning when WASM_ENABLE_AOT_STACK_FRAME is != 0. --- core/iwasm/compilation/aot_emit_function.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index 11129ac9c0..85a9239aa5 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -1407,7 +1407,9 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef *param_values = NULL, value_ret = NULL, func; LLVMValueRef import_func_idx, res; LLVMValueRef ext_ret, ext_ret_ptr, ext_ret_idx; +#if WASM_ENABLE_AOT_STACK_FRAME != 0 LLVMValueRef func_idx_ref; +#endif int32 i, j = 0, param_count, result_count, ext_ret_count; uint64 total_size; uint8 wasm_ret_type; From e9cc8731da157e03a2e07fc7aa9489e226210543 Mon Sep 17 00:00:00 2001 From: Benbuck Nason Date: Tue, 17 Sep 2024 19:24:30 -0700 Subject: [PATCH 29/69] Restore cmake hidden compile symbol visibility (#3796) This was originally fixed in #3655, but regressed in #3762 which removed the `-fvisibility=hidden` flag from the CMakeLists.txt file. This also removes extraneous ending whitespace from the file. --- CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0531ec411b..40658e9ac7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,7 +124,8 @@ include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) if (NOT WIN32) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security \ -ffunction-sections -fdata-sections \ - -Wno-unused-parameter -Wno-pedantic") + -Wno-unused-parameter -Wno-pedantic \ + -fvisibility=hidden") # Remove the extra spaces for better make log string (REGEX REPLACE " *" " " CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wno-unused") @@ -168,7 +169,7 @@ if (WAMR_BUILD_STATIC) endif () if (WIN32) - target_link_libraries(iwasm_static PRIVATE ntdll) + target_link_libraries(iwasm_static PRIVATE ntdll) endif() install (TARGETS iwasm_static ARCHIVE DESTINATION lib) @@ -190,7 +191,7 @@ if (WAMR_BUILD_SHARED) endif () if (WIN32) - target_link_libraries(iwasm_shared PRIVATE ntdll) + target_link_libraries(iwasm_shared PRIVATE ntdll) endif() install (TARGETS iwasm_shared LIBRARY DESTINATION lib) From 51a71092bf97116685986ebfcdf858e59a3834d0 Mon Sep 17 00:00:00 2001 From: Liangyu Zhang Date: Wed, 18 Sep 2024 11:02:10 +0800 Subject: [PATCH 30/69] Support dynamic aot debug (#3788) Enable dynamic aot debug feature which debugs the aot file and is able to set the break point and do single step. Refer to the README for the detailed steps. Signed-off-by: zhangliangyu3 --- build-scripts/config_common.cmake | 4 + core/config.h | 4 + core/iwasm/aot/aot_runtime.c | 18 +++ product-mini/platforms/nuttx/CMakeLists.txt | 6 + product-mini/platforms/nuttx/wamr.mk | 6 + product-mini/platforms/posix/main.c | 9 ++ test-tools/dynamic-aot-debug/README.md | 139 ++++++++++++++++++ .../dynamic-aot-debug/dynamic_aot_debug.py | 104 +++++++++++++ .../dynamic_aot_debug_workflow.svg | 17 +++ 9 files changed, 307 insertions(+) create mode 100644 test-tools/dynamic-aot-debug/README.md create mode 100644 test-tools/dynamic-aot-debug/dynamic_aot_debug.py create mode 100644 test-tools/dynamic-aot-debug/dynamic_aot_debug_workflow.svg diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 12fc06bd70..3d0d6bef76 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -404,6 +404,10 @@ endif () if (WAMR_BUILD_DEBUG_AOT EQUAL 1) message (" Debug AOT enabled") endif () +if (WAMR_BUILD_DYNAMIC_AOT_DEBUG EQUAL 1) + add_definitions (-DWASM_ENABLE_DYNAMIC_AOT_DEBUG=1) + message (" Dynamic AOT debug enabled") +endif () if (WAMR_BUILD_LOAD_CUSTOM_SECTION EQUAL 1) add_definitions (-DWASM_ENABLE_LOAD_CUSTOM_SECTION=1) message (" Load custom section enabled") diff --git a/core/config.h b/core/config.h index a25eb543ea..50f7989224 100644 --- a/core/config.h +++ b/core/config.h @@ -75,6 +75,10 @@ #define WASM_ENABLE_AOT 0 #endif +#ifndef WASM_ENABLE_DYNAMIC_AOT_DEBUG +#define WASM_ENABLE_DYNAMIC_AOT_DEBUG 0 +#endif + #ifndef WASM_ENABLE_WORD_ALIGN_READ #define WASM_ENABLE_WORD_ALIGN_READ 0 #endif diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 13664ca0ef..f1e63802a1 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -5072,6 +5072,18 @@ aot_const_str_set_insert(const uint8 *str, int32 len, AOTModule *module, return c_str; } +#if WASM_ENABLE_DYNAMIC_AOT_DEBUG != 0 +AOTModule *g_dynamic_aot_module = NULL; + +void __attribute__((noinline)) __enable_dynamic_aot_debug(void) +{ + /* empty implementation. */ +} + +void (*__enable_dynamic_aot_debug_ptr)(void) + __attribute__((visibility("default"))) = __enable_dynamic_aot_debug; +#endif + bool aot_set_module_name(AOTModule *module, const char *name, char *error_buf, uint32_t error_buf_size) @@ -5085,6 +5097,12 @@ aot_set_module_name(AOTModule *module, const char *name, char *error_buf, false, #endif error_buf, error_buf_size); +#if WASM_ENABLE_DYNAMIC_AOT_DEBUG != 0 + /* export g_dynamic_aot_module for dynamic aot debug */ + g_dynamic_aot_module = module; + /* trigger breakpoint __enable_dynamic_aot_debug */ + (*__enable_dynamic_aot_debug_ptr)(); +#endif return module->name != NULL; } diff --git a/product-mini/platforms/nuttx/CMakeLists.txt b/product-mini/platforms/nuttx/CMakeLists.txt index ca07a5d551..27ddd9fa91 100644 --- a/product-mini/platforms/nuttx/CMakeLists.txt +++ b/product-mini/platforms/nuttx/CMakeLists.txt @@ -49,6 +49,12 @@ else() add_definitions(-DWASM_ENABLE_WORD_ALIGN_READ=0) endif() +if(CONFIG_INTERPRETERS_WAMR_DYNAMIC_AOT_DEBUG) + add_definitions(-DWASM_ENABLE_DYNAMIC_AOT_DEBUG=1) +else() + add_definitions(-DWASM_ENABLE_DYNAMIC_AOT_DEBUG=0) +endif() + if(CONFIG_INTERPRETERS_WAMR_STACK_GUARD_SIZE) add_definitions(-DWASM_STACK_GUARD_SIZE=0) else() diff --git a/product-mini/platforms/nuttx/wamr.mk b/product-mini/platforms/nuttx/wamr.mk index 0ee76c7ddc..44d8694e57 100644 --- a/product-mini/platforms/nuttx/wamr.mk +++ b/product-mini/platforms/nuttx/wamr.mk @@ -148,6 +148,12 @@ else CFLAGS += -DWASM_ENABLE_WORD_ALIGN_READ=0 endif +ifeq ($(CONFIG_INTERPRETERS_WAMR_DYNAMIC_AOT_DEBUG),y) +CFLAGS += -DWASM_ENABLE_DYNAMIC_AOT_DEBUG=1 +else +CFLAGS += -DWASM_ENABLE_DYNAMIC_AOT_DEBUG=0 +endif + ifeq ($(CONFIG_INTERPRETERS_WAMR_MEM_DUAL_BUS_MIRROR),y) CFLAGS += -DWASM_MEM_DUAL_BUS_MIRROR=1 else diff --git a/product-mini/platforms/posix/main.c b/product-mini/platforms/posix/main.c index 14dc01f6bf..af50223a4f 100644 --- a/product-mini/platforms/posix/main.c +++ b/product-mini/platforms/posix/main.c @@ -928,6 +928,15 @@ main(int argc, char *argv[]) goto fail2; } +#if WASM_ENABLE_DYNAMIC_AOT_DEBUG != 0 + if (!wasm_runtime_set_module_name(wasm_module, wasm_file, error_buf, + sizeof(error_buf))) { + printf("set aot module name failed in dynamic aot debug mode, %s\n", + error_buf); + goto fail3; + } +#endif + #if WASM_ENABLE_LIBC_WASI != 0 libc_wasi_init(wasm_module, argc, argv, &wasi_parse_ctx); #endif diff --git a/test-tools/dynamic-aot-debug/README.md b/test-tools/dynamic-aot-debug/README.md new file mode 100644 index 0000000000..6325a803ab --- /dev/null +++ b/test-tools/dynamic-aot-debug/README.md @@ -0,0 +1,139 @@ +# Dynamic AOT Module Debugging + +> Note: Dynamic AOT debugging is experimental and only a few debugging capabilities are supported. + +This guide explains how to debug WAMR AOT modules with dynamic AOT features. Follow these steps to set up and run your debugging environment. + +## 1. Test source code + +The following c program file is used as a debugging test file. + +```bash +#include + +int main() { + printf("hello, world!\n"); + int a = 1024; + printf("a is %d\n",a); + int b = 42; + printf("b is %d\n",b); + return 0; +} +``` + +## 2. Build iwasm with dynamic aot debugging feature + +To enable dynamic AOT debugging, ensure the following +compile options are enabled when you [build iwasm](../../product-mini/README.md): + +```bash +cmake -DWAMR_BUILD_AOT=1 -DWAMR_BUILD_DYNAMIC_AOT_DEBUG=1 -DCMAKE_BUILD_TYPE=Debug +``` + +## 3. Build wamrc + +Developer may need to build out two versions of wamrc, one is to compile the wasm binary into the AOT file, the other is to compile the wasm binary into an object file. To build out the former, just build wamrc as normal, see [wamrc-compiler/README.md](../../wamr-compiler/README.md). To build out the latter, the `WAMR_BUILD_DEBUG_AOT` flag must be added to cmake, please refer to the first two steps in [doc/source_debugging_aot.md](../../doc/source_debugging_aot.md), and if you encounter the error “‘eLanguageTypeC17’ not declared in this scope”, you can bypass it by commenting out the case judgments. This will not affect the debugging results. + +## 4. Dynamic aot debugging and verification across various platforms + +You can adjust the compiler options for different architectures and instruction sets. + +### 4.1 Linux + +#### Compile test.c to test.wasm + +```bash +/opt/wasi-sdk/bin/clang -O0 -g -gdwarf-2 -o test.wasm test.c +``` + +#### Compile test.wasm to test.aot + +```bash +./wamrc --opt-level=0 -o test.aot test.wasm +``` + +#### Compile test.wasm to test object file + +> Note: please use the version wamrc which was built with `cmake -DWAMR_BUILD_DEBUG_AOT` flag. + +```bash +./wamrc --opt-level=0 --format=object -o test.obj test.wasm +``` + +#### Launch the program using gdbserver on the remote linux host + +```bash +cd ~/aot_debug # This directory contains iwasm and test.aot +gdbserver hostip:port ./iwasm test.aot +``` + +#### Local remote debugging + +```bash +expport OBJ_PATH=~/aot_debug +cd ~/aot_debug # This directory contains iwasm, test.c, test obj file and dynamic_aot_debug.py +gdb ./iwasm +(gdb) target remote hostip:port +(gdb) source dynamic_aot_debug.py +(gdb) c +(gdb) b test.c:main +(gdb) n +``` + +### 4.2 ARMv7 + +#### Compile test.c to test.wasm + +```bash +/opt/wasi-sdk/bin/clang -O0 -nostdlib -z stack-size=8192 -Wl,--initial-memory=65536 +-g -gdwarf-2 -o test.wasm test.c -Wl,--export=main -Wl,--export=__main_argc_argv +-Wl,--export=__heap_base -Wl,--export=__data_end -Wl,--no-entry -Wl,--allow-undefined +``` + +#### Compile test.wasm to test.aot + +```bash +./wamrc --opt-level=0 --target=thumbv7 --target-abi=gnueabihf --cpu=cortex-a7 +--cpu-features=-neon -o test.aot test.wasm +``` + +#### Compile test.wasm to test object file + +> Note: please use the version wamrc which was built with `cmake -DWAMR_BUILD_DEBUG_AOT` flag. + +```bash +./wamrc --opt-level=0 --format=object --target=thumbv7 --target-abi=gnueabihf +--cpu=cortex-a7 --cpu-features=-neon -o test.obj test.wasm +``` + +#### Start Emulator + +In Terminal 1, start the emulator in debug mode and launch the GDB server: + +```bash +# start emulator on debug mode, and will start gdb server, set port as 1234 +./emulator.sh vela -qemu -S -s +ap> iwasm test.aot +``` + +#### Start NuttX Using GDB + +In Terminal 2, set the path to your object file and start NuttX with GDB: + +```bash +# You can save test.obj file in this path +export OBJ_PATH=~/work/data/aot_debug +gdb-multiarch nuttx -ex "tar remote:1234" -ex "source dynamic_aot_debug.py" +``` + +In the GDB prompt: + +```bash +(gdb) c +(gdb) b test.c:main +(gdb) n +``` + +## 5. Workflow + +Refer to the workflow diagram (wasm-micro-runtime/test-tools/dynamic-aot-debug) for an overview of the debugging process. In addition, the implementation of this dynamic aot debugging solution is not complete yet. It only supports breakpoints and single-step execution, and it is not yet known to view detailed information such as variables. diff --git a/test-tools/dynamic-aot-debug/dynamic_aot_debug.py b/test-tools/dynamic-aot-debug/dynamic_aot_debug.py new file mode 100644 index 0000000000..1548954f4b --- /dev/null +++ b/test-tools/dynamic-aot-debug/dynamic_aot_debug.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 XiaoMi Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import os +import gdb + +# Get object file path from environment variable or use default value +path_objs = os.getenv("OBJ_PATH", "~/objects/") + +# Expand user directory symbol (~) +path_objs = os.path.expanduser(path_objs) +print(f"Object files will be loaded from: {path_objs} on localhost") + + +def add_symbol_with_aot_info(aot_module_info): + """Add symbol file with AOT information to GDB and list current breakpoints.""" + try: + text_addr = aot_module_info.get("code") + file_name = aot_module_info.get("name") + + if not text_addr or not file_name: + print("Error: 'code' or 'name' missing in AOT module info.") + return + + # Extract base file name without extension + file_name_without_extension, _ = os.path.splitext(file_name) + + # Remove directory part if present + file_name = os.path.basename(file_name_without_extension) + + # Add .obj extension to the file name + file_name = file_name + ".obj" + + # Construct the path for the symbol file + path_symfile = os.path.join(path_objs, file_name) + + # Construct the command to add the symbol file + cmd = f"add-symbol-file {path_symfile} {text_addr}" + gdb.execute(cmd) + + # Print current breakpoints + breakpoints = gdb.execute("info breakpoints", to_string=True) + print("Current breakpoints:", breakpoints) + + except gdb.error as e: + print(f"GDB error: {e}") + except Exception as e: + print(f"Unexpected error: {e}") + + +class ReadGDynamicAotModule(gdb.Command): + """Command to read the g_dynamic_aot_module structure and extract information.""" + + def __init__(self): + super(self.__class__, self).__init__("read_gda", gdb.COMMAND_USER) + + def invoke(self, args, from_tty): + """Retrieve and process the g_dynamic_aot_module structure.""" + try: + aot_module = gdb.parse_and_eval("g_dynamic_aot_module") + aot_module_info = {} + + # Ensure aot_module is a pointer and dereference it + if aot_module.type.code == gdb.TYPE_CODE_PTR: + aot_module = aot_module.dereference() + + # Check if it's a structure type + if aot_module.type.strip_typedefs().code == gdb.TYPE_CODE_STRUCT: + for field in aot_module.type.fields(): + field_name = field.name + var = aot_module[field_name] + + if field_name == "name": + aot_module_info["name"] = var.string() + elif field_name == "code": + aot_module_info["code"] = str(var) + + if "name" in aot_module_info and "code" in aot_module_info: + add_symbol_with_aot_info(aot_module_info) + else: + print("Could not find 'name' or 'code' in Aot_module.") + else: + print("Aot_module is not of struct type.") + else: + print("Aot_module is not a pointer type.") + except gdb.error as e: + print(f"An error occurred: {e}") + + +def init(): + """Initialize environment and set up debugger.""" + # Register the command to gdb + ReadGDynamicAotModule() + + # Set a breakpoint at function __enable_dynamic_aot_debug + breakpoint = gdb.Breakpoint("__enable_dynamic_aot_debug") + # Attach the self-defined command to the created breakpoint, read_gda means read global dynamic aot info. + breakpoint.commands = "read_gda" + + +init() diff --git a/test-tools/dynamic-aot-debug/dynamic_aot_debug_workflow.svg b/test-tools/dynamic-aot-debug/dynamic_aot_debug_workflow.svg new file mode 100644 index 0000000000..fc81cb87a5 --- /dev/null +++ b/test-tools/dynamic-aot-debug/dynamic_aot_debug_workflow.svg @@ -0,0 +1,17 @@ + + + + + + + + load python scriptPCrun python initset breakpoint $1 on __enable_dynamic_aot_debugRegister "read_gda" command into gdbBind readl_gda to breakpoint $1trigger breakpointexcute "read_gda" commandnoyesRead and parse code_addr and module name from g_dynamic_aot_moduleGet the file name, text segment address of the dynamically aot loaded moduleExecute add-symbol-file test -s text 0x408f10f8 to load the debugging information fileiwasm test.aotset breakpoint $2, b test.aot main contiuewasm_runtime_set_module_name__enable_dynamic_aot_debug_ptrtrigger breakpoint $1gdb continue ?run nuttxGDB run nuttx andconnect to gdb servertrigger breakpoint $2debug test.aot programnowaityes12nuttxexitcall load_aot_module_destroytest/obj filetest.wasmwamrccall aot_unloadWASM_ENABLE_DYNAMIC_AOT_DEBUG=1test.cwasi-SDKWAMR_BUILD_DEBUG_AOT=1aot_set_module_name \ No newline at end of file From 5e20cf383e90cb63750fc90408791c7ffba6fffa Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 18 Sep 2024 14:53:41 +0800 Subject: [PATCH 31/69] Refactor shared heap feature for interpreter mode (#3794) To add test cases and samples. --- core/iwasm/common/wasm_memory.c | 474 +++++++++--------- core/iwasm/common/wasm_shared_memory.c | 42 +- core/iwasm/interpreter/wasm_interp_classic.c | 209 +++++--- core/iwasm/interpreter/wasm_interp_fast.c | 166 ++++-- core/iwasm/interpreter/wasm_runtime.h | 4 +- .../libraries/thread-mgr/thread_manager.c | 9 + 6 files changed, 531 insertions(+), 373 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index dc9c4aa1be..48c87e7667 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -218,7 +218,6 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, fail4: wasm_munmap_linear_memory(heap->base_addr, size, size); - fail3: wasm_runtime_free(heap->heap_handle); fail2: @@ -227,87 +226,103 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, return NULL; } -bool -wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, - WASMSharedHeap *shared_heap) -{ -#if WASM_ENABLE_THREAD_MGR != 0 - return wasm_cluster_attach_shared_heap(module_inst, shared_heap); -#else - return wasm_runtime_attach_shared_heap_internal(module_inst, shared_heap); -#endif -} - bool wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, WASMSharedHeap *shared_heap) { - uint64 linear_mem_size = 0; - WASMMemoryInstance *memory = NULL; - WASMSharedHeap *heap = (WASMSharedHeap *)shared_heap; + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module_inst); + uint64 linear_mem_size; - if (module_inst->module_type == Wasm_Module_Bytecode) { - memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } + if (!memory) + return false; - // check if linear memory and shared heap are overlapped linear_mem_size = memory->memory_data_size; - if ((memory->is_memory64 && linear_mem_size > heap->start_off_mem64) - || (!memory->is_memory64 && linear_mem_size > heap->start_off_mem32)) { + /* check if linear memory and shared heap are overlapped */ + if ((memory->is_memory64 && linear_mem_size > shared_heap->start_off_mem64) + || (!memory->is_memory64 + && linear_mem_size > shared_heap->start_off_mem32)) { LOG_WARNING("Linear memory address is overlapped with shared heap"); return false; } +#if WASM_ENABLE_INTERP != 0 if (module_inst->module_type == Wasm_Module_Bytecode) { if (((WASMModuleInstance *)module_inst)->e->shared_heap) { LOG_WARNING("A shared heap is already attached"); return false; } - ((WASMModuleInstance *)module_inst)->e->shared_heap = heap; + ((WASMModuleInstance *)module_inst)->e->shared_heap = shared_heap; } - else if (module_inst->module_type == Wasm_Module_AoT) { +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { // TODO } +#endif return true; } -void -wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst) +bool +wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap) { #if WASM_ENABLE_THREAD_MGR != 0 - wasm_cluster_detach_shared_heap(module_inst); + return wasm_cluster_attach_shared_heap(module_inst, shared_heap); #else - wasm_runtime_detach_shared_heap_internal(module_inst); + return wasm_runtime_attach_shared_heap_internal(module_inst, shared_heap); #endif } void wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) { +#if WASM_ENABLE_INTERP != 0 if (module_inst->module_type == Wasm_Module_Bytecode) { ((WASMModuleInstance *)module_inst)->e->shared_heap = NULL; } - else if (module_inst->module_type == Wasm_Module_AoT) { +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { // TODO } +#endif } -static bool -is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, - bool is_memory64, uint64 app_offset, uint32 bytes) +void +wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_detach_shared_heap(module_inst); +#else + wasm_runtime_detach_shared_heap_internal(module_inst); +#endif +} + +static WASMSharedHeap * +get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) { - WASMSharedHeap *heap = NULL; +#if WASM_ENABLE_INTERP != 0 if (module_inst_comm->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; + return ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; } - else if (module_inst_comm->module_type == Wasm_Module_AoT) { +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst_comm->module_type == Wasm_Module_AoT) { // TODO + return NULL; } +#endif + return NULL; +} + +static bool +is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, + bool is_memory64, uint64 app_offset, uint32 bytes) +{ + WASMSharedHeap *heap = get_shared_heap(module_inst); if (!heap) { return false; @@ -325,21 +340,15 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, return true; } } + return false; } static bool -is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, +is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, uint8 *addr, uint32 bytes) { - WASMSharedHeap *heap = NULL; - - if (module_inst_comm->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; - } - else if (module_inst_comm->module_type == Wasm_Module_AoT) { - // TODO - } + WASMSharedHeap *heap = get_shared_heap(module_inst); if (heap && addr >= heap->base_addr && addr + bytes <= heap->base_addr + heap->size @@ -349,156 +358,72 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, return false; } -static uint64 -shared_heap_addr_native_to_app(WASMModuleInstanceCommon *module_inst, - WASMMemoryInstance *memory, void *addr) -{ - WASMSharedHeap *heap = NULL; - - if (module_inst->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } - - if (!heap) { - LOG_WARNING("Wasm module doesn't attach to a shared heap"); - return 0; - } - if (!addr) { - LOG_WARNING("Invalid address"); - return 0; - } - - if (memory && memory->is_memory64) { - return heap->start_off_mem64 + ((uint8 *)addr - heap->base_addr); - } - else if (memory && !memory->is_memory64) { - return heap->start_off_mem32 + ((uint8 *)addr - heap->base_addr); - } - return 0; -} - -static void * -shared_heap_addr_app_to_native(WASMModuleInstanceCommon *module_inst, - WASMMemoryInstance *memory, uint64 ptr) -{ - void *addr = NULL; - WASMSharedHeap *heap = NULL; - - if (module_inst->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } - - if (!heap) { - LOG_WARNING("Wasm module doesn't attach to a shared heap"); - return NULL; - } - - if (!memory) { - LOG_WARNING("Wasm memory is not initialized"); - return NULL; - } - - if (memory->is_memory64) { - addr = heap->base_addr + (ptr - heap->start_off_mem64); - } - else { - addr = heap->base_addr + (ptr - heap->start_off_mem32); - } - - return addr; -} - -static uint64 -shared_heap_get_addr_start(WASMSharedHeap *heap, WASMMemoryInstance *memory) -{ - uint64 shared_heap_start = 0; - - if (!heap || !memory) { - LOG_ERROR("Invalid heap or memory"); - return 0; - } - - if (memory && !memory->is_memory64) { - shared_heap_start = heap->start_off_mem32; - } - else if (memory && memory->is_memory64) { - shared_heap_start = heap->start_off_mem64; - } - - return shared_heap_start; -} - uint64 wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, uint64_t size, void **p_native_addr) { - WASMSharedHeap *heap = NULL; - WASMMemoryInstance *memory = NULL; + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module_inst); + WASMSharedHeap *shared_heap = get_shared_heap(module_inst); - if (module_inst->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; - memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } + if (!memory || !shared_heap) + return 0; - if (heap) { - *p_native_addr = mem_allocator_malloc(heap->heap_handle, size); + *p_native_addr = mem_allocator_malloc(shared_heap->heap_handle, size); + if (!*p_native_addr) + return 0; - return shared_heap_addr_native_to_app(module_inst, memory, - *p_native_addr); - } - else { - LOG_WARNING("Wasm module doesn't attach to a shared heap"); - } - return 0; + if (memory->is_memory64) + return shared_heap->start_off_mem64 + + ((uint8 *)*p_native_addr - shared_heap->base_addr); + else + return shared_heap->start_off_mem32 + + ((uint8 *)*p_native_addr - shared_heap->base_addr); } void -wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr) +wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr) { - WASMSharedHeap *heap = NULL; - WASMMemoryInstance *memory = NULL; - void *addr = NULL; - - if (module_inst->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; - memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module_inst); + WASMSharedHeap *shared_heap = get_shared_heap(module_inst); + uint8 *addr = NULL; - if (!heap) { - LOG_WARNING("Wasm module doesn't attach to a shared heap"); + if (!memory || !shared_heap) { return; } - addr = shared_heap_addr_app_to_native(module_inst, memory, ptr); - - if (heap) { - mem_allocator_free(heap->base_addr, addr); + if (memory->is_memory64) { + if (ptr < shared_heap->start_off_mem64) { /* ptr can not > UINT64_MAX */ + LOG_WARNING("The address to free isn't in shared heap"); + return; + } + addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem64); + } + else { + if (ptr < shared_heap->start_off_mem32 || ptr > UINT32_MAX) { + LOG_WARNING("The address to free isn't in shared heap"); + return; + } + addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem32); } + + mem_allocator_free(shared_heap->heap_handle, addr); } -#endif +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, const MemAllocOption *alloc_option) { bool ret = false; + #if WASM_ENABLE_SHARED_HEAP != 0 if (os_mutex_init(&shared_heap_list_lock)) { return false; } #endif + if (mem_alloc_type == Alloc_With_Pool) { ret = wasm_memory_init_with_pool(alloc_option->pool.heap_buf, alloc_option->pool.heap_size); @@ -516,6 +441,10 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, memory_mode = MEMORY_MODE_SYSTEM_ALLOCATOR; ret = true; } + else { + ret = false; + } + #if WASM_ENABLE_SHARED_HEAP != 0 if (!ret) { os_mutex_destroy(&shared_heap_list_lock); @@ -527,33 +456,27 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, #if WASM_ENABLE_SHARED_HEAP != 0 static void -wasm_runtime_shared_heap_destroy() +wasm_runtime_destroy_shared_heaps() { WASMSharedHeap *heap = shared_heap_list; WASMSharedHeap *cur; - int ret = 0; while (heap) { cur = heap; heap = heap->next; - ret = ret + mem_allocator_destroy(cur->heap_handle); + mem_allocator_destroy(cur->heap_handle); wasm_runtime_free(cur->heap_handle); wasm_munmap_linear_memory(cur->base_addr, cur->size, cur->size); wasm_runtime_free(cur); } - - if (ret != 0) { - LOG_ERROR("Memory leak detected in shared heap"); - } } #endif void wasm_runtime_memory_destroy(void) { - #if WASM_ENABLE_SHARED_HEAP != 0 - wasm_runtime_shared_heap_destroy(); + wasm_runtime_destroy_shared_heaps(); #endif if (memory_mode == MEMORY_MODE_POOL) { @@ -730,6 +653,13 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, goto fail; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_offset, size)) { + return true; + } +#endif + #if WASM_ENABLE_MEMORY64 != 0 if (memory_inst->is_memory64) max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; @@ -737,7 +667,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, /* boundary overflow check */ if (size > max_linear_memory_size || app_offset > max_linear_memory_size - size) { - goto shared_heap_bound_check; + goto fail; } SHARED_MEMORY_LOCK(memory_inst); @@ -749,13 +679,6 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); -shared_heap_bound_check: -#if WASM_ENABLE_SHARED_HEAP != 0 - if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_offset, size)) { - return true; - } -#endif fail: wasm_set_exception(module_inst, "out of bounds memory access"); return false; @@ -766,6 +689,7 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, uint64 app_str_offset) { WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; char *str, *str_end; @@ -776,22 +700,42 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, return true; } - if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, NULL, - &app_end_offset)) + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { goto fail; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_str_offset, 1)) { + WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + str = (char *)shared_heap->base_addr + + (memory_inst->is_memory64 + ? (app_str_offset - shared_heap->start_off_mem64) + : (app_str_offset - shared_heap->start_off_mem32)); + str_end = (char *)shared_heap->base_addr + shared_heap->size; + } + else +#endif + { + if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, + NULL, &app_end_offset)) + goto fail; #if WASM_ENABLE_MEMORY64 != 0 - if (module_inst->memories[0]->is_memory64) - max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; + if (memory_inst->is_memory64) + max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; #endif - /* boundary overflow check, max start offset can only be size - 1, while end - * offset can be size */ - if (app_str_offset >= max_linear_memory_size - || app_end_offset > max_linear_memory_size) - goto fail; + /* boundary overflow check, max start offset can be size - 1, while end + offset can be size */ + if (app_str_offset >= max_linear_memory_size + || app_end_offset > max_linear_memory_size) + goto fail; + + str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset); + str_end = str + (app_end_offset - app_str_offset); + } - str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset); - str_end = str + (app_end_offset - app_str_offset); while (str < str_end && *str != '\0') str++; if (str == str_end) @@ -833,6 +777,12 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, goto fail; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size)) { + return true; + } +#endif + SHARED_MEMORY_LOCK(memory_inst); if (memory_inst->memory_data <= addr @@ -841,13 +791,6 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, return true; } -#if WASM_ENABLE_SHARED_HEAP != 0 - else if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, - size)) { - SHARED_MEMORY_UNLOCK(memory_inst); - return true; - } -#endif SHARED_MEMORY_UNLOCK(memory_inst); fail: @@ -874,6 +817,23 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, return NULL; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_offset, 1)) { + WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + uint64 shared_heap_start = 0; + + if (memory_inst && !memory_inst->is_memory64) { + shared_heap_start = shared_heap->start_off_mem32; + } + else if (memory_inst && memory_inst->is_memory64) { + shared_heap_start = shared_heap->start_off_mem64; + } + + return shared_heap->base_addr + app_offset - shared_heap_start; + } +#endif + SHARED_MEMORY_LOCK(memory_inst); addr = memory_inst->memory_data + (uintptr_t)app_offset; @@ -884,19 +844,6 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); return addr; } -#if WASM_ENABLE_SHARED_HEAP != 0 - else if (is_app_addr_in_shared_heap(module_inst_comm, - memory_inst->is_memory64, - app_offset, 1)) { - uint64 heap_start = shared_heap_get_addr_start( - module_inst->e->shared_heap, memory_inst); - uint64 heap_offset = (uint64)app_offset - heap_start; - - addr = module_inst->e->shared_heap->base_addr + heap_offset; - SHARED_MEMORY_UNLOCK(memory_inst); - return addr; - } -#endif SHARED_MEMORY_UNLOCK(memory_inst); return NULL; } @@ -931,6 +878,22 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, return 0; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1)) { + WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + uint64 shared_heap_start = 0; + + if (memory_inst && !memory_inst->is_memory64) { + shared_heap_start = shared_heap->start_off_mem32; + } + else if (memory_inst && memory_inst->is_memory64) { + shared_heap_start = shared_heap->start_off_mem64; + } + + return shared_heap_start + (addr - shared_heap->base_addr); + } +#endif + SHARED_MEMORY_LOCK(memory_inst); if (bounds_checks) { @@ -940,17 +903,6 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); return ret; } - else { -#if WASM_ENABLE_SHARED_HEAP != 0 - uint64 shared_heap_start = shared_heap_get_addr_start( - module_inst->e->shared_heap, memory_inst); - ret = - (uint64)(addr - (uint8 *)module_inst->e->shared_heap->base_addr) - + shared_heap_start; - SHARED_MEMORY_UNLOCK(memory_inst); - return ret; -#endif - } } /* If bounds checks is disabled, return the offset directly */ else if (addr != NULL) { @@ -1039,6 +991,10 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst); uint8 *native_addr; bool bounds_checks; +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; + bool is_in_shared_heap = false; +#endif bh_assert(app_buf_addr <= UINTPTR_MAX && app_buf_size <= UINTPTR_MAX); @@ -1047,9 +1003,25 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, return false; } - native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, + memory_inst->is_memory64, app_buf_addr, + app_buf_size)) { + shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module_inst); + native_addr = shared_heap->base_addr + + (memory_inst->is_memory64 + ? (app_buf_addr - shared_heap->start_off_mem64) + : (app_buf_addr - shared_heap->start_off_mem32)); + is_in_shared_heap = true; + } + else +#endif + { + native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; + } - bounds_checks = is_bounds_checks_enabled((wasm_module_inst_t)module_inst); + bounds_checks = + is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst); if (!bounds_checks) { if (app_buf_addr == 0) { @@ -1058,6 +1030,24 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, goto success; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_in_shared_heap) { + const char *str, *str_end; + + /* The whole string must be in the linear memory */ + str = (const char *)native_addr; + str_end = (const char *)shared_heap->base_addr + shared_heap->size; + while (str < str_end && *str != '\0') + str++; + if (str == str_end) { + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; + } + else + goto success; + } +#endif + /* No need to check the app_offset and buf_size if memory access boundary check with hardware trap is enabled */ #ifndef OS_ENABLE_HW_BOUND_CHECK @@ -1205,7 +1195,7 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, #endif #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *heap; + WASMSharedHeap *shared_heap; #endif uint8 *memory_data_old, *memory_data_new, *heap_data_old; @@ -1241,16 +1231,20 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, total_size_new = num_bytes_per_page * (uint64)total_page_count; #if WASM_ENABLE_SHARED_HEAP != 0 - heap = module->e->shared_heap; - if (memory->is_memory64 && total_size_new > heap->start_off_mem64) { - LOG_WARNING("Linear memory address is overlapped with shared heap"); - ret = false; - goto return_func; - } - else if (!memory->is_memory64 && total_size_new > heap->start_off_mem32) { - LOG_WARNING("Linear memory address is overlapped with shared heap"); - ret = false; - goto return_func; + shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module); + if (shared_heap) { + if (memory->is_memory64 + && total_size_new > shared_heap->start_off_mem64) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } + else if (!memory->is_memory64 + && total_size_new > shared_heap->start_off_mem32) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } } #endif if (inc_page_count <= 0) diff --git a/core/iwasm/common/wasm_shared_memory.c b/core/iwasm/common/wasm_shared_memory.c index 9cfdd0926c..e0d86d6f40 100644 --- a/core/iwasm/common/wasm_shared_memory.c +++ b/core/iwasm/common/wasm_shared_memory.c @@ -243,6 +243,29 @@ map_try_release_wait_info(HashMap *wait_hash_map, AtomicWaitInfo *wait_info, destroy_wait_info(wait_info); } +#if WASM_ENABLE_SHARED_HEAP != 0 +static bool +is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, + uint8 *addr, uint32 bytes) +{ + WASMSharedHeap *shared_heap = NULL; + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + shared_heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } +#endif + + return shared_heap && addr >= shared_heap->base_addr + && addr + bytes <= shared_heap->base_addr + shared_heap->size; +} +#endif + uint32 wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, uint64 expect, int64 timeout, bool wait64) @@ -271,9 +294,17 @@ wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, } shared_memory_lock(module_inst->memories[0]); - if ((uint8 *)address < module_inst->memories[0]->memory_data - || (uint8 *)address + (wait64 ? 8 : 4) - > module_inst->memories[0]->memory_data_end) { + if ( +#if WASM_ENABLE_SHARED_HEAP != 0 + /* not in shared heap */ + !is_native_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, + address, wait64 ? 8 : 4) + && +#endif + /* and not in linear memory */ + ((uint8 *)address < module_inst->memories[0]->memory_data + || (uint8 *)address + (wait64 ? 8 : 4) + > module_inst->memories[0]->memory_data_end)) { shared_memory_unlock(module_inst->memories[0]); wasm_runtime_set_exception(module, "out of bounds memory access"); return -1; @@ -397,6 +428,11 @@ wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, shared_memory_lock(module_inst->memories[0]); out_of_bounds = +#if WASM_ENABLE_SHARED_HEAP != 0 + /* not in shared heap */ + !is_native_addr_in_shared_heap(module, address, 4) && +#endif + /* and not in linear memory */ ((uint8 *)address < module_inst->memories[0]->memory_data || (uint8 *)address + 4 > module_inst->memories[0]->memory_data_end); shared_memory_unlock(module_inst->memories[0]); diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index f39ae3468b..ecbe408ff9 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -46,105 +46,89 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if WASM_ENABLE_MEMORY64 == 0 +#if WASM_ENABLE_SHARED_HEAP != 0 +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 1) -#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ - || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ - || WASM_ENABLE_BULK_MEMORY != 0 -#define GOTO_OUT_OF_BOUNDS goto out_of_bounds; -#else -#define GOTO_OUT_OF_BOUNDS -#endif +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) -#if WASM_ENABLE_SHARED_HEAP != 0 -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ - do { \ - if (offset1 + bytes >= UINT32_MAX - module->shared_heap->size \ - && offset1 + bytes <= UINT32_MAX) { \ - uint64 heap_start = UINT32_MAX - module->shared_heap->size; \ - uint64 heap_offset = (uint64)offset1 - heap_start; \ - maddr = module->shared_heap->data + heap_offset; \ - } \ - else { \ - GOTO_OUT_OF_BOUNDS; \ - } \ - } while (0) +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else #else -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) #endif +#if WASM_ENABLE_MEMORY64 == 0 + #if (!defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0) #define CHECK_MEMORY_OVERFLOW(bytes) \ do { \ uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ /* If offset1 is in valid range, maddr must also \ be in valid range, no need to check it again. */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ + goto out_of_bounds; \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ /* App heap space is not valid space for \ bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ + goto out_of_bounds; \ } while (0) + #else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ -#define CHECK_MEMORY_OVERFLOW(bytes) \ - do { \ - uint64 offset1 = (uint64)offset + (uint64)addr; \ - maddr = memory->memory_data + offset1; \ + +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ } while (0) -#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ - do { \ - maddr = memory->memory_data + (uint32)(start); \ - uint64 offset1 = start; \ +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ } while (0) + #endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ #else /* else of WASM_ENABLE_MEMORY64 == 0 */ -#if WASM_ENABLE_SHARED_HEAP != 0 -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ - do { \ - if (offset1 + bytes >= UINT64_MAX - module->shared_heap->size \ - && offset1 + bytes <= UINT64_MAX) { \ - uint64 heap_start = UINT64_MAX - module->shared_heap->size; \ - uint64 heap_offset = (uint64)offset1 - heap_start; \ - maddr = module->shared_heap->data + heap_offset; \ - } \ - else \ - goto out_of_bounds; \ - } while (0) -#else -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) goto out_of_bounds; -#endif - -#define CHECK_MEMORY_OVERFLOW(bytes) \ - do { \ - uint64 offset1 = (uint64)offset + (uint64)addr; \ - /* If memory64 is enabled, offset1, offset1 + bytes can \ - * overflow */ \ - if (disable_bounds_checks \ - || (offset1 >= offset && offset1 + bytes >= offset1 \ - && offset1 + bytes <= get_linear_mem_size())) \ - maddr = memory->memory_data + offset1; \ - else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + /* If memory64 is enabled, offset1, offset1 + bytes can overflow */ \ + if (disable_bounds_checks \ + || (offset1 >= offset && offset1 + bytes >= offset1 \ + && offset1 + bytes <= get_linear_mem_size())) \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ } while (0) + #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ uint64 offset1 = (uint64)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ /* If memory64 is enabled, offset1 + bytes can overflow */ \ if (disable_bounds_checks \ || (offset1 + bytes >= offset1 \ @@ -153,7 +137,7 @@ typedef float64 CellType_F64; bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ + goto out_of_bounds; \ } while (0) #endif /* end of WASM_ENABLE_MEMORY64 == 0 */ @@ -1648,6 +1632,22 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (memory) is_memory64 = memory->is_memory64; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap = module->e->shared_heap; + uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; +#if WASM_ENABLE_MEMORY64 != 0 + uint64 shared_heap_start_off = + shared_heap ? (is_memory64 ? shared_heap->start_off_mem64 + : shared_heap->start_off_mem32) + : 0; + uint64 shared_heap_end_off = + shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0; +#else + uint64 shared_heap_start_off = + shared_heap ? shared_heap->start_off_mem32 : 0; + uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0; +#endif +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_MULTI_MEMORY != 0 uint32 memidx = 0; uint32 memidx_cached = (uint32)-1; @@ -3498,8 +3498,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, str_obj = (WASMString)wasm_stringref_obj_get_value( stringref_obj); - memory_inst = module->memories[mem_idx]; - maddr = memory_inst->memory_data + addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } if (opcode == WASM_OP_STRING_ENCODE_WTF16) { flag = WTF16; @@ -3666,8 +3673,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, addr = POP_I32(); stringview_wtf8_obj = POP_REF(); - memory_inst = module->memories[mem_idx]; - maddr = memory_inst->memory_data + addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } bytes_written = wasm_string_encode( (WASMString)wasm_stringview_wtf8_obj_get_value( @@ -5694,9 +5708,18 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr); #else - if ((uint64)(uint32)addr + bytes > linear_mem_size) - goto out_of_bounds; - maddr = memory->memory_data + (uint32)addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)addr, + bytes)) + shared_heap_addr_app_to_native((uint64)(uint32)addr, + maddr); + else +#endif + { + if ((uint64)(uint32)addr + bytes > linear_mem_size) + goto out_of_bounds; + maddr = memory->memory_data + (uint32)addr; + } #endif if (bh_bitmap_get_bit(module->e->common.data_dropped, @@ -5746,15 +5769,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_THREAD_MGR != 0 linear_mem_size = get_linear_mem_size(); #endif + + dlen = linear_mem_size - dst; + /* dst boundary check */ #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); -#else - if ((uint64)dst + len > linear_mem_size) - goto out_of_bounds; - mdst = memory->memory_data + dst; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) + dlen = shared_heap_end_off - dst + 1; #endif - dlen = linear_mem_size - dst; +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) { + shared_heap_addr_app_to_native((uint64)dst, mdst); + dlen = shared_heap_end_off - dst + 1; + } + else +#endif + { + if ((uint64)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + dst; + } +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_MULTI_MEMORY != 0 /* src memidx */ @@ -5770,9 +5808,16 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc); #else - if ((uint64)src + len > linear_mem_size) - goto out_of_bounds; - msrc = memory->memory_data + src; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)src, len)) + shared_heap_addr_app_to_native((uint64)src, msrc); + else +#endif + { + if ((uint64)src + len > linear_mem_size) + goto out_of_bounds; + msrc = memory->memory_data + src; + } #endif #if WASM_ENABLE_MEMORY64 == 0 @@ -5809,9 +5854,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #else - if ((uint64)(uint32)dst + len > linear_mem_size) - goto out_of_bounds; - mdst = memory->memory_data + (uint32)dst; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)dst, len)) + shared_heap_addr_app_to_native((uint64)(uint32)dst, + mdst); + else +#endif + { + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; + } #endif memset(mdst, fill_val, len); diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 84170f2135..82b6ff71bd 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -37,29 +37,20 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ - || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ - || WASM_ENABLE_BULK_MEMORY != 0 -#define GOTO_OUT_OF_BOUNDS goto out_of_bounds; -#else -#define GOTO_OUT_OF_BOUNDS -#endif - #if WASM_ENABLE_SHARED_HEAP != 0 -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ - do { \ - if (offset1 + bytes >= UINT32_MAX - module->e->shared_heap->size \ - && offset1 + bytes <= UINT32_MAX) { \ - uint64 heap_start = UINT32_MAX - module->e->shared_heap->size; \ - uint64 heap_offset = (uint64)offset1 - heap_start; \ - maddr = module->e->shared_heap->base_addr + heap_offset; \ - } \ - else { \ - GOTO_OUT_OF_BOUNDS; \ - } \ - } while (0) +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 1) + +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) + +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else #else -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) #endif #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ @@ -67,35 +58,39 @@ typedef float64 CellType_F64; #define CHECK_MEMORY_OVERFLOW(bytes) \ do { \ uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ /* If offset1 is in valid range, maddr must also \ be in valid range, no need to check it again. */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ + goto out_of_bounds; \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ /* App heap space is not valid space for \ bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ + goto out_of_bounds; \ } while (0) #else -#define CHECK_MEMORY_OVERFLOW(bytes) \ - do { \ - uint64 offset1 = (uint64)offset + (uint64)addr; \ - maddr = memory->memory_data + offset1; \ +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ } while (0) -#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ - do { \ - maddr = memory->memory_data + (uint32)(start); \ - uint64 offset1 = start; \ +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ } while (0) #endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ @@ -1542,6 +1537,24 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 bool is_return_call = false; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap = module->e ? module->e->shared_heap : NULL; + uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; + /* +#if WASM_ENABLE_MEMORY64 != 0 + uint64 shared_heap_start_off = + shared_heap ? (is_memory64 ? shared_heap->start_off_mem64 + : shared_heap->start_off_mem32) + : 0; + uint64 shared_heap_end_off = + shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0; +#else + */ /* TODO: uncomment the code when memory64 is enabled for fast-interp */ + uint64 shared_heap_start_off = + shared_heap ? shared_heap->start_off_mem32 : 0; + uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0; +/* #endif */ +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_LABELS_AS_VALUES != 0 #define HANDLE_OPCODE(op) &&HANDLE_##op @@ -2857,8 +2870,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, str_obj = (WASMString)wasm_stringref_obj_get_value( stringref_obj); - memory_inst = module->memories[mem_idx]; - maddr = memory_inst->memory_data + addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } if (opcode == WASM_OP_STRING_ENCODE_WTF16) { flag = WTF16; @@ -3025,8 +3045,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, addr = POP_I32(); stringview_wtf8_obj = POP_REF(); - memory_inst = module->memories[mem_idx]; - maddr = memory_inst->memory_data + addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } bytes_written = wasm_string_encode( (WASMString)wasm_stringview_wtf8_obj_get_value( @@ -5011,9 +5038,18 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr); #else - if ((uint64)(uint32)addr + bytes > linear_mem_size) - goto out_of_bounds; - maddr = memory->memory_data + (uint32)addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)addr, + bytes)) + shared_heap_addr_app_to_native((uint64)(uint32)addr, + maddr); + else +#endif + { + if ((uint64)(uint32)addr + bytes > linear_mem_size) + goto out_of_bounds; + maddr = memory->memory_data + (uint32)addr; + } #endif if (bh_bitmap_get_bit(module->e->common.data_dropped, segment)) { @@ -5046,6 +5082,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, { uint32 dst, src, len; uint8 *mdst, *msrc; + uint64 dlen; len = POP_I32(); src = POP_I32(); @@ -5055,22 +5092,43 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, linear_mem_size = get_linear_mem_size(); #endif + dlen = linear_mem_size - dst; + #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc); CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); -#else - if ((uint64)(uint32)src + len > linear_mem_size) - goto out_of_bounds; - msrc = memory->memory_data + (uint32)src; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) + dlen = shared_heap_end_off - dst + 1; +#endif +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)src, len)) + shared_heap_addr_app_to_native((uint64)src, msrc); + else +#endif + { + if ((uint64)(uint32)src + len > linear_mem_size) + goto out_of_bounds; + msrc = memory->memory_data + (uint32)src; + } - if ((uint64)(uint32)dst + len > linear_mem_size) - goto out_of_bounds; - mdst = memory->memory_data + (uint32)dst; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) { + shared_heap_addr_app_to_native((uint64)dst, mdst); + dlen = shared_heap_end_off - dst + 1; + } + else #endif + { + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; + } +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ /* allowing the destination and source to overlap */ - bh_memmove_s(mdst, (uint32)(linear_mem_size - dst), - msrc, len); + bh_memmove_s(mdst, (uint32)dlen, msrc, len); break; } case WASM_OP_MEMORY_FILL: @@ -5089,9 +5147,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #else - if ((uint64)(uint32)dst + len > linear_mem_size) - goto out_of_bounds; - mdst = memory->memory_data + (uint32)dst; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)dst, len)) + shared_heap_addr_app_to_native((uint64)(uint32)dst, + mdst); + else +#endif + { + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; + } #endif memset(mdst, fill_val, len); diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 3fa3682c03..3c4e5faa70 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -96,8 +96,8 @@ typedef union { typedef struct WASMSharedHeap { struct WASMSharedHeap *next; void *heap_handle; - uint8_t *base_addr; - uint32_t size; + uint8 *base_addr; + uint32 size; uint64 start_off_mem64; uint64 start_off_mem32; } WASMSharedHeap; diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index 27ff1441f9..46a1bb329a 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -1449,10 +1449,19 @@ wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst, bh_assert(cluster); os_mutex_lock(&cluster->lock); + /* Try attaching shared heap to this module instance first + to ensure that we can attach it to all other instances. */ + if (!wasm_runtime_attach_shared_heap_internal(module_inst, heap)) { + os_mutex_unlock(&cluster->lock); + return false; + } + /* Detach the shared heap so it can be attached again. */ + wasm_runtime_detach_shared_heap_internal(module_inst); traverse_list(&cluster->exec_env_list, attach_shared_heap_visitor, heap); os_mutex_unlock(&cluster->lock); } + return true; } From 21330990a8f5963dd09d81e491ca4a34f7196ab1 Mon Sep 17 00:00:00 2001 From: Anders Bakken Date: Thu, 19 Sep 2024 17:54:09 -0700 Subject: [PATCH 32/69] Add no_resolve to LoadArgs and wasm_runtime_resolve_symbols (#3790) Add no_resolve to LoadArgs and wasm_runtime_resolve_symbols so one can delay resolving of symbols. This is useful for inspecting the module between loading and instantiating. --- core/iwasm/aot/aot_loader.c | 150 ++++-------------------- core/iwasm/aot/aot_runtime.c | 122 +++++++++++++++++++ core/iwasm/aot/aot_runtime.h | 12 ++ core/iwasm/common/wasm_runtime_common.c | 16 +++ core/iwasm/include/wasm_c_api.h | 4 + core/iwasm/include/wasm_export.h | 11 ++ core/iwasm/interpreter/wasm_loader.c | 138 ++++------------------ core/iwasm/interpreter/wasm_runtime.c | 118 +++++++++++++++++++ core/iwasm/interpreter/wasm_runtime.h | 7 ++ 9 files changed, 337 insertions(+), 241 deletions(-) diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 0abafd9ddb..5c81318332 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -634,73 +634,6 @@ str2uint32(const char *buf, uint32 *p_res); static bool str2uint64(const char *buf, uint64 *p_res); -#if WASM_ENABLE_MULTI_MODULE != 0 -static void * -aot_loader_resolve_function(const AOTModule *module, const char *function_name, - const AOTFuncType *expected_function_type, - char *error_buf, uint32 error_buf_size); - -static void * -aot_loader_resolve_function_ex(const char *module_name, - const char *function_name, - const AOTFuncType *expected_function_type, - char *error_buf, uint32 error_buf_size) -{ - WASMModuleCommon *module_reg; - - module_reg = wasm_runtime_find_module_registered(module_name); - if (!module_reg || module_reg->module_type != Wasm_Module_AoT) { - LOG_DEBUG("can not find a module named %s for function %s", module_name, - function_name); - set_error_buf(error_buf, error_buf_size, "unknown import"); - return NULL; - } - return aot_loader_resolve_function((AOTModule *)module_reg, function_name, - expected_function_type, error_buf, - error_buf_size); -} - -static void * -aot_loader_resolve_function(const AOTModule *module, const char *function_name, - const AOTFuncType *expected_function_type, - char *error_buf, uint32 error_buf_size) -{ - void *function = NULL; - AOTExport *export = NULL; - AOTFuncType *target_function_type = NULL; - - export = loader_find_export((WASMModuleCommon *)module, module->name, - function_name, EXPORT_KIND_FUNC, error_buf, - error_buf_size); - if (!export) { - return NULL; - } - - /* resolve function type and function */ - if (export->index < module->import_func_count) { - target_function_type = module->import_funcs[export->index].func_type; - function = module->import_funcs[export->index].func_ptr_linked; - } - else { - target_function_type = - (AOTFuncType *)module - ->types[module->func_type_indexes[export->index - - module->import_func_count]]; - function = - (module->func_ptrs[export->index - module->import_func_count]); - } - /* check function type */ - if (!wasm_type_equal((WASMType *)expected_function_type, - (WASMType *)target_function_type, module->types, - module->type_count)) { - LOG_DEBUG("%s.%s failed the type check", module->name, function_name); - set_error_buf(error_buf, error_buf_size, "incompatible import type"); - return NULL; - } - return function; -} -#endif /* end of WASM_ENABLE_MULTI_MODULE */ - static bool load_native_symbol_section(const uint8 *buf, const uint8 *buf_end, AOTModule *module, bool is_load_from_file_buf, @@ -2285,19 +2218,13 @@ destroy_import_funcs(AOTImportFunc *import_funcs) static bool load_import_funcs(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, - bool is_load_from_file_buf, char *error_buf, + bool is_load_from_file_buf, bool no_resolve, char *error_buf, uint32 error_buf_size) { - char *module_name, *field_name; const uint8 *buf = *p_buf; AOTImportFunc *import_funcs; uint64 size; uint32 i; -#if WASM_ENABLE_MULTI_MODULE != 0 - AOTModule *sub_module = NULL; - AOTFunc *linked_func = NULL; - AOTFuncType *declare_func_type = NULL; -#endif /* Allocate memory */ size = sizeof(AOTImportFunc) * (uint64)module->import_func_count; @@ -2314,53 +2241,17 @@ load_import_funcs(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, return false; } -#if WASM_ENABLE_MULTI_MODULE != 0 - declare_func_type = - (AOTFuncType *)module->types[import_funcs[i].func_type_index]; - read_string(buf, buf_end, module_name); - read_string(buf, buf_end, field_name); - - import_funcs[i].module_name = module_name; - import_funcs[i].func_name = field_name; - linked_func = wasm_native_resolve_symbol( - module_name, field_name, declare_func_type, - &import_funcs[i].signature, &import_funcs[i].attachment, - &import_funcs[i].call_conv_raw); - if (!linked_func) { - sub_module = NULL; - if (!wasm_runtime_is_built_in_module(module_name)) { - sub_module = (AOTModule *)wasm_runtime_load_depended_module( - (WASMModuleCommon *)module, module_name, error_buf, - error_buf_size); - if (!sub_module) { - LOG_ERROR("failed to load sub module: %s", error_buf); - return false; - } - } - if (!sub_module) - linked_func = aot_loader_resolve_function_ex( - module_name, field_name, declare_func_type, error_buf, - error_buf_size); - else - linked_func = aot_loader_resolve_function( - sub_module, field_name, declare_func_type, error_buf, - error_buf_size); - } - import_funcs[i].func_ptr_linked = linked_func; - import_funcs[i].func_type = declare_func_type; - -#else import_funcs[i].func_type = (AOTFuncType *)module->types[import_funcs[i].func_type_index]; read_string(buf, buf_end, import_funcs[i].module_name); read_string(buf, buf_end, import_funcs[i].func_name); - module_name = import_funcs[i].module_name; - field_name = import_funcs[i].func_name; - import_funcs[i].func_ptr_linked = wasm_native_resolve_symbol( - module_name, field_name, import_funcs[i].func_type, - &import_funcs[i].signature, &import_funcs[i].attachment, - &import_funcs[i].call_conv_raw); -#endif + import_funcs[i].attachment = NULL; + import_funcs[i].signature = NULL; + import_funcs[i].call_conv_raw = false; + + if (!no_resolve) { + aot_resolve_import_func(module, &import_funcs[i]); + } #if WASM_ENABLE_LIBC_WASI != 0 if (!strcmp(import_funcs[i].module_name, "wasi_unstable") @@ -2378,7 +2269,7 @@ load_import_funcs(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, static bool load_import_func_info(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, bool is_load_from_file_buf, - char *error_buf, uint32 error_buf_size) + bool no_resolve, char *error_buf, uint32 error_buf_size) { const uint8 *buf = *p_buf; @@ -2387,7 +2278,7 @@ load_import_func_info(const uint8 **p_buf, const uint8 *buf_end, /* load import funcs */ if (module->import_func_count > 0 && !load_import_funcs(&buf, buf_end, module, is_load_from_file_buf, - error_buf, error_buf_size)) + no_resolve, error_buf, error_buf_size)) return false; *p_buf = buf; @@ -2514,7 +2405,7 @@ load_object_data_sections_info(const uint8 **p_buf, const uint8 *buf_end, static bool load_init_data_section(const uint8 *buf, const uint8 *buf_end, AOTModule *module, bool is_load_from_file_buf, - char *error_buf, uint32 error_buf_size) + bool no_resolve, char *error_buf, uint32 error_buf_size) { const uint8 *p = buf, *p_end = buf_end; @@ -2525,7 +2416,7 @@ load_init_data_section(const uint8 *buf, const uint8 *buf_end, error_buf, error_buf_size) || !load_global_info(&p, p_end, module, error_buf, error_buf_size) || !load_import_func_info(&p, p_end, module, is_load_from_file_buf, - error_buf, error_buf_size)) + no_resolve, error_buf, error_buf_size)) return false; /* load function count and start function index */ @@ -3819,7 +3710,7 @@ has_module_memory64(AOTModule *module) static bool load_from_sections(AOTModule *module, AOTSection *sections, - bool is_load_from_file_buf, char *error_buf, + bool is_load_from_file_buf, bool no_resolve, char *error_buf, uint32 error_buf_size) { AOTSection *section = sections; @@ -3852,8 +3743,8 @@ load_from_sections(AOTModule *module, AOTSection *sections, break; case AOT_SECTION_TYPE_INIT_DATA: if (!load_init_data_section(buf, buf_end, module, - is_load_from_file_buf, error_buf, - error_buf_size)) + is_load_from_file_buf, no_resolve, + error_buf, error_buf_size)) return false; break; case AOT_SECTION_TYPE_TEXT: @@ -4076,7 +3967,7 @@ aot_load_from_sections(AOTSection *section_list, char *error_buf, if (!module) return NULL; - if (!load_from_sections(module, section_list, false, error_buf, + if (!load_from_sections(module, section_list, false, false, error_buf, error_buf_size)) { aot_unload(module); return NULL; @@ -4246,7 +4137,8 @@ create_sections(AOTModule *module, const uint8 *buf, uint32 size, static bool load(const uint8 *buf, uint32 size, AOTModule *module, - bool wasm_binary_freeable, char *error_buf, uint32 error_buf_size) + bool wasm_binary_freeable, bool no_resolve, char *error_buf, + uint32 error_buf_size) { const uint8 *buf_end = buf + size; const uint8 *p = buf, *p_end = buf_end; @@ -4273,7 +4165,7 @@ load(const uint8 *buf, uint32 size, AOTModule *module, return false; ret = load_from_sections(module, section_list, !wasm_binary_freeable, - error_buf, error_buf_size); + no_resolve, error_buf, error_buf_size); if (!ret) { /* If load_from_sections() fails, then aot text is destroyed in destroy_sections() */ @@ -4321,8 +4213,8 @@ aot_load_from_aot_file(const uint8 *buf, uint32 size, const LoadArgs *args, return NULL; os_thread_jit_write_protect_np(false); /* Make memory writable */ - if (!load(buf, size, module, args->wasm_binary_freeable, error_buf, - error_buf_size)) { + if (!load(buf, size, module, args->wasm_binary_freeable, args->no_resolve, + error_buf, error_buf_size)) { aot_unload(module); return NULL; } diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index f1e63802a1..63a3c83c90 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -5111,3 +5111,125 @@ aot_get_module_name(AOTModule *module) { return module->name; } + +bool +aot_resolve_symbols(AOTModule *module) +{ + bool ret = true; + uint32 idx; + for (idx = 0; idx < module->import_func_count; ++idx) { + AOTImportFunc *aot_import_func = &module->import_funcs[idx]; + if (!aot_import_func->func_ptr_linked) { + if (!aot_resolve_import_func(module, aot_import_func)) { + LOG_WARNING("Failed to link function (%s, %s)", + aot_import_func->module_name, + aot_import_func->func_name); + ret = false; + } + } + } + return ret; +} + +#if WASM_ENABLE_MULTI_MODULE != 0 +static void * +aot_resolve_function(const AOTModule *module, const char *function_name, + const AOTFuncType *expected_function_type, char *error_buf, + uint32 error_buf_size); + +static void * +aot_resolve_function_ex(const char *module_name, const char *function_name, + const AOTFuncType *expected_function_type, + char *error_buf, uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_AoT) { + LOG_DEBUG("can not find a module named %s for function %s", module_name, + function_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + return aot_resolve_function((AOTModule *)module_reg, function_name, + expected_function_type, error_buf, + error_buf_size); +} + +static void * +aot_resolve_function(const AOTModule *module, const char *function_name, + const AOTFuncType *expected_function_type, char *error_buf, + uint32 error_buf_size) +{ + void *function = NULL; + AOTExport *export = NULL; + AOTFuncType *target_function_type = NULL; + + export = loader_find_export((WASMModuleCommon *)module, module->name, + function_name, EXPORT_KIND_FUNC, error_buf, + error_buf_size); + if (!export) { + return NULL; + } + + /* resolve function type and function */ + if (export->index < module->import_func_count) { + target_function_type = module->import_funcs[export->index].func_type; + function = module->import_funcs[export->index].func_ptr_linked; + } + else { + target_function_type = + (AOTFuncType *)module + ->types[module->func_type_indexes[export->index + - module->import_func_count]]; + function = + (module->func_ptrs[export->index - module->import_func_count]); + } + /* check function type */ + if (!wasm_type_equal((WASMType *)expected_function_type, + (WASMType *)target_function_type, module->types, + module->type_count)) { + LOG_DEBUG("%s.%s failed the type check", module->name, function_name); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + return function; +} +#endif /* end of WASM_ENABLE_MULTI_MODULE */ + +bool +aot_resolve_import_func(AOTModule *module, AOTImportFunc *import_func) +{ +#if WASM_ENABLE_MULTI_MODULE != 0 + char error_buf[128]; + AOTModule *sub_module = NULL; +#endif + import_func->func_ptr_linked = wasm_native_resolve_symbol( + import_func->module_name, import_func->func_name, + import_func->func_type, &import_func->signature, + &import_func->attachment, &import_func->call_conv_raw); +#if WASM_ENABLE_MULTI_MODULE != 0 + if (!import_func->func_ptr_linked) { + if (!wasm_runtime_is_built_in_module(import_func->module_name)) { + sub_module = (AOTModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)module, import_func->module_name, error_buf, + sizeof(error_buf)); + if (!sub_module) { + LOG_WARNING("Failed to load sub module: %s", error_buf); + } + if (!sub_module) + import_func->func_ptr_linked = aot_resolve_function_ex( + import_func->module_name, import_func->func_name, + import_func->func_type, error_buf, sizeof(error_buf)); + else + import_func->func_ptr_linked = aot_resolve_function( + sub_module, import_func->func_name, import_func->func_type, + error_buf, sizeof(error_buf)); + if (!import_func->func_ptr_linked) { + LOG_WARNING("Failed to link function: %s", error_buf); + } + } + } +#endif + return import_func->func_ptr_linked != NULL; +} diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index f13d7eefc0..9c73dd4017 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -492,6 +492,18 @@ aot_load_from_sections(AOTSection *section_list, char *error_buf, void aot_unload(AOTModule *module); +/** + * Resolve symbols for an AOT module + */ +bool +aot_resolve_symbols(AOTModule *module); + +/** + * Helper function to resolve a single function + */ +bool +aot_resolve_import_func(AOTModule *module, AOTImportFunc *import_func); + /** * Instantiate a AOT module. * diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 314dc7ddb1..667cbba03f 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -1484,6 +1484,22 @@ wasm_runtime_load_ex(uint8 *buf, uint32 size, const LoadArgs *args, error_buf_size); } +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_resolve_symbols(WASMModuleCommon *module) +{ +#if WASM_ENABLE_INTERP != 0 + if (module->module_type == Wasm_Module_Bytecode) { + return wasm_resolve_symbols((WASMModule *)module); + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module->module_type == Wasm_Module_AoT) { + return aot_resolve_symbols((AOTModule *)module); + } +#endif + return false; +} + WASMModuleCommon * wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf, uint32 error_buf_size) diff --git a/core/iwasm/include/wasm_c_api.h b/core/iwasm/include/wasm_c_api.h index 4994454bd8..9fc4601486 100644 --- a/core/iwasm/include/wasm_c_api.h +++ b/core/iwasm/include/wasm_c_api.h @@ -534,6 +534,10 @@ typedef struct LoadArgs { bool clone_wasm_binary; /* This option is only used by the AOT/wasm loader (see wasm_export.h) */ bool wasm_binary_freeable; + /* false by default, if true, don't resolve the symbols yet. The + wasm_runtime_load_ex has to be followed by a wasm_runtime_resolve_symbols + call */ + bool no_resolve; /* TODO: more fields? */ } LoadArgs; #endif /* LOAD_ARGS_OPTION_DEFINED */ diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 1a03f7280d..c9037b3cc1 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -252,6 +252,11 @@ typedef struct LoadArgs { const strings), making it possible to free the wasm binary buffer after loading. */ bool wasm_binary_freeable; + + /* false by default, if true, don't resolve the symbols yet. The + wasm_runtime_load_ex has to be followed by a wasm_runtime_resolve_symbols + call */ + bool no_resolve; /* TODO: more fields? */ } LoadArgs; #endif /* LOAD_ARGS_OPTION_DEFINED */ @@ -569,6 +574,12 @@ WASM_RUNTIME_API_EXTERN wasm_module_t wasm_runtime_load_ex(uint8_t *buf, uint32_t size, const LoadArgs *args, char *error_buf, uint32_t error_buf_size); +/** + * Resolve symbols for a previously loaded WASM module. Only useful when the + * module was loaded with LoadArgs::no_resolve set to true + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_resolve_symbols(wasm_module_t module); /** * Load a WASM module from a specified WASM or AOT section list. * diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 3a21b1fc6b..ff3501e3d0 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -2246,60 +2246,6 @@ wasm_loader_find_export(const WASMModule *module, const char *module_name, #endif #if WASM_ENABLE_MULTI_MODULE != 0 -static WASMFunction * -wasm_loader_resolve_function(const char *module_name, const char *function_name, - const WASMFuncType *expected_function_type, - char *error_buf, uint32 error_buf_size) -{ - WASMModuleCommon *module_reg; - WASMFunction *function = NULL; - WASMExport *export = NULL; - WASMModule *module = NULL; - WASMFuncType *target_function_type = NULL; - - module_reg = wasm_runtime_find_module_registered(module_name); - if (!module_reg || module_reg->module_type != Wasm_Module_Bytecode) { - LOG_DEBUG("can not find a module named %s for function %s", module_name, - function_name); - set_error_buf(error_buf, error_buf_size, "unknown import"); - return NULL; - } - - module = (WASMModule *)module_reg; - export = - wasm_loader_find_export(module, module_name, function_name, - EXPORT_KIND_FUNC, error_buf, error_buf_size); - if (!export) { - return NULL; - } - - /* resolve function type and function */ - if (export->index < module->import_function_count) { - target_function_type = - module->import_functions[export->index].u.function.func_type; - function = module->import_functions[export->index] - .u.function.import_func_linked; - } - else { - target_function_type = - module->functions[export->index - module->import_function_count] - ->func_type; - function = - module->functions[export->index - module->import_function_count]; - } - - /* check function type */ - if (!wasm_type_equal((WASMType *)expected_function_type, - (WASMType *)target_function_type, module->types, - module->type_count)) { - LOG_DEBUG("%s.%s failed the type check", module_name, function_name); - set_error_buf(error_buf, error_buf_size, "incompatible import type"); - return NULL; - } - - return function; -} - static WASMTable * wasm_loader_resolve_table(const char *module_name, const char *table_name, uint32 init_size, uint32 max_size, char *error_buf, @@ -2494,21 +2440,11 @@ static bool load_function_import(const uint8 **p_buf, const uint8 *buf_end, const WASMModule *parent_module, const char *sub_module_name, const char *function_name, - WASMFunctionImport *function, char *error_buf, - uint32 error_buf_size) + WASMFunctionImport *function, bool no_resolve, + char *error_buf, uint32 error_buf_size) { const uint8 *p = *p_buf, *p_end = buf_end; uint32 declare_type_index = 0; - WASMFuncType *declare_func_type = NULL; - WASMFunction *linked_func = NULL; -#if WASM_ENABLE_MULTI_MODULE != 0 - WASMModule *sub_module = NULL; - bool is_built_in_module = false; -#endif - const char *linked_signature = NULL; - void *linked_attachment = NULL; - bool linked_call_conv_raw = false; - bool is_native_symbol = false; read_leb_uint32(p, p_end, declare_type_index); *p_buf = p; @@ -2527,43 +2463,19 @@ load_function_import(const uint8 **p_buf, const uint8 *buf_end, parent_module->types, parent_module->type_count, declare_type_index); #endif - declare_func_type = + function->func_type = (WASMFuncType *)parent_module->types[declare_type_index]; - /* lookup registered native symbols first */ - linked_func = wasm_native_resolve_symbol( - sub_module_name, function_name, declare_func_type, &linked_signature, - &linked_attachment, &linked_call_conv_raw); - if (linked_func) { - is_native_symbol = true; - } -#if WASM_ENABLE_MULTI_MODULE != 0 - else { - if (!(is_built_in_module = - wasm_runtime_is_built_in_module(sub_module_name))) { - sub_module = (WASMModule *)wasm_runtime_load_depended_module( - (WASMModuleCommon *)parent_module, sub_module_name, error_buf, - error_buf_size); - } - if (is_built_in_module || sub_module) - linked_func = wasm_loader_resolve_function( - sub_module_name, function_name, declare_func_type, error_buf, - error_buf_size); - } -#endif - function->module_name = (char *)sub_module_name; function->field_name = (char *)function_name; - function->func_type = declare_func_type; - /* func_ptr_linked is for native registered symbol */ - function->func_ptr_linked = is_native_symbol ? linked_func : NULL; - function->signature = linked_signature; - function->attachment = linked_attachment; - function->call_conv_raw = linked_call_conv_raw; -#if WASM_ENABLE_MULTI_MODULE != 0 - function->import_module = is_native_symbol ? NULL : sub_module; - function->import_func_linked = is_native_symbol ? NULL : linked_func; -#endif + function->attachment = NULL; + function->signature = NULL; + function->call_conv_raw = false; + + /* lookup registered native symbols first */ + if (!no_resolve) { + wasm_resolve_import_func(parent_module, function); + } return true; fail: return false; @@ -3258,8 +3170,8 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, static bool load_import_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, - bool is_load_from_file_buf, char *error_buf, - uint32 error_buf_size) + bool is_load_from_file_buf, bool no_resolve, + char *error_buf, uint32 error_buf_size) { const uint8 *p = buf, *p_end = buf_end, *p_old; uint32 import_count, name_len, type_index, i, u32, flags; @@ -3442,9 +3354,10 @@ load_import_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, case IMPORT_KIND_FUNC: /* import function */ bh_assert(import_functions); import = import_functions++; - if (!load_function_import( - &p, p_end, module, sub_module_name, field_name, - &import->u.function, error_buf, error_buf_size)) { + if (!load_function_import(&p, p_end, module, + sub_module_name, field_name, + &import->u.function, no_resolve, + error_buf, error_buf_size)) { return false; } break; @@ -5760,7 +5673,7 @@ static void **handle_table; static bool load_from_sections(WASMModule *module, WASMSection *sections, bool is_load_from_file_buf, bool wasm_binary_freeable, - char *error_buf, uint32 error_buf_size) + bool no_resolve, char *error_buf, uint32 error_buf_size) { WASMExport *export; WASMSection *section = sections; @@ -5817,8 +5730,8 @@ load_from_sections(WASMModule *module, WASMSection *sections, break; case SECTION_TYPE_IMPORT: if (!load_import_section(buf, buf_end, module, - reuse_const_strings, error_buf, - error_buf_size)) + reuse_const_strings, no_resolve, + error_buf, error_buf_size)) return false; break; case SECTION_TYPE_FUNC: @@ -6343,7 +6256,7 @@ wasm_loader_load_from_sections(WASMSection *section_list, char *error_buf, if (!module) return NULL; - if (!load_from_sections(module, section_list, false, true, error_buf, + if (!load_from_sections(module, section_list, false, true, false, error_buf, error_buf_size)) { wasm_loader_unload(module); return NULL; @@ -6488,7 +6401,8 @@ static union { static bool load(const uint8 *buf, uint32 size, WASMModule *module, - bool wasm_binary_freeable, char *error_buf, uint32 error_buf_size) + bool wasm_binary_freeable, bool no_resolve, char *error_buf, + uint32 error_buf_size) { const uint8 *buf_end = buf + size; const uint8 *p = buf, *p_end = buf_end; @@ -6519,7 +6433,7 @@ load(const uint8 *buf, uint32 size, WASMModule *module, if (!create_sections(buf, size, §ion_list, error_buf, error_buf_size) || !load_from_sections(module, section_list, true, wasm_binary_freeable, - error_buf, error_buf_size)) { + no_resolve, error_buf, error_buf_size)) { destroy_sections(section_list); return false; } @@ -6695,8 +6609,8 @@ wasm_loader_load(uint8 *buf, uint32 size, module->load_size = size; #endif - if (!load(buf, size, module, args->wasm_binary_freeable, error_buf, - error_buf_size)) { + if (!load(buf, size, module, args->wasm_binary_freeable, args->no_resolve, + error_buf, error_buf_size)) { goto fail; } diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index e8f4c749e0..e4142ab88c 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -83,6 +83,124 @@ wasm_unload(WASMModule *module) wasm_loader_unload(module); } +bool +wasm_resolve_symbols(WASMModule *module) +{ + bool ret = true; + uint32 idx; + for (idx = 0; idx < module->import_function_count; ++idx) { + WASMFunctionImport *import = &module->import_functions[idx].u.function; + bool linked = import->func_ptr_linked; +#if WASM_ENABLE_MULTI_MODULE != 0 + if (import->import_func_linked) { + linked = true; + } +#endif + if (!linked && !wasm_resolve_import_func(module, import)) { + ret = false; + } + } + return ret; +} + +#if WASM_ENABLE_MULTI_MODULE != 0 +static WASMFunction * +wasm_resolve_function(const char *module_name, const char *function_name, + const WASMFuncType *expected_function_type, + char *error_buf, uint32 error_buf_size) +{ + WASMModuleCommon *module_reg; + WASMFunction *function = NULL; + WASMExport *export = NULL; + WASMModule *module = NULL; + WASMFuncType *target_function_type = NULL; + + module_reg = wasm_runtime_find_module_registered(module_name); + if (!module_reg || module_reg->module_type != Wasm_Module_Bytecode) { + LOG_DEBUG("can not find a module named %s for function %s", module_name, + function_name); + set_error_buf(error_buf, error_buf_size, "unknown import"); + return NULL; + } + + module = (WASMModule *)module_reg; + export = loader_find_export((WASMModuleCommon *)module, module_name, + function_name, EXPORT_KIND_FUNC, error_buf, + error_buf_size); + if (!export) { + return NULL; + } + + /* resolve function type and function */ + if (export->index < module->import_function_count) { + target_function_type = + module->import_functions[export->index].u.function.func_type; + function = module->import_functions[export->index] + .u.function.import_func_linked; + } + else { + target_function_type = + module->functions[export->index - module->import_function_count] + ->func_type; + function = + module->functions[export->index - module->import_function_count]; + } + + /* check function type */ + if (!wasm_type_equal((WASMType *)expected_function_type, + (WASMType *)target_function_type, module->types, + module->type_count)) { + LOG_DEBUG("%s.%s failed the type check", module_name, function_name); + set_error_buf(error_buf, error_buf_size, "incompatible import type"); + return NULL; + } + + return function; +} +#endif + +bool +wasm_resolve_import_func(const WASMModule *module, WASMFunctionImport *function) +{ +#if WASM_ENABLE_MULTI_MODULE != 0 + char error_buf[128]; + WASMModule *sub_module = NULL; +#endif + function->func_ptr_linked = wasm_native_resolve_symbol( + function->module_name, function->field_name, function->func_type, + &function->signature, &function->attachment, &function->call_conv_raw); + + if (function->func_ptr_linked) { + return true; + } + +#if WASM_ENABLE_MULTI_MODULE != 0 + if (!wasm_runtime_is_built_in_module(function->module_name)) { + sub_module = (WASMModule *)wasm_runtime_load_depended_module( + (WASMModuleCommon *)module, function->module_name, error_buf, + sizeof(error_buf)); + if (!sub_module) { + LOG_WARNING("failed to load sub module: %s", error_buf); + return false; + } + } + function->import_func_linked = wasm_resolve_function( + function->module_name, function->field_name, function->func_type, + error_buf, sizeof(error_buf)); + + if (function->import_func_linked) { + function->import_module = sub_module; + return true; + } + else { + LOG_WARNING("failed to link function (%s, %s): %s", + function->module_name, function->field_name, error_buf); + } +#endif + + return false; +} + static void * runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size) { diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index c430186952..e46b63cda1 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -513,6 +513,13 @@ wasm_load_from_sections(WASMSection *section_list, char *error_buf, void wasm_unload(WASMModule *module); +bool +wasm_resolve_symbols(WASMModule *module); + +bool +wasm_resolve_import_func(const WASMModule *module, + WASMFunctionImport *function); + WASMModuleInstance * wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, WASMExecEnv *exec_env_main, uint32 stack_size, From 4dacef2d600d3476cf8ec8dbe962cc540fbb258d Mon Sep 17 00:00:00 2001 From: WenLY1 <130950131+WenLY1@users.noreply.github.com> Date: Fri, 20 Sep 2024 14:24:38 +0800 Subject: [PATCH 33/69] shared heap: Fix some issues and add basic unit test case (#3801) Fix some issues and add basic unit test case for shared heap feature. Signed-off-by: wenlingyun1 --- core/iwasm/common/wasm_memory.c | 47 +++--- core/iwasm/common/wasm_memory.h | 3 +- core/iwasm/common/wasm_runtime_common.c | 39 ++--- core/iwasm/include/wasm_export.h | 30 ++-- core/iwasm/interpreter/wasm_runtime.h | 2 +- .../shared-heap/shared_heap_wrapper.c | 4 +- tests/unit/CMakeLists.txt | 1 + tests/unit/shared-heap/CMakeLists.txt | 59 ++++++++ tests/unit/shared-heap/shared_heap_test.cc | 142 ++++++++++++++++++ .../unit/shared-heap/wasm-apps/CMakeLists.txt | 39 +++++ tests/unit/shared-heap/wasm-apps/test.c | 22 +++ 11 files changed, 332 insertions(+), 56 deletions(-) create mode 100644 tests/unit/shared-heap/CMakeLists.txt create mode 100644 tests/unit/shared-heap/shared_heap_test.cc create mode 100644 tests/unit/shared-heap/wasm-apps/CMakeLists.txt create mode 100644 tests/unit/shared-heap/wasm-apps/test.c diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 48c87e7667..fe6b7a074e 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -148,22 +148,13 @@ static void wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size); -static void -set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) -{ - if (error_buf != NULL) { - snprintf(error_buf, error_buf_size, - "Operation of shared heap failed: %s", string); - } -} - static void * -runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size) +runtime_malloc(uint64 size) { void *mem; if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { - set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + LOG_WARNING("Allocate memory failed"); return NULL; } @@ -172,27 +163,32 @@ runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size) } WASMSharedHeap * -wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, - uint32 error_buf_size) +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) { uint64 heap_struct_size = sizeof(WASMSharedHeap); uint32 size = init_args->size; WASMSharedHeap *heap; - if (!(heap = runtime_malloc(heap_struct_size, error_buf, error_buf_size))) { + if (size == 0) { goto fail1; } + + if (!(heap = runtime_malloc(heap_struct_size))) { + goto fail1; + } + if (!(heap->heap_handle = - runtime_malloc(mem_allocator_get_heap_struct_size(), error_buf, - error_buf_size))) { + runtime_malloc(mem_allocator_get_heap_struct_size()))) { goto fail2; } + + size = align_uint(size, os_getpagesize()); + heap->size = size; heap->start_off_mem64 = UINT64_MAX - heap->size + 1; heap->start_off_mem32 = UINT32_MAX - heap->size + 1; - size = align_uint(size, os_getpagesize()); if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { - set_error_buf(error_buf, error_buf_size, "invalid size of shared heap"); + LOG_WARNING("Invalid size of shared heap"); goto fail3; } @@ -201,7 +197,7 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, } if (!mem_allocator_create_with_struct_and_pool( heap->heap_handle, heap_struct_size, heap->base_addr, size)) { - set_error_buf(error_buf, error_buf_size, "init share heap failed"); + LOG_WARNING("init share heap failed"); goto fail4; } @@ -365,20 +361,25 @@ wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, WASMMemoryInstance *memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); WASMSharedHeap *shared_heap = get_shared_heap(module_inst); + void *native_addr = NULL; if (!memory || !shared_heap) return 0; - *p_native_addr = mem_allocator_malloc(shared_heap->heap_handle, size); - if (!*p_native_addr) + native_addr = mem_allocator_malloc(shared_heap->heap_handle, size); + if (!native_addr) return 0; + if (p_native_addr) { + *p_native_addr = native_addr; + } + if (memory->is_memory64) return shared_heap->start_off_mem64 - + ((uint8 *)*p_native_addr - shared_heap->base_addr); + + ((uint8 *)native_addr - shared_heap->base_addr); else return shared_heap->start_off_mem32 - + ((uint8 *)*p_native_addr - shared_heap->base_addr); + + ((uint8 *)native_addr - shared_heap->base_addr); } void diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index be89772e37..cc6418e841 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -43,8 +43,7 @@ SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size) #if WASM_ENABLE_SHARED_HEAP != 0 WASMSharedHeap * -wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, - uint32 error_buf_size); +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); bool wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 314dc7ddb1..c827b28fe3 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -4498,6 +4498,11 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, uint32 *argv, uint32 argc, uint32 *argv_ret) { WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); +#if WASM_ENABLE_MEMORY64 != 0 + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module); + bool is_memory64 = memory ? memory->is_memory64 : false; +#endif typedef void (*NativeRawFuncPtr)(WASMExecEnv *, uint64 *); NativeRawFuncPtr invoke_native_raw = (NativeRawFuncPtr)func_ptr; uint64 argv_buf[16] = { 0 }, *argv1 = argv_buf, *argv_dst, size, arg_i64; @@ -4525,11 +4530,11 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, #endif { *(uint32 *)argv_dst = arg_i32 = *argv_src++; - /* TODO: memory64 if future there is a way for supporting - * wasm64 and wasm32 in libc at the same time, remove the - * macro control */ -#if WASM_ENABLE_MEMORY64 == 0 - if (signature) { + if (signature +#if WASM_ENABLE_MEMORY64 != 0 + && !is_memory64 +#endif + ) { if (signature[i + 1] == '*') { /* param is a pointer */ if (signature[i + 2] == '~') @@ -4558,7 +4563,6 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, module, (uint64)arg_i32); } } -#endif break; } case VALUE_TYPE_I64: @@ -4568,7 +4572,7 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, GET_I64_FROM_ADDR(argv_src)); argv_src += 2; arg_i64 = *argv_dst; - if (signature) { + if (signature && is_memory64) { /* TODO: memory64 pointer with length need a new symbol * to represent type i64, with '~' still represent i32 * length */ @@ -4729,9 +4733,6 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, fail: if (argv1 != argv_buf) wasm_runtime_free(argv1); -#if WASM_ENABLE_MEMORY64 == 0 - (void)arg_i64; -#endif return ret; } @@ -5655,6 +5656,11 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, uint32 *argv_ret) { WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); +#if WASM_ENABLE_MEMORY64 != 0 + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module); + bool is_memory64 = memory ? memory->is_memory64 : false; +#endif uint64 argv_buf[32] = { 0 }, *argv1 = argv_buf, *ints, *stacks, size, arg_i64; uint32 *argv_src = argv, i, argc1, n_ints = 0, n_stacks = 0; @@ -5720,11 +5726,11 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, { arg_i32 = *argv_src++; arg_i64 = arg_i32; - /* TODO: memory64 if future there is a way for supporting - * wasm64 and wasm32 in libc at the same time, remove the - * macro control */ -#if WASM_ENABLE_MEMORY64 == 0 - if (signature) { + if (signature +#if WASM_ENABLE_MEMORY64 != 0 + && !is_memory64 +#endif + ) { if (signature[i + 1] == '*') { /* param is a pointer */ if (signature[i + 2] == '~') @@ -5751,7 +5757,6 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, module, (uint64)arg_i32); } } -#endif if (n_ints < MAX_REG_INTS) ints[n_ints++] = arg_i64; else @@ -5763,7 +5768,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, { arg_i64 = GET_I64_FROM_ADDR(argv_src); argv_src += 2; - if (signature) { + if (signature && is_memory64) { /* TODO: memory64 pointer with length need a new symbol * to represent type i64, with '~' still represent i32 * length */ diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 5ce617ef16..ffdafcef0a 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -323,11 +323,9 @@ typedef enum { WASM_LOG_LEVEL_VERBOSE = 4 } log_level_t; -#if WASM_ENABLE_SHARED_HEAP != 0 typedef struct SharedHeapInitArgs { - uint32 size; + uint32_t size; } SharedHeapInitArgs; -#endif /** * Initialize the WASM runtime environment, and also initialize @@ -2119,21 +2117,21 @@ wasm_runtime_detect_native_stack_overflow_size(wasm_exec_env_t exec_env, WASM_RUNTIME_API_EXTERN bool wasm_runtime_is_underlying_binary_freeable(const wasm_module_t module); -#if WASM_ENABLE_SHARED_HEAP != 0 /** * Create a shared heap + * * @param init_args the initialization arguments - * @param error_buf buffer to output the error info if failed - * @param error_buf_size the size of the error buffer + * @return the shared heap created */ WASM_RUNTIME_API_EXTERN wasm_shared_heap_t -wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, - uint32 error_buf_size); +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); /** * Attach a shared heap to a module instance + * * @param module_inst the module instance * @param shared_heap the shared heap + * @return true if success, false if failed */ WASM_RUNTIME_API_EXTERN bool wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst, @@ -2141,6 +2139,7 @@ wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst, /** * Detach a shared heap from a module instance + * * @param module_inst the module instance */ WASM_RUNTIME_API_EXTERN void @@ -2148,22 +2147,29 @@ wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst); /** * Allocate memory from a shared heap + * * @param module_inst the module instance * @param size required memory size * @param p_native_addr native address of allocated memory + * + * @return return the allocated memory address, which re-uses part of the wasm + * address space and is in the range of [UINT32 - shared_heap_size + 1, UINT32] + * (when the wasm memory is 32-bit) or [UINT64 - shared_heap_size + 1, UINT64] + * (when the wasm memory is 64-bit). Note that it is not an absolute address. + * Return non-zero if success, zero if failed. */ -WASM_RUNTIME_API_EXTERN uint64 -wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64 size, +WASM_RUNTIME_API_EXTERN uint64_t +wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64_t size, void **p_native_addr); /** * Free the memory allocated from shared heap + * * @param module_inst the module instance * @param ptr the offset in wasm app */ WASM_RUNTIME_API_EXTERN void -wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr); -#endif +wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64_t ptr); #ifdef __cplusplus } diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 3c4e5faa70..d4905a10d6 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -97,7 +97,7 @@ typedef struct WASMSharedHeap { struct WASMSharedHeap *next; void *heap_handle; uint8 *base_addr; - uint32 size; + uint64 size; uint64 start_off_mem64; uint64 start_off_mem32; } WASMSharedHeap; diff --git a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c index 164568ac23..978687f3c3 100644 --- a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c +++ b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c @@ -31,8 +31,10 @@ shared_free_wrapper(wasm_exec_env_t exec_env, void *ptr) { wasm_module_inst_t module_inst = get_module_inst(exec_env); - if (!validate_native_addr(ptr, (uint64)sizeof(uint32))) + if (!validate_native_addr(ptr, (uint64)sizeof(uintptr_t))) { + LOG_WARNING("Invalid app address"); return; + } module_shared_free(addr_native_to_app(ptr)); } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 40c9bb6aed..9f7a69229f 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -50,3 +50,4 @@ add_subdirectory(linux-perf) add_subdirectory(gc) add_subdirectory(memory64) add_subdirectory(tid-allocator) +add_subdirectory(shared-heap) \ No newline at end of file diff --git a/tests/unit/shared-heap/CMakeLists.txt b/tests/unit/shared-heap/CMakeLists.txt new file mode 100644 index 0000000000..6baf420f89 --- /dev/null +++ b/tests/unit/shared-heap/CMakeLists.txt @@ -0,0 +1,59 @@ +# Copyright (C) 2024 Xiaomi Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) + +project(test-shared-heap) + +add_definitions(-DRUN_ON_LINUX) + +set(WAMR_BUILD_APP_FRAMEWORK 0) +set(WAMR_BUILD_AOT 0) +set(WAMR_BUILD_INTERP 1) +set(WAMR_BUILD_FAST_INTERP 1) +set(WAMR_BUILD_JIT 0) +set(WAMR_BUILD_MEMORY64 1) +set(WAMR_BUILD_SHARED_HEAP 1) + +# Compile wasm modules +add_subdirectory(wasm-apps) + +# if only load this CMake other than load it as subdirectory +include(../unit_common.cmake) + +set(LLVM_SRC_ROOT "${WAMR_ROOT_DIR}/core/deps/llvm") + +if (NOT EXISTS "${LLVM_SRC_ROOT}/build") + message(FATAL_ERROR "Cannot find LLVM dir: ${LLVM_SRC_ROOT}/build") +endif () + +set(CMAKE_PREFIX_PATH "${LLVM_SRC_ROOT}/build;${CMAKE_PREFIX_PATH}") +find_package(LLVM REQUIRED CONFIG) +include_directories(${LLVM_INCLUDE_DIRS}) +add_definitions(${LLVM_DEFINITIONS}) +message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") +message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") + +include(${IWASM_DIR}/compilation/iwasm_compl.cmake) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +file(GLOB_RECURSE source_all ${CMAKE_CURRENT_SOURCE_DIR}/*.cc) + +set(UNIT_SOURCE ${source_all}) + +aux_source_directory(. SRC_LIST) + +set(unit_test_sources + ${UNIT_SOURCE} + ${WAMR_RUNTIME_LIB_SOURCE} + ${UNCOMMON_SHARED_SOURCE} + ${SRC_LIST} + ) + +# Now simply link against gtest or gtest_main as needed. Eg +add_executable(shared_heap_test ${unit_test_sources}) + +target_link_libraries(shared_heap_test ${LLVM_AVAILABLE_LIBS} gtest_main) + +gtest_discover_tests(shared_heap_test) \ No newline at end of file diff --git a/tests/unit/shared-heap/shared_heap_test.cc b/tests/unit/shared-heap/shared_heap_test.cc new file mode 100644 index 0000000000..5e45d31119 --- /dev/null +++ b/tests/unit/shared-heap/shared_heap_test.cc @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2024 Xiaomi Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "test_helper.h" +#include "gtest/gtest.h" + +#include "bh_read_file.h" +#include "wasm_runtime_common.h" + +class shared_heap_test : public testing::Test +{ + protected: + virtual void SetUp() {} + static void SetUpTestCase() {} + virtual void TearDown() {} + WAMRRuntimeRAII<512 * 1024> runtime; +}; + +struct ret_env { + wasm_exec_env_t exec_env; + wasm_module_t wasm_module; + wasm_module_inst_t wasm_module_inst; + unsigned char *wasm_file_buf; + char error_buf[128]; +}; + +struct ret_env +load_wasm(char *wasm_file_tested, unsigned int app_heap_size) +{ + std::string wasm_mem_page = wasm_file_tested; + const char *wasm_file = strdup(wasm_mem_page.c_str()); + wasm_module_inst_t wasm_module_inst = nullptr; + wasm_module_t wasm_module = nullptr; + wasm_exec_env_t exec_env = nullptr; + unsigned char *wasm_file_buf = nullptr; + unsigned int wasm_file_size = 0; + unsigned int stack_size = 16 * 1024, heap_size = app_heap_size; + char error_buf[128] = { 0 }; + struct ret_env ret_module_env; + + memset(ret_module_env.error_buf, 0, 128); + wasm_file_buf = + (unsigned char *)bh_read_file_to_buffer(wasm_file, &wasm_file_size); + if (!wasm_file_buf) { + goto fail; + } + + wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, error_buf, + sizeof(error_buf)); + if (!wasm_module) { + memcpy(ret_module_env.error_buf, error_buf, 128); + goto fail; + } + + wasm_module_inst = wasm_runtime_instantiate( + wasm_module, stack_size, heap_size, error_buf, sizeof(error_buf)); + if (!wasm_module_inst) { + memcpy(ret_module_env.error_buf, error_buf, 128); + goto fail; + } + + exec_env = wasm_runtime_create_exec_env(wasm_module_inst, stack_size); + +fail: + ret_module_env.exec_env = exec_env; + ret_module_env.wasm_module = wasm_module; + ret_module_env.wasm_module_inst = wasm_module_inst; + ret_module_env.wasm_file_buf = wasm_file_buf; + + return ret_module_env; +} + +void +destroy_module_env(struct ret_env module_env) +{ + if (module_env.exec_env) { + wasm_runtime_destroy_exec_env(module_env.exec_env); + } + + if (module_env.wasm_module_inst) { + wasm_runtime_deinstantiate(module_env.wasm_module_inst); + } + + if (module_env.wasm_module) { + wasm_runtime_unload(module_env.wasm_module); + } + + if (module_env.wasm_file_buf) { + wasm_runtime_free(module_env.wasm_file_buf); + } +} + +TEST_F(shared_heap_test, test_shared_heap) +{ + struct ret_env tmp_module_env; + WASMFunctionInstanceCommon *func_test = nullptr; + bool ret = false; + uint32 argv[1] = { 65535 }; + const char *exception = nullptr; + SharedHeapInitArgs args; + WASMSharedHeap *shared_heap = nullptr; + + args.size = 1024; + shared_heap = wasm_runtime_create_shared_heap(&args); + tmp_module_env = load_wasm((char *)"test.wasm", 0); + + if (!shared_heap) { + printf("Failed to create shared heap\n"); + goto test_failed; + } + if (!wasm_runtime_attach_shared_heap(tmp_module_env.wasm_module_inst, shared_heap)) { + printf("Failed to attach shared heap\n"); + goto test_failed; + } + func_test = wasm_runtime_lookup_function( + tmp_module_env.wasm_module_inst, "test"); + if (!func_test) { + printf("\nFailed to wasm_runtime_lookup_function!\n"); + goto test_failed; + } + + ret = + wasm_runtime_call_wasm(tmp_module_env.exec_env, func_test, 1, argv); + if (!ret) { + printf("\nFailed to wasm_runtime_call_wasm!\n"); + const char *s = wasm_runtime_get_exception(tmp_module_env.wasm_module_inst); + printf("exception: %s\n", s); + goto test_failed; + } + + wasm_runtime_detach_shared_heap(tmp_module_env.wasm_module_inst); + + EXPECT_EQ(10, argv[0]); + + destroy_module_env(tmp_module_env); + return; +test_failed: + destroy_module_env(tmp_module_env); + EXPECT_EQ(1, 0); +} diff --git a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt new file mode 100644 index 0000000000..3627a2c144 --- /dev/null +++ b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt @@ -0,0 +1,39 @@ +# Copyright (C) 2024 Xiaomi Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(wasm-apps) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../..) + +set(CMAKE_SYSTEM_PROCESSOR wasm32) +set(CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set(WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set(CMAKE_C_FLAGS "-nostdlib -pthread -Qunused-arguments") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -z stack-size=8192 -nostdlib -O0") +set(CMAKE_C_COMPILER_TARGET "wasm32") +set(CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +set(DEFINED_SYMBOLS + "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") + +set(CMAKE_EXE_LINKER_FLAGS + "-Wl,--no-entry \ + -Wl,--initial-memory=65536 \ + -Wl,--export-all \ + -Wl,--allow-undefined" + ) + +add_executable(test.wasm test.c) +target_link_libraries(test.wasm) + +add_custom_command(TARGET test.wasm POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/test.wasm + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMENT "Copy test.wasm to the same directory of google test" + ) \ No newline at end of file diff --git a/tests/unit/shared-heap/wasm-apps/test.c b/tests/unit/shared-heap/wasm-apps/test.c new file mode 100644 index 0000000000..ce59903c6c --- /dev/null +++ b/tests/unit/shared-heap/wasm-apps/test.c @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2024 Xiaomi Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +extern void * +shared_malloc(int size); +extern void +shared_free(void *offset); + +int +test() +{ + int *ptr = (int *)shared_malloc(10); + + *ptr = 10; + int a = *ptr; + shared_free(ptr); + return a; +} \ No newline at end of file From c4aa1deda50d9428863fe73104bdfa6dd018129f Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Fri, 20 Sep 2024 16:13:20 +0800 Subject: [PATCH 34/69] Add shared heap sample (#3806) --- .github/scripts/codeql_buildscript.sh | 30 ++ .../compilation_on_android_ubuntu.yml | 8 + .github/workflows/compilation_on_macos.yml | 8 + .github/workflows/nightly_run.yml | 8 + build-scripts/config_common.cmake | 2 +- .../shared-heap/shared_heap_wrapper.c | 2 +- samples/shared-heap/CMakeLists.txt | 92 ++++++ samples/shared-heap/src/main.c | 309 ++++++++++++++++++ samples/shared-heap/wasm-apps/CMakeLists.txt | 43 +++ samples/shared-heap/wasm-apps/test1.c | 30 ++ samples/shared-heap/wasm-apps/test2.c | 18 + 11 files changed, 548 insertions(+), 2 deletions(-) create mode 100644 samples/shared-heap/CMakeLists.txt create mode 100644 samples/shared-heap/src/main.c create mode 100644 samples/shared-heap/wasm-apps/CMakeLists.txt create mode 100644 samples/shared-heap/wasm-apps/test1.c create mode 100644 samples/shared-heap/wasm-apps/test2.c diff --git a/.github/scripts/codeql_buildscript.sh b/.github/scripts/codeql_buildscript.sh index 3e523775f8..f1c412a82b 100755 --- a/.github/scripts/codeql_buildscript.sh +++ b/.github/scripts/codeql_buildscript.sh @@ -126,6 +126,16 @@ if [[ $? != 0 ]]; then exit 1; fi +# build iwasm with multi-memory enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MULTI_MEMORY=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with multi-memory enabled!" + exit 1; +fi + # build iwasm with hardware boundary check disabled cd ${WAMR_DIR}/product-mini/platforms/linux rm -rf build && mkdir build && cd build @@ -280,3 +290,23 @@ if [[ $? != 0 ]]; then echo "Failed to build iwasm with linux perf support enabled!" exit 1; fi + +# build iwasm with shared heap enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_SHARED_HEAP=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with shared heap enabled!" + exit 1; +fi + +# build iwasm with dynamic aot debug enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_DYNAMIC_AOT_DEBUG=1 +make -j +if [[ $? != 0 ]]; + echo "Failed to build iwasm dynamic aot debug enabled!" + exit 1; +fi diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 249a6f276c..27592fde6d 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -578,6 +578,14 @@ jobs: ./run.sh test1 ./run.sh test2 + - name: Build Sample [shared-heap] + run: | + cd samples/shared-heap + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./shared_heap_test + test: needs: [ diff --git a/.github/workflows/compilation_on_macos.yml b/.github/workflows/compilation_on_macos.yml index 3b92f45256..f92bba62f5 100644 --- a/.github/workflows/compilation_on_macos.yml +++ b/.github/workflows/compilation_on_macos.yml @@ -386,3 +386,11 @@ jobs: ./build.sh ./run.sh test1 ./run.sh test2 + + - name: Build Sample [shared-heap] + run: | + cd samples/shared-heap + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./shared_heap_test diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index f39085d09d..832c34d701 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -593,6 +593,14 @@ jobs: exit $? working-directory: ./wamr-app-framework/samples/simple + - name: Build Sample [shared-heap] + run: | + cd samples/shared-heap + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./shared_heap_test + test: needs: [ diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 5af944e46e..f91fbcdb04 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -498,7 +498,7 @@ if (WAMR_BUILD_MODULE_INST_CONTEXT EQUAL 1) message (" Module instance context enabled") endif () if (WAMR_BUILD_GC_HEAP_VERIFY EQUAL 1) - add_definitions (-DWASM_ENABLE_GC_VERIFY=1) + add_definitions (-DBH_ENABLE_GC_VERIFY=1) message (" GC heap verification enabled") endif () if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1) diff --git a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c index 978687f3c3..76e662dacd 100644 --- a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c +++ b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c @@ -54,4 +54,4 @@ get_lib_shared_heap_export_apis(NativeSymbol **p_shared_heap_apis) { *p_shared_heap_apis = native_symbols_shared_heap; return sizeof(native_symbols_shared_heap) / sizeof(NativeSymbol); -} \ No newline at end of file +} diff --git a/samples/shared-heap/CMakeLists.txt b/samples/shared-heap/CMakeLists.txt new file mode 100644 index 0000000000..382ddd5460 --- /dev/null +++ b/samples/shared-heap/CMakeLists.txt @@ -0,0 +1,92 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + project (shared_heap_test) +else() + project (shared_heap_test C ASM) +endif() + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" + +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Debug) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) +set (WAMR_BUILD_LIBC_BUILTIN 1) +set (WAMR_BUILD_LIBC_WASI 0) +set (WAMR_BUILD_SHARED_HEAP 1) +set (WAMR_BUILD_GC_HEAP_VERIFY 1) + +if (NOT MSVC) + # linker flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif () + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +################ application related ################ +include_directories(${CMAKE_CURRENT_LIST_DIR}/src) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (shared_heap_test src/main.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (shared_heap_test PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (APPLE) + target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread) +else () + target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread -lrt) +endif () + +add_subdirectory(wasm-apps) diff --git a/samples/shared-heap/src/main.c b/samples/shared-heap/src/main.c new file mode 100644 index 0000000000..416652a3e7 --- /dev/null +++ b/samples/shared-heap/src/main.c @@ -0,0 +1,309 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_platform.h" +#include "bh_read_file.h" + +typedef struct thread_arg { + bh_queue *queue; + wasm_module_inst_t module_inst; +} thread_arg; + +static void * +thread1_callback(void *arg) +{ + thread_arg *targ = arg; + wasm_module_inst_t module_inst = targ->module_inst; + bh_queue *queue = targ->queue; + wasm_exec_env_t exec_env; + wasm_function_inst_t my_shared_malloc_func; + wasm_function_inst_t my_shared_free_func; + uint32 i, argv[2]; + + /* lookup wasm functions */ + if (!(my_shared_malloc_func = + wasm_runtime_lookup_function(module_inst, "my_shared_malloc")) + || !(my_shared_free_func = + wasm_runtime_lookup_function(module_inst, "my_shared_free"))) { + printf("Failed to lookup function.\n"); + } + + /* create exec env */ + if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { + printf("Failed to create exec env.\n"); + return NULL; + } + + /* allocate memory with wasm_runtime_shared_heap_malloc and send it + to wasm app2 */ + for (i = 0; i < 5; i++) { + uint8 *buf; + uint64 offset; + + offset = wasm_runtime_shared_heap_malloc(module_inst, 1024 * (i + 1), + (void **)&buf); + + if (offset == 0) { + printf("Failed to allocate memory from shared heap\n"); + break; + } + + snprintf(buf, 1024, "Hello, this is buf %u allocated from shared heap", + i + 1); + + printf("wasm app1 send buf: %s\n\n", buf); + if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + printf("Failed to post message to queue\n"); + wasm_runtime_shared_heap_free(module_inst, offset); + break; + } + } + + /* allocate memory by calling my_shared_malloc function and send it + to wasm app2 */ + for (i = 5; i < 10; i++) { + uint8 *buf; + + argv[0] = 1024 * (i + 1); + argv[1] = i + 1; + wasm_runtime_call_wasm(exec_env, my_shared_malloc_func, 2, argv); + + if (wasm_runtime_get_exception(module_inst)) { + printf("Failed to call 'my_shared_malloc` function: %s\n", + wasm_runtime_get_exception(module_inst)); + break; + } + if (argv[0] == 0) { + printf("Failed to allocate memory from shared heap\n"); + break; + } + + buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); + + printf("wasm app1 send buf: %s\n\n", buf); + if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + printf("Failed to post message to queue\n"); + wasm_runtime_shared_heap_free(module_inst, argv[0]); + break; + } + } + + wasm_runtime_destroy_exec_env(exec_env); + + return NULL; +} + +static void +queue_callback(void *message, void *arg) +{ + bh_message_t msg = (bh_message_t)message; + wasm_exec_env_t exec_env = arg; + wasm_module_inst_t module_inst = wasm_runtime_get_module_inst(exec_env); + wasm_function_inst_t print_buf_func; + uint32 argv[2]; + + /* lookup wasm function */ + if (!(print_buf_func = + wasm_runtime_lookup_function(module_inst, "print_buf"))) { + printf("Failed to lookup function.\n"); + return; + } + + char *buf = bh_message_payload(msg); + printf("wasm app's native queue received buf: %s\n\n", buf); + + /* call wasm function */ + argv[0] = wasm_runtime_addr_native_to_app(module_inst, buf); + wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); +} + +static void * +thread2_callback(void *arg) +{ + thread_arg *targ = arg; + bh_queue *queue = targ->queue; + wasm_module_inst_t module_inst = targ->module_inst; + wasm_exec_env_t exec_env; + + /* create exec env */ + if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { + printf("Failed to create exec env.\n"); + return NULL; + } + + /* enter queue's message loop until bh_queue_exit_loop_run + is called */ + bh_queue_enter_loop_run(queue, queue_callback, exec_env); + + wasm_runtime_destroy_exec_env(exec_env); + + return NULL; +} + +static char global_heap_buf[512 * 1024]; + +int +main(int argc, char **argv) +{ + char *wasm_file1 = NULL, *wasm_file2 = NULL; + uint8 *wasm_file1_buf = NULL, *wasm_file2_buf = NULL; + uint32 wasm_file1_size, wasm_file2_size; + wasm_module_t wasm_module1 = NULL, wasm_module2 = NULL; + wasm_module_inst_t module_inst1 = NULL; + wasm_module_inst_t module_inst2 = NULL; + wasm_shared_heap_t shared_heap = NULL; + bh_queue *queue = NULL; + RuntimeInitArgs init_args; + SharedHeapInitArgs heap_init_args; + char error_buf[128] = { 0 }; + int ret = -1; + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + /* init wasm runtime */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + /* create queue */ + if (!(queue = bh_queue_create())) { + printf("Create queue failed.\n"); + goto fail; + } + + /* read wasm file */ + wasm_file1 = "./wasm-apps/test1.wasm"; + if (!(wasm_file1_buf = + bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { + printf("Open wasm file %s failed.\n", wasm_file1); + goto fail; + } + + /* load wasm file */ + wasm_module1 = wasm_runtime_load((uint8 *)wasm_file1_buf, wasm_file1_size, + error_buf, sizeof(error_buf)); + if (!wasm_module1) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* instantiate module */ + module_inst1 = wasm_runtime_instantiate(wasm_module1, 65536, 0, error_buf, + sizeof(error_buf)); + if (!module_inst1) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* read wasm file */ + wasm_file2 = "./wasm-apps/test2.wasm"; + if (!(wasm_file2_buf = + bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { + printf("Open wasm file %s failed.\n", wasm_file1); + goto fail; + } + + /* load wasm file */ + wasm_module2 = wasm_runtime_load((uint8 *)wasm_file2_buf, wasm_file2_size, + error_buf, sizeof(error_buf)); + if (!wasm_module2) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* instantiate module */ + module_inst2 = wasm_runtime_instantiate(wasm_module2, 65536, 0, error_buf, + sizeof(error_buf)); + if (!module_inst2) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* create shared heap */ + memset(&heap_init_args, 0, sizeof(heap_init_args)); + heap_init_args.size = 65536; + shared_heap = wasm_runtime_create_shared_heap(&heap_init_args); + if (!shared_heap) { + printf("Create shared heap failed. error: %s\n", error_buf); + goto fail; + } + + /* attach module instance 1 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(module_inst1, shared_heap)) { + printf("Attach shared heap failed.\n"); + goto fail; + } + + /* attach module instance 2 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(module_inst2, shared_heap)) { + printf("Attach shared heap failed.\n"); + goto fail; + } + + /* create thread 1 */ + struct thread_arg targ1 = { 0 }; + korp_tid tid1; + targ1.queue = queue; + targ1.module_inst = module_inst1; + if (os_thread_create(&tid1, thread1_callback, &targ1, + APP_THREAD_STACK_SIZE_DEFAULT)) { + printf("Failed to create thread 1\n"); + goto fail; + } + + /* create thread 2 */ + struct thread_arg targ2 = { 0 }; + korp_tid tid2; + targ2.queue = queue; + targ2.module_inst = module_inst2; + if (os_thread_create(&tid2, thread2_callback, &targ2, + APP_THREAD_STACK_SIZE_DEFAULT)) { + printf("Failed to create thread 2\n"); + os_thread_join(tid1, NULL); + goto fail; + } + + /* wait until all messages are post to wasm app2 and wasm app2 + handles all of them, then exit the queue message loop */ + usleep(2000); + bh_queue_exit_loop_run(queue); + + os_thread_join(tid1, NULL); + os_thread_join(tid2, NULL); + + ret = 0; + +fail: + if (module_inst2) + wasm_runtime_deinstantiate(module_inst2); + + if (module_inst1) + wasm_runtime_deinstantiate(module_inst1); + + if (wasm_module2) + wasm_runtime_unload(wasm_module2); + + if (wasm_module1) + wasm_runtime_unload(wasm_module1); + + if (wasm_file2_buf) + wasm_runtime_free(wasm_file2_buf); + + if (wasm_file1_buf) + wasm_runtime_free(wasm_file1_buf); + + if (queue) + bh_queue_destroy(queue); + + wasm_runtime_destroy(); + + return ret; +} diff --git a/samples/shared-heap/wasm-apps/CMakeLists.txt b/samples/shared-heap/wasm-apps/CMakeLists.txt new file mode 100644 index 0000000000..819c4aca36 --- /dev/null +++ b/samples/shared-heap/wasm-apps/CMakeLists.txt @@ -0,0 +1,43 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(wasm-apps) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +if (APPLE) + set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) + set (CMAKE_C_LINK_FLAGS "") + set (CMAKE_CXX_LINK_FLAGS "") +endif () + +set (CMAKE_SYSTEM_PROCESSOR wasm32) +set (CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set (WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set (CMAKE_C_FLAGS "-nostdlib -Qunused-arguments -z stack-size=32768") +set (CMAKE_C_COMPILER_TARGET "wasm32") +set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +set (DEFINED_SYMBOLS "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") + +set (CMAKE_EXE_LINKER_FLAGS + "-Wl,--initial-memory=65536, \ + -Wl,--no-entry,--strip-all, \ + -Wl,--export=__heap_base,--export=__data_end \ + -Wl,--export=__wasm_call_ctors \ + -Wl,--export=my_shared_malloc \ + -Wl,--export=my_shared_free \ + -Wl,--export=print_buf \ + -Wl,--allow-undefined" +) + +add_executable(test1.wasm test1.c) +target_link_libraries(test1.wasm) + +add_executable(test2.wasm test2.c) +target_link_libraries(test2.wasm) diff --git a/samples/shared-heap/wasm-apps/test1.c b/samples/shared-heap/wasm-apps/test1.c new file mode 100644 index 0000000000..9186df2ce3 --- /dev/null +++ b/samples/shared-heap/wasm-apps/test1.c @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +extern void * +shared_malloc(uint32_t size); +extern void +shared_free(void *ptr); + +void * +my_shared_malloc(uint32_t size, uint32_t index) +{ + char *buf = shared_malloc(size); + + if (buf) + snprintf(buf, 1024, "Hello, this is buf %u allocated from shared heap", + index); + + return buf; +} + +void +my_shared_free(void *ptr) +{ + shared_free(ptr); +} diff --git a/samples/shared-heap/wasm-apps/test2.c b/samples/shared-heap/wasm-apps/test2.c new file mode 100644 index 0000000000..12af9e52b9 --- /dev/null +++ b/samples/shared-heap/wasm-apps/test2.c @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +#include + +extern void +shared_free(void *ptr); + +void +print_buf(char *buf) +{ + printf("wasm app2's wasm func received buf: %s\n\n", buf); + shared_free(buf); +} From e87f7a920d568d769652dad7f67bb159aa2bce3a Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 25 Sep 2024 16:43:00 +0800 Subject: [PATCH 35/69] Fix Windows compile error when uvwasi is enabled (#3810) No need to compile win_file.c when uvwasi is enabled. --- core/iwasm/aot/aot_loader.c | 2 +- core/iwasm/common/wasm_runtime_common.c | 1 - core/shared/platform/windows/platform_internal.h | 9 +++++++-- core/shared/platform/windows/shared_platform.cmake | 3 +++ core/shared/platform/windows/win_socket.c | 4 ++-- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 5c81318332..0304bd0962 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -2520,7 +2520,7 @@ try_merge_data_and_text(const uint8 **buf, const uint8 **buf_end, /* order not essential just as compiler does: .text section first */ *buf = sections; *buf_end = sections + code_size; - bh_memcpy_s(sections, code_size, old_buf, code_size); + bh_memcpy_s(sections, (uint32)code_size, old_buf, (uint32)code_size); os_munmap(old_buf, code_size); sections += align_uint((uint32)code_size, page_size); diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 667cbba03f..453cf9e3c8 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -340,7 +340,6 @@ runtime_exception_handler(EXCEPTION_POINTERS *exce_info) PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord; uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1]; WASMModuleInstance *module_inst; - WASMMemoryInstance *memory_inst; WASMJmpBuf *jmpbuf_node; uint8 *mapped_mem_start_addr = NULL; uint8 *mapped_mem_end_addr = NULL; diff --git a/core/shared/platform/windows/platform_internal.h b/core/shared/platform/windows/platform_internal.h index ed021a9aa1..1cd8187afd 100644 --- a/core/shared/platform/windows/platform_internal.h +++ b/core/shared/platform/windows/platform_internal.h @@ -168,12 +168,13 @@ typedef struct windows_dir_stream { windows_handle *handle; } windows_dir_stream; -typedef windows_handle *os_file_handle; typedef windows_dir_stream *os_dir_stream; -#if WASM_ENABLE_UVWASI != 1 +#if WASM_ENABLE_UVWASI == 0 +typedef windows_handle *os_file_handle; typedef HANDLE os_raw_file_handle; #else +typedef uint32_t os_file_handle; typedef uint32_t os_raw_file_handle; #endif @@ -190,7 +191,11 @@ typedef uint32_t os_raw_file_handle; static inline os_file_handle os_get_invalid_handle(void) { +#if WASM_ENABLE_UVWASI == 0 return NULL; +#else + return -1; +#endif } #ifdef __cplusplus diff --git a/core/shared/platform/windows/shared_platform.cmake b/core/shared/platform/windows/shared_platform.cmake index 502a8a2ed9..c2d74463fb 100644 --- a/core/shared/platform/windows/shared_platform.cmake +++ b/core/shared/platform/windows/shared_platform.cmake @@ -15,6 +15,9 @@ file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c if (NOT WAMR_BUILD_LIBC_WASI EQUAL 1) list(REMOVE_ITEM source_all ${PLATFORM_SHARED_DIR}/win_file.c) +elseif (WAMR_BUILD_LIBC_UVWASI EQUAL 1) + # uvwasi doesn't need to compile win_file.c + list(REMOVE_ITEM source_all ${PLATFORM_SHARED_DIR}/win_file.c) else() include (${CMAKE_CURRENT_LIST_DIR}/../common/libc-util/platform_common_libc_util.cmake) set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE}) diff --git a/core/shared/platform/windows/win_socket.c b/core/shared/platform/windows/win_socket.c index b19e5b0976..8d61c45ccc 100644 --- a/core/shared/platform/windows/win_socket.c +++ b/core/shared/platform/windows/win_socket.c @@ -182,8 +182,8 @@ os_socket_accept(bh_socket_t server_sock, bh_socket_t *sock, void *addr, (*sock)->type = windows_handle_type_socket; (*sock)->access_mode = windows_access_mode_read | windows_access_mode_write; (*sock)->fdflags = 0; - (*sock)->raw.socket = - accept(server_sock->raw.socket, (struct sockaddr *)&addr_tmp, &len); + (*sock)->raw.socket = accept(server_sock->raw.socket, + (struct sockaddr *)&addr_tmp, (int *)&len); if ((*sock)->raw.socket == INVALID_SOCKET) { BH_FREE(*sock); From 86926aa9d2e643304864a5c33bda0eccc8494087 Mon Sep 17 00:00:00 2001 From: palchikov Date: Wed, 25 Sep 2024 11:59:52 +0300 Subject: [PATCH 36/69] Fix unused param warning when GC is enabled (#3814) --- core/iwasm/interpreter/wasm.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index e043465d44..0aefd30c3c 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -1238,6 +1238,9 @@ wasm_value_type_size_internal(uint8 value_type, uint8 pointer_size) else { bh_assert(0); } +#if WASM_ENABLE_GC == 0 + (void)pointer_size; +#endif return 0; } From 5ce6f90bd6442a7d30dd84e459a14ea7091a2d7f Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:52:00 +0800 Subject: [PATCH 37/69] Add scoreboard CI for supply-chain security (#3819) --- .github/workflows/supply_chain.yml | 65 ++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/supply_chain.yml diff --git a/.github/workflows/supply_chain.yml b/.github/workflows/supply_chain.yml new file mode 100644 index 0000000000..2307756abb --- /dev/null +++ b/.github/workflows/supply_chain.yml @@ -0,0 +1,65 @@ +# This workflow uses actions that are not certified by GitHub. They are provided +# by a third-party and are governed by separate terms of service, privacy +# policy, and support documentation. + +# Check current WASM Micro Runtime results here: https://securityscorecards.dev/viewer/?uri=github.com/bytecodealliance/wasm-micro-runtime + +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + # midnight UTC + schedule: + - cron: "0 0 * * *" + # allow to be triggered manually + workflow_dispatch: + +# Declare default permissions as read only. +permissions: + contents: read + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + if: github.repository == 'bytecodealliance/wasm-micro-runtime' + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + + steps: + - name: "Checkout code" + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + with: + results_file: results.sarif + results_format: sarif + + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + publish_results: true + + # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF + # format to the repository Actions tab. + - name: "Upload artifact" + uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard. + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@17573ee1cc1b9d061760f3a006fc4aac4f944fd5 # v2.2.4 + with: + sarif_file: results.sarif From 438b81bd04a64c79d11457923d46f08c8da9313b Mon Sep 17 00:00:00 2001 From: Kvencc <43509398+Kvencc@users.noreply.github.com> Date: Sat, 28 Sep 2024 21:55:09 +0800 Subject: [PATCH 38/69] Fix missing symbols when using aot mode on riscv platforms (#3812) Add symbol __atomic_compare_exchange_4 and __atomic_store_4. --- core/iwasm/aot/arch/aot_reloc_riscv.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/iwasm/aot/arch/aot_reloc_riscv.c b/core/iwasm/aot/arch/aot_reloc_riscv.c index 058ad0e10a..8df9f9f8ed 100644 --- a/core/iwasm/aot/arch/aot_reloc_riscv.c +++ b/core/iwasm/aot/arch/aot_reloc_riscv.c @@ -24,6 +24,7 @@ #undef NEED_SOFT_I32_DIV #undef NEED_SOFT_I64_MUL #undef NEED_SOFT_I64_DIV +#undef NEED_SOFT_ATOMIC #ifdef __riscv_flen #if __riscv_flen == 32 @@ -48,6 +49,10 @@ #define NEED_SOFT_I64_DIV #endif +#ifndef __riscv_atomic +#define NEED_SOFT_ATOMIC +#endif + /* clang-format off */ void __adddf3(void); void __addsf3(void); @@ -101,6 +106,9 @@ void __umoddi3(void); void __umodsi3(void); void __unorddf2(void); void __unordsf2(void); +bool __atomic_compare_exchange_4(volatile void *, void *, unsigned int, + bool, int, int); +void __atomic_store_4(volatile void *, unsigned int, int); /* clang-format on */ static SymbolMap target_sym_map[] = { @@ -127,6 +135,7 @@ static SymbolMap target_sym_map[] = { * to convert float and long long */ REG_SYM(__floatundisf), + REG_SYM(__floatdisf), #endif #ifdef NEED_SOFT_DP REG_SYM(__adddf3), @@ -175,6 +184,10 @@ static SymbolMap target_sym_map[] = { REG_SYM(__moddi3), REG_SYM(__udivdi3), REG_SYM(__umoddi3), +#endif +#ifdef NEED_SOFT_ATOMIC + REG_SYM(__atomic_compare_exchange_4), + REG_SYM(__atomic_store_4), #endif /* clang-format on */ }; From 9ba36e284c201b52716b314d088a68163a419f7c Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Sun, 29 Sep 2024 12:50:59 +0800 Subject: [PATCH 39/69] Implement shared heap for AOT (#3815) --- .../compilation_on_android_ubuntu.yml | 1 + .github/workflows/compilation_on_macos.yml | 1 + .github/workflows/nightly_run.yml | 1 + core/config.h | 4 +- core/iwasm/aot/aot_runtime.c | 21 ++ core/iwasm/aot/aot_runtime.h | 8 + core/iwasm/common/wasm_memory.c | 117 ++++++-- core/iwasm/common/wasm_memory.h | 4 + core/iwasm/common/wasm_runtime_common.c | 18 ++ core/iwasm/common/wasm_shared_memory.c | 7 +- core/iwasm/compilation/aot_emit_memory.c | 278 +++++++++++++++++- core/iwasm/compilation/aot_llvm.c | 78 +++++ core/iwasm/compilation/aot_llvm.h | 5 + core/iwasm/include/aot_comp_option.h | 1 + core/iwasm/interpreter/wasm_interp_classic.c | 10 +- core/iwasm/interpreter/wasm_loader.c | 3 + core/iwasm/interpreter/wasm_mini_loader.c | 3 + core/iwasm/interpreter/wasm_runtime.c | 8 + core/iwasm/interpreter/wasm_runtime.h | 11 +- .../shared-heap/shared_heap_wrapper.c | 8 +- .../libraries/thread-mgr/thread_manager.c | 10 - doc/build_wamr.md | 16 + samples/shared-heap/CMakeLists.txt | 37 ++- samples/shared-heap/src/main.c | 43 ++- samples/shared-heap/wasm-apps/CMakeLists.txt | 6 +- samples/shared-heap/wasm-apps/test1.c | 48 ++- samples/shared-heap/wasm-apps/test2.c | 4 +- tests/unit/shared-heap/wasm-apps/test.c | 10 +- wamr-compiler/main.c | 4 + 29 files changed, 684 insertions(+), 81 deletions(-) diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 27592fde6d..6be445944f 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -585,6 +585,7 @@ jobs: cmake .. cmake --build . --config Debug --parallel 4 ./shared_heap_test + ./shared_heap_test --aot test: needs: diff --git a/.github/workflows/compilation_on_macos.yml b/.github/workflows/compilation_on_macos.yml index f92bba62f5..5b1edac85b 100644 --- a/.github/workflows/compilation_on_macos.yml +++ b/.github/workflows/compilation_on_macos.yml @@ -394,3 +394,4 @@ jobs: cmake .. cmake --build . --config Debug --parallel 4 ./shared_heap_test + ./shared_heap_test --aot diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index 832c34d701..5149c830fa 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -600,6 +600,7 @@ jobs: cmake .. cmake --build . --config Debug --parallel 4 ./shared_heap_test + ./shared_heap_test --aot test: needs: diff --git a/core/config.h b/core/config.h index c49b63ca2d..6bab4da908 100644 --- a/core/config.h +++ b/core/config.h @@ -396,7 +396,9 @@ #define APP_HEAP_SIZE_DEFAULT (8 * 1024) #endif #define APP_HEAP_SIZE_MIN (256) -#define APP_HEAP_SIZE_MAX (512 * 1024 * 1024) +/* The ems memory allocator supports maximal heap size 1GB, + see ems_gc_internal.h */ +#define APP_HEAP_SIZE_MAX (1024 * 1024 * 1024) /* Default min/max gc heap size of each app */ #ifndef GC_HEAP_SIZE_DEFAULT diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 63a3c83c90..fae534b7ef 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -57,6 +57,9 @@ bh_static_assert(sizeof(AOTMemoryInstance) == 120); bh_static_assert(offsetof(AOTTableInstance, elems) == 24); bh_static_assert(offsetof(AOTModuleInstanceExtra, stack_sizes) == 0); +bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj) + == 8); +bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_start_off) == 16); bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3); @@ -1885,6 +1888,24 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, extra->stack_sizes = aot_get_data_section_addr(module, AOT_STACK_SIZES_SECTION_NAME, NULL); + /* + * The AOT code checks whether the n bytes to access are in shared heap + * by checking whether the beginning address meets: + * addr >= start_off && addr <= end_off - n-bytes + 1 + * where n is 1/2/4/8/16 and `end_off - n-bytes + 1` is constant, e.g., + * UINT32_MAX, UINT32_MAX-1, UINT32_MAX-3 for n = 1, 2 or 4 in 32-bit + * target. To simplify the check, when shared heap is disabled, we set + * the start off to UINT64_MAX in 64-bit target and UINT32_MAX in 32-bit + * target, so in the checking, the above formula will be false, we don't + * need to check whether the shared heap is enabled or not in the AOT + * code. + */ +#if UINTPTR_MAX == UINT64_MAX + extra->shared_heap_start_off.u64 = UINT64_MAX; +#else + extra->shared_heap_start_off.u32[0] = UINT32_MAX; +#endif + #if WASM_ENABLE_PERF_PROFILING != 0 total_size = sizeof(AOTFuncPerfProfInfo) * ((uint64)module->import_func_count + module->func_count); diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 2728e57ef6..bf5e4366c7 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -111,6 +111,14 @@ typedef struct AOTFunctionInstance { typedef struct AOTModuleInstanceExtra { DefPointer(const uint32 *, stack_sizes); + /* + * Adjusted shared heap based addr to simple the calculation + * in the aot code. The value is: + * shared_heap->base_addr - shared_heap->start_off + */ + DefPointer(uint8 *, shared_heap_base_addr_adj); + MemBound shared_heap_start_off; + WASMModuleInstanceExtraCommon common; AOTFunctionInstance **functions; uint32 function_count; diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index e70cc2a5b6..d2d89e5952 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -165,7 +165,7 @@ runtime_malloc(uint64 size) WASMSharedHeap * wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) { - uint64 heap_struct_size = sizeof(WASMSharedHeap); + uint64 heap_struct_size = sizeof(WASMSharedHeap), map_size; uint32 size = init_args->size; WASMSharedHeap *heap; @@ -192,7 +192,18 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) goto fail3; } - if (!(heap->base_addr = wasm_mmap_linear_memory(size, size))) { +#ifndef OS_ENABLE_HW_BOUND_CHECK + map_size = size; +#else + /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: + * ea = i + memarg.offset + * both i and memarg.offset are u32 in range 0 to 4G + * so the range of ea is 0 to 8G + */ + map_size = 8 * (uint64)BH_GB; +#endif + + if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) { goto fail3; } if (!mem_allocator_create_with_struct_and_pool( @@ -213,7 +224,7 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) return heap; fail4: - wasm_munmap_linear_memory(heap->base_addr, size, size); + wasm_munmap_linear_memory(heap->base_addr, size, map_size); fail3: wasm_runtime_free(heap->heap_handle); fail2: @@ -245,18 +256,52 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, #if WASM_ENABLE_INTERP != 0 if (module_inst->module_type == Wasm_Module_Bytecode) { - if (((WASMModuleInstance *)module_inst)->e->shared_heap) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + if (e->shared_heap) { LOG_WARNING("A shared heap is already attached"); return false; } - ((WASMModuleInstance *)module_inst)->e->shared_heap = shared_heap; - } + e->shared_heap = shared_heap; +#if WASM_ENABLE_JIT != 0 +#if UINTPTR_MAX == UINT64_MAX + if (memory->is_memory64) + e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; + else + e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u64; +#else + e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; #endif +#endif /* end of WASM_ENABLE_JIT != 0 */ + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + if (e->shared_heap) { + LOG_WARNING("A shared heap is already attached"); + return false; + } + e->shared_heap = shared_heap; +#if UINTPTR_MAX == UINT64_MAX + if (memory->is_memory64) + e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; + else + e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u64; +#else + e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; #endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ return true; } @@ -277,14 +322,32 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) { #if WASM_ENABLE_INTERP != 0 if (module_inst->module_type == Wasm_Module_Bytecode) { - ((WASMModuleInstance *)module_inst)->e->shared_heap = NULL; - } + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + e->shared_heap = NULL; +#if WASM_ENABLE_JIT != 0 +#if UINTPTR_MAX == UINT64_MAX + e->shared_heap_start_off.u64 = UINT64_MAX; +#else + e->shared_heap_start_off.u32[0] = UINT32_MAX; +#endif + e->shared_heap_base_addr_adj = NULL; #endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + e->shared_heap = NULL; +#if UINTPTR_MAX == UINT64_MAX + e->shared_heap_start_off.u64 = UINT64_MAX; +#else + e->shared_heap_start_off.u32[0] = UINT32_MAX; #endif + e->shared_heap_base_addr_adj = NULL; + } +#endif /* end of WASM_ENABLE_AOT != 0 */ } void @@ -307,13 +370,21 @@ get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) #endif #if WASM_ENABLE_AOT != 0 if (module_inst_comm->module_type == Wasm_Module_AoT) { - // TODO - return NULL; + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst_comm) + ->e; + return e->shared_heap; } #endif return NULL; } +WASMSharedHeap * +wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) +{ + return get_shared_heap(module_inst_comm); +} + static bool is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, bool is_memory64, uint64 app_offset, uint32 bytes) @@ -324,6 +395,10 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, return false; } + if (bytes == 0) { + bytes = 1; + } + if (!is_memory64) { if (app_offset >= heap->start_off_mem32 && app_offset <= UINT32_MAX - bytes + 1) { @@ -457,17 +532,23 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, #if WASM_ENABLE_SHARED_HEAP != 0 static void -wasm_runtime_destroy_shared_heaps() +destroy_shared_heaps() { WASMSharedHeap *heap = shared_heap_list; WASMSharedHeap *cur; + uint64 map_size; while (heap) { cur = heap; heap = heap->next; mem_allocator_destroy(cur->heap_handle); wasm_runtime_free(cur->heap_handle); - wasm_munmap_linear_memory(cur->base_addr, cur->size, cur->size); +#ifndef OS_ENABLE_HW_BOUND_CHECK + map_size = cur->size; +#else + map_size = 8 * (uint64)BH_GB; +#endif + wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size); wasm_runtime_free(cur); } } @@ -477,7 +558,7 @@ void wasm_runtime_memory_destroy(void) { #if WASM_ENABLE_SHARED_HEAP != 0 - wasm_runtime_destroy_shared_heaps(); + destroy_shared_heaps(); #endif if (memory_mode == MEMORY_MODE_POOL) { @@ -1178,7 +1259,7 @@ wasm_mremap_linear_memory(void *mapped_mem, uint64 old_size, uint64 new_size, } static void * -wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size) +wasm_mmap_linear_memory(uint64 map_size, uint64 commit_size) { return wasm_mremap_linear_memory(NULL, 0, map_size, commit_size); } diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index cc6418e841..bceea0ee43 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -54,9 +54,13 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, void wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst); + void wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst); +WASMSharedHeap * +wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm); + uint64 wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, uint64 size, void **p_native_addr); diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index dc20edba20..aa929e9716 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -185,6 +185,9 @@ static bool is_sig_addr_in_guard_pages(void *sig_addr, WASMModuleInstance *module_inst) { WASMMemoryInstance *memory_inst; +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; +#endif uint8 *mapped_mem_start_addr = NULL; uint8 *mapped_mem_end_addr = NULL; uint32 i; @@ -202,6 +205,21 @@ is_sig_addr_in_guard_pages(void *sig_addr, WASMModuleInstance *module_inst) } } +#if WASM_ENABLE_SHARED_HEAP != 0 + shared_heap = + wasm_runtime_get_shared_heap((WASMModuleInstanceCommon *)module_inst); + if (shared_heap) { + mapped_mem_start_addr = shared_heap->base_addr; + mapped_mem_end_addr = shared_heap->base_addr + 8 * (uint64)BH_GB; + if (mapped_mem_start_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr < mapped_mem_end_addr) { + /* The address which causes segmentation fault is inside + the shared heap's guard regions */ + return true; + } + } +#endif + return false; } diff --git a/core/iwasm/common/wasm_shared_memory.c b/core/iwasm/common/wasm_shared_memory.c index e0d86d6f40..2027a57200 100644 --- a/core/iwasm/common/wasm_shared_memory.c +++ b/core/iwasm/common/wasm_shared_memory.c @@ -8,6 +8,9 @@ #if WASM_ENABLE_THREAD_MGR != 0 #include "../libraries/thread-mgr/thread_manager.h" #endif +#if WASM_ENABLE_AOT != 0 +#include "../aot/aot_runtime.h" +#endif /* * Note: this lock can be per memory. @@ -257,7 +260,9 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, #endif #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { - // TODO + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + shared_heap = e->shared_heap; } #endif diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 806150ff19..869a1dbb27 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -118,10 +118,10 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { LLVMValueRef offset_const = MEMORY64_COND_VALUE(I64_CONST(offset), I32_CONST(offset)); - LLVMValueRef addr, maddr, offset1, cmp1, cmp2, cmp; + LLVMValueRef addr, maddr, maddr_phi = NULL, offset1, cmp1, cmp2, cmp; LLVMValueRef mem_base_addr, mem_check_bound; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef check_succ; + LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; AOTValue *aot_value_top; uint32 local_idx_of_aot_value = 0; uint64 const_value; @@ -131,6 +131,11 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, bool is_shared_memory = comp_ctx->comp_data->memories[0].flags & SHARED_MEMORY_FLAG; #endif +#if WASM_ENABLE_MEMORY64 == 0 + bool is_memory64 = false; +#else + bool is_memory64 = IS_MEMORY64; +#endif is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; @@ -268,8 +273,137 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* offset1 = offset + addr; */ + /* TODO: check whether integer overflow occurs when memory is 64-bit + and boundary check is enabled */ BUILD_OP(Add, offset_const, addr, offset1, "offset1"); + if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; + LLVMValueRef is_in_shared_heap, shared_heap_check_bound = NULL; + + /* Add basic blocks */ + ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap"); + ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); + ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); + + LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr); + LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, + app_addr_in_shared_heap); + LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + if (!(maddr_phi = + LLVMBuildPhi(comp_ctx->builder, + enable_segue ? INT8_PTR_TYPE_GS : INT8_PTR_TYPE, + "maddr_phi"))) { + aot_set_last_error("llvm build phi failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); + + if (!is_target_64bit) { + /* Check whether interger overflow occurs in addr + offset */ + LLVMBasicBlockRef check_integer_overflow_end; + ADD_BASIC_BLOCK(check_integer_overflow_end, + "check_integer_overflow_end"); + LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); + + BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, + cmp1, check_integer_overflow_end)) { + goto fail; + } + SET_BUILD_POS(check_integer_overflow_end); + } + + shared_heap_check_bound = + is_memory64 ? I64_CONST(UINT64_MAX - bytes + 1) + : (comp_ctx->pointer_size == sizeof(uint64) + ? I64_CONST(UINT32_MAX - bytes + 1) + : I32_CONST(UINT32_MAX - bytes + 1)); + CHECK_LLVM_CONST(shared_heap_check_bound); + + /* Check whether the bytes to access are in shared heap */ + if (!comp_ctx->enable_bound_check) { + /* Use IntUGT but not IntUGE to compare, since (1) in the ems + memory allocator, the hmu node includes hmu header and hmu + memory, only the latter is returned to the caller as the + allocated memory, the hmu header isn't returned so the + first byte of the shared heap won't be accesed, (2) using + IntUGT gets better performance than IntUGE in some cases */ + BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off, + is_in_shared_heap, "is_in_shared_heap"); + /* We don't check the shared heap's upper boundary if boundary + check isn't enabled, the runtime may also use the guard pages + of shared heap to check the boundary if hardware boundary + check feature is enabled. */ + } + else { + /* Use IntUGT but not IntUGE to compare, same as above */ + BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off, + cmp1, "cmp1"); + /* Check the shared heap's upper boundary if boundary check is + enabled */ + BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp2, + "cmp2"); + BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap"); + } + + if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap, + app_addr_in_shared_heap, app_addr_in_linear_mem)) { + aot_set_last_error("llvm build cond br failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap); + + /* Get native address inside shared heap */ + if (!(maddr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->shared_heap_base_addr_adj, + &offset1, 1, "maddr_shared_heap"))) { + aot_set_last_error("llvm build inbounds gep failed"); + goto fail; + } + + if (enable_segue) { + LLVMValueRef mem_base_addr_u64, maddr_u64, offset_to_mem_base; + + if (!(maddr_u64 = LLVMBuildPtrToInt(comp_ctx->builder, maddr, + I64_TYPE, "maddr_u64")) + || !(mem_base_addr_u64 = + LLVMBuildPtrToInt(comp_ctx->builder, mem_base_addr, + I64_TYPE, "mem_base_addr_u64"))) { + aot_set_last_error("llvm build ptr to int failed"); + goto fail; + } + if (!(offset_to_mem_base = + LLVMBuildSub(comp_ctx->builder, maddr_u64, + mem_base_addr_u64, "offset_to_mem_base"))) { + aot_set_last_error("llvm build sub failed"); + goto fail; + } + if (!(maddr = LLVMBuildIntToPtr( + comp_ctx->builder, offset_to_mem_base, INT8_PTR_TYPE_GS, + "maddr_shared_heap_segue"))) { + aot_set_last_error("llvm build int to ptr failed."); + goto fail; + } + } + + LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1); + + if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { + aot_set_last_error("llvm build br failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + } + if (comp_ctx->enable_bound_check && !(is_local_of_aot_value && aot_checked_addr_list_find(func_ctx, local_idx_of_aot_value, @@ -305,10 +439,16 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); } else { - /* Check integer overflow */ - BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2"); - BUILD_OP(Or, cmp1, cmp2, cmp, "cmp"); + if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + /* Check integer overflow has been checked above */ + BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); + } + else { + /* Check integer overflow */ + BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); + BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2"); + BUILD_OP(Or, cmp1, cmp2, cmp, "cmp"); + } } /* Add basic blocks */ @@ -354,7 +494,19 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } } - return maddr; + + if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); + if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { + aot_set_last_error("llvm build br failed"); + goto fail; + } + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + return maddr_phi; + } + else + return maddr; fail: return NULL; } @@ -985,10 +1137,15 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef offset, LLVMValueRef bytes) { LLVMValueRef maddr, max_addr, cmp; - LLVMValueRef mem_base_addr; + LLVMValueRef mem_base_addr, maddr_phi = NULL; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef check_succ; + LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; LLVMValueRef mem_size; +#if WASM_ENABLE_MEMORY64 == 0 + bool is_memory64 = false; +#else + bool is_memory64 = IS_MEMORY64; +#endif /* Get memory base address and memory data size */ #if WASM_ENABLE_SHARED_MEMORY != 0 @@ -1053,9 +1210,96 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, offset = LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset"); bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len"); + if (!offset || !bytes) { + aot_set_last_error("llvm build zext failed."); + goto fail; + } + /* TODO: check whether integer overflow occurs when memory is 64-bit + and boundary check is enabled */ BUILD_OP(Add, offset, bytes, max_addr, "max_addr"); + + if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; + LLVMValueRef shared_heap_start_off, shared_heap_check_bound; + LLVMValueRef max_offset, cmp1, cmp2, is_in_shared_heap; + + /* Add basic blocks */ + ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap"); + ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); + ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); + + LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr); + LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, + app_addr_in_shared_heap); + LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE, + "maddr_phi"))) { + aot_set_last_error("llvm build phi failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); + + shared_heap_start_off = func_ctx->shared_heap_start_off; + if (comp_ctx->pointer_size == sizeof(uint32)) { + if (!(shared_heap_start_off = + LLVMBuildZExt(comp_ctx->builder, shared_heap_start_off, + I64_TYPE, "shared_heap_start_off_u64"))) { + aot_set_last_error("llvm build zext failed"); + goto fail; + } + } + shared_heap_check_bound = + is_memory64 ? I64_CONST(UINT64_MAX) : I64_CONST(UINT32_MAX); + CHECK_LLVM_CONST(shared_heap_check_bound); + + /* Check whether the bytes to access are in shared heap */ + if (!comp_ctx->enable_bound_check) { + /* Use IntUGT but not IntUGE to compare, same as the check + in aot_check_memory_overflow */ + BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off, + is_in_shared_heap, "is_in_shared_heap"); + } + else { + BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off, + cmp1, "cmp1"); + BUILD_OP(Add, max_addr, I64_NEG_ONE, max_offset, "max_offset"); + BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_check_bound, cmp2, + "cmp2"); + BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap"); + } + + if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap, + app_addr_in_shared_heap, app_addr_in_linear_mem)) { + aot_set_last_error("llvm build cond br failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap); + + /* Get native address inside shared heap */ + if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->shared_heap_base_addr_adj, + &offset, 1, "maddr_shared_heap"))) { + aot_set_last_error("llvm build inbounds gep failed"); + goto fail; + } + LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1); + + if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { + aot_set_last_error("llvm build br failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + } + BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, check_succ)) { @@ -1068,11 +1312,23 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("llvm build add failed."); goto fail; } - return maddr; + + if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); + if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { + aot_set_last_error("llvm build br failed"); + goto fail; + } + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + return maddr_phi; + } + else + return maddr; fail: return NULL; } -#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 or WASM_ENABLE_STRINGREF != 0 */ +#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 || WASM_ENABLE_STRINGREF != 0 */ #if WASM_ENABLE_BULK_MEMORY != 0 bool diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 820a55e965..fb1c4308b2 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1518,6 +1518,75 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } +static bool +create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef offset, base_addr_p, start_off_p, cmp; + uint32 offset_u32; + + /* Load aot_inst->e->shared_heap_base_addr_adj */ + offset_u32 = get_module_inst_extra_offset(comp_ctx); +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->is_jit_mode) + offset_u32 += + offsetof(WASMModuleInstanceExtra, shared_heap_base_addr_adj); + else +#endif + offset_u32 += + offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj); + offset = I32_CONST(offset_u32); + CHECK_LLVM_CONST(offset); + + if (!(base_addr_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, 1, + "shared_heap_base_addr_adj_p"))) { + aot_set_last_error("llvm build inbounds gep failed"); + return false; + } + if (!(func_ctx->shared_heap_base_addr_adj = + LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, base_addr_p, + "shared_heap_base_addr_adj"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + + /* Load aot_inst->e->shared_heap_start_off */ + offset_u32 = get_module_inst_extra_offset(comp_ctx); +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->is_jit_mode) + offset_u32 += offsetof(WASMModuleInstanceExtra, shared_heap_start_off); + else +#endif + offset_u32 += offsetof(AOTModuleInstanceExtra, shared_heap_start_off); + offset = I32_CONST(offset_u32); + CHECK_LLVM_CONST(offset); + + if (!(start_off_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, 1, + "shared_heap_start_off_p"))) { + aot_set_last_error("llvm build inbounds gep failed"); + return false; + } + if (!(func_ctx->shared_heap_start_off = LLVMBuildLoad2( + comp_ctx->builder, + comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE, + start_off_p, "shared_heap_start_off"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + + if (!(cmp = LLVMBuildIsNotNull(comp_ctx->builder, + func_ctx->shared_heap_base_addr_adj, + "has_shared_heap"))) { + aot_set_last_error("llvm build is not null failed"); + return false; + } + + return true; +fail: + return false; +} + static bool create_cur_exception(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { @@ -1808,6 +1877,12 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx, goto fail; } + /* Load shared heap, shared heap start off mem32 or mem64 */ + if (comp_ctx->enable_shared_heap + && !create_shared_heap_info(comp_ctx, func_ctx)) { + goto fail; + } + return func_ctx; fail: @@ -2619,6 +2694,9 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) if (option->enable_gc) comp_ctx->enable_gc = true; + if (option->enable_shared_heap) + comp_ctx->enable_shared_heap = true; + comp_ctx->opt_level = option->opt_level; comp_ctx->size_level = option->size_level; diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 43212e5027..0dce988bc9 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -242,6 +242,9 @@ typedef struct AOTFuncContext { bool mem_space_unchanged; AOTCheckedAddrList checked_addr_list; + LLVMValueRef shared_heap_base_addr_adj; + LLVMValueRef shared_heap_start_off; + LLVMBasicBlockRef got_exception_block; LLVMBasicBlockRef func_return_block; LLVMValueRef exception_id_phi; @@ -467,6 +470,8 @@ typedef struct AOTCompContext { /* Enable GC */ bool enable_gc; + bool enable_shared_heap; + uint32 opt_level; uint32 size_level; diff --git a/core/iwasm/include/aot_comp_option.h b/core/iwasm/include/aot_comp_option.h index 98f33a1608..4df80ec58c 100644 --- a/core/iwasm/include/aot_comp_option.h +++ b/core/iwasm/include/aot_comp_option.h @@ -71,6 +71,7 @@ typedef struct AOTCompOption { bool enable_llvm_pgo; bool enable_stack_estimation; bool quick_invoke_c_api_import; + bool enable_shared_heap; char *use_prof_file; uint32_t opt_level; uint32_t size_level; diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index ecbe408ff9..cb385ce9cb 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -47,8 +47,14 @@ typedef float64 CellType_F64; #endif #if WASM_ENABLE_SHARED_HEAP != 0 -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && (app_addr) >= shared_heap_start_off \ +#if WASM_ENABLE_MULTI_MEMORY != 0 +/* Only enable shared heap for the default memory */ +#define is_default_memory (memidx == 0) +#else +#define is_default_memory true +#endif +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ && (app_addr) <= shared_heap_end_off - bytes + 1) #define shared_heap_addr_app_to_native(app_addr, native_addr) \ diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index ff3501e3d0..e61e7677dd 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -5330,6 +5330,9 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, option.enable_memory_profiling = true; option.enable_stack_estimation = true; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + option.enable_shared_heap = true; +#endif module->comp_ctx = aot_create_comp_context(module->comp_data, &option); if (!module->comp_ctx) { diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 968eaf0096..ff0a4f403c 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -2158,6 +2158,9 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, option.enable_memory_profiling = true; option.enable_stack_estimation = true; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + option.enable_shared_heap = true; +#endif module->comp_ctx = aot_create_comp_context(module->comp_data, &option); if (!module->comp_ctx) { diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index e4142ab88c..d474f6eae6 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -2791,6 +2791,14 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, } } +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 +#if UINTPTR_MAX == UINT64_MAX + module_inst->e->shared_heap_start_off.u64 = UINT64_MAX; +#else + module_inst->e->shared_heap_start_off.u32[0] = UINT32_MAX; +#endif +#endif + #if WASM_ENABLE_GC != 0 /* Initialize the table data with init expr */ for (i = 0; i < module->table_count; i++) { diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 48c333f4cb..2644b00162 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -92,7 +92,6 @@ typedef union { uint32 u32[2]; } MemBound; -#if WASM_ENABLE_SHARED_HEAP != 0 typedef struct WASMSharedHeap { struct WASMSharedHeap *next; void *heap_handle; @@ -101,7 +100,6 @@ typedef struct WASMSharedHeap { uint64 start_off_mem64; uint64 start_off_mem32; } WASMSharedHeap; -#endif struct WASMMemoryInstance { /* Module type */ @@ -366,6 +364,15 @@ typedef struct WASMModuleInstanceExtra { #if WASM_ENABLE_SHARED_HEAP != 0 WASMSharedHeap *shared_heap; +#if WASM_ENABLE_JIT != 0 + /* + * Adjusted shared heap based addr to simple the calculation + * in the aot code. The value is: + * shared_heap->base_addr - shared_heap->start_off + */ + uint8 *shared_heap_base_addr_adj; + MemBound shared_heap_start_off; +#endif #endif #if WASM_ENABLE_DEBUG_INTERP != 0 \ diff --git a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c index 76e662dacd..b7b78307ae 100644 --- a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c +++ b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c @@ -20,14 +20,14 @@ /* clang-format on */ static uint32 -shared_malloc_wrapper(wasm_exec_env_t exec_env, uint32 size) +shared_heap_malloc_wrapper(wasm_exec_env_t exec_env, uint32 size) { wasm_module_inst_t module_inst = get_module_inst(exec_env); return (uint32)module_shared_malloc((uint64)size, NULL); } static void -shared_free_wrapper(wasm_exec_env_t exec_env, void *ptr) +shared_heap_free_wrapper(wasm_exec_env_t exec_env, void *ptr) { wasm_module_inst_t module_inst = get_module_inst(exec_env); @@ -45,8 +45,8 @@ shared_free_wrapper(wasm_exec_env_t exec_env, void *ptr) /* clang-format on */ static NativeSymbol native_symbols_shared_heap[] = { - REG_NATIVE_FUNC(shared_malloc, "(i)i"), - REG_NATIVE_FUNC(shared_free, "(*)"), + REG_NATIVE_FUNC(shared_heap_malloc, "(i)i"), + REG_NATIVE_FUNC(shared_heap_free, "(*)"), }; uint32 diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index 46a1bb329a..55e0526c3b 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -1428,16 +1428,6 @@ wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst, { WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst); - if (module_inst->module_type == Wasm_Module_Bytecode) { - if (((WASMModuleInstance *)module_inst)->e->shared_heap) { - LOG_WARNING("A shared heap is already attached"); - return false; - } - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } - if (exec_env == NULL) { /* Maybe threads have not been started yet. */ return wasm_runtime_attach_shared_heap_internal(module_inst, heap); diff --git a/doc/build_wamr.md b/doc/build_wamr.md index 2adb17f6a1..4537ee0841 100644 --- a/doc/build_wamr.md +++ b/doc/build_wamr.md @@ -300,6 +300,22 @@ Currently we only profile the memory consumption of module, module_instance and wasm_runtime_get_context ``` +#### **Shared heap among wasm apps and host native** +- **WAMR_BUILD_SHARED_HEAP**=1/0, default to disable if not set +> Note: If it is enabled, allow to create one or more shared heaps, and attach one to a module instance, the belows APIs ared provided: +```C + wasm_runtime_create_shared_heap + wasm_runtime_attach_shared_heap + wasm_runtime_detach_shared_heap + wasm_runtime_shared_heap_malloc + wasm_runtime_shared_heap_free +``` +And the wasm app can calls below APIs to allocate/free memory from/to the shared heap if it is attached to the app's module instance: +```C + void *shared_heap_malloc(); + void shared_heap_free(void *ptr); +``` + **Combination of configurations:** We can combine the configurations. For example, if we want to disable interpreter, enable AOT and WASI, we can run command: diff --git a/samples/shared-heap/CMakeLists.txt b/samples/shared-heap/CMakeLists.txt index 382ddd5460..6346d077e7 100644 --- a/samples/shared-heap/CMakeLists.txt +++ b/samples/shared-heap/CMakeLists.txt @@ -48,6 +48,7 @@ if (NOT CMAKE_BUILD_TYPE) endif () set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_FAST_INTERP 1) set (WAMR_BUILD_AOT 1) set (WAMR_BUILD_JIT 0) set (WAMR_BUILD_LIBC_BUILTIN 1) @@ -72,7 +73,11 @@ endif () set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) -add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) +add_library(vmlib STATIC ${WAMR_RUNTIME_LIB_SOURCE}) +if (MSVC) + target_compile_definitions(vmlib PRIVATE WASM_API_EXTERN=) +endif() +target_link_libraries(vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) ################ application related ################ include_directories(${CMAKE_CURRENT_LIST_DIR}/src) @@ -90,3 +95,33 @@ else () endif () add_subdirectory(wasm-apps) + +if (WAMR_BUILD_AOT EQUAL 1) + set (WAMR_COMPILER_DIR ${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build) + message (CHECK_START "Detecting WAMR_COMPILER at ${WAMR_COMPILER_DIR}") + find_file (WAMR_COMPILER + wamrc + PATHS "${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH + ) + if (WAMR_COMPILER) + message (CHECK_PASS "found") + else() + message (CHECK_FAIL "not found") + endif() + if (NOT EXISTS ${WAMR_COMPILER}) + message (FATAL_ERROR "Please build wamrc under ${WAMR_ROOT_DIR}/wamr-compiler") + else() + message (STATUS "WAMR_COMPILER is ${WAMR_COMPILER}") + endif() + + add_custom_target( + wasm_to_aot + ALL + DEPENDS wasm-apps/test1.wasm wasm-apps/test2.wasm ${WAMR_COMPILER} + COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test1.aot wasm-apps/test1.wasm + COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test2.aot wasm-apps/test2.wasm + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) +endif() diff --git a/samples/shared-heap/src/main.c b/samples/shared-heap/src/main.c index 416652a3e7..f4024f08c2 100644 --- a/samples/shared-heap/src/main.c +++ b/samples/shared-heap/src/main.c @@ -19,15 +19,15 @@ thread1_callback(void *arg) wasm_module_inst_t module_inst = targ->module_inst; bh_queue *queue = targ->queue; wasm_exec_env_t exec_env; - wasm_function_inst_t my_shared_malloc_func; - wasm_function_inst_t my_shared_free_func; + wasm_function_inst_t my_shared_heap_malloc_func; + wasm_function_inst_t my_shared_heap_free_func; uint32 i, argv[2]; /* lookup wasm functions */ - if (!(my_shared_malloc_func = - wasm_runtime_lookup_function(module_inst, "my_shared_malloc")) - || !(my_shared_free_func = - wasm_runtime_lookup_function(module_inst, "my_shared_free"))) { + if (!(my_shared_heap_malloc_func = wasm_runtime_lookup_function( + module_inst, "my_shared_heap_malloc")) + || !(my_shared_heap_free_func = wasm_runtime_lookup_function( + module_inst, "my_shared_heap_free"))) { printf("Failed to lookup function.\n"); } @@ -62,17 +62,17 @@ thread1_callback(void *arg) } } - /* allocate memory by calling my_shared_malloc function and send it + /* allocate memory by calling my_shared_heap_malloc function and send it to wasm app2 */ for (i = 5; i < 10; i++) { uint8 *buf; argv[0] = 1024 * (i + 1); argv[1] = i + 1; - wasm_runtime_call_wasm(exec_env, my_shared_malloc_func, 2, argv); + wasm_runtime_call_wasm(exec_env, my_shared_heap_malloc_func, 2, argv); if (wasm_runtime_get_exception(module_inst)) { - printf("Failed to call 'my_shared_malloc` function: %s\n", + printf("Failed to call 'my_shared_heap_malloc' function: %s\n", wasm_runtime_get_exception(module_inst)); break; } @@ -118,6 +118,10 @@ queue_callback(void *message, void *arg) /* call wasm function */ argv[0] = wasm_runtime_addr_native_to_app(module_inst, buf); wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); + if (wasm_runtime_get_exception(module_inst)) { + printf("Failed to call 'print_buf' function: %s\n", + wasm_runtime_get_exception(module_inst)); + } } static void * @@ -159,8 +163,17 @@ main(int argc, char **argv) RuntimeInitArgs init_args; SharedHeapInitArgs heap_init_args; char error_buf[128] = { 0 }; + bool aot_mode = false; int ret = -1; + if (argc > 1 && !strcmp(argv[1], "--aot")) + aot_mode = true; + + if (!aot_mode) + printf("Test shared heap in interpreter mode\n\n"); + else + printf("Test shared heap in AOT mode\n\n"); + memset(&init_args, 0, sizeof(RuntimeInitArgs)); init_args.mem_alloc_type = Alloc_With_Pool; @@ -180,7 +193,10 @@ main(int argc, char **argv) } /* read wasm file */ - wasm_file1 = "./wasm-apps/test1.wasm"; + if (!aot_mode) + wasm_file1 = "./wasm-apps/test1.wasm"; + else + wasm_file1 = "./wasm-apps/test1.aot"; if (!(wasm_file1_buf = bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { printf("Open wasm file %s failed.\n", wasm_file1); @@ -204,7 +220,10 @@ main(int argc, char **argv) } /* read wasm file */ - wasm_file2 = "./wasm-apps/test2.wasm"; + if (!aot_mode) + wasm_file2 = "./wasm-apps/test2.wasm"; + else + wasm_file2 = "./wasm-apps/test2.aot"; if (!(wasm_file2_buf = bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { printf("Open wasm file %s failed.\n", wasm_file1); @@ -273,7 +292,7 @@ main(int argc, char **argv) /* wait until all messages are post to wasm app2 and wasm app2 handles all of them, then exit the queue message loop */ - usleep(2000); + usleep(10000); bh_queue_exit_loop_run(queue); os_thread_join(tid1, NULL); diff --git a/samples/shared-heap/wasm-apps/CMakeLists.txt b/samples/shared-heap/wasm-apps/CMakeLists.txt index 819c4aca36..c0010af6a8 100644 --- a/samples/shared-heap/wasm-apps/CMakeLists.txt +++ b/samples/shared-heap/wasm-apps/CMakeLists.txt @@ -26,12 +26,12 @@ set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") set (DEFINED_SYMBOLS "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") set (CMAKE_EXE_LINKER_FLAGS - "-Wl,--initial-memory=65536, \ + "-O0 -Wl,--initial-memory=65536, \ -Wl,--no-entry,--strip-all, \ -Wl,--export=__heap_base,--export=__data_end \ -Wl,--export=__wasm_call_ctors \ - -Wl,--export=my_shared_malloc \ - -Wl,--export=my_shared_free \ + -Wl,--export=my_shared_heap_malloc \ + -Wl,--export=my_shared_heap_free \ -Wl,--export=print_buf \ -Wl,--allow-undefined" ) diff --git a/samples/shared-heap/wasm-apps/test1.c b/samples/shared-heap/wasm-apps/test1.c index 9186df2ce3..c8fe0c7553 100644 --- a/samples/shared-heap/wasm-apps/test1.c +++ b/samples/shared-heap/wasm-apps/test1.c @@ -5,26 +5,56 @@ #include #include +#include extern void * -shared_malloc(uint32_t size); +shared_heap_malloc(uint32_t size); extern void -shared_free(void *ptr); +shared_heap_free(void *ptr); void * -my_shared_malloc(uint32_t size, uint32_t index) +my_shared_heap_malloc(uint32_t size, uint32_t index) { - char *buf = shared_malloc(size); + char *buf1 = NULL, *buf2 = NULL, *buf; - if (buf) - snprintf(buf, 1024, "Hello, this is buf %u allocated from shared heap", - index); + buf1 = shared_heap_malloc(128); + if (!buf1) + return NULL; + buf1[0] = 'H'; + buf1[1] = 'e'; + buf1[2] = 'l'; + buf1[3] = 'l'; + buf1[4] = 'o'; + buf1[5] = ','; + buf1[6] = ' '; + + buf2 = shared_heap_malloc(128); + if (!buf2) { + shared_heap_free(buf1); + return NULL; + } + + snprintf(buf2, 128, "this is buf %u allocated from shared heap", index); + + buf = shared_heap_malloc(size); + if (!buf) { + shared_heap_free(buf1); + shared_heap_free(buf2); + return NULL; + } + + memset(buf, 0, size); + memcpy(buf, buf1, strlen(buf1)); + memcpy(buf + strlen(buf1), buf2, strlen(buf2)); + + shared_heap_free(buf1); + shared_heap_free(buf2); return buf; } void -my_shared_free(void *ptr) +my_shared_heap_free(void *ptr) { - shared_free(ptr); + shared_heap_free(ptr); } diff --git a/samples/shared-heap/wasm-apps/test2.c b/samples/shared-heap/wasm-apps/test2.c index 12af9e52b9..b63efcd1a2 100644 --- a/samples/shared-heap/wasm-apps/test2.c +++ b/samples/shared-heap/wasm-apps/test2.c @@ -8,11 +8,11 @@ #include extern void -shared_free(void *ptr); +shared_heap_free(void *ptr); void print_buf(char *buf) { printf("wasm app2's wasm func received buf: %s\n\n", buf); - shared_free(buf); + shared_heap_free(buf); } diff --git a/tests/unit/shared-heap/wasm-apps/test.c b/tests/unit/shared-heap/wasm-apps/test.c index ce59903c6c..b83ee64ffa 100644 --- a/tests/unit/shared-heap/wasm-apps/test.c +++ b/tests/unit/shared-heap/wasm-apps/test.c @@ -6,17 +6,17 @@ #include extern void * -shared_malloc(int size); +shared_heap_malloc(int size); extern void -shared_free(void *offset); +shared_heap_free(void *offset); int test() { - int *ptr = (int *)shared_malloc(10); + int *ptr = (int *)shared_heap_malloc(10); *ptr = 10; int a = *ptr; - shared_free(ptr); + shared_heap_free(ptr); return a; -} \ No newline at end of file +} diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index 8dca712cd1..3efe344e6a 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -206,6 +206,7 @@ print_help() printf(" --enable-linux-perf Enable linux perf support\n"); #endif printf(" --mllvm=