From 583fa52226af336611bb74af8010c2e0664cc1bb Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Thu, 18 Jul 2024 21:15:12 +0900 Subject: [PATCH 01/58] spec-test-script: Add xtensa case (#3643) --- tests/wamr-test-suites/spec-test-script/all.py | 1 + tests/wamr-test-suites/spec-test-script/runtest.py | 3 +++ tests/wamr-test-suites/test_wamr.sh | 9 +++++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/wamr-test-suites/spec-test-script/all.py b/tests/wamr-test-suites/spec-test-script/all.py index 56ec16bee9..f1611c6fe5 100644 --- a/tests/wamr-test-suites/spec-test-script/all.py +++ b/tests/wamr-test-suites/spec-test-script/all.py @@ -66,6 +66,7 @@ def get_iwasm_cmd(platform: str) -> str: "RISCV64_LP64D", "THUMBV7", "THUMBV7_VFP", + "XTENSA", ] def ignore_the_case( diff --git a/tests/wamr-test-suites/spec-test-script/runtest.py b/tests/wamr-test-suites/spec-test-script/runtest.py index 10e25832a1..145e81dab5 100755 --- a/tests/wamr-test-suites/spec-test-script/runtest.py +++ b/tests/wamr-test-suites/spec-test-script/runtest.py @@ -58,6 +58,7 @@ "riscv64": ["--target=riscv64", "--target-abi=lp64", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c"], "riscv64_lp64f": ["--target=riscv64", "--target-abi=lp64f", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c,+f"], "riscv64_lp64d": ["--target=riscv64", "--target-abi=lp64d", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c,+f,+d"], + "xtensa": ["--target=xtensa"], } def debug(data): @@ -1185,6 +1186,8 @@ def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r): elif opts.target.startswith("riscv64"): cmd = "qemu-system-riscv64 -semihosting -M virt,aclint=on -cpu rv64 -smp 1 -nographic -bios none -kernel".split() cmd.append(opts.qemu_firmware) + elif opts.target.startswith("xtensa"): + cmd = f"qemu-system-xtensa -semihosting -nographic -serial mon:stdio -machine esp32s3 -drive file={opts.qemu_firmware},if=mtd,format=raw".split() else: raise Exception("Unknwon target for QEMU: %s" % opts.target) diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index c21da0a9fc..ee0dd7bbc0 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -80,7 +80,7 @@ WAMRC_CMD="" # prod/testsuite-all branch WASI_TESTSUITE_COMMIT="ee807fc551978490bf1c277059aabfa1e589a6c2" TARGET_LIST=("AARCH64" "AARCH64_VFP" "ARMV7" "ARMV7_VFP" "THUMBV7" "THUMBV7_VFP" \ - "RISCV32" "RISCV32_ILP32F" "RISCV32_ILP32D" "RISCV64" "RISCV64_LP64F" "RISCV64_LP64D") + "RISCV32" "RISCV32_ILP32F" "RISCV32_ILP32D" "RISCV64" "RISCV64_LP64F" "RISCV64_LP64D" "XTENSA") REQUIREMENT_NAME="" # Initialize an empty array for subrequirement IDs SUBREQUIREMENT_IDS=() @@ -792,9 +792,14 @@ function build_wamrc() return fi + BUILD_LLVM_SH=build_llvm.sh + if [ ${TARGET} = "XTENSA" ]; then + BUILD_LLVM_SH=build_llvm_xtensa.sh + fi + echo "Build wamrc for spec test under aot compile type" cd ${WAMR_DIR}/wamr-compiler \ - && ./build_llvm.sh \ + && ./${BUILD_LLVM_SH} \ && if [ -d build ]; then rm -r build/*; else mkdir build; fi \ && cd build \ && cmake .. -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE} \ From 0d8ffebd3952a52ca1f2a7ee163549202c3890ed Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Fri, 19 Jul 2024 11:45:55 +0800 Subject: [PATCH 02/58] spec-test-script/runtest.py: Move "--size-level=1" to common place for RISCV64 (#3644) --- tests/wamr-test-suites/spec-test-script/runtest.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/wamr-test-suites/spec-test-script/runtest.py b/tests/wamr-test-suites/spec-test-script/runtest.py index 145e81dab5..2713c49405 100755 --- a/tests/wamr-test-suites/spec-test-script/runtest.py +++ b/tests/wamr-test-suites/spec-test-script/runtest.py @@ -55,9 +55,10 @@ "riscv32": ["--target=riscv32", "--target-abi=ilp32", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c"], "riscv32_ilp32f": ["--target=riscv32", "--target-abi=ilp32f", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c,+f"], "riscv32_ilp32d": ["--target=riscv32", "--target-abi=ilp32d", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c,+f,+d"], - "riscv64": ["--target=riscv64", "--target-abi=lp64", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c"], - "riscv64_lp64f": ["--target=riscv64", "--target-abi=lp64f", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c,+f"], - "riscv64_lp64d": ["--target=riscv64", "--target-abi=lp64d", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c,+f,+d"], + # RISCV64 requires -mcmodel=medany, which can be set by --size-level=1 + "riscv64": ["--target=riscv64", "--target-abi=lp64", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c", "--size-level=1"], + "riscv64_lp64f": ["--target=riscv64", "--target-abi=lp64f", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c,+f", "--size-level=1"], + "riscv64_lp64d": ["--target=riscv64", "--target-abi=lp64d", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c,+f,+d", "--size-level=1"], "xtensa": ["--target=xtensa"], } @@ -1140,10 +1141,6 @@ def compile_wasm_to_aot(wasm_tempfile, aot_tempfile, runner, opts, r, output = ' if opts.qemu or opts.memory64: cmd.append("--bounds-checks=1") - # RISCV64 requires -mcmodel=medany, which can be set by --size-level=1 - if test_target.startswith("riscv64"): - cmd.append("--size-level=1") - cmd += ["-o", aot_tempfile, wasm_tempfile] log("Running: %s" % " ".join(cmd)) From b05fdfd3babd634a510573c9f08b851cb0de6d7e Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Mon, 22 Jul 2024 14:20:08 +0900 Subject: [PATCH 03/58] spec-test-script/runtest.py: Use a shorter timeout when expected to fail (#3647) This is a band-aid fix; ideally we should wait for the expected failure message directly with a timeout, not the successful prompt as we currently do. --- tests/wamr-test-suites/spec-test-script/runtest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/wamr-test-suites/spec-test-script/runtest.py b/tests/wamr-test-suites/spec-test-script/runtest.py index 2713c49405..8e53ddbc76 100755 --- a/tests/wamr-test-suites/spec-test-script/runtest.py +++ b/tests/wamr-test-suites/spec-test-script/runtest.py @@ -273,6 +273,8 @@ def assert_prompt(runner, prompts, timeout, is_need_execute_result): help="change to the directory before running tests") parser.add_argument('--start-timeout', default=30, type=int, help="default timeout for initial prompt") +parser.add_argument('--start-fail-timeout', default=2, type=int, + help="default timeout for initial prompt (when expected to fail)") parser.add_argument('--test-timeout', default=20, type=int, help="default timeout for each individual test action") parser.add_argument('--no-pty', action='store_true', @@ -1230,7 +1232,7 @@ def test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile, if test_aot: r = compile_wasm_to_aot(wasm_tempfile, aot_tempfile, True, opts, r) try: - assert_prompt(r, ['Compile success'], opts.start_timeout, True) + assert_prompt(r, ['Compile success'], opts.start_fail_timeout, True) except: _, exc, _ = sys.exc_info() if (r.buf.find(expected) >= 0): @@ -1251,7 +1253,7 @@ def test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile, if loadable: # Wait for the initial prompt try: - assert_prompt(r, ['webassembly> '], opts.start_timeout, True) + assert_prompt(r, ['webassembly> '], opts.start_fail_timeout, True) except: _, exc, _ = sys.exc_info() if (r.buf.find(expected) >= 0): @@ -1331,7 +1333,7 @@ def test_assert_with_exception(form, wast_tempfile, wasm_tempfile, aot_tempfile, if test_aot: r = compile_wasm_to_aot(wasm_tempfile, aot_tempfile, True, opts, r) try: - assert_prompt(r, ['Compile success'], opts.start_timeout, True) + assert_prompt(r, ['Compile success'], opts.start_fail_timeout, True) except: _, exc, _ = sys.exc_info() if (r.buf.find(error_msg) >= 0): From 50f28495a1704ac6a11b33bde89b7f333a1048a4 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Mon, 22 Jul 2024 14:30:00 +0900 Subject: [PATCH 04/58] Remove a few hardcoded spec test knowledge from the core library (#3648) Tweak the stack sizes in the spec test runner instead. --- core/iwasm/aot/aot_runtime.c | 11 +---------- core/iwasm/interpreter/wasm_runtime.c | 11 +---------- .../spec-test-script/runtest.py | 18 ++++++++++++------ 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index bfe691ea28..7e8799e05f 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1748,16 +1748,7 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, /* Initialize the thread related data */ if (stack_size == 0) stack_size = DEFAULT_WASM_STACK_SIZE; -#if WASM_ENABLE_SPEC_TEST != 0 -#if WASM_ENABLE_TAIL_CALL == 0 - if (stack_size < 128 * 1024) - stack_size = 128 * 1024; -#else - /* Some tail-call cases require large operand stack */ - if (stack_size < 10 * 1024 * 1024) - stack_size = 10 * 1024 * 1024; -#endif -#endif + module_inst->default_wasm_stack_size = stack_size; extra->stack_sizes = diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 0d4d0b37d4..f3b9b2e822 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -2990,16 +2990,7 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, /* Initialize the thread related data */ if (stack_size == 0) stack_size = DEFAULT_WASM_STACK_SIZE; -#if WASM_ENABLE_SPEC_TEST != 0 -#if WASM_ENABLE_TAIL_CALL == 0 - if (stack_size < 128 * 1024) - stack_size = 128 * 1024; -#else - /* Some tail-call cases require large operand stack */ - if (stack_size < 10 * 1024 * 1024) - stack_size = 10 * 1024 * 1024; -#endif -#endif + module_inst->default_wasm_stack_size = stack_size; if (module->malloc_function != (uint32)-1) { diff --git a/tests/wamr-test-suites/spec-test-script/runtest.py b/tests/wamr-test-suites/spec-test-script/runtest.py index 8e53ddbc76..b7595fb0b5 100755 --- a/tests/wamr-test-suites/spec-test-script/runtest.py +++ b/tests/wamr-test-suites/spec-test-script/runtest.py @@ -1161,13 +1161,19 @@ def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r): if opts.qemu: tmpfile = f"/tmp/{os.path.basename(tmpfile)}" - if opts.verbose: - cmd_iwasm = [opts.interpreter, "--heap-size=0", "-v=5", "--repl", tmpfile] - else: - cmd_iwasm = [opts.interpreter, "--heap-size=0", "--repl", tmpfile] - + cmd_iwasm = [opts.interpreter, "--heap-size=0", "--repl"] if opts.multi_module: - cmd_iwasm.insert(1, "--module-path=" + (tempfile.gettempdir() if not opts.qemu else "/tmp" )) + cmd_iwasm.append("--module-path=" + (tempfile.gettempdir() if not opts.qemu else "/tmp" )) + if opts.gc: + # our tail-call implementation is known broken. + # work it around by using a huge stack. + # cf. https://github.com/bytecodealliance/wasm-micro-runtime/issues/2231 + cmd_iwasm.append("--stack-size=10485760") # 10MB (!) + else: + cmd_iwasm.append("--stack-size=131072") # 128KB + if opts.verbose: + cmd_iwasm.append("-v=5") + cmd_iwasm.append(tmpfile) if opts.qemu: if opts.qemu_firmware == '': From 058bc47102ea0a97159ef7f9c4c1d471af6a8e9a Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Mon, 22 Jul 2024 17:16:41 +0800 Subject: [PATCH 05/58] [wasi-nn] Add a new wasi-nn backend openvino (#3603) --- .gitignore | 3 +- build-scripts/config_common.cmake | 13 + core/iwasm/libraries/wasi-nn/README.md | 15 +- .../libraries/wasi-nn/cmake/wasi_nn.cmake | 71 ++- .../libraries/wasi-nn/include/wasi_nn_types.h | 4 + .../wasi-nn/src/utils/wasi_nn_app_native.c | 2 +- core/iwasm/libraries/wasi-nn/src/wasi_nn.c | 70 +-- .../libraries/wasi-nn/src/wasi_nn_openvino.c | 559 ++++++++++++++++++ .../libraries/wasi-nn/src/wasi_nn_openvino.h | 36 ++ .../wasi-nn/test/Dockerfile.wasi-nn-smoke | 73 ++- .../wasi-nn/test/bump_wasi_nn_to_0_6_0.patch | 47 ++ .../libraries/wasi-nn/test/run_smoke_test.py | 289 +++++++-- 12 files changed, 1036 insertions(+), 146 deletions(-) create mode 100644 core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.c create mode 100644 core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.h create mode 100644 core/iwasm/libraries/wasi-nn/test/bump_wasi_nn_to_0_6_0.patch diff --git a/.gitignore b/.gitignore index b85dd392c8..8a8d9ac288 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,8 @@ .venv /.idea **/cmake-build-*/ -**/*build/ +**/*build*/ +!/build-scripts *.obj *.a *.so diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 28ace98350..8a1002235b 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -434,6 +434,19 @@ endif () if (WAMR_BUILD_WASI_NN EQUAL 1) message (" WASI-NN enabled") add_definitions (-DWASM_ENABLE_WASI_NN=1) + # Variant backends + if (NOT WAMR_BUILD_WASI_NN_TFLITE EQUAL 1 AND NOT WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1) + message (FATAL_ERROR " Need to select a backend for WASI-NN") + endif () + if (WAMR_BUILD_WASI_NN_TFLITE EQUAL 1) + message (" WASI-NN backend tflite enabled") + add_definitions (-DWASM_ENABLE_WASI_NN_TFLITE) + endif () + if (WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1) + message (" WASI-NN backend openvino enabled") + add_definitions (-DWASM_ENABLE_WASI_NN_OPENVINO) + endif () + # Variant devices if (WAMR_BUILD_WASI_NN_ENABLE_GPU EQUAL 1) message (" WASI-NN: GPU enabled") add_definitions (-DWASM_ENABLE_WASI_NN_GPU=1) diff --git a/core/iwasm/libraries/wasi-nn/README.md b/core/iwasm/libraries/wasi-nn/README.md index 7eb156ffc5..e0d3a25cee 100644 --- a/core/iwasm/libraries/wasi-nn/README.md +++ b/core/iwasm/libraries/wasi-nn/README.md @@ -19,6 +19,13 @@ $ cmake -DWAMR_BUILD_WASI_NN=1 ... > ![Caution] > If enable `WAMR_BUID_WASI_NN`, iwasm will link a shared WAMR library instead of a static one. Wasi-nn backends will be loaded dynamically at runtime. Users shall specify the path of the backend library and register it to the iwasm runtime with `--native-lib=`. All shared libraries should be placed in the `LD_LIBRARY_PATH`. +#### Compilation options + +- `WAMR_BUILD_WASI_NN`. enable wasi-nn support. can't work alone. need to identify a backend. Match legacy wasi-nn spec naming convention. use `wasi_nn` as import module names. +- `WAMR_BUILD_WASI_EPHEMERAL_NN`. Match latest wasi-nn spec naming convention. use `wasi_ephemeral_nn` as import module names. +- `WAMR_BUILD_WASI_NN_TFLITE`. identify the backend as TensorFlow Lite. +- `WAMR_BUILD_WASI_NN_OPENVINO`. identify the backend as OpenVINO. + ### Wasm The definition of functions provided by WASI-NN (Wasm imports) is in the header file [wasi_nn.h](_core/iwasm/libraries/wasi-nn/wasi_nn.h_). By only including this file in a WASM application you will bind WASI-NN into your module. @@ -37,6 +44,12 @@ typedef enum { fp16 = 0, fp32, up8, ip32 } tensor_type; It is required to recompile the Wasm application if you want to switch between the two sets of functions. +#### Openvino + +If you're planning to use OpenVINO backends, the first step is to install OpenVINO on your computer. To do this correctly, please follow the official installation guide which you can find at this link: https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-linux.html. + +After you've installed OpenVINO, you'll need to let cmake system know where to find it. You can do this by setting an environment variable named `OpenVINO_DIR`. This variable should point to the place on your computer where OpenVINO is installed. By setting this variable, your system will be able to locate and use OpenVINO when needed. You can find installation path by running the following command if using APT `$dpkg -L openvino`. The path should be _/opt/intel/openvino/_ or _/usr/lib/openvino_. + ## Tests To run the tests we assume that the current directory is the root of the repository. @@ -167,7 +180,7 @@ Due to the different requirements of each backend, we'll use a Docker container $ pwd /workspaces/wasm-micro-runtime/ -$ docker build -t wasi-nn-smoke:v1.0 -f Dockerfile.wasi-nn-smoke . +$ docker build -t wasi-nn-smoke:v1.0 -f ./core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke . ``` #### Execute diff --git a/core/iwasm/libraries/wasi-nn/cmake/wasi_nn.cmake b/core/iwasm/libraries/wasi-nn/cmake/wasi_nn.cmake index 172205790d..49286f2f4e 100644 --- a/core/iwasm/libraries/wasi-nn/cmake/wasi_nn.cmake +++ b/core/iwasm/libraries/wasi-nn/cmake/wasi_nn.cmake @@ -3,13 +3,30 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) -# Find tensorflow-lite -find_package(tensorflow_lite REQUIRED) +if(WAMR_BUILD_WASI_NN_TFLITE EQUAL 1) + # Find tensorflow-lite + find_package(tensorflow_lite REQUIRED) +endif() -set(WASI_NN_ROOT ${CMAKE_CURRENT_LIST_DIR}/..) +if(WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1) + if(NOT DEFINED ENV{OpenVINO_DIR}) + message(FATAL_ERROR + "OpenVINO_DIR is not defined. " + "Please follow https://docs.openvino.ai/2024/get-started/install-openvino.html," + "install openvino, and set environment variable OpenVINO_DIR." + "Like OpenVINO_DIR=/usr/lib/openvino-2023.2/ cmake ..." + "Or OpenVINO_DIR=/opt/intel/openvino/ cmake ..." + ) + endif() + + list(APPEND CMAKE_MODULE_PATH $ENV{OpenVINO_DIR}) + # Find OpenVINO + find_package(OpenVINO REQUIRED COMPONENTS Runtime) +endif() # # wasi-nn general +set(WASI_NN_ROOT ${CMAKE_CURRENT_LIST_DIR}/..) add_library( wasi-nn-general SHARED @@ -37,20 +54,34 @@ target_compile_definitions( # # wasi-nn backends -add_library( - wasi-nn-tflite - SHARED - ${WASI_NN_ROOT}/src/wasi_nn_tensorflowlite.cpp -) -#target_link_options( -# wasi-nn-tflite -# PRIVATE -# -Wl,--whole-archive libwasi-nn-general.a -# -Wl,--no-whole-archive -#) -target_link_libraries( - wasi-nn-tflite - PUBLIC - tensorflow-lite - wasi-nn-general -) + +# - tflite +if(WAMR_BUILD_WASI_NN_TFLITE EQUAL 1) + add_library( + wasi-nn-tflite + SHARED + ${WASI_NN_ROOT}/src/wasi_nn_tensorflowlite.cpp + ) + target_link_libraries( + wasi-nn-tflite + PUBLIC + tensorflow-lite + wasi-nn-general + ) +endif() + +# - openvino +if(WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1) + add_library( + wasi-nn-openvino + SHARED + ${WASI_NN_ROOT}/src/wasi_nn_openvino.c + ) + target_link_libraries( + wasi-nn-openvino + PUBLIC + openvino::runtime + openvino::runtime::c + wasi-nn-general + ) +endif() \ No newline at end of file diff --git a/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h b/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h index 75f14eb705..71c530ea59 100644 --- a/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h +++ b/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h @@ -164,6 +164,10 @@ typedef struct { bool wasi_nn_register_backend(api_function apis); +void +wasi_nn_dump_tensor_dimension(tensor_dimensions *dim, int32_t output_len, + char *output); + #ifdef __cplusplus } #endif diff --git a/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c b/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c index 07516f34d3..6e91c949bf 100644 --- a/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c +++ b/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c @@ -88,7 +88,7 @@ graph_builder_array_app_native(wasm_module_inst_t instance, } NN_DBG_PRINTF("Graph builder %d contains %d elements", i, - builder->size); + builder[i].size); } builder_array->buf = builder; diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn.c b/core/iwasm/libraries/wasi-nn/src/wasi_nn.c index de931b41b1..2d987d3b4a 100644 --- a/core/iwasm/libraries/wasi-nn/src/wasi_nn.c +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn.c @@ -26,15 +26,17 @@ // so, just keep one `api_function` is enough static api_function lookup = { 0 }; -#define call_wasi_nn_func(wasi_error, func, ...) \ - do { \ - if (lookup.func) { \ - wasi_error = lookup.func(__VA_ARGS__); \ - } \ - else { \ - NN_ERR_PRINTF("Error: %s is not registered", #func); \ - wasi_error = unsupported_operation; \ - } \ +#define call_wasi_nn_func(wasi_error, func, ...) \ + do { \ + if (lookup.func) { \ + wasi_error = lookup.func(__VA_ARGS__); \ + if (wasi_error != success) \ + NN_ERR_PRINTF("Error %s: %d", #func, wasi_error); \ + } \ + else { \ + NN_ERR_PRINTF("Error %s is not registered", #func); \ + wasi_error = unsupported_operation; \ + } \ } while (0) static HashMap *hashmap; @@ -68,7 +70,9 @@ key_equal_func(void *key1, void *key2) static void key_destroy_func(void *key1) -{} +{ + /* key type is wasm_module_inst_t*. do nothing */ +} static void value_destroy_func(void *value) @@ -79,7 +83,8 @@ value_destroy_func(void *value) static WASINNContext * wasi_nn_initialize_context() { - NN_DBG_PRINTF("Initializing wasi-nn context"); + NN_DBG_PRINTF("[WASI NN] INIT..."); + WASINNContext *wasi_nn_ctx = (WASINNContext *)wasm_runtime_malloc(sizeof(WASINNContext)); if (wasi_nn_ctx == NULL) { @@ -92,7 +97,6 @@ wasi_nn_initialize_context() wasi_nn_error res; call_wasi_nn_func(res, init, &wasi_nn_ctx->backend_ctx); if (res != success) { - NN_ERR_PRINTF("Error while initializing backend"); wasm_runtime_free(wasi_nn_ctx); return NULL; } @@ -103,7 +107,7 @@ wasi_nn_initialize_context() static bool wasi_nn_initialize() { - NN_DBG_PRINTF("Initializing wasi-nn"); + NN_DBG_PRINTF("[WASI NN General] Initializing wasi-nn"); // hashmap { instance: wasi_nn_ctx } hashmap = bh_hash_map_create(HASHMAP_INITIAL_SIZE, true, hash_func, key_equal_func, key_destroy_func, @@ -133,13 +137,15 @@ wasm_runtime_get_wasi_nn_ctx(wasm_module_inst_t instance) return NULL; } } - NN_DBG_PRINTF("Returning ctx"); + return wasi_nn_ctx; } static void wasi_nn_ctx_destroy(WASINNContext *wasi_nn_ctx) { + NN_DBG_PRINTF("[WASI NN] DEINIT..."); + if (wasi_nn_ctx == NULL) { NN_ERR_PRINTF( "Error when deallocating memory. WASI-NN context is NULL"); @@ -152,9 +158,6 @@ wasi_nn_ctx_destroy(WASINNContext *wasi_nn_ctx) /* only one backend can be registered */ wasi_nn_error res; call_wasi_nn_func(res, deinit, wasi_nn_ctx->backend_ctx); - if (res != success) { - NN_ERR_PRINTF("Error while destroyging backend"); - } wasm_runtime_free(wasi_nn_ctx); } @@ -191,7 +194,7 @@ wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder, graph_encoding encoding, execution_target target, graph *g) #endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ { - NN_DBG_PRINTF("Running wasi_nn_load [encoding=%d, target=%d]...", encoding, + NN_DBG_PRINTF("[WASI NN] LOAD [encoding=%d, target=%d]...", encoding, target); wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); @@ -222,7 +225,6 @@ wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder, WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); call_wasi_nn_func(res, load, wasi_nn_ctx->backend_ctx, &builder_native, encoding, target, g); - NN_DBG_PRINTF("wasi_nn_load finished with status %d [graph=%d]", res, *g); if (res != success) goto fail; @@ -241,7 +243,7 @@ wasi_nn_error wasi_nn_load_by_name(wasm_exec_env_t exec_env, char *name, uint32_t name_len, graph *g) { - NN_DBG_PRINTF("Running wasi_nn_load_by_name ..."); + NN_DBG_PRINTF("[WASI NN] LOAD_BY_NAME %s...", name); wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); if (!instance) { @@ -261,7 +263,6 @@ wasi_nn_load_by_name(wasm_exec_env_t exec_env, char *name, uint32_t name_len, wasi_nn_error res; call_wasi_nn_func(res, load_by_name, wasi_nn_ctx->backend_ctx, name, name_len, g); - NN_DBG_PRINTF("wasi_nn_load_by_name finished with status %d", *g); if (res != success) return res; @@ -274,7 +275,7 @@ wasi_nn_error wasi_nn_init_execution_context(wasm_exec_env_t exec_env, graph g, graph_execution_context *ctx) { - NN_DBG_PRINTF("Running wasi_nn_init_execution_context [graph=%d]...", g); + NN_DBG_PRINTF("[WASI NN] INIT_EXECUTION_CONTEXT..."); wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); if (!instance) { @@ -295,9 +296,6 @@ wasi_nn_init_execution_context(wasm_exec_env_t exec_env, graph g, call_wasi_nn_func(res, init_execution_context, wasi_nn_ctx->backend_ctx, g, ctx); - NN_DBG_PRINTF( - "wasi_nn_init_execution_context finished with status %d [ctx=%d]", res, - *ctx); return res; } @@ -305,8 +303,7 @@ wasi_nn_error wasi_nn_set_input(wasm_exec_env_t exec_env, graph_execution_context ctx, uint32_t index, tensor_wasm *input_tensor) { - NN_DBG_PRINTF("Running wasi_nn_set_input [ctx=%d, index=%d]...", ctx, - index); + NN_DBG_PRINTF("[WASI NN] SET_INPUT [ctx=%d, index=%d]...", ctx, index); wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); if (!instance) { @@ -331,14 +328,13 @@ wasi_nn_set_input(wasm_exec_env_t exec_env, graph_execution_context ctx, if (input_tensor_native.dimensions) wasm_runtime_free(input_tensor_native.dimensions); - NN_DBG_PRINTF("wasi_nn_set_input finished with status %d", res); return res; } wasi_nn_error wasi_nn_compute(wasm_exec_env_t exec_env, graph_execution_context ctx) { - NN_DBG_PRINTF("Running wasi_nn_compute [ctx=%d]...", ctx); + NN_DBG_PRINTF("[WASI NN] COMPUTE [ctx=%d]...", ctx); wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); if (!instance) { @@ -352,7 +348,6 @@ wasi_nn_compute(wasm_exec_env_t exec_env, graph_execution_context ctx) return res; call_wasi_nn_func(res, compute, wasi_nn_ctx->backend_ctx, ctx); - NN_DBG_PRINTF("wasi_nn_compute finished with status %d", res); return res; } @@ -368,8 +363,7 @@ wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx, uint32_t *output_tensor_size) #endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ { - NN_DBG_PRINTF("Running wasi_nn_get_output [ctx=%d, index=%d]...", ctx, - index); + NN_DBG_PRINTF("[WASI NN] GET_OUTPUT [ctx=%d, index=%d]...", ctx, index); wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); if (!instance) { @@ -396,8 +390,6 @@ wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx, call_wasi_nn_func(res, get_output, wasi_nn_ctx->backend_ctx, ctx, index, output_tensor, output_tensor_size); #endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ - NN_DBG_PRINTF("wasi_nn_get_output finished with status %d [data_size=%d]", - res, *output_tensor_size); return res; } @@ -435,7 +427,7 @@ get_wasi_nn_export_apis(NativeSymbol **p_native_symbols) __attribute__((used)) uint32_t get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols) { - NN_DBG_PRINTF("--|> get_native_lib"); + NN_DBG_PRINTF("[Native Register] get_native_lib"); #if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 *p_module_name = "wasi_ephemeral_nn"; @@ -449,7 +441,7 @@ get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols) __attribute__((used)) int init_native_lib() { - NN_DBG_PRINTF("--|> init_native_lib"); + NN_DBG_PRINTF("[Native Register] init_native_lib"); if (!wasi_nn_initialize()) return 1; @@ -460,7 +452,7 @@ init_native_lib() __attribute__((used)) void deinit_native_lib() { - NN_DBG_PRINTF("--|> deinit_native_lib"); + NN_DBG_PRINTF("[Native Register] deinit_native_lib"); wasi_nn_destroy(); } @@ -468,7 +460,7 @@ deinit_native_lib() __attribute__((used)) bool wasi_nn_register_backend(api_function apis) { - NN_DBG_PRINTF("--|> wasi_nn_register_backend"); + NN_DBG_PRINTF("[Native Register] wasi_nn_register_backend"); lookup = apis; return true; -} \ No newline at end of file +} diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.c b/core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.c new file mode 100644 index 0000000000..678673e8ba --- /dev/null +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.c @@ -0,0 +1,559 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasi_nn_types.h" +#include "wasi_nn_openvino.h" +#include "logger.h" +#include "bh_platform.h" + +#include "openvino/c/openvino.h" + +/* + * refer to + * https://docs.openvino.ai/2024/openvino-workflow/running-inference/integrate-openvino-with-your-application.html + * + * Steps about integrating OpenVINO are: + * + * 1. Create OpenVINO Runtime Core + * 2. Compile Model + * 3. Create Inference Request + * 4. Set Inputs + * 5. Start Inference + * 6. Process Inference Results + * + * from 4. to 6. is the Inference Loop + */ + +typedef struct { + ov_core_t *core; + /* keep input model files */ + void *weight_data; + ov_tensor_t *weights_tensor; + ov_model_t *model; + /* add prepostprocess */ + ov_model_t *new_model; + ov_compiled_model_t *compiled_model; + ov_infer_request_t *infer_request; + ov_tensor_t *input_tensor; +} OpenVINOContext; + +/* + * BE AWARE OF "goto fail" + */ +#define CHECK_OV_STATUS(status, error_code) \ + do { \ + ov_status_e s = status; \ + if (s != OK) { \ + NN_ERR_PRINTF("return status \"%s\", line %d", \ + ov_get_error_info(s), __LINE__); \ + error_code = runtime_error; \ + goto fail; \ + } \ + } while (0) + +static void +dump_ov_shape_t(const ov_shape_t *shape, int32_t output_len, char *output) +{ + int ret = 0; + + ret = snprintf(output, output_len, "%ld,[", shape->rank); + if (!ret) + return; + + output_len -= ret; + output += ret; + + for (unsigned i = 0; i < shape->rank && output_len; i++) { + ret = snprintf(output, output_len, " %ld", shape->dims[i]); + if (!ret) + return; + + output_len -= ret; + output += ret; + } + + snprintf(output, output_len, "]"); + return; +} + +#ifndef NDEBUG +static void +print_model_input_output_info(ov_model_t *model) +{ + wasi_nn_error ov_error; + char *friendly_name = NULL; + size_t input_size = 0; + ov_output_const_port_t *input_port = NULL; + ov_shape_t input_shape = { 0 }; + ov_element_type_e input_type; + char shape_info[64] = { 0 }; + ov_output_const_port_t *output_port = NULL; + ov_shape_t output_shape = { 0 }; + ov_element_type_e output_type; + + CHECK_OV_STATUS(ov_model_get_friendly_name(model, &friendly_name), + ov_error); + NN_DBG_PRINTF("model name: %s", friendly_name); + + ov_model_inputs_size(model, &input_size); + for (unsigned i = 0; i < input_size; i++) { + CHECK_OV_STATUS(ov_model_const_input_by_index(model, i, &input_port), + ov_error); + CHECK_OV_STATUS(ov_const_port_get_shape(input_port, &input_shape), + ov_error); + CHECK_OV_STATUS(ov_port_get_element_type(input_port, &input_type), + ov_error); + + dump_ov_shape_t(&input_shape, 60, shape_info); + NN_DBG_PRINTF("model input[%u]. element_type: %d, shape: %s", i, + input_type, shape_info); + + ov_shape_free(&input_shape); + memset(&input_shape, 0, sizeof(input_shape)); + ov_output_const_port_free(input_port); + input_port = NULL; + } + + size_t output_size = 0; + ov_model_outputs_size(model, &output_size); + for (unsigned i = 0; i < output_size; i++) { + CHECK_OV_STATUS(ov_model_const_output_by_index(model, i, &output_port), + ov_error); + CHECK_OV_STATUS(ov_const_port_get_shape(output_port, &output_shape), + ov_error); + CHECK_OV_STATUS(ov_port_get_element_type(output_port, &output_type), + ov_error); + + dump_ov_shape_t(&output_shape, 60, shape_info); + NN_DBG_PRINTF("model output[%u]. element_type: %d, shape: %s", i, + output_type, shape_info); + + ov_shape_free(&output_shape); + memset(&output_shape, 0, sizeof(output_shape)); + ov_output_const_port_free(output_port); + output_port = NULL; + } + +fail: + if (friendly_name) + ov_free(friendly_name); + ov_shape_free(&input_shape); + if (input_port) + ov_output_const_port_free(input_port); + ov_shape_free(&output_shape); + if (output_port) + ov_output_const_port_free(output_port); + return; +} +#endif + +static ov_element_type_e +wasi_nn_tensor_type_to_openvino_element_type(tensor_type wasi_nn_type) +{ + switch (wasi_nn_type) { + case fp16: + return F16; + case fp32: + return F32; + case fp64: + return F64; + case bf16: + return BF16; + case u8: + return U8; + case i32: + return I32; + case i64: + return I64; + default: + break; + } + + NN_ERR_PRINTF("%d is an undefined tensor type", wasi_nn_type); + return UNDEFINED; +} + +static wasi_nn_error +uint32_array_to_int64_array(uint32_t array_size, uint32_t *src, int64_t **dst) +{ + *dst = malloc(array_size * sizeof(int64_t)); + if (!(*dst)) + return runtime_error; + + for (unsigned i = 0; i < array_size; i++) { + (*dst)[i] = src[i]; + } + + return success; +} + +wasi_nn_error +openvino_load(void *ctx, graph_builder_array *builder, graph_encoding encoding, + execution_target target, graph *g) +{ + OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + wasi_nn_error ret = unsupported_operation; + + if (encoding != openvino) { + NN_ERR_PRINTF("Unexpected encoding %d.", encoding); + return invalid_argument; + } + + /*FIXME: unblock non-cpu device after supporting */ + if (target != cpu) { + NN_ERR_PRINTF("Unexpected device %d.", target); + return invalid_argument; + } + + if (builder->size != 2) { + NN_ERR_PRINTF("Unexpected builder format."); + return invalid_argument; + } + + /* + * The first builder is the XML file. + * The second builder is the weight file. + */ + graph_builder xml = builder->buf[0]; + graph_builder weight = builder->buf[1]; + + /* if xml is a String with a model in IR */ + if (!(xml.buf[xml.size] == '\0' && xml.buf[xml.size - 1] != '\0')) { + NN_ERR_PRINTF("Invalid xml string."); + return invalid_argument; + } + + /* transfer weight to an ov tensor */ + { + ov_ctx->weight_data = malloc(weight.size); + if (!ov_ctx->weight_data) + goto fail; + memcpy(ov_ctx->weight_data, weight.buf, weight.size); + + ov_element_type_e type = U8; + int64_t dims[1] = { weight.size }; + ov_shape_t shape = { 1, dims }; + CHECK_OV_STATUS(ov_tensor_create_from_host_ptr(type, shape, + ov_ctx->weight_data, + &ov_ctx->weights_tensor), + ret); + } + + /* load model from buffer */ + CHECK_OV_STATUS(ov_core_read_model_from_memory_buffer( + ov_ctx->core, (char *)xml.buf, xml.size, + ov_ctx->weights_tensor, &ov_ctx->model), + ret); +#ifndef NDEBUG + print_model_input_output_info(ov_ctx->model); +#endif + + ret = success; +fail: + return ret; +} + +wasi_nn_error +openvino_load_by_name(void *ctx, const char *filename, uint32_t filename_len, + graph *g) +{ + OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + wasi_nn_error ret = unsupported_operation; + + CHECK_OV_STATUS( + ov_core_read_model(ov_ctx->core, filename, NULL, &ov_ctx->model), ret); + + ret = success; +fail: + return ret; +} + +wasi_nn_error +openvino_init_execution_context(void *ctx, graph g, + graph_execution_context *exec_ctx) +{ + return success; +} + +wasi_nn_error +openvino_set_input(void *ctx, graph_execution_context exec_ctx, uint32_t index, + tensor *wasi_nn_tensor) +{ + OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + wasi_nn_error ret = unsupported_operation; + ov_shape_t input_shape = { 0 }; + int64_t *ov_dims = NULL; + + ov_preprocess_prepostprocessor_t *ppp = NULL; + ov_preprocess_input_info_t *input_info = NULL; + ov_preprocess_input_tensor_info_t *input_tensor_info = NULL; + ov_layout_t *input_layout = NULL; + ov_preprocess_preprocess_steps_t *input_process = NULL; + ov_preprocess_input_model_info_t *p_input_model = NULL; + ov_layout_t *model_layout = NULL; + ov_preprocess_output_info_t *output_info = NULL; + ov_preprocess_output_tensor_info_t *output_tensor_info = NULL; + + /* wasi_nn_tensor -> ov_tensor */ + { + ret = uint32_array_to_int64_array(wasi_nn_tensor->dimensions->size, + wasi_nn_tensor->dimensions->buf, + &ov_dims); + if (ret != success) + goto fail; + + /* NCHW -> NHWC */ + if (wasi_nn_tensor->dimensions->size == 4 || ov_dims[1] == 3) { + /* N */ + /* H */ + ov_dims[1] = ov_dims[2]; + /* W */ + ov_dims[2] = ov_dims[3]; + /* C */ + ov_dims[3] = 3; + } + + CHECK_OV_STATUS(ov_shape_create(wasi_nn_tensor->dimensions->size, + ov_dims, &input_shape), + ret); + + ov_element_type_e input_type = + wasi_nn_tensor_type_to_openvino_element_type(wasi_nn_tensor->type); + if (input_type == UNDEFINED) + goto fail; + + char shape_info[64] = { 0 }; + dump_ov_shape_t(&input_shape, 60, shape_info); + NN_DBG_PRINTF("input tensor. element_type: %d, shape: %s", input_type, + shape_info); + + CHECK_OV_STATUS(ov_tensor_create_from_host_ptr(input_type, input_shape, + wasi_nn_tensor->data, + &ov_ctx->input_tensor), + ret); + } + + /* set preprocess based on wasi_nn_tensor */ + { + CHECK_OV_STATUS( + ov_preprocess_prepostprocessor_create(ov_ctx->model, &ppp), ret); + + /* reuse user' created tensor's info */ + CHECK_OV_STATUS(ov_preprocess_prepostprocessor_get_input_info_by_index( + ppp, index, &input_info), + ret); + CHECK_OV_STATUS(ov_preprocess_input_info_get_tensor_info( + input_info, &input_tensor_info), + ret); + CHECK_OV_STATUS(ov_preprocess_input_tensor_info_set_from( + input_tensor_info, ov_ctx->input_tensor), + ret); + /* ! HAS TO BE NHWC. Match previous layout conversion */ + CHECK_OV_STATUS(ov_layout_create("NHWC", &input_layout), ret); + CHECK_OV_STATUS(ov_preprocess_input_tensor_info_set_layout( + input_tensor_info, input_layout), + ret); + + /* add RESIZE */ + CHECK_OV_STATUS(ov_preprocess_input_info_get_preprocess_steps( + input_info, &input_process), + ret); + CHECK_OV_STATUS( + ov_preprocess_preprocess_steps_resize(input_process, RESIZE_LINEAR), + ret); + + /* input model */ + CHECK_OV_STATUS( + ov_preprocess_input_info_get_model_info(input_info, &p_input_model), + ret); + // TODO: what if not? + CHECK_OV_STATUS(ov_layout_create("NCHW", &model_layout), ret); + CHECK_OV_STATUS(ov_preprocess_input_model_info_set_layout(p_input_model, + model_layout), + ret); + + /* output -> F32(possibility) */ + CHECK_OV_STATUS(ov_preprocess_prepostprocessor_get_output_info_by_index( + ppp, index, &output_info), + ret); + CHECK_OV_STATUS(ov_preprocess_output_info_get_tensor_info( + output_info, &output_tensor_info), + ret); + CHECK_OV_STATUS( + ov_preprocess_output_set_element_type(output_tensor_info, F32), + ret); + + CHECK_OV_STATUS( + ov_preprocess_prepostprocessor_build(ppp, &ov_ctx->new_model), ret); + } + + CHECK_OV_STATUS(ov_core_compile_model(ov_ctx->core, ov_ctx->new_model, + "CPU", 0, &ov_ctx->compiled_model), + ret); + + CHECK_OV_STATUS(ov_compiled_model_create_infer_request( + ov_ctx->compiled_model, &ov_ctx->infer_request), + ret); + + /* install ov_tensor -> infer_request */ + CHECK_OV_STATUS(ov_infer_request_set_input_tensor_by_index( + ov_ctx->infer_request, index, ov_ctx->input_tensor), + ret); + ret = success; + +fail: + if (ov_dims) + free(ov_dims); + ov_shape_free(&input_shape); + if (ppp) + ov_preprocess_prepostprocessor_free(ppp); + if (input_info) + ov_preprocess_input_info_free(input_info); + if (input_tensor_info) + ov_preprocess_input_tensor_info_free(input_tensor_info); + if (input_layout) + ov_layout_free(input_layout); + if (input_process) + ov_preprocess_preprocess_steps_free(input_process); + if (p_input_model) + ov_preprocess_input_model_info_free(p_input_model); + if (model_layout) + ov_layout_free(model_layout); + if (output_info) + ov_preprocess_output_info_free(output_info); + if (output_tensor_info) + ov_preprocess_output_tensor_info_free(output_tensor_info); + + return ret; +} + +wasi_nn_error +openvino_compute(void *ctx, graph_execution_context exec_ctx) +{ + OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + wasi_nn_error ret = unsupported_operation; + + CHECK_OV_STATUS(ov_infer_request_infer(ov_ctx->infer_request), ret); + ret = success; +fail: + return ret; +} + +wasi_nn_error +openvino_get_output(void *ctx, graph_execution_context exec_ctx, uint32_t index, + tensor_data output_tensor, uint32_t *output_tensor_size) +{ + OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + wasi_nn_error ret = unsupported_operation; + ov_tensor_t *ov_tensor = NULL; + void *data = NULL; + size_t byte_size = 0; + + CHECK_OV_STATUS(ov_infer_request_get_output_tensor_by_index( + ov_ctx->infer_request, index, &ov_tensor), + ret); + + CHECK_OV_STATUS(ov_tensor_get_byte_size(ov_tensor, &byte_size), ret); + + CHECK_OV_STATUS(ov_tensor_data(ov_tensor, &data), ret); + + memcpy(output_tensor, data, byte_size); + + *output_tensor_size = (uint32_t)byte_size; + + ret = success; + +fail: + if (ov_tensor) + ov_tensor_free(ov_tensor); + return ret; +} + +wasi_nn_error +openvino_initialize(void **ctx) +{ + ov_version_t version; + OpenVINOContext *ov_ctx = NULL; + wasi_nn_error ret = unsupported_operation; + + if (!ctx) { + ret = invalid_argument; + goto fail; + } + + /* Get OpenVINO runtime version */ + CHECK_OV_STATUS(ov_get_openvino_version(&version), ret); + NN_INFO_PRINTF("OpenVINO INFO:"); + NN_INFO_PRINTF(" Description : %s", version.description); + NN_INFO_PRINTF(" Build Number: %s", version.buildNumber); + ov_version_free(&version); + + ov_ctx = (OpenVINOContext *)os_malloc(sizeof(OpenVINOContext)); + if (!ov_ctx) { + NN_ERR_PRINTF("Allocate for OpenVINOContext failed"); + ret = runtime_error; + goto fail; + } + + memset(ov_ctx, 0, sizeof(OpenVINOContext)); + + /* Initialize OpenVINO Runtime Core */ + CHECK_OV_STATUS(ov_core_create(&ov_ctx->core), ret); + + *ctx = (void *)ov_ctx; + return success; +fail: + openvino_destroy((void *)ov_ctx); + return ret; +} + +wasi_nn_error +openvino_destroy(void *ctx) +{ + OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + + if (!ov_ctx) + return invalid_argument; + + if (ov_ctx->weight_data) + free(ov_ctx->weight_data); + + if (ov_ctx->weights_tensor) + ov_tensor_free(ov_ctx->weights_tensor); + + if (ov_ctx->input_tensor) + ov_tensor_free(ov_ctx->input_tensor); + + if (ov_ctx->infer_request) + ov_infer_request_free(ov_ctx->infer_request); + + if (ov_ctx->compiled_model) + ov_compiled_model_free(ov_ctx->compiled_model); + + if (ov_ctx->model) + ov_model_free(ov_ctx->model); + + if (ov_ctx->core) + ov_core_free(ov_ctx->core); + + os_free(ov_ctx); + return success; +} + +__attribute__((constructor(200))) void +openvino_register_backend() +{ + api_function apis = { + .load = openvino_load, + .load_by_name = openvino_load_by_name, + .init_execution_context = openvino_init_execution_context, + .set_input = openvino_set_input, + .compute = openvino_compute, + .get_output = openvino_get_output, + .init = openvino_initialize, + .deinit = openvino_destroy, + }; + wasi_nn_register_backend(apis); +} \ No newline at end of file diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.h b/core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.h new file mode 100644 index 0000000000..798c69a0d1 --- /dev/null +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef WASI_NN_OPENVINO_HPP +#define WASI_NN_OPENVINO_HPP + +#include "wasi_nn_types.h" + +wasi_nn_error +openvino_load(void *ctx, graph_builder_array *builder, graph_encoding encoding, + execution_target target, graph *g); + +wasi_nn_error +openvino_init_execution_context(void *ctx, graph g, + graph_execution_context *exec_ctx); + +wasi_nn_error +openvino_set_input(void *ctx, graph_execution_context exec_ctx, uint32_t index, + tensor *input_tensor); + +wasi_nn_error +openvino_compute(void *ctx, graph_execution_context exec_ctx); + +wasi_nn_error +openvino_get_output(void *ctx, graph_execution_context exec_ctx, uint32_t index, + tensor_data output_tensor, uint32_t *output_tensor_size); + +wasi_nn_error +openvino_initialize(void **ctx); + +wasi_nn_error +openvino_destroy(void *ctx); + +#endif /* WASI_NN_OPENVINO_HPP */ \ No newline at end of file diff --git a/core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke b/core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke index 997080c903..4378ac3953 100644 --- a/core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke +++ b/core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke @@ -1,6 +1,8 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# hadolint global ignore=DL3003,DL3008,DL3009,DL3059 + FROM mcr.microsoft.com/devcontainers/rust:1-1-bullseye@sha256:ddc1ee022d327f024c07484c9333db3fbbfd504bc096cdb66635653a2bebb33e ARG DEBIAN_FRONTEND=noninteractive @@ -8,7 +10,10 @@ ENV TZ=Asian/Shanghai # hadolint ignore=DL3009 RUN apt-get update \ - && apt-get upgrade -y + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends cmake + +RUN rustup target add wasm32-wasi # # Openvino @@ -17,33 +22,27 @@ RUN apt-get update \ # - https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_from_archive_linux.html # - https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-linux.html # -# FIXME: upgrade to 2024.1 or latest after wasi-nn(rust binding) is ready -WORKDIR /opt/intel -RUN wget -q https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3.2/linux/l_openvino_toolkit_ubuntu20_2022.3.2.9279.e2c7e4d7b4d_x86_64.tgz -RUN tar -xf l_openvino_toolkit_ubuntu20_2022.3.2.9279.e2c7e4d7b4d_x86_64.tgz \ - && rm l_openvino_toolkit_ubuntu20_2022.3.2.9279.e2c7e4d7b4d_x86_64.tgz \ - && mv l_openvino_toolkit_ubuntu20_2022.3.2.9279.e2c7e4d7b4d_x86_64 /opt/intel/openvino - -WORKDIR /opt/intel/openvino -RUN ./install_dependencies/install_openvino_dependencies.sh -y \ - && ./setupvars.sh - -# -# wasmtime -WORKDIR /opt -RUN wget -q https://github.com/bytecodealliance/wasmtime/releases/download/v21.0.0/wasmtime-v21.0.0-x86_64-linux.tar.xz -RUN tar -xf wasmtime-v21.0.0-x86_64-linux.tar.xz \ - && rm wasmtime-v21.0.0-x86_64-linux.tar.xz \ - && ln -sf "$(realpath ./wasmtime-v21.0.0-x86_64-linux/wasmtime)" /usr/local/bin/wasmtime +RUN wget -q https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ + && echo "deb https://apt.repos.intel.com/openvino/2023 ubuntu20 main" | tee /etc/apt/sources.list.d/intel-openvino-2023.list +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get install --no-install-recommends -y openvino-2023.2.0 # # wasi-nn # compilation requirements -RUN rustup target add wasm32-wasi wasm32-unknown-unknown WORKDIR /workspaces/wasi-nn RUN git clone --depth 1 https://github.com/bytecodealliance/wasi-nn.git . -# hadolint ignore=DL3059 -#RUN ./build.sh rust + +WORKDIR /workspaces/wasi-nn/rust/examples/classification-example/ +RUN cargo build --target=wasm32-wasi + +WORKDIR /workspaces/wasi-nn/rust/examples/classification-example/build +RUN cp ../target/wasm32-wasi/debug/wasi-nn-example.wasm . \ + && wget -q --no-clobber https://github.com/intel/openvino-rs/raw/main/crates/openvino/tests/fixtures/mobilenet/mobilenet.xml \ + && wget -q --no-clobber https://github.com/intel/openvino-rs/raw/main/crates/openvino/tests/fixtures/mobilenet/mobilenet.bin # There are model files(mobilenet*) and wasm files(wasi-nn-example.wasm) in the directory, # /workspaces/wasi-nn/rust/examples/classification-example/build @@ -52,14 +51,35 @@ RUN git clone --depth 1 https://github.com/bytecodealliance/wasi-nn.git . WORKDIR /tmp RUN wget -q https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh \ && chmod a+x ./install.sh -RUN ./install.sh -p /opt/wasmedge --plugins wasi_nn-tensorflowlite +# RUN ./install.sh -p /opt/wasmedge --plugins wasi_nn-tensorflowlite wasi_nn-openvino +RUN ./install.sh -r yes -D -p /opt/wasmedge --plugins wasi_nn-openvino --dist ubuntu20.04 \ + && /opt/wasmedge/bin/wasmedge --version ENV PATH=/opt/wasmedge/bin:${PATH} -ENV WASMEDGE_LIB_DIR=/opt/wasmedge/lib +# ENV WASMEDGE_LIB_DIR=/opt/wasmedge/lib # # wasmedge-wasinn-examples WORKDIR /workspaces/wasmedge-wasinn-examples RUN git clone --depth 1 https://github.com/second-state/WasmEdge-WASINN-examples.git . +COPY core/iwasm/libraries/wasi-nn/test/bump_wasi_nn_to_0_6_0.patch . +RUN git apply ./bump_wasi_nn_to_0_6_0.patch +# recompile with wasi-nn 0.6.0 +RUN cd openvino-mobilenet-image/rust && cargo build --target=wasm32-wasi +RUN cd openvino-mobilenet-raw/rust && cargo build --target=wasm32-wasi +RUN cd openvino-road-segmentation-adas/openvino-road-seg-adas && cargo build --target=wasm32-wasi +RUN cd tflite-birds_v1-image/rust && cargo build --target=wasm32-wasi + +# preparation +RUN cd openvino-mobilenet-image \ + && ./download_mobilenet.sh . \ + && ls -l mobilenet.xml mobilenet.bin + +RUN cd openvino-mobilenet-raw \ + && ./download_mobilenet.sh . \ + && ls -l mobilenet.xml mobilenet.bin tensor-1x224x224x3-f32.bgr + +# RUN apt update \ +# && apt install -y valgrind # # iwasm. build from source @@ -67,7 +87,10 @@ WORKDIR /workspaces/wamr COPY . . WORKDIR /workspaces/wamr/product-mini/platforms/linux -RUN cmake -S . -B build -DWAMR_BUILD_WASI_NN=1 -DWAMR_BUILD_WASI_EPHEMERAL_NN=1 \ +RUN OpenVINO_DIR=/usr/lib/openvino-2023.2.0 \ + cmake -S . -B build \ + -DWAMR_BUILD_WASI_NN=1 -DWAMR_BUILD_WASI_EPHEMERAL_NN=1 \ + -DWAMR_BUILD_WASI_NN_OPENVINO=1 -DWAMR_BUILD_WASI_NN_TFLITE=1 \ && cmake --build build RUN ln -sf "$(realpath ./build/iwasm)" /usr/local/bin/iwasm diff --git a/core/iwasm/libraries/wasi-nn/test/bump_wasi_nn_to_0_6_0.patch b/core/iwasm/libraries/wasi-nn/test/bump_wasi_nn_to_0_6_0.patch new file mode 100644 index 0000000000..46e152b278 --- /dev/null +++ b/core/iwasm/libraries/wasi-nn/test/bump_wasi_nn_to_0_6_0.patch @@ -0,0 +1,47 @@ +diff --git a/openvino-mobilenet-image/rust/Cargo.toml b/openvino-mobilenet-image/rust/Cargo.toml +index d09e0a4..c7083fb 100644 +--- a/openvino-mobilenet-image/rust/Cargo.toml ++++ b/openvino-mobilenet-image/rust/Cargo.toml +@@ -8,6 +8,6 @@ publish = false + + [dependencies] + image = { version = "0.23.14", default-features = false, features = ["gif", "jpeg", "ico", "png", "pnm", "tga", "tiff", "webp", "bmp", "hdr", "dxt", "dds", "farbfeld"] } +-wasi-nn = { version = "0.4.0" } ++wasi-nn = { version = "0.6.0" } + + [workspace] +diff --git a/openvino-mobilenet-raw/rust/Cargo.toml b/openvino-mobilenet-raw/rust/Cargo.toml +index 8eab25b..3f00aec 100644 +--- a/openvino-mobilenet-raw/rust/Cargo.toml ++++ b/openvino-mobilenet-raw/rust/Cargo.toml +@@ -7,6 +7,6 @@ edition = "2021" + publish = false + + [dependencies] +-wasi-nn = { version = "0.4.0" } ++wasi-nn = { version = "0.6.0" } + + [workspace] +diff --git a/openvino-road-segmentation-adas/openvino-road-seg-adas/Cargo.toml b/openvino-road-segmentation-adas/openvino-road-seg-adas/Cargo.toml +index 998f391..93f91e0 100644 +--- a/openvino-road-segmentation-adas/openvino-road-seg-adas/Cargo.toml ++++ b/openvino-road-segmentation-adas/openvino-road-seg-adas/Cargo.toml +@@ -5,5 +5,5 @@ name = "openvino-road-seg-adas" + version = "0.2.0" + + [dependencies] +-wasi-nn = "0.4.0" ++wasi-nn = "0.6.0" + image = { version = "0.23.14", default-features = false, features = ["gif", "jpeg", "ico", "png", "pnm", "tga", "tiff", "webp", "bmp", "hdr", "dxt", "dds", "farbfeld"] } +diff --git a/tflite-birds_v1-image/rust/Cargo.toml b/tflite-birds_v1-image/rust/Cargo.toml +index 572ecb9..9e89e87 100644 +--- a/tflite-birds_v1-image/rust/Cargo.toml ++++ b/tflite-birds_v1-image/rust/Cargo.toml +@@ -8,6 +8,6 @@ publish = false + + [dependencies] + image = { version = "0.23.14", default-features = false, features = ["gif", "jpeg", "ico", "png", "pnm", "tga", "tiff", "webp", "bmp", "hdr", "dxt", "dds", "farbfeld"] } +-wasi-nn = "0.4.0" ++wasi-nn = "0.6.0" + + [workspace] diff --git a/core/iwasm/libraries/wasi-nn/test/run_smoke_test.py b/core/iwasm/libraries/wasi-nn/test/run_smoke_test.py index 09e775be8f..08b19624a4 100644 --- a/core/iwasm/libraries/wasi-nn/test/run_smoke_test.py +++ b/core/iwasm/libraries/wasi-nn/test/run_smoke_test.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # +from dataclasses import dataclass from pathlib import Path from pprint import pprint import re @@ -13,72 +14,151 @@ from typing import List +@dataclass +class WasmEdgeExampleResult: + class_id: int + possibility: float + + +def execute_once( + runtime_bin: str, + runtime_args: List[str], + wasm_file: str, + wasm_args: List[str], + cwd: Path, +) -> str: + cmd = [runtime_bin] + cmd.extend(runtime_args) + cmd.append(wasm_file) + cmd.extend(wasm_args) + + # print(f'Execute: {" ".join(cmd)}') + + p = subprocess.run( + cmd, + cwd=cwd, + capture_output=True, + check=True, + text=True, + universal_newlines=True, + ) + return p.stdout + + +def execute_openvino_road_segmentation_adas_once( + runtime_bin: str, runtime_args: List[str], cwd: Path +) -> str: + """ + execute openvino-road-segmentation-adas with iwasm and wasmedge + """ + + wasm_file = ( + "./openvino-road-seg-adas/target/wasm32-wasi/debug/openvino-road-seg-adas.wasm" + ) + wasm_args = [ + "./model/road-segmentation-adas-0001.xml", + "./model/road-segmentation-adas-0001.bin", + "./image/empty_road_mapillary.jpg", + ] + return execute_once(runtime_bin, runtime_args, wasm_file, wasm_args, cwd) + + +def execute_openvino_mobilenet_raw_once( + runtime_bin: str, runtime_args: List[str], cwd: Path +) -> str: + """ + execute openvino-mobilenet-image with iwasm and wasmedge + """ + + wasm_file = "./rust/target/wasm32-wasi/debug/wasmedge-wasinn-example-mobilenet.wasm" + wasm_args = [ + "mobilenet.xml", + "mobilenet.bin", + "./tensor-1x224x224x3-f32.bgr", + ] + return execute_once(runtime_bin, runtime_args, wasm_file, wasm_args, cwd) + + +def execute_openvino_mobilenet_image_once( + runtime_bin: str, runtime_args: List[str], cwd: Path +) -> str: + """ + execute openvino-mobilenet-image with iwasm and wasmedge + """ + + wasm_file = ( + "./rust/target/wasm32-wasi/debug/wasmedge-wasinn-example-mobilenet-image.wasm" + ) + wasm_args = [ + "mobilenet.xml", + "mobilenet.bin", + "input.jpg", + ] + return execute_once(runtime_bin, runtime_args, wasm_file, wasm_args, cwd) + + def execute_tflite_birds_v1_image_once( runtime_bin: str, runtime_args: List[str], cwd: Path ) -> str: """ - execute tflite_birds_v1_image example with - - ``` - iwasm --native-lib=somewhere/libwasi-nn-tflite.so --map-dir=.:. \ - ./wasmedge-wasinn-example-tflite-bird-image.wasm \ - lite-model_aiy_vision_classifier_birds_V1_3.tflite \ - bird.jpg - ``` - - or - - ``` - wasmedge --dir=.:. \ - ./wasmedge-wasinn-example-tflite-bird-image.wasm \ - lite-model_aiy_vision_classifier_birds_V1_3.tflite \ - bird.jpg - ``` - - assumption: - - under the right directory, tflite-birds_v1-image - - every materials are ready + execute openvino-mobilenet-image with iwasm and wasmedge """ - wasm_file = "./wasmedge-wasinn-example-tflite-bird-image.wasm" + wasm_file = ( + "rust/target/wasm32-wasi/debug/wasmedge-wasinn-example-tflite-bird-image.wasm" + ) wasm_args = ["lite-model_aiy_vision_classifier_birds_V1_3.tflite", "bird.jpg"] + return execute_once(runtime_bin, runtime_args, wasm_file, wasm_args, cwd) - cmd = [runtime_bin] - cmd.extend(runtime_args) - cmd.append(wasm_file) - cmd.extend(wasm_args) - try: - p = subprocess.run( - cmd, - cwd=cwd, - capture_output=True, - check=True, - text=True, - universal_newlines=True, - ) - return p.stdout - except subprocess.CalledProcessError as e: - print(e.stderr) - print() - print(e.stdout) - - -def filter_output_tflite_birds_v1_image(output: str) -> List[str]: +def filter_output(output: str) -> List[WasmEdgeExampleResult]: """ - not all output is needed for comparision + not all output is required for comparison - pick lines like: " 1.) [526](136)Cathartes burrovianus" + pick lines like: " 1.) [166](198)Aix galericulata" """ filtered = [] - PATTERN = re.compile(r"^\s+\d\.\)\s+\[\d+\]\(\d+\)\w+") + PATTERN = re.compile(r"^\s+\d\.\)\s+\[(\d+)\]\(([.0-9]+)\)\w+") for line in output.split("\n"): - if PATTERN.search(line): - filtered.append(line.strip()) + m = PATTERN.search(line) + if m: + class_id, possibility = m.groups() + filtered.append(WasmEdgeExampleResult(class_id, possibility)) + assert len(filtered) return filtered +def compare_output( + iwasm_output: List[WasmEdgeExampleResult], + wasmedge_output: List[WasmEdgeExampleResult], +) -> bool: + """ + only compare top 2 and ignore possibility + """ + return (iwasm_output[0].class_id, iwasm_output[1].class_id) == ( + wasmedge_output[0].class_id, + wasmedge_output[1].class_id, + ) + + +def summarizer_result( + example_name: str, + iwasm_output: List[WasmEdgeExampleResult], + wasmedge_output: List[WasmEdgeExampleResult], +): + if compare_output(iwasm_output, wasmedge_output): + print(f"- {example_name}. PASS") + return + + print(f"- {example_name}. FAILED") + print("------------------------------------------------------------") + pprint(iwasm_output) + print("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<") + pprint(wasmedge_output) + print("------------------------------------------------------------") + + def execute_tflite_birds_v1_image(iwasm_bin: str, wasmedge_bin: str, cwd: Path): iwasm_output = execute_tflite_birds_v1_image_once( iwasm_bin, @@ -88,33 +168,124 @@ def execute_tflite_birds_v1_image(iwasm_bin: str, wasmedge_bin: str, cwd: Path): ], cwd, ) - iwasm_output = filter_output_tflite_birds_v1_image(iwasm_output) + iwasm_output = filter_output(iwasm_output) wasmedge_output = execute_tflite_birds_v1_image_once( wasmedge_bin, ["--dir=.:."], cwd ) - wasmedge_output = filter_output_tflite_birds_v1_image(wasmedge_output) + wasmedge_output = filter_output(wasmedge_output) + + summarizer_result("tf_lite_birds_v1_image", iwasm_output, wasmedge_output) + + +def execute_openvino_mobilenet_image(iwasm_bin: str, wasmedge_bin: str, cwd: Path): + iwasm_output = execute_openvino_mobilenet_image_once( + iwasm_bin, + [ + "--native-lib=/workspaces/wamr/product-mini/platforms/linux/build/libwasi-nn-openvino.so", + "--map-dir=.:.", + ], + cwd, + ) + iwasm_output = filter_output(iwasm_output) + + wasmedge_output = execute_openvino_mobilenet_image_once( + wasmedge_bin, ["--dir=.:."], cwd + ) + wasmedge_output = filter_output(wasmedge_output) + + summarizer_result("openvino_mobile_image", iwasm_output, wasmedge_output) + + +def execute_openvino_mobilenet_raw(iwasm_bin: str, wasmedge_bin: str, cwd: Path): + iwasm_output = execute_openvino_mobilenet_raw_once( + iwasm_bin, + [ + "--native-lib=/workspaces/wamr/product-mini/platforms/linux/build/libwasi-nn-openvino.so", + "--map-dir=.:.", + ], + cwd, + ) + iwasm_output = filter_output(iwasm_output) + + wasmedge_output = execute_openvino_mobilenet_raw_once( + wasmedge_bin, ["--dir=.:."], cwd + ) + wasmedge_output = filter_output(wasmedge_output) + + summarizer_result("openvino_mobile_raw", iwasm_output, wasmedge_output) + + +def execute_openvino_road_segmentation_adas( + iwasm_bin: str, wasmedge_bin: str, cwd: Path +): + def filter_output(output: str) -> str: + """ + focus on lines: + The size of the output buffer is 7340032 bytes + dump tensor to "wasinn-openvino-inference-output-1x4x512x896xf32.tensor" + """ + for line in output.split("\n"): + if "The size of the output buffer is" in line: + dump_tensor_size = int(line.split(" ")[-2]) + continue + + if "dump tensor to " in line: + dump_tensor_file = line.split(" ")[-1] + continue - if iwasm_output == wasmedge_output: - print("- tflite_birds_v1_image. PASS") + return (dump_tensor_file, dump_tensor_size) + + iwasm_output = execute_openvino_road_segmentation_adas_once( + iwasm_bin, + [ + "--native-lib=/workspaces/wamr/product-mini/platforms/linux/build/libwasi-nn-openvino.so", + "--map-dir=.:.", + ], + cwd, + ) + iwasm_tensor_file, iwasm_tensor_size = filter_output(iwasm_output) + + wasmedge_output = execute_openvino_road_segmentation_adas_once( + wasmedge_bin, ["--dir=.:."], cwd + ) + wasmedge_tensor_file, wasmedge_tensor_size = filter_output(wasmedge_output) + + # TODO: binary compare? + if iwasm_tensor_size == wasmedge_tensor_size: + print(f"- openvino_road_segmentation_adas. PASS") return - print("- tflite_birds_v1_image. FAILED") + print(f"- openvino_road_segmentation_adas. FAILED") print("------------------------------------------------------------") - pprint(iwasm_output) + print(f"FILE:{iwasm_tensor_file}, SIZE:{iwasm_tensor_size}") print("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<") - pprint(wasmedge_output) + print(f"FILE:{wasmedge_tensor_file}, SIZE:{wasmedge_tensor_size}") print("------------------------------------------------------------") -def execute_wasmedge_wasinn_exmaples(iwasm_bin: str, wasmedge_bin: str): +def execute_wasmedge_wasinn_examples(iwasm_bin: str, wasmedge_bin: str): assert Path.cwd().name == "wasmedge-wasinn-examples" assert shutil.which(iwasm_bin) assert shutil.which(wasmedge_bin) - tflite_birds_v1_image_dir = Path.cwd().joinpath("./tflite-birds_v1-image") - execute_tflite_birds_v1_image(iwasm_bin, wasmedge_bin, tflite_birds_v1_image_dir) + # TODO: keep commenting until https://github.com/bytecodealliance/wasm-micro-runtime/pull/3597 is merged + # tflite_birds_v1_image_dir = Path.cwd().joinpath("./tflite-birds_v1-image") + # execute_tflite_birds_v1_image(iwasm_bin, wasmedge_bin, tflite_birds_v1_image_dir) + + openvino_mobile_image_dir = Path.cwd().joinpath("./openvino-mobilenet-image") + execute_openvino_mobilenet_image(iwasm_bin, wasmedge_bin, openvino_mobile_image_dir) + + openvino_mobile_raw_dir = Path.cwd().joinpath("./openvino-mobilenet-raw") + execute_openvino_mobilenet_raw(iwasm_bin, wasmedge_bin, openvino_mobile_raw_dir) + + openvino_road_segmentation_adas_dir = Path.cwd().joinpath( + "./openvino-road-segmentation-adas" + ) + execute_openvino_road_segmentation_adas( + iwasm_bin, wasmedge_bin, openvino_road_segmentation_adas_dir + ) if __name__ == "__main__": - execute_wasmedge_wasinn_exmaples("iwasm", "wasmedge") + execute_wasmedge_wasinn_examples("iwasm", "wasmedge") From b086d5820adec3ff47b03b817b307139bc254801 Mon Sep 17 00:00:00 2001 From: Benbuck Nason Date: Mon, 22 Jul 2024 18:14:49 -0700 Subject: [PATCH 06/58] Change log of import function to be consistent (#3656) --- core/iwasm/common/wasm_native.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/iwasm/common/wasm_native.c b/core/iwasm/common/wasm_native.c index 40152dfa51..cde40825e5 100644 --- a/core/iwasm/common/wasm_native.c +++ b/core/iwasm/common/wasm_native.c @@ -230,7 +230,7 @@ wasm_native_resolve_symbol(const char *module_name, const char *field_name, #if WASM_ENABLE_WAMR_COMPILER == 0 /* Output warning except running aot compiler */ LOG_WARNING("failed to check signature '%s' and resolve " - "pointer params for import function (%s %s)\n", + "pointer params for import function (%s, %s)\n", signature, module_name, field_name); #endif return NULL; From 5e7d3ed59b014e14eb684a19378b76e85b2dc650 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Tue, 23 Jul 2024 16:34:47 +0800 Subject: [PATCH 07/58] Add APIs into wasm_c_api.h to summary wasm function execution duration (#3639) - `wasm_instance_sum_wasm_exec_time()` -> `wasm_runtime_sum_wasm_exec_time()` - `wasm_instance_get_wasm_func_exec_time()` -> `wasm_runtime_get_wasm_func_exec_time()` --- core/iwasm/common/wasm_c_api.c | 21 +++++++++++++++++++++ core/iwasm/include/wasm_c_api.h | 5 +++++ 2 files changed, 26 insertions(+) diff --git a/core/iwasm/common/wasm_c_api.c b/core/iwasm/common/wasm_c_api.c index 4fd6626531..01109658ff 100644 --- a/core/iwasm/common/wasm_c_api.c +++ b/core/iwasm/common/wasm_c_api.c @@ -5379,3 +5379,24 @@ wasm_extern_new_empty(wasm_store_t *store, wasm_externkind_t extern_kind) LOG_ERROR("Don't support linking table and memory for now"); return NULL; } + +double +wasm_instance_sum_wasm_exec_time(const wasm_instance_t *instance) +{ +#if WASM_ENABLE_PERF_PROFILING != 0 + return wasm_runtime_sum_wasm_exec_time(instance->inst_comm_rt); +#else + return -1.0; +#endif +} + +double +wasm_instance_get_wasm_func_exec_time(const wasm_instance_t *instance, + const char *name) +{ +#if WASM_ENABLE_PERF_PROFILING != 0 + return wasm_runtime_get_wasm_func_exec_time(instance->inst_comm_rt, name); +#else + return -1.0; +#endif +} diff --git a/core/iwasm/include/wasm_c_api.h b/core/iwasm/include/wasm_c_api.h index 3f1d2b64ad..4994454bd8 100644 --- a/core/iwasm/include/wasm_c_api.h +++ b/core/iwasm/include/wasm_c_api.h @@ -691,6 +691,11 @@ WASM_API_EXTERN own wasm_instance_t* wasm_instance_new_with_args_ex( WASM_API_EXTERN void wasm_instance_exports(const wasm_instance_t*, own wasm_extern_vec_t* out); +// Return total wasm functions' execution time in ms +WASM_API_EXTERN double wasm_instance_sum_wasm_exec_time(const wasm_instance_t*); +// Return execution time in ms of a given wasm function with +// func_name. If the function is not found, return 0. +WASM_API_EXTERN double wasm_instance_get_wasm_func_exec_time(const wasm_instance_t*, const char *); /////////////////////////////////////////////////////////////////////////////// // Convenience From 5744e1191636494608fb69652d763ac7abd947a2 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 23 Jul 2024 18:04:16 +0900 Subject: [PATCH 08/58] spec-test-script: Make case_last_words larger (#3651) The current size is usually too small to contain useful info for post mortem investigations. --- tests/wamr-test-suites/spec-test-script/all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/wamr-test-suites/spec-test-script/all.py b/tests/wamr-test-suites/spec-test-script/all.py index f1611c6fe5..60ab02eff6 100644 --- a/tests/wamr-test-suites/spec-test-script/all.py +++ b/tests/wamr-test-suites/spec-test-script/all.py @@ -247,7 +247,7 @@ def test_case( if verbose_flag: print(output, end="") else: - if len(case_last_words) == 16: + if len(case_last_words) == 1024: case_last_words.pop(0) case_last_words.append(output) From d7521eea4c7266807c7e023cca8e83d9d9179b0c Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 23 Jul 2024 18:59:31 +0900 Subject: [PATCH 09/58] libc-builtin: Fix a printf format (#3652) --- core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c b/core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c index f4cf6b8efc..8db888fe3d 100644 --- a/core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c +++ b/core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c @@ -1005,7 +1005,7 @@ print_i32_wrapper(wasm_exec_env_t exec_env, int32 i32) static void print_i64_wrapper(wasm_exec_env_t exec_env, int64 i64) { - os_printf("in specttest.print_i64(%" PRId32 ")\n", i64); + os_printf("in specttest.print_i64(%" PRId64 ")\n", i64); } static void From 79dacfcead5a3ed0f24f4a58f07f8554fc488ad8 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Wed, 24 Jul 2024 00:08:26 +0900 Subject: [PATCH 10/58] spec-test-script/runtest.py: Reduce stack size for aot w/o gc (#3653) --- tests/wamr-test-suites/spec-test-script/runtest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/wamr-test-suites/spec-test-script/runtest.py b/tests/wamr-test-suites/spec-test-script/runtest.py index b7595fb0b5..709ca01711 100755 --- a/tests/wamr-test-suites/spec-test-script/runtest.py +++ b/tests/wamr-test-suites/spec-test-script/runtest.py @@ -1170,7 +1170,13 @@ def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r): # cf. https://github.com/bytecodealliance/wasm-micro-runtime/issues/2231 cmd_iwasm.append("--stack-size=10485760") # 10MB (!) else: - cmd_iwasm.append("--stack-size=131072") # 128KB + if opts.aot: + # Note: aot w/o gc doesn't require the interpreter stack at all. + # Note: 1 is the minimum value we can specify because 0 means + # the default. + cmd_iwasm.append("--stack-size=1") + else: + cmd_iwasm.append("--stack-size=131072") # 128KB if opts.verbose: cmd_iwasm.append("-v=5") cmd_iwasm.append(tmpfile) From 5be8f3580d3bb446160a482e1163e06544889677 Mon Sep 17 00:00:00 2001 From: Benbuck Nason Date: Tue, 23 Jul 2024 20:37:41 -0700 Subject: [PATCH 11/58] Set compile symbol visibility to hidden in cmake (#3655) Set compile symbol visibility to hidden in cmake of root folder and product-mini/platforms/linux. --- CMakeLists.txt | 2 +- product-mini/platforms/linux/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bedf0af6c8..c7f7666897 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,7 +121,7 @@ set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wshadow -Wno-unused-parameter") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wshadow -Wno-unused-parameter -fvisibility=hidden") # set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wconversion -Wsign-conversion") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wno-unused") diff --git a/product-mini/platforms/linux/CMakeLists.txt b/product-mini/platforms/linux/CMakeLists.txt index 2e37b75f92..d7ba865a0a 100644 --- a/product-mini/platforms/linux/CMakeLists.txt +++ b/product-mini/platforms/linux/CMakeLists.txt @@ -145,7 +145,7 @@ set_target_properties (vmlib PROPERTIES POSITION_INDEPENDENT_CODE ON) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wshadow") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wshadow -fvisibility=hidden") # set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wconversion -Wsign-conversion") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wno-unused") From b300797b71c36c6e5dacbacf200664f7dfe8dc7e Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Wed, 24 Jul 2024 21:25:45 +0900 Subject: [PATCH 12/58] wamrc: Add --mllvm= option (#3658) This allows users to specify llvm command line options, similarly to clang's -mllvm option. My motivations: * -debug and friends * -mtext-section-literals for xtensa --- wamr-compiler/main.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index ae24ee5bc0..7f1c34f664 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -9,6 +9,8 @@ #include "wasm_export.h" #include "aot_export.h" +#include + #if BH_HAS_DLFCN #include @@ -195,6 +197,7 @@ print_help() #if WASM_ENABLE_LINUX_PERF != 0 printf(" --enable-linux-perf Enable linux perf support\n"); #endif + printf(" --mllvm=