From 6d96bfc895b22777fd31b8d1f67e6e5897ceebc2 Mon Sep 17 00:00:00 2001 From: Maksim Litskevich Date: Mon, 17 Jul 2023 13:52:40 +0100 Subject: [PATCH] add initial stress test --- .../compilation_on_android_ubuntu.yml | 4 +- .github/workflows/nightly_run.yml | 4 +- .../libraries/lib-wasi-threads/test/build.sh | 5 +- .../lib-wasi-threads/test/manifest.json | 3 + .../libraries/lib-wasi-threads/test/skip.json | 5 + .../lib-wasi-threads/test/spawn_stress_test.c | 114 ++++++++++++++++++ .../wasi-test-script/run_wasi_tests.sh | 40 ++++-- 7 files changed, 161 insertions(+), 14 deletions(-) create mode 100644 core/iwasm/libraries/lib-wasi-threads/test/manifest.json create mode 100644 core/iwasm/libraries/lib-wasi-threads/test/skip.json create mode 100644 core/iwasm/libraries/lib-wasi-threads/test/spawn_stress_test.c diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index f965d20068..5a0939b738 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -527,7 +527,7 @@ jobs: working-directory: ./core/iwasm/libraries/lib-socket/test/ - name: run tests - timeout-minutes: 10 + timeout-minutes: 20 run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} working-directory: ./tests/wamr-test-suites @@ -543,7 +543,7 @@ jobs: sudo apt install -y g++-multilib lib32gcc-9-dev - name: run tests x86_32 - timeout-minutes: 10 + timeout-minutes: 20 if: env.TEST_ON_X86_32 == 'true' run: ./test_wamr.sh ${{ env.X86_32_TARGET_TEST_OPTIONS }} ${{ matrix.test_option }} -t ${{ matrix.running_mode }} working-directory: ./tests/wamr-test-suites diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index 4664ad636f..cea5dccb17 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -595,7 +595,7 @@ jobs: working-directory: ./core/iwasm/libraries/lib-socket/test/ - name: run tests - timeout-minutes: 10 + timeout-minutes: 20 run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} working-directory: ./tests/wamr-test-suites @@ -611,7 +611,7 @@ jobs: sudo apt install -y g++-multilib lib32gcc-9-dev - name: run tests x86_32 - timeout-minutes: 10 + timeout-minutes: 20 if: env.TEST_ON_X86_32 == 'true' run: ./test_wamr.sh ${{ env.X86_32_TARGET_TEST_OPTIONS }} ${{ matrix.test_option }} -t ${{ matrix.running_mode }} working-directory: ./tests/wamr-test-suites \ No newline at end of file diff --git a/core/iwasm/libraries/lib-wasi-threads/test/build.sh b/core/iwasm/libraries/lib-wasi-threads/test/build.sh index 32586c20c8..611a3bd432 100755 --- a/core/iwasm/libraries/lib-wasi-threads/test/build.sh +++ b/core/iwasm/libraries/lib-wasi-threads/test/build.sh @@ -9,10 +9,13 @@ set -eo pipefail CC=${CC:=/opt/wasi-sdk/bin/clang} WAMR_DIR=../../../../.. +# Stress tests names +thread_start_file_exclusions=("spawn_stress_test.wasm" "linear_memory_size_update.wasm") + for test_c in *.c; do test_wasm="$(basename $test_c .c).wasm" - if [ $test_wasm = "linear_memory_size_update.wasm" ]; then + if [[ " ${thread_start_file_exclusions[@]} " =~ " ${test_wasm} " ]] ; then thread_start_file="" else thread_start_file=$WAMR_DIR/samples/wasi-threads/wasm-apps/wasi_thread_start.S diff --git a/core/iwasm/libraries/lib-wasi-threads/test/manifest.json b/core/iwasm/libraries/lib-wasi-threads/test/manifest.json new file mode 100644 index 0000000000..cd2cc7636c --- /dev/null +++ b/core/iwasm/libraries/lib-wasi-threads/test/manifest.json @@ -0,0 +1,3 @@ +{ + "name": "lib-wasi-threads tests" +} diff --git a/core/iwasm/libraries/lib-wasi-threads/test/skip.json b/core/iwasm/libraries/lib-wasi-threads/test/skip.json new file mode 100644 index 0000000000..87db7667cf --- /dev/null +++ b/core/iwasm/libraries/lib-wasi-threads/test/skip.json @@ -0,0 +1,5 @@ +{ + "lib-wasi-threads tests": { + "spawn_stress_test": "Stress tests are incompatible with the other part and executed differently" + } +} diff --git a/core/iwasm/libraries/lib-wasi-threads/test/spawn_stress_test.c b/core/iwasm/libraries/lib-wasi-threads/test/spawn_stress_test.c new file mode 100644 index 0000000000..35175fc3e3 --- /dev/null +++ b/core/iwasm/libraries/lib-wasi-threads/test/spawn_stress_test.c @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef __wasi__ +#error This example only compiles to WASM/WASI target +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +enum CONSTANTS { + NUM_ITER = 100000, + NUM_RETRY = 5, + MAX_NUM_THREADS = 8, +}; + +unsigned prime_numbers_count = 0; + +bool +is_prime(unsigned int num) +{ + for (unsigned int i = 2; i <= (unsigned int)(sqrt(num)); ++i) { + if (num % i == 0) { + return false; + } + } + + return true; +} + +void * +check_if_prime(void *value) +{ + unsigned int *num = (unsigned int *)(value); + usleep(10000); + if (is_prime(*num)) { + __atomic_fetch_add(&prime_numbers_count, 1, __ATOMIC_SEQ_CST); + } + return NULL; +} + +unsigned int +validate() +{ + unsigned int counter = 0; + for (unsigned int i = 2; i <= NUM_ITER; ++i) { + counter += is_prime(i); + } + + return counter; +} + +void +spawn_thread(pthread_t *thread, unsigned int *arg) +{ + int status_code = -1; + for (int tries = 0; status_code != 0 && tries < NUM_RETRY; ++tries) { + status_code = pthread_create(thread, NULL, &check_if_prime, arg); + assert(status_code == 0 || status_code == EAGAIN); + if (status_code == EAGAIN) { + usleep(2000); + } + } + + assert(status_code == 0 && "Thread creation should succeed"); +} + +int +main(int argc, char **argv) +{ + pthread_t threads[MAX_NUM_THREADS]; + unsigned int args[MAX_NUM_THREADS]; + double percentage = 0.1; + + for (unsigned int factorised_number = 2; factorised_number < NUM_ITER; + ++factorised_number) { + if (factorised_number > NUM_ITER * percentage) { + fprintf(stderr, "Stress test is %d%% finished\n", + (unsigned int)(percentage * 100)); + percentage += 0.1; + } + + unsigned int thread_num = factorised_number % MAX_NUM_THREADS; + if (threads[thread_num] != 0) { + assert(pthread_join(threads[thread_num], NULL) == 0); + } + + args[thread_num] = factorised_number; + + usleep(2000); + spawn_thread(&threads[thread_num], &args[thread_num]); + assert(threads[thread_num] != 0); + } + + for (int i = 0; i < MAX_NUM_THREADS; ++i) { + assert(threads[i] == 0 || pthread_join(threads[i], NULL) == 0); + } + + // Check the test results + assert( + prime_numbers_count == validate() + && "Answer mismatch between tested code and reference implementation"); + + fprintf(stderr, "Stress test finished successfully\n"); + return 0; +} diff --git a/tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh b/tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh index 7738b5080a..9f13ee0860 100755 --- a/tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh +++ b/tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh @@ -14,6 +14,7 @@ readonly WAMR_DIR="${WORK_DIR}/../../../.." readonly IWASM_CMD="${WORK_DIR}/../../../../product-mini/platforms/${PLATFORM}/build/iwasm \ --allow-resolve=google-public-dns-a.google.com \ --addr-pool=::1/128,127.0.0.1/32" +readonly IWASM_CMD_STRESS="${IWASM_CMD} --max-threads=8" readonly WAMRC_CMD="${WORK_DIR}/../../../../wamr-compiler/build/wamrc" readonly C_TESTS="tests/c/testsuite/" readonly ASSEMBLYSCRIPT_TESTS="tests/assemblyscript/testsuite/" @@ -24,6 +25,11 @@ readonly LIB_SOCKET_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-socket/test/" run_aot_tests () { local tests=("$@") for test_wasm in ${tests[@]}; do + local extra_stress_flags="" + if [[ "$test_wasm" =~ "stress" ]]; then + extra_stress_flags="--max-threads=8" + fi + test_aot="${test_wasm%.wasm}.aot" test_json="${test_wasm%.wasm}.json" @@ -41,7 +47,7 @@ run_aot_tests () { expected=$(jq .exit_code ${test_json}) fi - ${IWASM_CMD} $test_aot + ${IWASM_CMD} $extra_stress_flags $test_aot ret=${PIPESTATUS[0]} @@ -55,15 +61,31 @@ run_aot_tests () { if [[ $MODE != "aot" ]];then python3 -m venv wasi-env && source wasi-env/bin/activate python3 -m pip install -r test-runner/requirements.txt + + # Stress test requires max-threads=8 so it's run separately + if [[ -e "${THREAD_INTERNAL_TESTS}spawn_stress_test.wasm" ]]; then + ${IWASM_CMD_STRESS} ${THREAD_INTERNAL_TESTS}spawn_stress_test.wasm + ret=${PIPESTATUS[0]} + if [ "${ret}" -ne 0 ]; then + echo "Stress test spawn_stress_test FAILED with code " ${ret} + exit_code=${ret} + fi + fi + TEST_RUNTIME_EXE="${IWASM_CMD}" python3 test-runner/wasi_test_runner.py \ - -r adapters/wasm-micro-runtime.py \ - -t \ - ${C_TESTS} \ - ${ASSEMBLYSCRIPT_TESTS} \ - ${THREAD_PROPOSAL_TESTS} \ - ${THREAD_INTERNAL_TESTS} \ - ${LIB_SOCKET_TESTS} - exit_code=${PIPESTATUS[0]} + -r adapters/wasm-micro-runtime.py \ + -t \ + ${C_TESTS} \ + ${ASSEMBLYSCRIPT_TESTS} \ + ${THREAD_PROPOSAL_TESTS} \ + ${THREAD_INTERNAL_TESTS} \ + ${LIB_SOCKET_TESTS} \ + --exclude-filter "${THREAD_INTERNAL_TESTS}skip.json" + + ret=${PIPESTATUS[0]} + if [ "${ret}" -ne 0 ]; then + exit_code=${ret} + fi deactivate else target_option=""