Skip to content

Commit

Permalink
add initial stress test
Browse files Browse the repository at this point in the history
  • Loading branch information
Zzzabiyaka committed Jul 21, 2023
1 parent 81f0371 commit 850f2f3
Show file tree
Hide file tree
Showing 6 changed files with 236 additions and 18 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/nightly_run.yml
Original file line number Diff line number Diff line change
Expand Up @@ -586,7 +586,7 @@ jobs:

- name: Build WASI thread tests
if: matrix.test_option == '$WASI_TEST_OPTIONS'
run: bash build.sh
run: bash build.sh --enable_stress_tests
working-directory: ./core/iwasm/libraries/lib-wasi-threads/test/

- name: build socket api tests
Expand All @@ -595,7 +595,7 @@ jobs:
working-directory: ./core/iwasm/libraries/lib-socket/test/

- name: run tests
timeout-minutes: 10
timeout-minutes: 45
run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }}
working-directory: ./tests/wamr-test-suites

Expand Down
62 changes: 48 additions & 14 deletions core/iwasm/libraries/lib-wasi-threads/test/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,36 @@ set -eo pipefail
CC=${CC:=/opt/wasi-sdk/bin/clang}
WAMR_DIR=../../../../..

rm -f *.aot
rm -f *.wasm

# Set default values
enable_stress_tests=false

# Function to display usage information
function display_usage() {
echo "Usage: $0 [--enable_stress_tests]"
echo "Options:"
echo " --enable_stress_tests Enable stress tests"
}

# Parse command line arguments
while [[ $# -gt 0 ]]; do
case "$1" in
--enable_stress_tests)
enable_stress_tests=true
;;
*)
display_usage
exit 1
;;
esac
shift
done

# Stress tests names (to include only in nightly build)
stress_tests=("spawn_stress_test.wasm")

for test_c in *.c; do
test_wasm="$(basename $test_c .c).wasm"

Expand All @@ -18,18 +48,22 @@ for test_c in *.c; do
thread_start_file=$WAMR_DIR/samples/wasi-threads/wasm-apps/wasi_thread_start.S
fi

echo "Compiling $test_c to $test_wasm"
$CC \
-target wasm32-wasi-threads \
-pthread -ftls-model=local-exec \
-z stack-size=32768 \
-Wl,--export=__heap_base \
-Wl,--export=__data_end \
-Wl,--shared-memory,--max-memory=1966080 \
-Wl,--export=wasi_thread_start \
-Wl,--export=malloc \
-Wl,--export=free \
-I $WAMR_DIR/samples/wasi-threads/wasm-apps \
$thread_start_file \
$test_c -o $test_wasm
if [[ "$enable_stress_tests" = true ]] || ! [[ " ${stress_tests[@]} " =~ " ${test_wasm} " ]]; then
echo "Compiling $test_c to $test_wasm"
$CC \
-target wasm32-wasi-threads \
-pthread -ftls-model=local-exec \
-z stack-size=32768 \
-Wl,--export=__heap_base \
-Wl,--export=__data_end \
-Wl,--shared-memory,--max-memory=1966080 \
-Wl,--export=wasi_thread_start \
-Wl,--export=malloc \
-Wl,--export=free \
-I $WAMR_DIR/samples/wasi-threads/wasm-apps \
$thread_start_file \
$test_c -o $test_wasm
else
echo "Test $test_wasm is skipped because stress tests are not enabled"
fi
done
3 changes: 3 additions & 0 deletions core/iwasm/libraries/lib-wasi-threads/test/manifest.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"name": "lib-wasi-threads tests"
}
5 changes: 5 additions & 0 deletions core/iwasm/libraries/lib-wasi-threads/test/skip.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"lib-wasi-threads tests": {
"create_threads_until_limit": "Stress tests are incompatible with this test"
}
}
148 changes: 148 additions & 0 deletions core/iwasm/libraries/lib-wasi-threads/test/spawn_stress_test.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
/*
* Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/

#ifndef __wasi__
#error This example only compiles to WASM/WASI target
#endif

#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <stdbool.h>
#include <unistd.h>
#include <math.h>
#include <pthread.h>


#include "wasi_thread_start.h"

enum CONSTANTS {
NUM_ITER = 100000,
NUM_RETRY = 5,
SECOND = 1000 * 1000 * 1000, /* 1 second */
TIMEOUT = 50LL * SECOND,
MAX_NUM_THREADS = 8,
};

typedef struct {
start_args_t base;
int th_done;
int thread_id;
int value;
} shared_t;

unsigned int factorised_number = 1;
unsigned int prime_numbers = 0;
double percentage = 0.1;

// Inited threads - Deinited threads. Should always be zero in the end. Otherwise it's a leak.
int thread_init_delta = 0;

shared_t threads[MAX_NUM_THREADS];

bool is_prime(unsigned int num) {
for (unsigned int i = 2; i <= (unsigned int)(sqrt(num)); ++i) {
if (num % i == 0) {
return false;
}
}

return true;
}

void
__wasi_thread_start_C(int thread_id, int *start_arg)
{
shared_t *data = (shared_t *)start_arg;
__builtin_wasm_memory_atomic_wait32(NULL, 0, SECOND);

if (is_prime(data->value)) {
__atomic_fetch_add(&prime_numbers, 1, __ATOMIC_SEQ_CST);
}

__atomic_store_n(&data->th_done, 1, __ATOMIC_SEQ_CST);
__builtin_wasm_memory_atomic_notify(&data->th_done, 1);
}

unsigned int validate() {
unsigned int counter = 0;
for (unsigned int i = 2; i <= factorised_number; ++i) {
counter += is_prime(i);
}

return counter;
}

int
main(int argc, char **argv)
{
for (int i = 0; i < MAX_NUM_THREADS; ++i) {
// Init a new thread
{
assert(start_args_init(&threads[i].base));
factorised_number++;
threads[i].value = factorised_number;
thread_init_delta++;
}

threads[i].thread_id = __wasi_thread_spawn(&threads[i]);
assert(threads[i].thread_id > 0 && "Thread creation should succeed");
}

while (factorised_number < NUM_ITER) {
if (factorised_number > (unsigned int)(NUM_ITER * percentage)) {
printf("Stress test %d %% finished\n", (unsigned int)(percentage * 100));
percentage += 0.1;
}

for (int j = 0; j < MAX_NUM_THREADS && factorised_number < NUM_ITER; ++j) {
assert(__builtin_wasm_memory_atomic_wait32(&threads[j].th_done, 0, TIMEOUT) != 2);
// Deinit the expired thread
{
start_args_deinit(&threads[j].base);
thread_init_delta--;
}

// Invalidated the expired thread
{
memset(&threads[j], 0, sizeof(threads[j]));
}

// Init a new thread
{
assert(start_args_init(&threads[j].base));
thread_init_delta++;
}

factorised_number++;
threads[j].value = factorised_number;
for (int z = 0; factorised_number < NUM_ITER && z < NUM_RETRY && threads[j].thread_id < 0; z++) {
threads[j].thread_id = __wasi_thread_spawn(&threads[j]);
if (threads[j].thread_id < 0)
__builtin_wasm_memory_atomic_wait32(NULL, 0, SECOND);
}
assert((factorised_number == NUM_ITER || threads[j].thread_id > 0) && "Thread creation should succeed");
}
}

// Cleanup
for (int i = 0; i < MAX_NUM_THREADS; ++i) {
if (threads[i].thread_id > 0) {
assert(__builtin_wasm_memory_atomic_wait32(&threads[i].th_done, 0, TIMEOUT) != 2);
}
if (threads[i].base.stack != NULL) {
start_args_deinit(&threads[i].base);
thread_init_delta--;
}
}


// Check the test results
assert(prime_numbers == validate() && "Answer mismatch between tested code and reference implementation");
assert(factorised_number == NUM_ITER && "Counter and iteration count mismatch");
assert(thread_init_delta == 0 && "Init/Deinit threads is not matched. Check the code for memory leaks");

return 0;
}
32 changes: 30 additions & 2 deletions tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ readonly WORK_DIR=$PWD
readonly PLATFORM=$(uname -s | tr A-Z a-z)
readonly WAMR_DIR="${WORK_DIR}/../../../.."
readonly IWASM_CMD="${WORK_DIR}/../../../../product-mini/platforms/${PLATFORM}/build/iwasm"
readonly IWASM_CMD_STRESS="${WORK_DIR}/../../../../product-mini/platforms/${PLATFORM}/build/iwasm --max-threads=8"
readonly WAMRC_CMD="${WORK_DIR}/../../../../wamr-compiler/build/wamrc"
readonly C_TESTS="tests/c/testsuite/"
readonly ASSEMBLYSCRIPT_TESTS="tests/assemblyscript/testsuite/"
Expand All @@ -22,6 +23,11 @@ readonly LIB_SOCKET_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-socket/test/"
run_aot_tests () {
local tests=("$@")
for test_wasm in ${tests[@]}; do
local extra_stress_flags=""
if [[ "$test_wasm" =~ "stress" ]]; then
extra_stress_flags="--max-threads=8"
fi

test_aot="${test_wasm%.wasm}.aot"
test_json="${test_wasm%.wasm}.json"

Expand All @@ -38,8 +44,9 @@ run_aot_tests () {
if [ -f ${test_json} ]; then
expected=$(jq .exit_code ${test_json})
fi


${IWASM_CMD} $test_aot
${IWASM_CMD} $extra_stress_flags $test_aot

ret=${PIPESTATUS[0]}

Expand All @@ -53,14 +60,35 @@ run_aot_tests () {
if [[ $MODE != "aot" ]];then
python3 -m venv wasi-env && source wasi-env/bin/activate
python3 -m pip install -r test-runner/requirements.txt
TEST_RUNTIME_EXE="${IWASM_CMD}" python3 test-runner/wasi_test_runner.py \

# if we have stress test then the build.sh was called with --enable_stress_tests,
# so we disable the test that is bounded to max-threads=4 in iwasm, and increasing
# the number of threads to 8

if [[ -e "${THREAD_INTERNAL_TESTS}spawn_stress_test.wasm" ]]; then
TEST_RUNTIME_EXE="${IWASM_CMD_STRESS}" python3 test-runner/wasi_test_runner.py \
-r adapters/wasm-micro-runtime.py \
-t \
${C_TESTS} \
${ASSEMBLYSCRIPT_TESTS} \
${THREAD_PROPOSAL_TESTS} \
${THREAD_INTERNAL_TESTS} \
${LIB_SOCKET_TESTS} \
--exclude-filter "${THREAD_INTERNAL_TESTS}skip.json" \

else

TEST_RUNTIME_EXE="${IWASM_CMD}" python3 test-runner/wasi_test_runner.py \
-r adapters/wasm-micro-runtime.py \
-t \
${C_TESTS} \
${ASSEMBLYSCRIPT_TESTS} \
${THREAD_PROPOSAL_TESTS} \
${THREAD_INTERNAL_TESTS} \
${LIB_SOCKET_TESTS} \

fi

exit_code=${PIPESTATUS[0]}
deactivate
else
Expand Down

0 comments on commit 850f2f3

Please sign in to comment.