diff --git a/.github/workflows/daily-runtime-validation.yml b/.github/workflows/daily-runtime-validation.yml index f21e6b708..ae5096aa6 100644 --- a/.github/workflows/daily-runtime-validation.yml +++ b/.github/workflows/daily-runtime-validation.yml @@ -51,7 +51,7 @@ jobs: continue-on-error: true run: | python3 test-runner/wasi_test_runner.py \ - -r ./adapters/${{ matrix.runtime }}.sh \ + -r ./adapters/${{ matrix.runtime }}.py \ --json-output-location results.json \ -t tests/assemblyscript/testsuite \ -t tests/c/testsuite diff --git a/README.md b/README.md index e5739d009..29e86149a 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ python3 -m pip install -r test-runner/requirements.txt python3 test-runner/wasi_test_runner.py \ -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \ ./tests/c/testsuite/ \ - -r adapters/wasmtime.sh # path to a runtime adapter + -r adapters/wasmtime.py # path to a runtime adapter ``` Optionally you can specify test cases to skip with the `--exclude-filter` option. @@ -45,7 +45,7 @@ python3 test-runner/wasi_test_runner.py -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \ ./tests/c/testsuite/ \ --exclude-filter examples/skip.json \ - -r adapters/wasmtime.sh # path to a runtime adapter + -r adapters/wasmtime.py # path to a runtime adapter ``` The default executable in the adapter used for test execution can be @@ -55,7 +55,7 @@ overridden using `TEST_RUNTIME_EXE` variable. This only works with adapters defi ```bash TEST_RUNTIME_EXE="wasmtime --wasm-features all" python3 test-runner/wasi_test_runner.py \ -t ./tests/assemblyscript/testsuite/ \ - -r adapters/wasmtime.sh + -r adapters/wasmtime.py ``` ## Contributing diff --git a/adapters/wasm-micro-runtime.py b/adapters/wasm-micro-runtime.py new file mode 100755 index 000000000..ed7b49d8e --- /dev/null +++ b/adapters/wasm-micro-runtime.py @@ -0,0 +1,29 @@ +import argparse +import subprocess +import sys +import os +import shlex + +# shlex.split() splits according to shell quoting rules +IWASM = shlex.split(os.getenv("TEST_RUNTIME_EXE", "iwasm")) + +parser = argparse.ArgumentParser() +parser.add_argument("--version", action="store_true") +parser.add_argument("--test-file", action="store") +parser.add_argument("--arg", action="append", default=[]) +parser.add_argument("--env", action="append", default=[]) +parser.add_argument("--dir", action="append", default=[]) + +args = parser.parse_args() + +if args.version: + subprocess.run(IWASM + ["--version"]) + sys.exit(0) + +TEST_FILE = args.test_file +PROG_ARGS = args.arg +ENV_ARGS = [f"--env={i}" for i in args.env] +DIR_ARGS = [f"--dir={i}" for i in args.dir] + +r = subprocess.run(IWASM + ENV_ARGS + DIR_ARGS + [TEST_FILE] + PROG_ARGS) +sys.exit(r.returncode) diff --git a/adapters/wasm-micro-runtime.sh b/adapters/wasm-micro-runtime.sh deleted file mode 100755 index c500dee3b..000000000 --- a/adapters/wasm-micro-runtime.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# WARNING: this adapter assumes iwasm from the latest wasm-micro-runtime. -# Namely the change to propagate WASI exit code: -# https://github.com/bytecodealliance/wasm-micro-runtime/pull/1748 - -TEST_FILE= -ARGS=() -DIR=() -ENV=() - -IWASM="${TEST_RUNTIME_EXE:-iwasm}" - -while [[ $# -gt 0 ]]; do - case $1 in - --version) - ${IWASM} --version - exit 0 - ;; - --test-file) - TEST_FILE="$2" - shift - shift - ;; - --arg) - ARGS+=("$2") - shift - shift - ;; - --dir) - DIR+=("--dir=$2") - shift - shift - ;; - --env) - ENV+=("--env=$2") - shift - shift - ;; - *) - echo "Unknown option $1" - exit 1 - ;; - esac -done - -${IWASM} "${DIR[@]}" "${ENV[@]}" ${TEST_FILE} "${ARGS[@]}" diff --git a/adapters/wasmtime.py b/adapters/wasmtime.py new file mode 100755 index 000000000..399eedaa1 --- /dev/null +++ b/adapters/wasmtime.py @@ -0,0 +1,30 @@ +import argparse +import subprocess +import sys +import os +import shlex + +# shlex.split() splits according to shell quoting rules +WASMTIME = shlex.split(os.getenv("TEST_RUNTIME_EXE", "wasmtime")) + +parser = argparse.ArgumentParser() +parser.add_argument("--version", action="store_true") +parser.add_argument("--test-file", action="store") +parser.add_argument("--arg", action="append", default=[]) +parser.add_argument("--env", action="append", default=[]) +parser.add_argument("--dir", action="append", default=[]) + +args = parser.parse_args() + +if args.version: + # ensure no args when version is queried + subprocess.run(WASMTIME[0:1] + ["--version"]) + sys.exit(0) + +TEST_FILE = args.test_file +PROG_ARGS = args.arg +ENV_ARGS = [j for i in args.env for j in ["--env", i]] +DIR_ARGS = [j for i in args.dir for j in ["--dir", i]] + +r = subprocess.run(WASMTIME + ENV_ARGS + DIR_ARGS + [TEST_FILE] + PROG_ARGS) +sys.exit(r.returncode) diff --git a/adapters/wasmtime.sh b/adapters/wasmtime.sh deleted file mode 100755 index 22b2b7cf7..000000000 --- a/adapters/wasmtime.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -TEST_FILE= -ARGS=() -PROG_ARGS=() - -WASMTIME="${TEST_RUNTIME_EXE:-wasmtime}" - -while [[ $# -gt 0 ]]; do - case $1 in - --version) - wasmtime -V - exit 0 - ;; - --test-file) - TEST_FILE="$2" - shift - shift - ;; - --arg) - PROG_ARGS+=("$2") - shift - shift - ;; - --env) - ARGS+=("--env" "$2") - shift - shift - ;; - --dir) - ARGS+=("--dir" "$2") - shift - shift - ;; - *) - echo "Unknown option $1" - exit 1 - ;; - esac -done - -$WASMTIME $TEST_FILE "${ARGS[@]}" "${PROG_ARGS[@]}" \ No newline at end of file diff --git a/doc/adapters.md b/doc/adapters.md index 86c6f53a1..84010f298 100644 --- a/doc/adapters.md +++ b/doc/adapters.md @@ -4,7 +4,7 @@ The test runner is designed to support different types of WebAssembly runtimes. WASI test runner is designed to support as many WASM runtimes as possible, therefore runtime peculiarities aren't hardcoded in the runner. -In order to integrate WASM runtime with a test runner, the user has to provide a `runtime adapter`. It's an executable file that takes command-line arguments and translates them to a runtime call. +In order to integrate WASM runtime with a test runner, the user has to provide a `runtime adapter`. It's a *Python script* that takes command-line arguments and translates them to a runtime call. The reason for using a Python script over a generic "she-bang" executable, is to ensure cross-platform compatibility, such an executable is expected to be a python a script. ## Interface The adapter executable must accept the following command line parameters and execute actions associated with those parameters: @@ -37,14 +37,14 @@ and check if the exit code is equal to `13`. There are also two test cases in As Print runtime version: ```bash -$ ./adapter.sh --version +$ ./adapter.py --version wasmtime-cli 1.0.1 ``` Run WASM module: ```bash -$ ./adapter.sh --arg a1 --arg a2 --env E1=env1 --env E2=env2 --test-file test.wasm +$ ./adapter.py --arg a1 --arg a2 --env E1=env1 --env E2=env2 --test-file test.wasm # Expected to start test.wasm module with E1=env1 and E2=env2 # environment variables defined and arguments a1, a2 passed to # the module. diff --git a/test-runner/requirements/common.txt b/test-runner/requirements/common.txt index 4ae79866f..42f010a66 100644 --- a/test-runner/requirements/common.txt +++ b/test-runner/requirements/common.txt @@ -1 +1 @@ -colorama==0.4.3 +colorama>=0.4.3 diff --git a/test-runner/wasi_test_runner/runtime_adapter.py b/test-runner/wasi_test_runner/runtime_adapter.py index 9a12f2909..940c2e2c3 100644 --- a/test-runner/wasi_test_runner/runtime_adapter.py +++ b/test-runner/wasi_test_runner/runtime_adapter.py @@ -1,4 +1,5 @@ import subprocess +import sys from pathlib import Path from typing import Dict, NamedTuple, List @@ -16,7 +17,7 @@ def __init__(self, adapter_path: str) -> None: def get_version(self) -> RuntimeVersion: output = ( - subprocess.check_output([self._adapter_path, "--version"], encoding="UTF-8") + subprocess.check_output([sys.executable, self._adapter_path, "--version"], encoding="UTF-8") .strip() .split(" ") ) @@ -31,6 +32,7 @@ def run_test( ) -> Output: args = ( [ + sys.executable, self._adapter_path, "--test-file", self._abs(test_path),