Skip to content

Commit 1747886

Browse files
tolauwaetolauwae
committed
Revert "Remove deprecated built-in spectests"
This reverts commit a433012.
1 parent d038faf commit 1747886

30 files changed

+924
-226
lines changed

‎.github/workflows/test.yml

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,45 @@
1-
name: Tests
1+
name: WASM Spectests
22
on: [ push ]
33

4+
concurrency:
5+
group: ${{ github.workflow }}-${{ github.ref }}
6+
cancel-in-progress: true # Cancel in-flight jobs for the same branch or PR
7+
48
jobs:
59
# Stage 1: unit test vm machinery
610

7-
unit:
8-
name: Run VM unit tests
9-
runs-on: ${{matrix.os}}
10-
strategy:
11-
fail-fast: false
12-
matrix:
13-
os: [ ubuntu-latest, macos-latest ]
11+
test:
12+
name: Run official testsuite
13+
runs-on: ubuntu-latest
1414
if: github.event.pull_request.draft == false
1515
steps:
1616
- uses: actions/checkout@v4
1717
with:
1818
submodules: recursive
19+
- uses: actions/setup-python@v4
20+
with:
21+
python-version: "3.x"
1922

20-
- name: Create build folder
21-
run: mkdir build-unit-tests
23+
- name: Build warduino cli
24+
run: cmake . -D BUILD_EMULATOR=ON ; cmake --build .
25+
26+
- name: Build WABT # Build latest version
27+
run: |
28+
git clone --recursive https://github.com/WebAssembly/wabt
29+
cd wabt
30+
git submodule update --init
31+
mkdir build; cd build
32+
cmake ..
33+
cmake --build .
34+
35+
- name: Verify wat2wasm
36+
run: ./wabt/build/wat2wasm --version
2237

23-
- name: Build unit tests
24-
run: cmake .. -D BUILD_UNITTEST=ON ; cmake --build .
25-
working-directory: build-unit-tests
38+
- name: Install Python dependencies
39+
run: pip install -r requirements.txt
2640

27-
- name: Run unit tests
28-
run: ctest -VV
29-
working-directory: build-unit-tests
41+
- name: Clone and run tests
42+
run: ./tests/integration/run_spec_tests.py --compiler "./wabt/build/wat2wasm" --interpreter "./wdcli" --ignore "./tests/integration/ignore.txt"
3043

3144
# Stage 2: integration testing with official Wasm specification test suite
3245

‎.github/workflows/unittest.yml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: VM Tests
2+
on: [ push, pull_request ]
3+
4+
concurrency:
5+
group: ${{ github.workflow }}-${{ github.ref }}
6+
cancel-in-progress: true # Cancel in-flight jobs for the same branch or PR
7+
8+
jobs:
9+
test:
10+
name: Run unit tests
11+
runs-on: ${{matrix.os}}
12+
strategy:
13+
fail-fast: false
14+
matrix:
15+
os: [ ubuntu-latest, macos-latest ]
16+
if: github.event.pull_request.draft == false
17+
steps:
18+
- uses: actions/checkout@v3
19+
with:
20+
submodules: recursive
21+
22+
- name: Create build folder
23+
run: mkdir build-unit-tests
24+
25+
- name: Build unit tests
26+
run: cmake .. -D BUILD_UNITTEST=ON ; cmake --build .
27+
working-directory: build-unit-tests
28+
29+
- name: Run unit tests
30+
run: ctest -VV
31+
working-directory: build-unit-tests

‎CMakeLists.txt

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,11 @@ if (BUILD_EMULATOR)
5757
src/Edward/RFC.cpp
5858
)
5959

60+
set(TEST_FRAMEWORK
61+
tests/integration/wasm_tests.cpp
62+
tests/integration/assertion.cpp
63+
tests/integration/sexpr-parser/src/sexpr.c)
64+
6065
add_definitions(-DINFO=0)
6166
add_definitions(-DDEBUG=0)
6267
add_definitions(-DTRACE=0)
@@ -68,7 +73,7 @@ if (BUILD_EMULATOR)
6873
endif (CMAKE_COMPILER_IS_GNUCXX)
6974

7075
# WARDuino CLI
71-
add_executable(wdcli platforms/CLI-Emulator/main.cpp ${SOURCE_FILES})
76+
add_executable(wdcli platforms/CLI-Emulator/main.cpp ${SOURCE_FILES} ${TEST_FRAMEWORK})
7277
target_link_libraries(wdcli PRIVATE Threads::Threads)
7378
target_include_directories(wdcli PRIVATE ${EXTERNAL_LIB_HEADERS} "${PROJECT_BINARY_DIR}/include")
7479
endif (BUILD_EMULATOR)
@@ -115,7 +120,7 @@ if (BUILD_UNITTEST)
115120
set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
116121
FetchContent_MakeAvailable(googletest)
117122

118-
set(PATH_TO_UNIT_TEST ${PROJECT_SOURCE_DIR}/tests/unit)
123+
set(PATH_TO_UNIT_TEST ${PROJECT_SOURCE_DIR}/tests/vm_unit_tests)
119124

120125
aux_source_directory(${PATH_TO_UNIT_TEST}/shared/ SHARED_SRC)
121126

‎platforms/CLI-Emulator/main.cpp

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
#include "../../src/Debug/debugger.h"
1515
#include "../../src/Utils/macros.h"
16+
#include "../../tests/integration/wasm_tests.h"
1617
#include "warduino/config.h"
1718

1819
// Constants
@@ -39,6 +40,15 @@ void print_help() {
3940
fprintf(stdout,
4041
" --loop Let the runtime loop infinitely on exceptions "
4142
"(default: false)\n");
43+
fprintf(stdout,
44+
" --test Run in test mode with .wast file with module "
45+
"as argument\n");
46+
fprintf(stdout,
47+
" --asserts Name of file containing asserts to run against "
48+
"loaded module\n");
49+
fprintf(stdout,
50+
" --watcompiler Command to compile Wat files to Wasm "
51+
"binaries (default: wat2wasm)\n");
4252
fprintf(stdout,
4353
" --no-debug Run without debug thread"
4454
"(default: false)\n");
@@ -253,6 +263,7 @@ int main(int argc, const char *argv[]) {
253263
ARGV_SHIFT(); // Skip command name
254264

255265
bool return_exception = true;
266+
bool run_tests = false;
256267
bool no_debug = false;
257268
bool no_socket = false;
258269
const char *socket = "8192";
@@ -265,6 +276,9 @@ int main(int argc, const char *argv[]) {
265276
const char *fname = nullptr;
266277
std::vector<StackValue> arguments = std::vector<StackValue>();
267278

279+
const char *asserts_file = nullptr;
280+
const char *watcompiler = "wat2wasm";
281+
268282
if (argc > 0 && argv[0][0] != '-') {
269283
ARGV_GET(file_name);
270284

@@ -289,6 +303,13 @@ int main(int argc, const char *argv[]) {
289303
return 0;
290304
} else if (!strcmp("--loop", arg)) {
291305
return_exception = false;
306+
} else if (!strcmp("--test", arg)) {
307+
run_tests = true;
308+
ARGV_GET(file_name);
309+
} else if (!strcmp("--asserts", arg)) {
310+
ARGV_GET(asserts_file);
311+
} else if (!strcmp("--watcompiler", arg)) {
312+
ARGV_GET(watcompiler);
292313
} else if (!strcmp("--no-debug", arg)) {
293314
no_debug = true;
294315
} else if (!strcmp("--no-socket", arg)) {
@@ -338,6 +359,11 @@ int main(int argc, const char *argv[]) {
338359

339360
m->warduino = wac;
340361

362+
if (run_tests) {
363+
dbg_info("=== STARTING SPEC TESTS ===\n");
364+
return run_wasm_test(*wac, file_name, asserts_file, watcompiler);
365+
}
366+
341367
if (initiallyPaused) {
342368
wac->debugger->pauseRuntime(m);
343369
}

‎tests/integration/assertion.cpp

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#include "assertion.h"
2+
3+
#include <cstdlib>
4+
5+
Result *makeEmptyResult() {
6+
auto *result = (Result *)malloc(sizeof(Result));
7+
result->type = VOID;
8+
result->value = nullptr;
9+
return result;
10+
}
11+
12+
Result *makeValueResult(Value *val) {
13+
auto *result = (Result *)malloc(sizeof(Result));
14+
result->type = VAL;
15+
result->value = val;
16+
return result;
17+
}
18+
19+
Action *makeInvokeAction(char *name, Value *expr) {
20+
auto *action = (Action *)malloc(sizeof(Action));
21+
action->type = INVOKE;
22+
action->name = name;
23+
action->expr = expr;
24+
return action;
25+
}
26+
27+
Assertion *makeAssertionReturn(Action *action, Result *result) {
28+
auto *assertion = (Assertion *)malloc(sizeof(Assertion));
29+
assertion->type = RETURN;
30+
assertion->action = action;
31+
assertion->result = result;
32+
return assertion;
33+
}
34+
35+
Assertion *makeAssertionTrap(Action *action, char *result) {
36+
auto *assertion = (Assertion *)malloc(sizeof(Assertion));
37+
assertion->type = TRAP;
38+
assertion->action = action;
39+
assertion->failure = result;
40+
return assertion;
41+
}
42+
43+
Assertion *makeAssertionExhaustion(Action *action) {
44+
return makeAssertionTrap(action, "call stack exhausted");
45+
}
46+
47+
Value *makeUI64(uint64_t num) {
48+
auto *value = (Value *)malloc(sizeof(Value));
49+
value->type = UI64;
50+
value->uint64 = num;
51+
return value;
52+
}
53+
54+
Value *makeI64(int64_t num) {
55+
auto *value = (Value *)malloc(sizeof(Value));
56+
value->type = I64V;
57+
value->int64 = num;
58+
return value;
59+
}
60+
61+
Value *makeI32(int32_t num) {
62+
auto *value = (Value *)malloc(sizeof(Value));
63+
value->type = I32V;
64+
value->int32 = num;
65+
return value;
66+
}
67+
68+
Value *makeF32(float num) {
69+
auto *value = (Value *)malloc(sizeof(Value));
70+
value->type = F32V;
71+
value->f32 = num;
72+
return value;
73+
}
74+
75+
Value *makeF64(double num) {
76+
auto *value = (Value *)malloc(sizeof(Value));
77+
value->type = F64V;
78+
value->f64 = num;
79+
return value;
80+
}

‎tests/integration/assertion.h

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
#ifndef ASSERTION_H
2+
#define ASSERTION_H
3+
4+
#include <stdint.h>
5+
6+
/*
7+
cmd:
8+
<module> ;; define, validate, and initialize
9+
module ( register <string> <name>? ) ;; register module for imports
10+
<action> ;; perform action and print results
11+
<assertion> ;; assert result of an action
12+
<meta> ;; meta command
13+
14+
module:
15+
...
16+
( module <name>? binary <string>* ) ;; module in binary format (may be
17+
malformed) ( module <name>? quote <string>* ) ;; module quoted in text
18+
(may be malformed)
19+
20+
action:
21+
( invoke <name>? <string> <expr>* ) ;; invoke function export
22+
( get <name>? <string> ) ;; get global export
23+
24+
assertion:
25+
( assert_return <action> <result>* ) ;; assert action has expected
26+
results ( assert_trap <action> <failure> ) ;; assert action traps with
27+
given failure string ( assert_exhaustion <action> <failure> ) ;; assert action
28+
exhausts system resources ( assert_malformed <module> <failure> ) ;; assert
29+
module cannot be decoded with given failure string ( assert_invalid <module>
30+
<failure> ) ;; assert module is invalid with given failure string (
31+
assert_unlinkable <module> <failure> ) ;; assert module fails to link (
32+
assert_trap <module> <failure> ) ;; assert module traps on instantiation
33+
34+
result:
35+
( <val_type>.const <numpat> )
36+
37+
numpat:
38+
<value> ;; literal result
39+
nan:canonical ;; NaN in canonical form
40+
nan:arithmetic ;; NaN with 1 in MSB of payload
41+
42+
meta:
43+
( script <name>? <script> ) ;; name a subscript
44+
( input <name>? <string> ) ;; read script or module from file
45+
( output <name>? <string>? ) ;; output module to stout or file
46+
*/
47+
48+
typedef enum {
49+
RETURN,
50+
TRAP,
51+
EXHAUSTION,
52+
MALFORMED,
53+
INVALID,
54+
UNLINKABLE
55+
} AssertionType;
56+
typedef enum { INVOKE, GET } ActionType;
57+
typedef enum { UI32, I32V, UI64, I64V, F32V, F64V } ValueType;
58+
59+
typedef struct {
60+
ValueType type;
61+
union {
62+
uint32_t uint32;
63+
int32_t int32;
64+
uint64_t uint64;
65+
int64_t int64;
66+
float f32;
67+
double f64;
68+
};
69+
} Value;
70+
71+
typedef enum {
72+
NAN_R,
73+
VAL,
74+
VOID,
75+
} ResultType;
76+
77+
typedef struct {
78+
ResultType type;
79+
union {
80+
Value *value;
81+
};
82+
} Result;
83+
84+
typedef struct {
85+
ActionType type;
86+
char *name;
87+
Value *expr;
88+
} Action;
89+
90+
typedef struct {
91+
AssertionType type;
92+
union {
93+
Action *action;
94+
char *module;
95+
};
96+
union {
97+
Result *result;
98+
char *failure;
99+
};
100+
} Assertion;
101+
102+
Action *makeInvokeAction(char *name, Value *expr);
103+
104+
Assertion *makeAssertionReturn(Action *action, Result *result);
105+
106+
Assertion *makeAssertionTrap(Action *action, char *result);
107+
108+
Assertion *makeAssertionExhaustion(Action *action);
109+
110+
Result *makeEmptyResult();
111+
112+
Result *makeValueResult(Value *val);
113+
114+
Value *makeUI64(uint64_t value);
115+
116+
Value *makeI64(int64_t value);
117+
118+
Value *makeI32(int32_t value);
119+
120+
Value *makeF32(float num);
121+
122+
Value *makeF64(double num);
123+
124+
#endif

0 commit comments

Comments
 (0)