Skip to content

Commit 1649793

Browse files
wingowingo
committed
test runner: Report actual argv on errors
Instead of having the adapter files be top-level python scripts, now they are modules with exported functions. One of those functions computes the argv to execute when running a test. Also, clean up directories before and after each test, in addition to before running the suite. This makes one failure to clean up not cause other cascading failures, and makes it more reliable to be able to run test cases outside the test-runner harness.
1 parent 52c5c5e commit 1649793

File tree

7 files changed

+208
-112
lines changed

7 files changed

+208
-112
lines changed

‎adapters/wasm-micro-runtime.py‎

Lines changed: 24 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,35 @@
1-
import argparse
21
import subprocess
3-
import sys
42
import os
53
import shlex
4+
from pathlib import Path
5+
from typing import Dict, List, Tuple
66

77
# shlex.split() splits according to shell quoting rules
88
IWASM = shlex.split(os.getenv("IWASM", "iwasm"))
99

10-
parser = argparse.ArgumentParser()
11-
parser.add_argument("--version", action="store_true")
12-
parser.add_argument("--test-file", action="store")
13-
parser.add_argument("--arg", action="append", default=[])
14-
parser.add_argument("--env", action="append", default=[])
15-
parser.add_argument("--dir", action="append", default=[])
1610

17-
args = parser.parse_args()
11+
def get_name() -> str:
12+
return "wamr"
1813

19-
if args.version:
20-
subprocess.run(IWASM + ["--version"])
21-
sys.exit(0)
2214

23-
TEST_FILE = args.test_file
24-
PROG_ARGS = args.arg
25-
ENV_ARGS = [f"--env={i}" for i in args.env]
26-
DIR_ARGS = [f"--map-dir={i}" for i in args.dir]
15+
def get_version() -> str:
16+
# ensure no args when version is queried
17+
result = subprocess.run(IWASM + ["--version"],
18+
encoding="UTF-8", capture_output=True,
19+
check=True)
20+
output = result.stdout.splitlines()[0].split(" ")
21+
return output[1]
2722

28-
r = subprocess.run(IWASM + ENV_ARGS + DIR_ARGS + [TEST_FILE] + PROG_ARGS)
29-
sys.exit(r.returncode)
23+
24+
def compute_argv(test_path: str,
25+
args: List[str],
26+
env: Dict[str, str],
27+
dirs: List[Tuple[Path, str]]) -> List[str]:
28+
argv = [] + IWASM
29+
for k, v in env.items():
30+
argv += ["--env", f"{k}={v}"]
31+
for host, guest in dirs:
32+
argv += ["--map-dir", f"{host}::{guest}"] # noqa: E231
33+
argv += [test_path]
34+
argv += args
35+
return argv

‎adapters/wasmtime.py‎

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,35 @@
1-
import argparse
21
import subprocess
3-
import sys
42
import os
53
import shlex
4+
from pathlib import Path
5+
from typing import Dict, List, Tuple
66

77
# shlex.split() splits according to shell quoting rules
88
WASMTIME = shlex.split(os.getenv("WASMTIME", "wasmtime"))
99

10-
parser = argparse.ArgumentParser()
11-
parser.add_argument("--version", action="store_true")
12-
parser.add_argument("--test-file", action="store")
13-
parser.add_argument("--arg", action="append", default=[])
14-
parser.add_argument("--env", action="append", default=[])
15-
parser.add_argument("--dir", action="append", default=[])
1610

17-
args = parser.parse_args()
11+
def get_name() -> str:
12+
return "wasmtime"
1813

19-
if args.version:
14+
15+
def get_version() -> str:
2016
# ensure no args when version is queried
21-
subprocess.run(WASMTIME[0:1] + ["--version"])
22-
sys.exit(0)
17+
result = subprocess.run(WASMTIME[0:1] + ["--version"],
18+
encoding="UTF-8", capture_output=True,
19+
check=True)
20+
output = result.stdout.splitlines()[0].split(" ")
21+
return output[1]
2322

24-
TEST_FILE = args.test_file
25-
PROG_ARGS = args.arg
26-
ENV_ARGS = [j for i in args.env for j in ["--env", i]]
27-
DIR_ARGS = [j for i in args.dir for j in ["--dir", i]]
2823

29-
r = subprocess.run(WASMTIME + ENV_ARGS + DIR_ARGS + [TEST_FILE] + PROG_ARGS)
30-
sys.exit(r.returncode)
24+
def compute_argv(test_path: str,
25+
args: List[str],
26+
env: Dict[str, str],
27+
dirs: List[Tuple[Path, str]]) -> List[str]:
28+
argv = [] + WASMTIME
29+
for k, v in env.items():
30+
argv += ["--env", f"{k}={v}"]
31+
for host, guest in dirs:
32+
argv += ["--dir", f"{host}::{guest}"] # noqa: E231
33+
argv += [test_path]
34+
argv += args
35+
return argv

‎adapters/wizard.py‎

Lines changed: 32 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,44 @@
1-
import argparse
21
import subprocess
3-
import sys
42
import os
53
import shlex
4+
from pathlib import Path
5+
from typing import Dict, List, Tuple
66

77
# shlex.split() splits according to shell quoting rules
88
WIZARD = shlex.split(os.getenv("WIZARD", "wizeng.x86-64-linux"))
99

10-
parser = argparse.ArgumentParser()
11-
parser.add_argument("--version", action="store_true")
12-
parser.add_argument("--test-file", action="store")
13-
parser.add_argument("--arg", action="append", default=[])
14-
parser.add_argument("--env", action="append", default=[])
15-
parser.add_argument("--dir", action="append", default=[])
1610

17-
args = parser.parse_args()
11+
def get_name() -> str:
12+
return "wizard"
1813

19-
if args.version:
14+
15+
def get_version() -> str:
2016
# ensure no args when version is queried
21-
subprocess.run(WIZARD[0:1] + ["-version"])
22-
sys.exit(0)
17+
output = ""
18+
try:
19+
result = subprocess.run(WIZARD[0:1] + ["--version"],
20+
encoding="UTF-8", capture_output=True,
21+
check=False)
22+
output = result.stdout;
23+
except subprocess.CalledProcessError as e:
24+
# https://github.com/titzer/wizard-engine/issues/483
25+
if e.returncode != 3:
26+
raise e
27+
output = result.stdout
28+
output = output.splitlines()[0].split(" ")
29+
return output[1]
2330

24-
TEST_FILE = args.test_file
25-
PROG_ARGS = args.arg
26-
ENV_ARGS = None if len(args.env) == 0 else f'--env={",".join(args.env)}'
27-
DIR_ARGS = None if len(args.dir) == 0 else f'--dir={",".join(args.dir)}'
2831

29-
r = subprocess.run([arg for arg in WIZARD + [ENV_ARGS, DIR_ARGS, TEST_FILE] + PROG_ARGS if arg])
30-
sys.exit(r.returncode)
32+
def compute_argv(test_path: str,
33+
args: List[str],
34+
env: Dict[str, str],
35+
dirs: List[Tuple[Path, str]]) -> List[str]:
36+
argv = [] + WIZARD
37+
for k, v in env.items():
38+
argv += [f"--env={k}={v}"]
39+
for host, guest in dirs:
40+
# FIXME: https://github.com/titzer/wizard-engine/issues/482
41+
argv += [f"--dir={host}"]
42+
argv += [test_path]
43+
argv += args
44+
return argv

‎run-tests‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ from pathlib import Path
99
sys.path.insert(0, str(Path(__file__).parent / "test-runner"))
1010

1111
from wasi_test_runner.harness import run_tests
12-
from wasi_test_runner.runtime_adapter import RuntimeAdapter
12+
from wasi_test_runner import runtime_adapter
1313

1414
parser = argparse.ArgumentParser(
1515
description="WASI test runner"
@@ -42,7 +42,9 @@ parser.add_argument(
4242
help="Print more information about test results.",
4343
)
4444

45-
def find_test_dirs(root):
45+
def find_test_dirs():
46+
root = Path(__file__).parent / "tests"
47+
root = root.relative_to(Path('.').absolute(), walk_up=True)
4648
test_dirs = []
4749
for root, dirs, files in root.walk(on_error=print):
4850
if "manifest.json" in files:
@@ -53,32 +55,32 @@ def find_runtime_adapters(root, verbose=False):
5355
print(f"Detecting WASI runtime availability:")
5456
adapters = []
5557
for candidate in root.glob("*.py"):
56-
adapter = RuntimeAdapter(candidate)
5758
try:
59+
adapter = runtime_adapter.RuntimeAdapter(candidate)
5860
print(f" {candidate.name}: {adapter.get_version()}")
5961
adapters.append(adapter)
60-
except subprocess.CalledProcessError as e:
61-
print(f" {candidate.name}: unavailable; pass `--runtime {candidate}` to debug.")
62+
except runtime_adapter.LegacyRuntimeAdapterError:
63+
print(f" {candidate} is too old; update to new module format.")
64+
except runtime_adapter.UnavailableRuntimeAdapterError:
65+
print(f" {candidate.name} unavailable; pass `--runtime {candidate}` to debug.")
6266
print("")
6367
if len(adapters) == 0:
6468
print("Error: No WASI runtimes found")
6569
sys.exit(1)
6670
return adapters
6771

6872
options = parser.parse_args()
69-
test_suite = find_test_dirs(Path(__file__).parent / "tests")
73+
test_suite = find_test_dirs()
7074
if options.runtime_adapter:
71-
runtime_adapters = [RuntimeAdapter(options.runtime_adapter)]
72-
# Ensure it works.
7375
try:
74-
runtime_adapters[0].get_version()
75-
except subprocess.CalledProcessError as e:
76-
print(f"Error: failed to load {options.runtime_adapter}:")
77-
print(f" Failed command line: {' '.join(e.cmd)}")
78-
if e.stdout.strip() != "":
79-
print(f" stdout:\n{e.stdout}")
80-
if e.stderr.strip() != "":
81-
print(f" stderr:\n{e.stderr}")
76+
runtime_adapters = [
77+
runtime_adapter.RuntimeAdapter(options.runtime_adapter)
78+
]
79+
except runtime_adapter.LegacyRuntimeAdapterError as e:
80+
print(f"Error: {e.adapter_path} is too old; update to new module format.")
81+
sys.exit(1)
82+
except runtime_adapter.UnavailableRuntimeAdapterError as e:
83+
print(f"Error: failed to load {e.adapter_path}: {e.error}")
8284
sys.exit(1)
8385
else:
8486
runtime_adapters = find_runtime_adapters(Path(__file__).parent / "adapters")

‎test-runner/tests/test_test_suite_runner.py‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
from typing import Any
2+
from pathlib import Path
23
from unittest.mock import ANY, MagicMock, Mock, patch, mock_open
34

45
import wasi_test_runner.test_case as tc
@@ -10,9 +11,9 @@ def get_mock_open() -> Mock:
1011
def open_mock(filename: str, *_args: Any, **_kwargs: Any) -> Any:
1112
file_content = {
1213
"my-path/manifest.json": '{"name": "test-suite"}',
13-
"test1.json": '{"dirs": [".", "deep/dir"]}',
14-
"test2.json": '{"exit_code": 1, "args": ["a", "b"]}',
15-
"test3.json": '{"stdout": "output", "env": {"x": "1"}}',
14+
"my-path/test1.json": '{"dirs": [".", "deep/dir"]}',
15+
"my-path/test2.json": '{"exit_code": 1, "args": ["a", "b"]}',
16+
"my-path/test3.json": '{"stdout": "output", "env": {"x": "1"}}',
1617
}
1718
if filename in file_content:
1819
return mock_open(read_data=file_content[filename]).return_value
@@ -26,7 +27,10 @@ def open_mock(filename: str, *_args: Any, **_kwargs: Any) -> Any:
2627
@patch("builtins.open", get_mock_open())
2728
@patch("os.path.exists", Mock(return_value=True))
2829
def test_runner_end_to_end() -> None:
29-
test_paths = ["test1.wasm", "test2.wasm", "test3.wasm"]
30+
test_suite_dir = "my-path"
31+
test_suite_name = "test-suite"
32+
test_files = ["test1.wasm", "test2.wasm", "test3.wasm"]
33+
test_paths = [Path(test_suite_dir) / f for f in test_files]
3034

3135
failures = [tc.Failure("a", "b"), tc.Failure("x", "y"), tc.Failure("x", "z")]
3236

@@ -75,9 +79,8 @@ def test_runner_end_to_end() -> None:
7579
filt.should_skip.return_value = (False, None)
7680
filters = [filt]
7781

78-
test_suite_dir = "my-path"
79-
test_suite_name = "test-suite"
80-
with patch("glob.glob", return_value=test_paths):
82+
with (patch("glob.glob", return_value=[str(p) for p in test_paths]),
83+
patch("wasi_test_runner.test_suite_runner._cleanup_test_output")):
8184
suite = tsr.run_tests_from_test_suite(test_suite_dir, runtime,
8285
validators, # type: ignore
8386
reporters, # type: ignore
@@ -93,8 +96,9 @@ def test_runner_end_to_end() -> None:
9396
# Assert test runner calls
9497
assert runtime.run_test.call_count == 3
9598
for test_path, config in zip(test_paths, expected_config):
99+
expected_dirs = [(Path(test_suite_dir) / d, d) for d in config.dirs]
96100
runtime.compute_argv.assert_any_call(
97-
test_path, config.args, config.env, config.dirs
101+
str(test_path), config.args, config.env, expected_dirs
98102
)
99103
runtime.run_test.assert_called_with(expected_argv)
100104

0 commit comments

Comments
 (0)