Improve test runner ergonomics

.github/workflows/compile-tests.yml

@@ -7,6 +7,8 @@ on:
   pull_request:
     branches:
       - '*'
+    paths:
+      - 'tests/**'
 
 jobs:
   build_assemblyscript:

README.md

@@ -44,32 +44,31 @@ executor is quite simple; see the [specification] document for the details and t
 5. Execute the test suites from this repository:
 
    ```bash
-   python3 test-runner/wasi_test_runner.py                                                  \
-      -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \
-         ./tests/c/testsuite/wasm32-wasip1                                                 \
-         ./tests/rust/testsuite/                                                           \
-      -r adapters/wasmtime.py # path to a runtime adapter
+   ./run-tests
    ```
 
-Optionally you can specify test cases to skip with the `--exclude-filter` option.
+By default, the test runner will detect available WASI runtimes from
+those available in [adapters/](adapters/), and will run tests on all
+available runtimes.  Pass `--runtime` to instead use a specific runtime.
+
+```
+./run-tests --runtime adapters/wasmtime.py
+```
+
+Running tests will invoke the WASI runtime's binary in a subprocess:
+`wasmtime` for `adapters/wasmtime.py`, `iwasm` for
+`adapters/wasm-micro-runtime.py`, and so on.  These binaries can be
+overridden by setting corresponding environment variables (`WASMTIME`,
+`IWASM`, etc):
 
-```bash
-python3 test-runner/wasi_test_runner.py                                                  \
-    -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \
-       ./tests/c/testsuite/wasm32-wasip1                                                 \
-       ./tests/rust/testsuite/                                                           \
-    --exclude-filter examples/skip.json                                                  \
-    -r adapters/wasmtime.py # path to a runtime adapter
+```
+WASMTIME="wasmtime --wasm-features all" ./run-tests
 ```
 
-The default executable in the adapter used for test execution can be
-overridden using `TEST_RUNTIME_EXE` variable. This only works with adapters defined in
-[adapters/](adapters/), and might not work with 3rd party adapters.
+Optionally you can specify test cases to skip with the `--exclude-filter` option.
 
 ```bash
-TEST_RUNTIME_EXE="wasmtime --wasm-features all" python3 test-runner/wasi_test_runner.py                                                  \
-    -t ./tests/assemblyscript/testsuite/ \
-    -r adapters/wasmtime.py
+./run-tests --exclude-filter examples/skip.json                                                  \
 ```
 
 ## Contributing

adapters/wasm-micro-runtime.py

@@ -5,7 +5,7 @@
 import shlex
 
 # shlex.split() splits according to shell quoting rules
-IWASM = shlex.split(os.getenv("TEST_RUNTIME_EXE", "iwasm"))
+IWASM = shlex.split(os.getenv("IWASM", "iwasm"))
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--version", action="store_true")
@@ -23,7 +23,7 @@
 TEST_FILE = args.test_file
 PROG_ARGS = args.arg
 ENV_ARGS = [f"--env={i}" for i in args.env]
-DIR_ARGS = [f"--dir={i}" for i in args.dir]
+DIR_ARGS = [f"--map-dir={i}" for i in args.dir]
 
 r = subprocess.run(IWASM + ENV_ARGS + DIR_ARGS + [TEST_FILE] + PROG_ARGS)
 sys.exit(r.returncode)

adapters/wasmtime.py

@@ -5,7 +5,7 @@
 import shlex
 
 # shlex.split() splits according to shell quoting rules
-WASMTIME = shlex.split(os.getenv("TEST_RUNTIME_EXE", "wasmtime"))
+WASMTIME = shlex.split(os.getenv("WASMTIME", "wasmtime"))
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--version", action="store_true")

adapters/wizard.py

@@ -5,7 +5,7 @@
 import shlex
 
 # shlex.split() splits according to shell quoting rules
-WIZARD = shlex.split(os.getenv("TEST_RUNTIME_EXE", "wizeng.x86-64-linux"))
+WIZARD = shlex.split(os.getenv("WIZARD", "wizeng.x86-64-linux"))
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--version", action="store_true")

run-tests

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import argparse
+import subprocess
+import sys
+from typing import List
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent / "test-runner"))
+
+from wasi_test_runner.harness import run_tests
+from wasi_test_runner.runtime_adapter import RuntimeAdapter
+
+parser = argparse.ArgumentParser(
+    description="WASI test runner"
+)
+
+parser.add_argument(
+    "-f",
+    "--exclude-filter",
+    action="append",
+    default=[],
+    help="Path to JSON file indicating tests to exclude.",
+)
+parser.add_argument(
+    "-r", "--runtime-adapter", help="Path to a runtime adapter."
+)
+parser.add_argument(
+    "--json-output-location",
+    help="JSON test result destination. If not specified, JSON output won't be generated.",
+)
+parser.add_argument(
+    "--disable-colors",
+    action="store_true",
+    default=False,
+    help="Disables color for console output reporter.",
+)
+parser.add_argument(
+    "--verbose",
+    action="store_true",
+    default=False,
+    help="Print more information about test results.",
+)
+
+def find_test_dirs(root):
+    test_dirs = []
+    for root, dirs, files in root.walk(on_error=print):
+        if "manifest.json" in files:
+            test_dirs.append(root)
+    return test_dirs
+
+def find_runtime_adapters(root, verbose=False):
+    print(f"Detecting WASI runtime availability:")
+    adapters = []
+    for candidate in root.glob("*.py"):
+        adapter = RuntimeAdapter(candidate)
+        try:
+            print(f"  {candidate.name}: {adapter.get_version()}")
+            adapters.append(adapter)
+        except subprocess.CalledProcessError as e:
+            print(f"  {candidate.name}: unavailable; pass `--runtime {candidate}` to debug.")
+    print("")
+    if len(adapters) == 0:
+        print("Error: No WASI runtimes found")
+        sys.exit(1)
+    return adapters
+
+options = parser.parse_args()
+test_suite = find_test_dirs(Path(__file__).parent / "tests")
+if options.runtime_adapter:
+    runtime_adapters = [RuntimeAdapter(options.runtime_adapter)]
+    # Ensure it works.
+    try:
+        runtime_adapters[0].get_version()
+    except subprocess.CalledProcessError as e:
+        print(f"Error: failed to load {options.runtime_adapter}:")
+        print(f"  Failed command line: {' '.join(e.cmd)}")
+        if e.stdout.strip() != "":
+            print(f"  stdout:\n{e.stdout}")
+        if e.stderr.strip() != "":
+            print(f"  stderr:\n{e.stderr}")
+        sys.exit(1)
+else:
+    runtime_adapters = find_runtime_adapters(Path(__file__).parent / "adapters")
+
+exclude_filters = [JSONTestExcludeFilter(filt) for filt in options.exclude_filter]
+
+sys.exit(run_tests(runtime_adapters, test_suite,
+                   color=not options.disable_colors,
+                   json_log_file=options.json_output_location,
+                   exclude_filters=exclude_filters))

test-runner/requirements/dev.txt

@@ -1,6 +1,6 @@
 -r common.txt
 flake8==5.0.4
 mypy==0.991
-pylint==2.14.3
+pylint==3.3.8
 pytest==6.2.5
 coverage==6.3.3

test-runner/tests/test_test_suite.py

@@ -2,12 +2,14 @@
 
 import wasi_test_runner.test_case as tc
 import wasi_test_runner.test_suite as ts
+from wasi_test_runner.runtime_adapter import RuntimeVersion
 
 
 def create_test_case(name: str, is_executed: bool, is_failed: bool) -> tc.TestCase:
     failures = [tc.Failure("a", "b")] if is_failed else []
     return tc.TestCase(
         name,
+        ["test-runtime-exe", name],
         tc.Config(),
         tc.Result(tc.Output(0, "", ""), is_executed, failures),
         1.0,
@@ -17,6 +19,7 @@ def create_test_case(name: str, is_executed: bool, is_failed: bool) -> tc.TestCa
 def test_test_suite_should_return_correct_count() -> None:
     suite = ts.TestSuite(
         "suite",
+        RuntimeVersion("test-runtime", "3.14"),
         10.0,
         datetime.now(),
         [

test-runner/tests/test_test_suite_runner.py

@@ -3,6 +3,7 @@
 
 import wasi_test_runner.test_case as tc
 import wasi_test_runner.test_suite_runner as tsr
+from wasi_test_runner.runtime_adapter import RuntimeVersion
 
 
 def get_mock_open() -> Mock:
@@ -45,15 +46,23 @@ def test_runner_end_to_end() -> None:
         tc.Config(stdout="output", env={"x": "1"}),
     ]
 
+    runtime_name = "rt1"
+    runtime_version_str = "4.2"
+    runtime_version = RuntimeVersion(runtime_name, runtime_version_str)
+
+    expected_argv = [runtime_name, "<test>"]
     expected_test_cases = [
-        tc.TestCase(test_name, config, result, ANY)
+        tc.TestCase(test_name, expected_argv, config, result, ANY)
         for config, test_name, result in zip(
             expected_config, ["test1", "test2", "test3"], expected_results
         )
     ]
 
     runtime = Mock()
+    runtime.get_name.return_value = runtime_name
+    runtime.get_version.return_value = runtime_version
     runtime.run_test.side_effect = outputs
+    runtime.compute_argv.return_value = expected_argv
 
     validators = [
         Mock(side_effect=[None, None, failures[0]]),
@@ -66,11 +75,16 @@ def test_runner_end_to_end() -> None:
     filt.should_skip.return_value = (False, None)
     filters = [filt]
 
+    test_suite_dir = "my-path"
+    test_suite_name = "test-suite"
     with patch("glob.glob", return_value=test_paths):
-        suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters, filters)  # type: ignore
+        suite = tsr.run_tests_from_test_suite(test_suite_dir, runtime,
+                                              validators,  # type: ignore
+                                              reporters,   # type: ignore
+                                              filters)     # type: ignore
 
     # Assert manifest was read correctly
-    assert suite.name == "test-suite"
+    assert suite.name == test_suite_name
 
     # Assert test cases
     assert suite.test_count == 3
@@ -79,15 +93,18 @@ def test_runner_end_to_end() -> None:
     # Assert test runner calls
     assert runtime.run_test.call_count == 3
     for test_path, config in zip(test_paths, expected_config):
-        runtime.run_test.assert_any_call(
+        runtime.compute_argv.assert_any_call(
             test_path, config.args, config.env, config.dirs
         )
+        runtime.run_test.assert_called_with(expected_argv)
 
     # Assert reporters calls
     for reporter in reporters:
         assert reporter.report_test.call_count == 3
         for test_case in expected_test_cases:
-            reporter.report_test.assert_any_call(test_case)
+            reporter.report_test.assert_any_call(test_suite_name,
+                                                 runtime_version,
+                                                 test_case)
 
     # Assert validators calls
     for validator in validators:
@@ -99,7 +116,8 @@ def test_runner_end_to_end() -> None:
     for filt in filters:
         assert filt.should_skip.call_count == 3
         for test_case in expected_test_cases:
-            filt.should_skip.assert_any_call(suite.name, test_case.name)
+            filt.should_skip.assert_any_call(runtime_version, suite.name,
+                                             test_case.name)
 
 
 @patch("os.path.exists", Mock(return_value=False))

test-runner/wasi_test_runner/__main__.py

@@ -49,7 +49,8 @@ def main() -> int:
 
     options = parser.parse_args()
 
-    reporters: List[TestReporter] = [ConsoleTestReporter(not options.disable_colors)]
+    reporters: List[TestReporter] = [ConsoleTestReporter(not options.disable_colors,
+                                                         verbose=True)]
     if options.json_output_location:
         reporters.append(JSONTestReporter(options.json_output_location))
 
@@ -60,7 +61,7 @@ def main() -> int:
         filters.append(JSONTestExcludeFilter(filt))
 
     return run_all_tests(
-        RuntimeAdapter(options.runtime_adapter),
+        [RuntimeAdapter(options.runtime_adapter)],
         options.test_suite,
         validators,
         reporters,

test-runner/wasi_test_runner/filters.py

@@ -4,11 +4,13 @@
 
 import json
 
+from .runtime_adapter import RuntimeVersion
+
 
 class TestFilter(ABC):
     @abstractmethod
     def should_skip(
-        self, test_suite_name: str, test_name: str
+        self, runtime: RuntimeVersion, test_suite_name: str, test_name: str
     ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
         pass
 
@@ -19,7 +21,7 @@ def __init__(self, filename: str) -> None:
             self.filter_dict = json.load(file)
 
     def should_skip(
-        self, test_suite_name: str, test_name: str
+        self, runtime: RuntimeVersion, test_suite_name: str, test_name: str
     ) -> Union[Tuple[Literal[True], str], Tuple[Literal[False], Literal[None]]]:
         test_suite_filter = self.filter_dict.get(test_suite_name)
         if test_suite_filter is None: