test runner: Report actual argv on errors

wingo · wingo · commit 064f5ef35704 · 2025-09-19T12:20:13.000+02:00
Instead of having the adapter files be top-level python scripts, now
they are modules with exported functions.  One of those functions
computes the argv to execute when running a test.

Also, clean up directories before and after each test, in addition to
before running the suite.  This makes one failure to clean up not cause
other cascading failures, and makes it more reliable to be able to run
test cases outside the test-runner harness.
diff --git a/adapters/wasm-micro-runtime.py b/adapters/wasm-micro-runtime.py
@@ -1,29 +1,35 @@
-import argparse
 import subprocess
-import sys
 import os
 import shlex
+from pathlib import Path
+from typing import Dict, List, Tuple
 
 # shlex.split() splits according to shell quoting rules
 IWASM = shlex.split(os.getenv("IWASM", "iwasm"))
 
-parser = argparse.ArgumentParser()
-parser.add_argument("--version", action="store_true")
-parser.add_argument("--test-file", action="store")
-parser.add_argument("--arg", action="append", default=[])
-parser.add_argument("--env", action="append", default=[])
-parser.add_argument("--dir", action="append", default=[])
 
-args = parser.parse_args()
+def get_name() -> str:
+    return "wamr"
 
-if args.version:
-    subprocess.run(IWASM + ["--version"])
-    sys.exit(0)
 
-TEST_FILE = args.test_file
-PROG_ARGS = args.arg
-ENV_ARGS = [f"--env={i}" for i in args.env]
-DIR_ARGS = [f"--map-dir={i}" for i in args.dir]
+def get_version() -> str:
+    # ensure no args when version is queried
+    result = subprocess.run(IWASM + ["--version"],
+                            encoding="UTF-8", capture_output=True,
+                            check=True)
+    output = result.stdout.splitlines()[0].split(" ")
+    return output[1]
 
-r = subprocess.run(IWASM + ENV_ARGS + DIR_ARGS + [TEST_FILE] + PROG_ARGS)
-sys.exit(r.returncode)
+
+def compute_argv(test_path: str,
+                 args: List[str],
+                 env: Dict[str, str],
+                 dirs: List[Tuple[Path, str]]) -> List[str]:
+    argv = [] + IWASM
+    for k, v in env.items():
+        argv += ["--env", f"{k}={v}"]
+    for host, guest in dirs:
+        argv += ["--map-dir", f"{host}::{guest}"]  # noqa: E231
+    argv += [test_path]
+    argv += args
+    return argv
diff --git a/adapters/wasmtime.py b/adapters/wasmtime.py
@@ -1,30 +1,35 @@
-import argparse
 import subprocess
-import sys
 import os
 import shlex
+from pathlib import Path
+from typing import Dict, List, Tuple
 
 # shlex.split() splits according to shell quoting rules
 WASMTIME = shlex.split(os.getenv("WASMTIME", "wasmtime"))
 
-parser = argparse.ArgumentParser()
-parser.add_argument("--version", action="store_true")
-parser.add_argument("--test-file", action="store")
-parser.add_argument("--arg", action="append", default=[])
-parser.add_argument("--env", action="append", default=[])
-parser.add_argument("--dir", action="append", default=[])
 
-args = parser.parse_args()
+def get_name() -> str:
+    return "wasmtime"
 
-if args.version:
+
+def get_version() -> str:
     # ensure no args when version is queried
-    subprocess.run(WASMTIME[0:1] + ["--version"])
-    sys.exit(0)
+    result = subprocess.run(WASMTIME[0:1] + ["--version"],
+                            encoding="UTF-8", capture_output=True,
+                            check=True)
+    output = result.stdout.splitlines()[0].split(" ")
+    return output[1]
 
-TEST_FILE = args.test_file
-PROG_ARGS = args.arg
-ENV_ARGS = [j for i in args.env for j in ["--env", i]]
-DIR_ARGS = [j for i in args.dir for j in ["--dir", i]]
 
-r = subprocess.run(WASMTIME + ENV_ARGS + DIR_ARGS + [TEST_FILE] + PROG_ARGS)
-sys.exit(r.returncode)
+def compute_argv(test_path: str,
+                 args: List[str],
+                 env: Dict[str, str],
+                 dirs: List[Tuple[Path, str]]) -> List[str]:
+    argv = [] + WASMTIME
+    for k, v in env.items():
+        argv += ["--env", f"{k}={v}"]
+    for host, guest in dirs:
+        argv += ["--dir", f"{host}::{guest}"]  # noqa: E231
+    argv += [test_path]
+    argv += args
+    return argv
diff --git a/adapters/wizard.py b/adapters/wizard.py
@@ -1,30 +1,44 @@
-import argparse
 import subprocess
-import sys
 import os
 import shlex
+from pathlib import Path
+from typing import Dict, List, Tuple
 
 # shlex.split() splits according to shell quoting rules
 WIZARD = shlex.split(os.getenv("WIZARD", "wizeng.x86-64-linux"))
 
-parser = argparse.ArgumentParser()
-parser.add_argument("--version", action="store_true")
-parser.add_argument("--test-file", action="store")
-parser.add_argument("--arg", action="append", default=[])
-parser.add_argument("--env", action="append", default=[])
-parser.add_argument("--dir", action="append", default=[])
 
-args = parser.parse_args()
+def get_name() -> str:
+    return "wizard"
 
-if args.version:
+
+def get_version() -> str:
     # ensure no args when version is queried
-    subprocess.run(WIZARD[0:1] + ["-version"])
-    sys.exit(0)
+    output = ""
+    try:
+        result = subprocess.run(WIZARD[0:1] + ["--version"],
+                                encoding="UTF-8", capture_output=True,
+                                check=False)
+        output = result.stdout;
+    except subprocess.CalledProcessError as e:
+        # https://github.com/titzer/wizard-engine/issues/483
+        if e.returncode != 3:
+            raise e
+        output = result.stdout
+    output = output.splitlines()[0].split(" ")
+    return output[1]
 
-TEST_FILE = args.test_file
-PROG_ARGS = args.arg
-ENV_ARGS = None if len(args.env) == 0 else f'--env={",".join(args.env)}'
-DIR_ARGS = None if len(args.dir) == 0 else f'--dir={",".join(args.dir)}'
 
-r = subprocess.run([arg for arg in WIZARD + [ENV_ARGS, DIR_ARGS, TEST_FILE] + PROG_ARGS if arg])
-sys.exit(r.returncode)
+def compute_argv(test_path: str,
+                 args: List[str],
+                 env: Dict[str, str],
+                 dirs: List[Tuple[Path, str]]) -> List[str]:
+    argv = [] + WIZARD
+    for k, v in env.items():
+        argv += [f"--env={k}={v}"]
+    for host, guest in dirs:
+        # FIXME: https://github.com/titzer/wizard-engine/issues/482
+        argv += [f"--dir={host}"]
+    argv += [test_path]
+    argv += args
+    return argv
diff --git a/run-tests b/run-tests
@@ -9,7 +9,7 @@ from pathlib import Path
 sys.path.insert(0, str(Path(__file__).parent / "test-runner"))
 
 from wasi_test_runner.harness import run_tests
-from wasi_test_runner.runtime_adapter import RuntimeAdapter
+from wasi_test_runner import runtime_adapter
 
 parser = argparse.ArgumentParser(
     description="WASI test runner"
@@ -53,12 +53,14 @@ def find_runtime_adapters(root, verbose=False):
     print(f"Detecting WASI runtime availability:")
     adapters = []
     for candidate in root.glob("*.py"):
-        adapter = RuntimeAdapter(candidate)
         try:
+            adapter = runtime_adapter.RuntimeAdapter(candidate)
             print(f"  {candidate.name}: {adapter.get_version()}")
             adapters.append(adapter)
-        except subprocess.CalledProcessError as e:
-            print(f"  {candidate.name}: unavailable; pass `--runtime {candidate}` to debug.")
+        except runtime_adapter.LegacyRuntimeAdapterError:
+            print(f"  {candidate} is too old; update to new module format.")
+        except runtime_adapter.UnavailableRuntimeAdapterError:
+            print(f"  {candidate.name} unavailable; pass `--runtime {candidate}` to debug.")
     print("")
     if len(adapters) == 0:
         print("Error: No WASI runtimes found")
@@ -68,17 +70,15 @@ def find_runtime_adapters(root, verbose=False):
 options = parser.parse_args()
 test_suite = find_test_dirs(Path(__file__).parent / "tests")
 if options.runtime_adapter:
-    runtime_adapters = [RuntimeAdapter(options.runtime_adapter)]
-    # Ensure it works.
     try:
-        runtime_adapters[0].get_version()
-    except subprocess.CalledProcessError as e:
-        print(f"Error: failed to load {options.runtime_adapter}:")
-        print(f"  Failed command line: {' '.join(e.cmd)}")
-        if e.stdout.strip() != "":
-            print(f"  stdout:\n{e.stdout}")
-        if e.stderr.strip() != "":
-            print(f"  stderr:\n{e.stderr}")
+        runtime_adapters = [
+            runtime_adapter.RuntimeAdapter(options.runtime_adapter)
+        ]
+    except runtime_adapter.LegacyRuntimeAdapterError as e:
+        print(f"Error: {e.adapter_path} is too old; update to new module format.")
+        sys.exit(1)
+    except runtime_adapter.UnavailableRuntimeAdapterError as e:
+        print(f"Error: failed to load {e.adapter_path}: {e.error}")
         sys.exit(1)
 else:
     runtime_adapters = find_runtime_adapters(Path(__file__).parent / "adapters")
diff --git a/test-runner/wasi_test_runner/runtime_adapter.py b/test-runner/wasi_test_runner/runtime_adapter.py
@@ -1,7 +1,8 @@
+import importlib.util
 import subprocess
 import sys
 from pathlib import Path
-from typing import Dict, NamedTuple, List
+from typing import Dict, NamedTuple, List, Tuple, Any
 
 from .test_case import Output
 
@@ -14,31 +15,88 @@ def __str__(self) -> str:
         return f"{self.name} {self.version}"
 
 
+class RuntimeAdapterError(Exception):
+    adapter_path: str
+
+    def __init__(self, adapter_path: str) -> None:
+        self.adapter_path = adapter_path
+
+
+class LegacyRuntimeAdapterError(RuntimeAdapterError):
+    adapter_path: str
+
+
+class UnavailableRuntimeAdapterError(RuntimeAdapterError):
+    error: Exception
+
+    def __init__(self, adapter_path: str, error: Exception) -> None:
+        RuntimeAdapterError.__init__(self, adapter_path)
+        self.error = error
+
+
+def _assert_not_legacy_adapter(adapter_path: str) -> None:
+    """
+    Raise an exception if the python file at ADAPTER_PATH isn't
+    loadable as a normal Python module.
+    """
+    argv = [sys.executable, adapter_path, "--version"]
+    try:
+        result = subprocess.run(argv, encoding="UTF-8", check=True,
+                                capture_output=True)
+    except subprocess.CalledProcessError as e:
+        if 'FileNotFoundError' in e.stderr:
+            # The adapter is valid Python.  Running it tries to spawn
+            # the engine subprocess, but couldn't find the binary.  This
+            # indicates a legacy adapter.py.
+            raise LegacyRuntimeAdapterError(adapter_path) from e
+        # Some other error running adapter.py; could be a legacy
+        # adapter, could just be a typo.  Propagate the error.
+        raise UnavailableRuntimeAdapterError(adapter_path, e) from e
+    assert result.stderr == "", result.stderr
+    if result.stdout:
+        # Running the adapter as a subprocess succeeded and produced
+        # --version output: the engine is available but the adapter is
+        # legacy.
+        raise LegacyRuntimeAdapterError(adapter_path)
+    # Otherwise if loading the file produces no output, then we assume
+    # it's a module and not legacy.
+
+
+def _load_adapter_as_module(adapter_path: str) -> Any:
+    path = Path(adapter_path)
+    spec = importlib.util.spec_from_file_location(path.name, path)
+    assert spec is not None
+    assert spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
 class RuntimeAdapter:
     def __init__(self, adapter_path: str) -> None:
-        self._adapter_path = adapter_path
-        self._cached_version: RuntimeVersion | None = None
+        _assert_not_legacy_adapter(adapter_path)
+        self._adapter = _load_adapter_as_module(adapter_path)
+        try:
+            name = self._adapter.get_name()
+            version = self._adapter.get_version()
+        except subprocess.CalledProcessError as e:
+            raise UnavailableRuntimeAdapterError(adapter_path, e) from e
+        except FileNotFoundError as e:
+            raise UnavailableRuntimeAdapterError(adapter_path, e) from e
+        self._version = RuntimeVersion(name, version)
 
     def get_version(self) -> RuntimeVersion:
-        if self._cached_version is None:
-            argv = [sys.executable, self._adapter_path, "--version"]
-            result = subprocess.run(argv, encoding="UTF-8", capture_output=True,
-                                    check=True)
-            output = result.stdout.strip().split(" ")
-            self._cached_version = RuntimeVersion(output[0], output[1])
-        return self._cached_version
+        return self._version
 
     def compute_argv(self, test_path: str, args: List[str],
                      env_variables: Dict[str, str],
-                     dirs: List[str]) -> List[str]:
-        argv = [sys.executable, self._adapter_path]
-        argv += ["--test-file", test_path]
-        for d in dirs:
-            argv += ["--dir", f"{Path(test_path).parent / d}::{d}"]  # noqa: E231
-        for k, v in env_variables.items():
-            argv += ["--env", f"{k}={v}"]
-        for a in args:
-            argv += ["--arg", a]
+                     dirs: List[Tuple[Path, str]]) -> List[str]:
+        argv = self._adapter.compute_argv(test_path=test_path,
+                                          args=args,
+                                          env=env_variables,
+                                          dirs=dirs)
+        assert isinstance(argv, list)
+        assert all(isinstance(arg, str) for arg in argv)
         return argv
 
     def run_test(self, argv: List[str]) -> Output:
diff --git a/test-runner/wasi_test_runner/test_suite_runner.py b/test-runner/wasi_test_runner/test_suite_runner.py
