Merge pull request #152 from wingo/real-argv

Report actual argv on errors

Author: wingo

adapters/wasm-micro-runtime.py

@@ -1,29 +1,35 @@
-import argparse
 import subprocess
-import sys
 import os
 import shlex
+from pathlib import Path
+from typing import Dict, List, Tuple
 
 # shlex.split() splits according to shell quoting rules
 IWASM = shlex.split(os.getenv("IWASM", "iwasm"))
 
-parser = argparse.ArgumentParser()
-parser.add_argument("--version", action="store_true")
-parser.add_argument("--test-file", action="store")
-parser.add_argument("--arg", action="append", default=[])
-parser.add_argument("--env", action="append", default=[])
-parser.add_argument("--dir", action="append", default=[])
 
-args = parser.parse_args()
+def get_name() -> str:
+    return "wamr"
 
-if args.version:
-    subprocess.run(IWASM + ["--version"])
-    sys.exit(0)
 
-TEST_FILE = args.test_file
-PROG_ARGS = args.arg
-ENV_ARGS = [f"--env={i}" for i in args.env]
-DIR_ARGS = [f"--map-dir={i}" for i in args.dir]
+def get_version() -> str:
+    # ensure no args when version is queried
+    result = subprocess.run(IWASM + ["--version"],
+                            encoding="UTF-8", capture_output=True,
+                            check=True)
+    output = result.stdout.splitlines()[0].split(" ")
+    return output[1]
 
-r = subprocess.run(IWASM + ENV_ARGS + DIR_ARGS + [TEST_FILE] + PROG_ARGS)
-sys.exit(r.returncode)
+
+def compute_argv(test_path: str,
+                 args: List[str],
+                 env: Dict[str, str],
+                 dirs: List[Tuple[Path, str]]) -> List[str]:
+    argv = [] + IWASM
+    for k, v in env.items():
+        argv += ["--env", f"{k}={v}"]
+    for host, guest in dirs:
+        argv += ["--map-dir", f"{host}::{guest}"]  # noqa: E231
+    argv += [test_path]
+    argv += args
+    return argv

adapters/wasmtime.py

@@ -1,30 +1,35 @@
-import argparse
 import subprocess
-import sys
 import os
 import shlex
+from pathlib import Path
+from typing import Dict, List, Tuple
 
 # shlex.split() splits according to shell quoting rules
 WASMTIME = shlex.split(os.getenv("WASMTIME", "wasmtime"))
 
-parser = argparse.ArgumentParser()
-parser.add_argument("--version", action="store_true")
-parser.add_argument("--test-file", action="store")
-parser.add_argument("--arg", action="append", default=[])
-parser.add_argument("--env", action="append", default=[])
-parser.add_argument("--dir", action="append", default=[])
 
-args = parser.parse_args()
+def get_name() -> str:
+    return "wasmtime"
 
-if args.version:
+
+def get_version() -> str:
     # ensure no args when version is queried
-    subprocess.run(WASMTIME[0:1] + ["--version"])
-    sys.exit(0)
+    result = subprocess.run(WASMTIME[0:1] + ["--version"],
+                            encoding="UTF-8", capture_output=True,
+                            check=True)
+    output = result.stdout.splitlines()[0].split(" ")
+    return output[1]
 
-TEST_FILE = args.test_file
-PROG_ARGS = args.arg
-ENV_ARGS = [j for i in args.env for j in ["--env", i]]
-DIR_ARGS = [j for i in args.dir for j in ["--dir", i]]
 
-r = subprocess.run(WASMTIME + ENV_ARGS + DIR_ARGS + [TEST_FILE] + PROG_ARGS)
-sys.exit(r.returncode)
+def compute_argv(test_path: str,
+                 args: List[str],
+                 env: Dict[str, str],
+                 dirs: List[Tuple[Path, str]]) -> List[str]:
+    argv = [] + WASMTIME
+    for k, v in env.items():
+        argv += ["--env", f"{k}={v}"]
+    for host, guest in dirs:
+        argv += ["--dir", f"{host}::{guest}"]  # noqa: E231
+    argv += [test_path]
+    argv += args
+    return argv

adapters/wizard.py

@@ -1,30 +1,44 @@
-import argparse
 import subprocess
-import sys
 import os
 import shlex
+from pathlib import Path
+from typing import Dict, List, Tuple
 
 # shlex.split() splits according to shell quoting rules
 WIZARD = shlex.split(os.getenv("WIZARD", "wizeng.x86-64-linux"))
 
-parser = argparse.ArgumentParser()
-parser.add_argument("--version", action="store_true")
-parser.add_argument("--test-file", action="store")
-parser.add_argument("--arg", action="append", default=[])
-parser.add_argument("--env", action="append", default=[])
-parser.add_argument("--dir", action="append", default=[])
 
-args = parser.parse_args()
+def get_name() -> str:
+    return "wizard"
 
-if args.version:
+
+def get_version() -> str:
     # ensure no args when version is queried
-    subprocess.run(WIZARD[0:1] + ["-version"])
-    sys.exit(0)
+    output = ""
+    try:
+        result = subprocess.run(WIZARD[0:1] + ["--version"],
+                                encoding="UTF-8", capture_output=True,
+                                check=False)
+        output = result.stdout;
+    except subprocess.CalledProcessError as e:
+        # https://github.com/titzer/wizard-engine/issues/483
+        if e.returncode != 3:
+            raise e
+        output = result.stdout
+    output = output.splitlines()[0].split(" ")
+    return output[1]
 
-TEST_FILE = args.test_file
-PROG_ARGS = args.arg
-ENV_ARGS = None if len(args.env) == 0 else f'--env={",".join(args.env)}'
-DIR_ARGS = None if len(args.dir) == 0 else f'--dir={",".join(args.dir)}'
 
-r = subprocess.run([arg for arg in WIZARD + [ENV_ARGS, DIR_ARGS, TEST_FILE] + PROG_ARGS if arg])
-sys.exit(r.returncode)
+def compute_argv(test_path: str,
+                 args: List[str],
+                 env: Dict[str, str],
+                 dirs: List[Tuple[Path, str]]) -> List[str]:
+    argv = [] + WIZARD
+    for k, v in env.items():
+        argv += [f"--env={k}={v}"]
+    for host, guest in dirs:
+        # FIXME: https://github.com/titzer/wizard-engine/issues/482
+        argv += [f"--dir={host}"]
+    argv += [test_path]
+    argv += args
+    return argv

run-tests

@@ -9,7 +9,7 @@ from pathlib import Path
 sys.path.insert(0, str(Path(__file__).parent / "test-runner"))
 
 from wasi_test_runner.harness import run_tests
-from wasi_test_runner.runtime_adapter import RuntimeAdapter
+from wasi_test_runner import runtime_adapter
 
 parser = argparse.ArgumentParser(
     description="WASI test runner"
@@ -42,7 +42,9 @@ parser.add_argument(
     help="Print more information about test results.",
 )
 
-def find_test_dirs(root):
+def find_test_dirs():
+    root = Path(__file__).parent / "tests"
+    root = root.relative_to(Path('.').absolute(), walk_up=True)
     test_dirs = []
     for root, dirs, files in root.walk(on_error=print):
         if "manifest.json" in files:
@@ -53,32 +55,32 @@ def find_runtime_adapters(root, verbose=False):
     print(f"Detecting WASI runtime availability:")
     adapters = []
     for candidate in root.glob("*.py"):
-        adapter = RuntimeAdapter(candidate)
         try:
+            adapter = runtime_adapter.RuntimeAdapter(candidate)
             print(f"  {candidate.name}: {adapter.get_version()}")
             adapters.append(adapter)
-        except subprocess.CalledProcessError as e:
-            print(f"  {candidate.name}: unavailable; pass `--runtime {candidate}` to debug.")
+        except runtime_adapter.LegacyRuntimeAdapterError:
+            print(f"  {candidate} is too old; update to new module format.")
+        except runtime_adapter.UnavailableRuntimeAdapterError:
+            print(f"  {candidate.name} unavailable; pass `--runtime {candidate}` to debug.")
     print("")
     if len(adapters) == 0:
         print("Error: No WASI runtimes found")
         sys.exit(1)
     return adapters
 
 options = parser.parse_args()
-test_suite = find_test_dirs(Path(__file__).parent / "tests")
+test_suite = find_test_dirs()
 if options.runtime_adapter:
-    runtime_adapters = [RuntimeAdapter(options.runtime_adapter)]
-    # Ensure it works.
     try:
-        runtime_adapters[0].get_version()
-    except subprocess.CalledProcessError as e:
-        print(f"Error: failed to load {options.runtime_adapter}:")
-        print(f"  Failed command line: {' '.join(e.cmd)}")
-        if e.stdout.strip() != "":
-            print(f"  stdout:\n{e.stdout}")
-        if e.stderr.strip() != "":
-            print(f"  stderr:\n{e.stderr}")
+        runtime_adapters = [
+            runtime_adapter.RuntimeAdapter(options.runtime_adapter)
+        ]
+    except runtime_adapter.LegacyRuntimeAdapterError as e:
+        print(f"Error: {e.adapter_path} is too old; update to new module format.")
+        sys.exit(1)
+    except runtime_adapter.UnavailableRuntimeAdapterError as e:
+        print(f"Error: failed to load {e.adapter_path}: {e.error}")
         sys.exit(1)
 else:
     runtime_adapters = find_runtime_adapters(Path(__file__).parent / "adapters")

test-runner/tests/test_test_suite_runner.py

@@ -1,4 +1,5 @@
 from typing import Any
+from pathlib import Path
 from unittest.mock import ANY, MagicMock, Mock, patch, mock_open
 
 import wasi_test_runner.test_case as tc
@@ -10,9 +11,9 @@ def get_mock_open() -> Mock:
     def open_mock(filename: str, *_args: Any, **_kwargs: Any) -> Any:
         file_content = {
             "my-path/manifest.json": '{"name": "test-suite"}',
-            "test1.json": '{"dirs": [".", "deep/dir"]}',
-            "test2.json": '{"exit_code": 1, "args": ["a", "b"]}',
-            "test3.json": '{"stdout": "output", "env": {"x": "1"}}',
+            "my-path/test1.json": '{"dirs": [".", "deep/dir"]}',
+            "my-path/test2.json": '{"exit_code": 1, "args": ["a", "b"]}',
+            "my-path/test3.json": '{"stdout": "output", "env": {"x": "1"}}',
         }
         if filename in file_content:
             return mock_open(read_data=file_content[filename]).return_value
@@ -26,7 +27,10 @@ def open_mock(filename: str, *_args: Any, **_kwargs: Any) -> Any:
 @patch("builtins.open", get_mock_open())
 @patch("os.path.exists", Mock(return_value=True))
 def test_runner_end_to_end() -> None:
-    test_paths = ["test1.wasm", "test2.wasm", "test3.wasm"]
+    test_suite_dir = "my-path"
+    test_suite_name = "test-suite"
+    test_files = ["test1.wasm", "test2.wasm", "test3.wasm"]
+    test_paths = [Path(test_suite_dir) / f for f in test_files]
 
     failures = [tc.Failure("a", "b"), tc.Failure("x", "y"), tc.Failure("x", "z")]
 
@@ -75,9 +79,8 @@ def test_runner_end_to_end() -> None:
     filt.should_skip.return_value = (False, None)
     filters = [filt]
 
-    test_suite_dir = "my-path"
-    test_suite_name = "test-suite"
-    with patch("glob.glob", return_value=test_paths):
+    with (patch("glob.glob", return_value=[str(p) for p in test_paths]),
+          patch("wasi_test_runner.test_suite_runner._cleanup_test_output")):
         suite = tsr.run_tests_from_test_suite(test_suite_dir, runtime,
                                               validators,  # type: ignore
                                               reporters,   # type: ignore
@@ -93,8 +96,9 @@ def test_runner_end_to_end() -> None:
     # Assert test runner calls
     assert runtime.run_test.call_count == 3
     for test_path, config in zip(test_paths, expected_config):
+        expected_dirs = [(Path(test_suite_dir) / d, d) for d in config.dirs]
         runtime.compute_argv.assert_any_call(
-            test_path, config.args, config.env, config.dirs
+            str(test_path), config.args, config.env, expected_dirs
         )
         runtime.run_test.assert_called_with(expected_argv)