tests: Add a python runner for tests that doesn't rely on ATF

tests/basic.toml

@@ -0,0 +1,2 @@
+[noargs]
+  exitcode = 1

tests/runner.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2016 pkgconf authors (see AUTHORS).
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# This software is provided 'as is' and without any warranty, express or
+# implied.  In no event shall the authors be liable for any damages arising
+# from the use of this software.
+
+"""Simple runner for test cases."""
+
+from __future__ import annotations
+import asyncio
+import argparse
+import dataclasses
+import sys
+import typing
+
+try:
+	import tomllib
+except ImportError:
+	import tomli as tomllib
+
+if typing.TYPE_CHECKING:
+
+	class TestDefinition(typing.TypedDict, total=False):
+		exitcode: int
+
+	class Arguments(typing.Protocol):
+		suite: str
+		pkgconf: str
+		verbose: bool
+
+
+@dataclasses.dataclass
+class Result:
+
+	name: str
+	success: bool = True
+	reason: typing.Optional[str] = None
+
+
+async def run_test(pkgconf: str, name: str, test: TestDefinition, verbose: bool) -> Result:
+	args: typing.List[str] = []
+
+	proc = await asyncio.create_subprocess_exec(
+		pkgconf, *args,
+		stdout=asyncio.subprocess.PIPE,
+		stderr=asyncio.subprocess.PIPE)
+
+	out, err = await proc.communicate()
+
+	result = Result(name)
+
+	if (ret := test.get('exitcode')) is not None and proc.returncode != ret:
+		result.success = False
+		result.reason = f"Return code was {proc.returncode}, but expected {ret}"
+
+	if verbose:
+		if result.success:
+			print(f"{name}: passed")
+		else:
+			print(f"{name}: failed\n  reason: {result.reason}")
+
+	return result
+
+
+async def run(args: Arguments) -> None:
+	with open(args.suite, 'rb') as f:
+		suite: typing.Dict[str, TestDefinition] = tomllib.load(f)
+
+	results = await asyncio.gather(
+		*[run_test(args.pkgconf, n, s, args.verbose) for n, s in suite.items()]
+	)
+
+	return all(r.success for r in results)
+
+
+def main() -> None:
+	parser = argparse.ArgumentParser()
+	parser.add_argument('suite', help="A toml test suite definition")
+	parser.add_argument('pkgconf', help="Path to built pkgconf executable")
+	parser.add_argument('-v', '--verbose', action='store_true',
+						help="Print more information while running")
+	args: Arguments = parser.parse_args()
+
+	loop = asyncio.new_event_loop()
+	success = loop.run_until_complete(run(args))
+
+	sys.exit(int(not success))
+
+
+if __name__ == "__main__":
+	main()