pkgconf/tests/runner.py

#!/usr/bin/env python3

# Copyright (c) 2016 pkgconf authors (see AUTHORS).
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# This software is provided 'as is' and without any warranty, express or
# implied.  In no event shall the authors be liable for any damages arising
# from the use of this software.

"""Simple runner for test cases."""

from __future__ import annotations
import argparse
import asyncio
import dataclasses
import os
import sys
import typing

try:
	import tomllib
except ImportError:
	import tomli as tomllib

if typing.TYPE_CHECKING:

	class TestDefinition(typing.TypedDict, total=False):
		exitcode: int
		stdout: str
		stderr: str
		args: typing.List[str]
		env: typing.Dict[str, str]

	class Arguments(typing.Protocol):
		suites: typing.List[str]
		pkgconf: str
		verbose: bool


TEST_ROOT = os.path.dirname(os.path.abspath(__file__))


@dataclasses.dataclass
class Result:

	name: str
	success: bool = False
	reason: typing.Optional[str] = None


def interpolate(input: str) -> str:
	return input.format(
		test_root=TEST_ROOT,
	)


async def run_test(pkgconf: str, name: str, test: TestDefinition, verbose: bool) -> Result:

	env: typing.Dict[str, str] = {}
	if (renv := test.get('env', None)) is not None:
		env = {k: interpolate(v) for k, v in renv.items()}

	proc = await asyncio.create_subprocess_exec(
		pkgconf, *test.get('args', []),
		stdout=asyncio.subprocess.PIPE if 'stdout' in test is not None else asyncio.subprocess.DEVNULL,
		stderr=asyncio.subprocess.PIPE if 'stderr' in test is not None else asyncio.subprocess.DEVNULL,
		env=env)

	rout, rerr = await proc.communicate()
	out = rout.decode() if rout is not None else ''
	err = rerr.decode() if rerr is not None else ''

	result = Result(name)

	if (ret := test.get('exitcode', 0)) and proc.returncode != ret:
		result.reason = f"Return code was {proc.returncode}, but expected {ret}"
	elif (exp := test.get('stdout')) is not None and out != exp:
		result.reason = f"Stdout was {out}, but expected {exp}"
	elif (exp := test.get('stderr')) is not None and err != exp:
		result.reason = f"Stdout was {err}, but expected {err}"
	else:
		result.success = True

	if verbose:
		if result.success:
			print(f"{name}: passed")
		else:
			print(f"{name}: failed\n  reason: {result.reason}")

	return result


async def run(args: Arguments) -> None:
	tests: typing.List[typing.Coroutine[typing.Any, typing.Any, Result]] = []
	for suite in args.suites:
		with open(suite, 'rb') as f:
			suite: typing.Dict[str, TestDefinition] = tomllib.load(f)

		tests.extend(
			[run_test(args.pkgconf, n, s, args.verbose) for n, s in suite.items()])

	results = await asyncio.gather(*tests)

	return all(r.success for r in results)


def main() -> None:
	parser = argparse.ArgumentParser()
	parser.add_argument('pkgconf', help="Path to built pkgconf executable")
	parser.add_argument('suites', nargs="+", help="One or more toml test suite definition")
	parser.add_argument('-v', '--verbose', action='store_true',
						help="Print more information while running")
	args: Arguments = parser.parse_args()

	loop = asyncio.new_event_loop()
	success = loop.run_until_complete(run(args))

	sys.exit(int(not success))


if __name__ == "__main__":
	main()