From c75ce379100993f41a7502986146e735367cf8aa Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Tue, 3 Nov 2020 17:41:58 +1100 Subject: [PATCH] tests/run-multitests.py: Add a -p flag to run permutations of instances. Signed-off-by: Jim Mussared --- tests/run-multitests.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/run-multitests.py b/tests/run-multitests.py index 5b8e70e921..cdb2730eda 100755 --- a/tests/run-multitests.py +++ b/tests/run-multitests.py @@ -6,6 +6,7 @@ import sys, os, time, re, select import argparse +import itertools import subprocess import tempfile @@ -418,6 +419,13 @@ def main(): cmd_parser.add_argument( "-i", "--instance", action="append", default=[], help="instance(s) to run the tests on" ) + cmd_parser.add_argument( + "-p", + "--permutations", + type=int, + default=1, + help="repeat the test with this many permutations of the instance order", + ) cmd_parser.add_argument("files", nargs="+", help="input test files") cmd_args = cmd_parser.parse_args() @@ -450,8 +458,14 @@ def main(): for _ in range(max_instances - len(instances_test)): instances_test.append(PyInstanceSubProcess([MICROPYTHON])) + all_pass = True try: - all_pass = run_tests(test_files, instances_truth, instances_test) + for i, instances_test_permutation in enumerate(itertools.permutations(instances_test)): + if i >= cmd_args.permutations: + break + + all_pass &= run_tests(test_files, instances_truth, instances_test_permutation) + finally: for i in instances_truth: i.close()