From a5e632d98f5ec03d3b2730711115da83c02009f6 Mon Sep 17 00:00:00 2001 From: Robin Dunn Date: Fri, 10 Aug 2012 04:54:43 +0000 Subject: [PATCH] Make it possible to run each test suite in a new process, while still collecting and combining all the results so it looks like all tests were run in a single pass. This slows down the test run quite a bit, but protects against tests causing problems for each other (which was happening more often on OSX.) Using nosetest's multi-process feature wasn't workable in this case because new wx.App's could not be created in the child multiprocess.Process instances on Macs because of how they are forked. We need a totally new process (using the Framework Python) to make it work. git-svn-id: https://svn.wxwidgets.org/svn/wx/wxPython/Phoenix/trunk@72317 c3d73ce0-8a6f-49c7-b76d-6d57e0e08775 --- build.py | 1 + unittests/do-runtests.py | 68 ++++++++++++++++++ unittests/runtests.py | 150 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 212 insertions(+), 7 deletions(-) create mode 100644 unittests/do-runtests.py diff --git a/build.py b/build.py index b02089a4..ab147664 100755 --- a/build.py +++ b/build.py @@ -212,6 +212,7 @@ def setPythonVersion(args): PYTHON_ARCH = runcmd('%s -c "import platform; print(platform.architecture()[0])"' % PYTHON, True, False) msg('Python\'s architecture is %s' % PYTHON_ARCH) + os.environ['PYTHON'] = PYTHON diff --git a/unittests/do-runtests.py b/unittests/do-runtests.py new file mode 100644 index 00000000..d93a9ae9 --- /dev/null +++ b/unittests/do-runtests.py @@ -0,0 +1,68 @@ +#--------------------------------------------------------------------------- +# Name: unittests/do-runtests.py +# Author: Robin Dunn +# +# Created: 6-Aug-2012 +# Copyright: (c) 2012 by Total Control Software +# License: wxWindows License +#--------------------------------------------------------------------------- + +""" +Run the unittests given on the command line while using a custom TestResults +class, and output the results in a format that can be integrated into another +TestResults (in the calling process.) See runtests.py for more information +and also for the code that calls this script via subprocess. +""" + +import sys +import os +import unittest +import unittest.runner +from StringIO import StringIO +import pickle + +g_testResult = None + +# make sure the phoenix dir is on the path +if os.path.dirname(__file__): + phoenixDir = os.path.abspath(os.path.dirname(__file__)+'/..') +else: # run as main? + d = os.path.dirname(sys.argv[0]) + if not d: d = '.' + phoenixDir = os.path.abspath(d+'/..') +sys.path.insert(0, phoenixDir) + + +class MyTestResult(unittest.TextTestResult): + def stopTestRun(self): + self.output = self.stream.getvalue() + + def getResultsMsg(self): + def fixList(src): + return [(self.getDescription(test), err) for test, err in src] + msg = dict() + msg['output'] = self.output + msg['testsRun'] = self.testsRun + msg['failures'] = fixList(self.failures) + msg['errors'] = fixList(self.errors) + msg['skipped'] = fixList(self.skipped) + msg['expectedFailures'] = fixList(self.expectedFailures) + msg['unexpectedSuccesses'] = fixList(self.unexpectedSuccesses) + return msg + + +class MyTestRunner(unittest.TextTestRunner): + def _makeResult(self): + global g_testResult + if g_testResult is None: + self.stream = unittest.runner._WritelnDecorator(StringIO()) + g_testResult = MyTestResult(self.stream, self.descriptions, self.verbosity) + return g_testResult + + +if __name__ == '__main__': + unittest.main(module=None, exit=False, testRunner=MyTestRunner) + msg = g_testResult.getResultsMsg() + text = pickle.dumps(msg) + sys.stdout.write(text) + \ No newline at end of file diff --git a/unittests/runtests.py b/unittests/runtests.py index 0151ec19..11f28e21 100755 --- a/unittests/runtests.py +++ b/unittests/runtests.py @@ -1,25 +1,161 @@ +#--------------------------------------------------------------------------- +# Name: unittests/runtests.py +# Author: Robin Dunn +# +# Created: 3-Dec-2010 +# Copyright: (c) 2010 by Total Control Software +# License: wxWindows License +#--------------------------------------------------------------------------- + +""" +This script will find and run all of the Phoenix test cases. We use custom +TestSuite and other customized unittest classes so we can run each test +module in a separate process. This helps to isolate the test cases from each +other so they don't stomp on each other too much. + +Currently the process granularity is the TestSuite created when the tests in +a single module are loaded, which makes it essentially the same as when +running that module standalone. More granularity is possible by using +separate processes for each TestCase.testMethod, but I haven't seen the need +for that yet. + +See do-runtests.py for the script that is run in the child processes. +""" + + import sys import os +import glob +import subprocess +import imp_unittest, unittest -# make sure our development dir is on the path +# make sure the phoenix dir is on the path if os.path.dirname(__file__): phoenixDir = os.path.abspath(os.path.dirname(__file__)+'/..') else: # run as main? d = os.path.dirname(sys.argv[0]) if not d: d = '.' phoenixDir = os.path.abspath(d+'/..') - -# in case phoenixDir not in sys.path: sys.path.insert(0, phoenixDir) -# stuff for debugging import wx print("wx.version: " + wx.version()) print("pid: " + str(os.getpid())) #print("executable: " + sys.executable); raw_input("Press Enter...") -import imp_unittest, unittest +import wtc -args = sys.argv[:1] + 'discover -p test_*.py -s unittests -t .'.split() + sys.argv[1:] -unittest.main( argv=args ) +#--------------------------------------------------------------------------- +def getTestName(test): + cls = test.__class__ + return "%s.%s.%s" % (cls.__module__, cls.__name__, test._testMethodName) + + +class MyTestSuite(unittest.TestSuite): + """ + Override run() to run the TestCases in a new process. + """ + def run(self, result, debug=False): + if self._tests and isinstance(self._tests[0], unittest.TestSuite): + # self is a suite of suites, recurse down another level + return unittest.TestSuite.run(self, result, debug) + elif self._tests and not isinstance(self._tests[0], wtc.WidgetTestCase): + # we can run normal test cases in this process + return unittest.TestSuite.run(self, result, debug) + else: + # Otherwise we want to run these tests in a new process, + # get the names of all the test cases in this test suite + testNames = list() + for test in self: + name = getTestName(test) + testNames.append(name) + + # build the command to be run + PYTHON = os.environ.get('PYTHON', sys.executable) + runner = os.path.join(phoenixDir, 'unittests', 'do-runtests.py') + cmd = [PYTHON, '-u', runner] + if result.verbosity > 1: + cmd.append('--verbose') + elif result.verbosity < 1: + cmd.append('--quiet') + cmd += testNames + + # run it + sp = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE)#, stderr=subprocess.STDOUT) + output = sp.stdout.read() + if sys.version_info > (3,): + output = output.decode('ascii') + output = output.rstrip() + rval = sp.wait() + if rval: + print("Command '%s' failed with exit code %d." % (cmd, rval)) + sys.exit(rval) + + # Unpickle the output and copy it to result. It should be a + # dictionary with info from the TestResult class used in the + # child process. + import pickle + msg = pickle.loads(output) + result.stream.write(msg['output']) + result.stream.flush() + result.testsRun += msg['testsRun'] + result.failures += msg['failures'] + result.errors += msg['errors'] + result.skipped += msg['skipped'] + result.expectedFailures += msg['expectedFailures'] + result.unexpectedSuccesses += msg['unexpectedSuccesses'] + + return result + + + +class MyTestResult(unittest.TextTestResult): + def __init__(self, stream, descriptions, verbosity): + super(MyTestResult, self).__init__(stream, descriptions, verbosity) + self.verbosity = verbosity + + def getDescription(self, test): + """ + Override getDescription() to be able to deal with the test already + being converted to a string. + """ + if isinstance(test, basestring): + return test + return super(MyTestResult, self).getDescription(test) + + + +class MyTestLoader(unittest.TestLoader): + suiteClass = MyTestSuite + +class MyTestRunner(unittest.TextTestRunner): + resultclass = MyTestResult + +#--------------------------------------------------------------------------- + +if __name__ == '__main__': + if '--single-process' in sys.argv: + sys.argv.remove('--single-process') + args = sys.argv[:1] + 'discover -p test_*.py -s unittests -t .'.split() + sys.argv[1:] + unittest.main(argv=args) + + else: + # The discover option doesn't use my my custom loader or suite + # classes, so we'll do the finding of the test files in this case. + #names = ['test_gdicmn', 'test_panel', 'test_msgdlg', 'test_uiaction'] + names = glob.glob(os.path.join('unittests', 'test_*.py')) + names = [os.path.splitext(os.path.basename(n))[0] for n in names] + args = sys.argv + names + unittest.main(argv=args, module=None, + testRunner=MyTestRunner, testLoader=MyTestLoader()) + + + + #loader = MyTestLoader() + #suite = unittest.TestSuite() + #for name in ['test_panel', 'test_msgdlg']: + # suite.addTests(loader.loadTestsFromName(name)) + #runner = MyTestRunner() + #runner.run(suite) +