Make it possible to run each test suite in a new process, while still collecting and combining all the results so it looks like all tests were run in a single pass. This slows down the test run quite a bit, but protects against tests causing problems for each other (which was happening more often on OSX.) Using nosetest's multi-process feature wasn't workable in this case because new wx.App's could not be created in the child multiprocess.Process instances on Macs because of how they are forked. We need a totally new process (using the Framework Python) to make it work.

git-svn-id: https://svn.wxwidgets.org/svn/wx/wxPython/Phoenix/trunk@72317 c3d73ce0-8a6f-49c7-b76d-6d57e0e08775

build.py

@@ -212,6 +212,7 @@ def setPythonVersion(args):
     PYTHON_ARCH = runcmd('%s -c "import platform; print(platform.architecture()[0])"' 
                          % PYTHON, True, False)
     msg('Python\'s architecture is %s' % PYTHON_ARCH)
+    os.environ['PYTHON'] = PYTHON
         
 
 

unittests/do-runtests.py

@@ -0,0 +1,68 @@
+#---------------------------------------------------------------------------
+# Name:        unittests/do-runtests.py
+# Author:      Robin Dunn
+#
+# Created:     6-Aug-2012
+# Copyright:   (c) 2012 by Total Control Software
+# License:     wxWindows License
+#---------------------------------------------------------------------------
+
+"""
+Run the unittests given on the command line while using a custom TestResults
+class, and output the results in a format that can be integrated into another
+TestResults (in the calling process.) See runtests.py for more information
+and also for the code that calls this script via subprocess.
+"""
+
+import sys
+import os
+import unittest
+import unittest.runner
+from StringIO import StringIO
+import pickle
+    
+g_testResult = None
+
+# make sure the phoenix dir is on the path
+if os.path.dirname(__file__):
+    phoenixDir = os.path.abspath(os.path.dirname(__file__)+'/..')
+else:  # run as main?
+    d = os.path.dirname(sys.argv[0])
+    if not d: d = '.'
+    phoenixDir = os.path.abspath(d+'/..')
+sys.path.insert(0, phoenixDir)
+
+
+class MyTestResult(unittest.TextTestResult):
+    def stopTestRun(self):
+        self.output = self.stream.getvalue()
+        
+    def getResultsMsg(self):
+        def fixList(src):
+            return [(self.getDescription(test), err) for test, err in src]
+        msg = dict()
+        msg['output'] = self.output
+        msg['testsRun'] = self.testsRun
+        msg['failures'] = fixList(self.failures)
+        msg['errors'] = fixList(self.errors)
+        msg['skipped'] = fixList(self.skipped)
+        msg['expectedFailures'] = fixList(self.expectedFailures)
+        msg['unexpectedSuccesses'] = fixList(self.unexpectedSuccesses)
+        return msg
+
+
+class MyTestRunner(unittest.TextTestRunner):
+    def _makeResult(self):
+        global g_testResult
+        if g_testResult is None:
+            self.stream = unittest.runner._WritelnDecorator(StringIO())
+            g_testResult = MyTestResult(self.stream, self.descriptions, self.verbosity)
+        return g_testResult
+    
+
+if __name__ == '__main__':
+    unittest.main(module=None, exit=False, testRunner=MyTestRunner)
+    msg = g_testResult.getResultsMsg()
+    text = pickle.dumps(msg)
+    sys.stdout.write(text)
+    

unittests/runtests.py

@@ -1,25 +1,161 @@
+#---------------------------------------------------------------------------
+# Name:        unittests/runtests.py
+# Author:      Robin Dunn
+#
+# Created:     3-Dec-2010
+# Copyright:   (c) 2010 by Total Control Software
+# License:     wxWindows License
+#---------------------------------------------------------------------------
+
+"""
+This script will find and run all of the Phoenix test cases. We use custom
+TestSuite and other customized unittest classes so we can run each test
+module in a separate process. This helps to isolate the test cases from each
+other so they don't stomp on each other too much. 
+
+Currently the process granularity is the TestSuite created when the tests in
+a single module are loaded, which makes it essentially the same as when
+running that module standalone. More granularity is possible by using
+separate processes for each TestCase.testMethod, but I haven't seen the need
+for that yet.
+
+See do-runtests.py for the script that is run in the child processes.
+"""
+
+
 import sys
 import os
+import glob
+import subprocess
+import imp_unittest, unittest
 
-# make sure our development dir is on the path
+# make sure the phoenix dir is on the path
 if os.path.dirname(__file__):
     phoenixDir = os.path.abspath(os.path.dirname(__file__)+'/..')
 else:  # run as main?
     d = os.path.dirname(sys.argv[0])
     if not d: d = '.'
     phoenixDir = os.path.abspath(d+'/..')
-
-# in case phoenixDir not in sys.path:
 sys.path.insert(0, phoenixDir)
 
-# stuff for debugging
 import wx
 print("wx.version: " + wx.version())
 print("pid: " + str(os.getpid()))
 #print("executable: " + sys.executable); raw_input("Press Enter...")
 
-import imp_unittest, unittest
+import wtc
 
-args = sys.argv[:1] + 'discover -p test_*.py -s unittests -t .'.split() + sys.argv[1:]
-unittest.main( argv=args )
+#---------------------------------------------------------------------------
 
+def getTestName(test):
+    cls = test.__class__
+    return "%s.%s.%s" % (cls.__module__, cls.__name__, test._testMethodName)
+
+
+class MyTestSuite(unittest.TestSuite):
+    """
+    Override run() to run the TestCases in a new process.
+    """
+    def run(self, result, debug=False):
+        if self._tests and isinstance(self._tests[0], unittest.TestSuite):
+            # self is a suite of suites, recurse down another level
+            return unittest.TestSuite.run(self, result, debug)
+        elif self._tests and not isinstance(self._tests[0], wtc.WidgetTestCase):
+            # we can run normal test cases in this process
+            return unittest.TestSuite.run(self, result, debug)
+        else:
+            # Otherwise we want to run these tests in a new process,
+            # get the names of all the test cases in this test suite
+            testNames = list()
+            for test in self:
+                name = getTestName(test)
+                testNames.append(name)
+                
+            # build the command to be run
+            PYTHON = os.environ.get('PYTHON', sys.executable)
+            runner = os.path.join(phoenixDir, 'unittests', 'do-runtests.py')
+            cmd = [PYTHON, '-u', runner] 
+            if result.verbosity > 1:
+                cmd.append('--verbose')
+            elif result.verbosity < 1:
+                cmd.append('--quiet')
+            cmd += testNames
+            
+            # run it
+            sp = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE)#, stderr=subprocess.STDOUT)
+            output = sp.stdout.read()
+            if sys.version_info > (3,):
+                output = output.decode('ascii') 
+            output = output.rstrip()
+            rval = sp.wait()
+            if rval:
+                print("Command '%s' failed with exit code %d." % (cmd, rval))
+                sys.exit(rval)
+                        
+            # Unpickle the output and copy it to result. It should be a
+            # dictionary with info from the TestResult class used in the
+            # child process.
+            import pickle
+            msg = pickle.loads(output)
+            result.stream.write(msg['output'])
+            result.stream.flush()
+            result.testsRun += msg['testsRun']
+            result.failures += msg['failures']
+            result.errors += msg['errors']
+            result.skipped += msg['skipped']
+            result.expectedFailures += msg['expectedFailures']
+            result.unexpectedSuccesses += msg['unexpectedSuccesses']
+            
+            return result
+                
+            
+
+class MyTestResult(unittest.TextTestResult):
+    def __init__(self, stream, descriptions, verbosity):
+        super(MyTestResult, self).__init__(stream, descriptions, verbosity)
+        self.verbosity = verbosity
+    
+    def getDescription(self, test):
+        """
+        Override getDescription() to be able to deal with the test already
+        being converted to a string.
+        """
+        if isinstance(test, basestring):
+            return test
+        return super(MyTestResult, self).getDescription(test)
+    
+    
+    
+class MyTestLoader(unittest.TestLoader):
+    suiteClass = MyTestSuite
+
+class MyTestRunner(unittest.TextTestRunner):
+    resultclass = MyTestResult
+
+#---------------------------------------------------------------------------
+
+if __name__ == '__main__':
+    if '--single-process' in sys.argv:
+        sys.argv.remove('--single-process')
+        args = sys.argv[:1] + 'discover -p test_*.py -s unittests -t .'.split() + sys.argv[1:]
+        unittest.main(argv=args)
+
+    else:
+        # The discover option doesn't use my my custom loader or suite
+        # classes, so we'll do the finding of the test files in this case.
+        #names = ['test_gdicmn', 'test_panel', 'test_msgdlg', 'test_uiaction']
+        names = glob.glob(os.path.join('unittests', 'test_*.py'))
+        names = [os.path.splitext(os.path.basename(n))[0] for n in names]
+        args = sys.argv + names
+        unittest.main(argv=args, module=None, 
+                      testRunner=MyTestRunner, testLoader=MyTestLoader())
+
+
+        
+    #loader = MyTestLoader()
+    #suite = unittest.TestSuite()
+    #for name in ['test_panel', 'test_msgdlg']:
+    #    suite.addTests(loader.loadTestsFromName(name))
+    #runner = MyTestRunner()
+    #runner.run(suite)
+