From 304245ce439fe49e788d7d2dab8690f4f67a56fd Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 2 Dec 2019 10:59:29 -0700 Subject: Fix some test fixture-handling nits. dir-fixture() did not concatenate a source list into a path as the testing doc says it should. fix-fixture() did not either, and the doc didn't say so; now it does (docstring and external testing doc). There was a way to slip through both dir_fixtures and file_fixture with the path invalid - if you have fixture dirs defined, and the dir/file is not found in them, you'll come out of the loop set to the last fixture dir and not try the current dir. This doesn't break anything, just leads to a somewhat misleading message if the fixture really isn't found - the traceback indicates the fixture was not found in whatever the last passed-in fixture dir was. Now it looks like it was not found in the source testing dir. The message can still be improved to be more descriptive. A couple of minor Py2 removals. Testing doc didn't mention FIXTURE_DIRS, so this was added. A bunch of other doc fiddling. Signed-off-by: Mats Wichmann --- testing/framework/TestCmd.py | 111 ++++++++++++------------- testing/framework/test-framework.rst | 152 ++++++++++++++++++++++++----------- 2 files changed, 156 insertions(+), 107 deletions(-) diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py index ed8b4b4..9b5df8e 100644 --- a/testing/framework/TestCmd.py +++ b/testing/framework/TestCmd.py @@ -298,6 +298,7 @@ import re import shutil import signal import stat +import subprocess import sys import tempfile import threading @@ -306,7 +307,6 @@ import traceback import types -IS_PY3 = sys.version_info[0] == 3 IS_WINDOWS = sys.platform == 'win32' IS_64_BIT = sys.maxsize > 2**32 IS_PYPY = hasattr(sys, 'pypy_translation_info') @@ -729,23 +729,6 @@ else: default_sleep_seconds = 1 -import subprocess - -try: - subprocess.Popen.terminate -except AttributeError: - if sys.platform == 'win32': - import win32process - - def terminate(self): - win32process.TerminateProcess(self._handle, 1) - else: - def terminate(self): - os.kill(self.pid, signal.SIGTERM) - method = types.MethodType(terminate, None, subprocess.Popen) - setattr(subprocess.Popen, 'terminate', method) - - # From Josiah Carlson, # ASPN : Python Cookbook : Module to allow Asynchronous subprocess use on Windows and Posix platforms # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/440554 @@ -1025,12 +1008,11 @@ class TestCmd(object): self.condition = 'no_result' self.workdir_set(workdir) self.subdir(subdir) - self.fixture_dirs = [] try: self.fixture_dirs = (os.environ['FIXTURE_DIRS']).split(os.pathsep) except KeyError: - pass + self.fixture_dirs = [] def __del__(self): @@ -1051,7 +1033,7 @@ class TestCmd(object): def canonicalize(self, path): if is_List(path): - path = os.path.join(*tuple(path)) + path = os.path.join(*path) if not os.path.isabs(path): path = os.path.join(self.workdir, path) return path @@ -1312,7 +1294,7 @@ class TestCmd(object): file = self.canonicalize(file) if mode[0] != 'r': raise ValueError("mode must begin with 'r'") - if IS_PY3 and 'b' not in mode: + if 'b' not in mode: with open(file, mode, newline=newline) as f: return f.read() else: @@ -1360,23 +1342,31 @@ class TestCmd(object): return result def dir_fixture(self, srcdir, dstdir=None): - """Copies the contents of the specified folder srcdir from - the directory of the called script, to the current - working directory. - - The srcdir name may be a list, in which case the elements are - concatenated with the os.path.join() method. The dstdir is - assumed to be under the temporary working directory, it gets - created automatically, if it does not already exist. + """ Copies the contents of the fixture dir to the test dir. + + If srcdir is an absolute path, it is tried directly, else + the fixture_dirs are prepended to it and tried in succession. + To tightly control the search order, the harness can be called + with FIXTURE_DIRS also including the test source directory + in the desired place, it will otherwise be tried last. + + srcdir may be a list, in which case the elements are first + joined into a pathname. + + If a dstdir is supplied, it is taken to be under the temporary + working dir. dstdir is created automatically if needed. """ + if is_List(srcdir): + srcdir = os.path.join(*srcdir) + spath = srcdir if srcdir and self.fixture_dirs and not os.path.isabs(srcdir): for dir in self.fixture_dirs: spath = os.path.join(dir, srcdir) if os.path.isdir(spath): break - else: - spath = srcdir + else: + spath = srcdir if dstdir: dstdir = self.canonicalize(dstdir) @@ -1403,23 +1393,33 @@ class TestCmd(object): shutil.copy(epath, dpath) def file_fixture(self, srcfile, dstfile=None): - """Copies the file srcfile from the directory of - the called script, to the current working directory. + """ Copies a fixture file to the test dir, optionally renaming. + + If srcfile is an absolute path, it is tried directly, else + the fixture_dirs are prepended to it and tried in succession. + To tightly control the search order, the harness can be called + with FIXTURE_DIRS also including the test source directory + in the desired place, it will otherwise be tried last. + + srcfile may be a list, in which case the elements are first + joined into a pathname. - The dstfile is assumed to be under the temporary working - directory unless it is an absolute path name. - If dstfile is specified its target directory gets created - automatically, if it does not already exist. + dstfile is assumed to be under the temporary working directory + unless it is an absolute path name. Any directory components + of dstfile are created automatically if needed. """ - srcpath, srctail = os.path.split(srcfile) + if is_List(srcfile): + srcdir = os.path.join(*srcfile) - if srcpath and (not self.fixture_dirs or os.path.isabs(srcpath)): - spath = srcfile - else: + srcpath, srctail = os.path.split(srcfile) + spath = srcfile + if srcfile and self.fixture_dirs and not os.path.isabs(srcfile): for dir in self.fixture_dirs: spath = os.path.join(dir, srcfile) if os.path.isfile(spath): break + else: + spath = srcfile if not dstfile: if srctail: @@ -1435,8 +1435,8 @@ class TestCmd(object): dstlist = dstlist[1:] for idx in range(len(dstlist)): self.subdir(dstlist[:idx + 1]) - dpath = os.path.join(self.workdir, dstfile) + shutil.copy(spath, dpath) def start(self, program=None, @@ -1445,8 +1445,7 @@ class TestCmd(object): universal_newlines=None, timeout=_Null, **kw): - """ - Starts a program or script for the test environment. + """ Starts a program or script for the test environment. The specified program will have the original directory prepended unless it is enclosed in a [list]. @@ -1478,7 +1477,7 @@ class TestCmd(object): self.timer = threading.Timer(float(timeout), self._timeout) self.timer.start() - if IS_PY3 and sys.platform == 'win32': + if sys.platform == 'win32': # Set this otherwist stdout/stderr pipes default to # windows default locale cp1252 which will throw exception # if using non-ascii characters. @@ -1517,15 +1516,10 @@ class TestCmd(object): if not stream: return stream - # TODO: Run full tests on both platforms and see if this fixes failures # It seems that py3.6 still sets text mode if you set encoding. - elif sys.version_info[0] == 3: # TODO and sys.version_info[1] < 6: - stream = stream.decode('utf-8', errors='replace') - stream = stream.replace('\r\n', '\n') - elif sys.version_info[0] == 2: - stream = stream.replace('\r\n', '\n') - - return stream + # was: if IS_PY3: # TODO and sys.version_info[1] < 6: + stream = stream.decode('utf-8', errors='replace') + return stream.replace('\r\n', '\n') def finish(self, popen=None, **kw): """ @@ -1587,7 +1581,8 @@ class TestCmd(object): if is_List(stdin): stdin = ''.join(stdin) - if stdin and IS_PY3:# and sys.version_info[1] < 6: + # TODO: was: if stdin and IS_PY3:# and sys.version_info[1] < 6: + if stdin: stdin = to_bytes(stdin) # TODO(sgk): figure out how to re-use the logic in the .finish() @@ -1687,7 +1682,7 @@ class TestCmd(object): if sub is None: continue if is_List(sub): - sub = os.path.join(*tuple(sub)) + sub = os.path.join(*sub) new = os.path.join(self.workdir, sub) try: os.mkdir(new) @@ -1786,7 +1781,7 @@ class TestCmd(object): """Find an executable file. """ if is_List(file): - file = os.path.join(*tuple(file)) + file = os.path.join(*file) if not os.path.isabs(file): file = where_is(file, path, pathext) return file @@ -1808,7 +1803,7 @@ class TestCmd(object): the temporary working directory name with the specified arguments using the os.path.join() method. """ - return os.path.join(self.workdir, *tuple(args)) + return os.path.join(self.workdir, *args) def readable(self, top, read=1): """Make the specified directory tree readable (read == 1) diff --git a/testing/framework/test-framework.rst b/testing/framework/test-framework.rst index a0fe861..965efd4 100644 --- a/testing/framework/test-framework.rst +++ b/testing/framework/test-framework.rst @@ -113,14 +113,14 @@ You may specifically list one or more tests to be run:: $ python runtest.py src/engine/SCons/BuilderTests.py $ python runtest.py test/option-j.py test/Program.py -Folder names are allowed arguments as well, so you can do:: +Folder names are allowed in the test list as well, so you can do:: $ python runtest.py test/SWIG to run all SWIG tests only. You can also use the ``-f`` option to execute just the tests listed in -a specified text file:: +a test list file:: $ cat testlist.txt test/option-j.py @@ -136,9 +136,11 @@ If more than one test is run, the ``runtest.py`` script prints a summary of how many tests passed, failed, or yielded no result, and lists any unsuccessful tests. -The above invocations all test directly the files underneath the ``src/`` -subdirectory, and do not require that a packaging build be performed -first. The ``runtest.py`` script supports additional options to run +The above invocations all test against the scons files underneath the ``src/`` +subdirectory, and do not require that a packaging build of SCons be performed +first. This is the most common mode: make some changes, and test in +place the effects. +The ``runtest.py`` script supports additional options to run tests against unpacked packages in the ``build/test-*/`` subdirectories. If you are testing a separate Tool outside of the SCons source tree, you @@ -149,26 +151,33 @@ have to call the ``runtest.py`` script in *external* (stand-alone) mode:: This ensures that the testing framework doesn't try to access SCons classes needed for some of the *internal* test cases. -Note, that the actual tests are carried out in a temporary folder each, -which gets deleted afterwards. This ensures that your source directories -don't get clobbered with temporary files from the test runs. It also -means that you can't simply change into a folder to "debug things" after -a test has gone wrong. For a way around this, check out the ``PRESERVE`` -environment variable. It can be seen in action in -`How to convert old tests`_ below. +Note that as each test is run, it is executed in a temporary directory +created just for that test, which is by default removed when the +test is complete. This ensures that your source directories +don't get clobbered with temporary files and changes from the test runs. +If the test itself needs to know the directory, it can be obtained +as ``test.workdir``, or more commonly by calling ``test.workpath``, +a function which takes a path-component argument and returns the path to +that path-component in the testing directory. + +The use of an ephemeral test directory means that you can't simply change +into a folder to "debug things" after a test has gone wrong. +For a way around this, check out the ``PRESERVE`` environment variable. +It can be seen in action in `How to Convert Old Tests to Use Fixures`_ below. Not Running Tests ================= If you simply want to check which tests would get executed, you can call -the ``runtest.py`` script with the ``-l`` option:: +the ``runtest.py`` script with the ``-l`` option combined with whichever +test finding options (see below) you intend to use. Example: - $ python runtest.py -l + $ python runtest.py -l test/scons-time -Then there is also the ``-n`` option, which prints the command line for -each single test, but doesn't actually execute them:: +``runtest.py`` also has ``-n`` option, which prints the command line for +each test which would have been run, but doesn't actually execute them:: - $ python runtest.py -n + $ python runtest.py -n -a Finding Tests ============= @@ -182,7 +191,7 @@ directory. It then dives into the ``src`` and ``test`` folders, where it tries to find filenames ``*Test.py`` - for the ``src`` directory + for the ``src`` folder ``*.py`` for the ``test`` folder @@ -216,6 +225,8 @@ a simple "Hello, world!" example:: """) test.write('hello.c', """\ + #include + int main(int argc, char *argv[]) { @@ -259,7 +270,7 @@ a simple "Hello, world!" example:: ``test.write('hello.c', ...)`` This lines creates an ``hello.c`` file in the temporary directory. - Note that we have to escape the ``\\n`` in the + Note that we have to escape the newline in the ``"Hello, world!\\n"`` string so that it ends up as a single backslash in the ``hello.c`` file on disk. @@ -282,7 +293,9 @@ a simple "Hello, world!" example:: non-zero, or there is any error output. ``test.pass_test()`` - This is always the last line in a test script. It prints ``PASSED`` + This is always the last line in a test script. If we get to + this line, it means we haven't bailed out on a failure or skip, + so the result was good. It prints ``PASSED`` on the screen and makes sure we exit with a ``0`` status to indicate the test passed. As a side effect of destroying the ``test`` object, the created temporary directory will be removed. @@ -295,14 +308,21 @@ on the fly by the test program. We give a filename to the ``TestSCons.write()`` method, and a string holding its contents, and it gets written to the test folder right before starting.. -This technique can still be seen throughout most of the end-to-end tests, -but there is a better way. To create a test, you need to create the -files that will be used, then when they work reasonably, they need to -be pasted into the script. The process repeats for maintenance. Once -a test gets more complex and/or grows many steps, the test script gets -harder to read. Why not keep the files as is? - -In testing parlance, a fixture is a repeatable test setup. The scons +This simple technique can be seen throughout most of the end-to-end +tests as it was the original technique provided to test developers, +but it is definitely not the preferred way to write a new test. +To develop this way, you first need to create the necessary files and +get them to work, then convert them to an embedded string form, which may +involve lots of extra escaping. These embedded files are then tricky +to maintain. As a test grows multiple steps, it becomes less easy to +read, since the embedded strings aren't quite the final files, and +the volume of test code obscures the flow of the testing steps. +Additionally, as SCons moves more to the use of code checkers and +formatters to detect problems and keep a standard coding style for +better readability, note that these techniques don't look inside +strings, so they're either left out or lots of manual work has to be done. + +In testing parlance, a fixture is a repeatable test setup. The SCons test harness allows the use of saved files or directories to be used in that sense: "the fixture for this test is foo", instead of writing a whole bunch of strings to create files. Since these setups can be @@ -314,22 +334,28 @@ them, see instructions in the above section named "Finding Tests". Directory Fixtures ################## -The function ``dir_fixture(self, srcdir, dstdir=None)`` in the ``TestCmd`` -class copies the contents of the specified folder ``srcdir`` from +The test harness method ``dir_fixture(srcdir, [dstdir])`` +copies the contents of the specified folder ``srcdir`` from the directory of the called test script to the current temporary test directory. The ``srcdir`` name may be a list, in which case the elements are concatenated with the ``os.path.join()`` method. The ``dstdir`` is assumed to be under the temporary working directory, it gets created automatically, if it does not already exist. +If ``srcdir`` represents an absolute path, it is used as-is. Otherwise, +if the harness was invoked with the environment variable ``FIXTURE_DIRS`` +set, the test instance will present that list of directories to search +as ``self.fixture_dirs``, each of these are additionally searched for +a directory with the name of ``srcdir``. + A short syntax example:: test = TestSCons.TestSCons() test.dir_fixture('image') test.run() -would copy all files and subfolders from the local ``image`` folder, -to the temporary directory for the current test. +would copy all files and subfolders from the local ``image`` folder +to the temporary directory for the current test, then run it. To see a real example for this in action, refer to the test named ``test/packaging/convenience-functions/convenience-functions.py``. @@ -337,13 +363,17 @@ To see a real example for this in action, refer to the test named File Fixtures ############# -Like for directory fixtures, ``file_fixture(self, srcfile, dstfile=None)`` +Similarly, the method ``file_fixture(srcfile, [dstfile])`` copies the file ``srcfile`` from the directory of the called script, to the temporary test directory. The ``dstfile`` is assumed to be under the temporary working directory, unless it is an absolute path name. If ``dstfile`` is specified, its target directory gets created automatically if it doesn't already exist. +If ``srcfile`` represents an absolute path, it is used as-is. Otherwise, +any passed in fixture directories are used as additional places to +search, as for the ``dir_fixture`` case. + With the following code:: test = TestSCons.TestSCons() @@ -368,7 +398,7 @@ list of available Tools, though not all may have tests yet. How to Convert Old Tests to Use Fixures ####################################### -Tests using the inline ``TestSCons.write()`` method can easily be +Tests using the inline ``TestSCons.write()`` method can fairly easily be converted to the fixture based approach. For this, we need to get at the files as they are written to each temporary test folder. @@ -383,9 +413,9 @@ So, you should be able to give the commands:: assuming Linux and a bash-like shell. For a Windows ``cmd`` shell, use ``set PRESERVE=1`` (that will leave it set for the duration of the -``cmd`` session, unless manually deleted). +``cmd`` session, unless manually cleared). -The output should then look something like this:: +The output will then look something like this:: 1/1 (100.00%) /usr/bin/python -tt test/packaging/sandbox-test.py PASSED @@ -399,6 +429,19 @@ tedious ``TestSCons.write()`` statements and replace them by a single Finally, don't forget to clean up and remove the temporary test directory. ``;)`` +For more complex testing scenarios you can use ``file_fixture`` with +the option to rename (that is, supplying a second argument, which is +the name to give the fixture file being copied). For example some test +files write multiple ``SConstruct`` files across the full run. +These files can be given different names - perhaps using a sufffix - +and then sucessively copied to the final name as needed:: + + test.file_fixture('fixture/SConstruct.part1', 'SConstruct') + # more setup, then run test + test.file_fixture('fixture/SConstruct.part2', 'SConstruct') + # etc. + + When Not to Use a Fixture ######################### @@ -413,30 +456,33 @@ kind of usage that does not lend itself to a fixture:: test.write('SConstruct', """ cc = Environment().Dictionary('CC') - env = Environment(LINK = r'%(_python_)s mylink.py', - LINKFLAGS = [], - CC = r'%(_python_)s mycc.py', - CXX = cc, - CXXFLAGS = []) - env.Program(target = 'test1', source = 'test1.c') + env = Environment(LINK=r'%(_python_)s mylink.py', + LINKFLAGS=[], + CC=r'%(_python_)s mycc.py', + CXX=cc, + CXXFLAGS=[]) + env.Program(target='test1', source='test1.c') """ % locals()) Here the value of ``_python_`` is picked out of the script's -``locals`` dictionary and interpolated into the string that -will be written to ``SConstruct``. +``locals`` dictionary - which works because we've set it above - +and interpolated using a mapping key into the string that will +be written to ``SConstruct``. A fixture would be hard to use +here because we don't know the value of `_python_` until runtime. The other files created in this test may still be candidates for -use in a fixture, however. +use as fixture files, however. Debugging End-to-End Tests ========================== Most of the end to end tests have expectations for standard output -and error from the test runs. The expectation could be either +and error embedded in the tests. The expectation could be either that there is nothing on that stream, or that it will contain very specific text which the test matches against. So adding -``print()`` calls, or ``sys,stderr.write()`` or similar will -emit data that the tests do not expect, and cause further failures. +``print()`` calls, or ``sys.stderr.write()`` or similar will +emit data that the tests do not expect, and thus cause further +failures - possibly even obscuring the original error. Say you have three different tests in a script, and the third one is unexpectedly failing. You add some debug prints to the part of scons that is involved, and now the first test of the @@ -445,7 +491,15 @@ to the third test you were trying to debug. Still, there are some techniques to help debugging. -Probably the most effective technique is to use the internal +The first step should be to run the tests so the harness +emits more information, without forcing more information into +the test stdout/stderr which will confuse result evaulation. +``runtest.py`` has several verbose levels which can be used +for this purpose: + + python runtest.py --verbose=2 test/foo.py + +You can also use the internal ``SCons.Debug.Trace()`` function, which prints output to ``/dev/tty`` on Linux/UNIX systems and ``con`` on Windows systems, so you can see what's going on. -- cgit v0.12 From ec9f68cb44bfe788c509d1de52c3f674a0fe4655 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 27 Feb 2020 07:08:32 -0700 Subject: [PR #3571] fix error and improve docs sider spotted a cut-n-paste error introduced in the PR (srcdir set instead of srcfile) framework doc now uses directory consistently (over folder, which is a desktop concept rather than a filesystem concept) tweaked wording a bit more Signed-off-by: Mats Wichmann --- testing/framework/TestCmd.py | 2 +- testing/framework/test-framework.rst | 101 ++++++++++++++++++----------------- 2 files changed, 53 insertions(+), 50 deletions(-) diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py index 9b5df8e..01d12cb 100644 --- a/testing/framework/TestCmd.py +++ b/testing/framework/TestCmd.py @@ -1409,7 +1409,7 @@ class TestCmd(object): of dstfile are created automatically if needed. """ if is_List(srcfile): - srcdir = os.path.join(*srcfile) + srcfile = os.path.join(*srcfile) srcpath, srctail = os.path.split(srcfile) spath = srcfile diff --git a/testing/framework/test-framework.rst b/testing/framework/test-framework.rst index 965efd4..ce2e3c9 100644 --- a/testing/framework/test-framework.rst +++ b/testing/framework/test-framework.rst @@ -34,10 +34,10 @@ There are three types of SCons tests: *External Tests* For the support of external Tools (in the form of packages, preferably), the testing framework is extended so it can run in standalone mode. - You can start it from the top-level folder of your Tool's source tree, + You can start it from the top-level directory of your Tool's source tree, where it then finds all Python scripts (``*.py``) underneath the local ``test/`` directory. This implies that Tool tests have to be kept in - a folder named ``test``, like for the SCons core. + a directory named ``test``, like for the SCons core. Contrasting End-to-End and Unit Tests @@ -138,8 +138,8 @@ unsuccessful tests. The above invocations all test against the scons files underneath the ``src/`` subdirectory, and do not require that a packaging build of SCons be performed -first. This is the most common mode: make some changes, and test in -place the effects. +first. This is the most common mode: make some changes, and test the +effects in place. The ``runtest.py`` script supports additional options to run tests against unpacked packages in the ``build/test-*/`` subdirectories. @@ -156,12 +156,12 @@ created just for that test, which is by default removed when the test is complete. This ensures that your source directories don't get clobbered with temporary files and changes from the test runs. If the test itself needs to know the directory, it can be obtained -as ``test.workdir``, or more commonly by calling ``test.workpath``, +as ``test.workdir``, or more commonly by calling ``test.workpath()``, a function which takes a path-component argument and returns the path to that path-component in the testing directory. The use of an ephemeral test directory means that you can't simply change -into a folder to "debug things" after a test has gone wrong. +into a directory to "debug things" after a test has gone wrong. For a way around this, check out the ``PRESERVE`` environment variable. It can be seen in action in `How to Convert Old Tests to Use Fixures`_ below. @@ -174,8 +174,8 @@ test finding options (see below) you intend to use. Example: $ python runtest.py -l test/scons-time -``runtest.py`` also has ``-n`` option, which prints the command line for -each test which would have been run, but doesn't actually execute them:: +``runtest.py`` also has a ``-n`` option, which prints the command line for +each test which would have been run, but doesn't actually run them:: $ python runtest.py -n -a @@ -187,25 +187,25 @@ When started in *standard* mode:: $ python runtest.py -a ``runtest.py`` assumes that it is run from the SCons top-level source -directory. It then dives into the ``src`` and ``test`` folders, where -it tries to find filenames +directory. It then dives into the ``src`` and ``test`` directories, +where it tries to find filenames ``*Test.py`` - for the ``src`` folder + for the ``src`` directory (unit tests) ``*.py`` - for the ``test`` folder + for the ``test`` directory (end-to-end tests) When using fixtures, you may quickly end up in a position where you have -supporting Python script files in a subfolder, but they shouldn't get +supporting Python script files in a subdirectory which shouldn't be picked up as test scripts. In this case you have two options: -#. Add a file with the name ``sconstest.skip`` to your subfolder. This - lets ``runtest.py`` skip the contents of the directory completely. -#. Create a file ``.exclude_tests`` in each folder in question, and in - it list line-by-line the files to get excluded from testing. +#. Add a file with the name ``sconstest.skip`` to your subdirectory. This + tells ``runtest.py`` to skip the contents of the directory completely. +#. Create a file ``.exclude_tests`` in each directory in question, and in + it list line-by-line the files to exclude from testing. -The same rules apply when testing external Tools by using the ``-e`` +The same rules apply when testing external Tools when using the ``-e`` option. @@ -263,13 +263,12 @@ a simple "Hello, world!" example:: ``test.write('SConstruct', ...)`` This line creates an ``SConstruct`` file in the temporary directory, to be used as input to the ``scons`` run(s) that we're testing. - Note the use of the Python triple-quote syntax for the contents - of the ``SConstruct`` file. Because input files for tests are all - created from in-line data like this, the tests can sometimes get - a little confusing to read, because some of the Python code is found + Note the use of the Python triple-quoted string for the contents + of the ``SConstruct`` file (and see the next section for an + alternative approach). ``test.write('hello.c', ...)`` - This lines creates an ``hello.c`` file in the temporary directory. + This line creates an ``hello.c`` file in the temporary directory. Note that we have to escape the newline in the ``"Hello, world!\\n"`` string so that it ends up as a single backslash in the ``hello.c`` file on disk. @@ -286,7 +285,7 @@ a simple "Hello, world!" example:: ``test.run(program='./hello', stdout="Hello, world!\n")`` This shows use of the ``TestSCons.run()`` method to execute a program other than ``scons``, in this case the ``hello`` program we just - presumably built. The ``stdout=`` keyword argument also tells the + built. The ``stdout=`` keyword argument also tells the ``TestSCons.run()`` method to fail if the program output does not match the expected string ``"Hello, world!\n"``. Like the previous ``test.run()`` line, it will also fail the test if the exit status is @@ -306,21 +305,21 @@ Working with Fixtures In the simple example above, the files to set up the test are created on the fly by the test program. We give a filename to the ``TestSCons.write()`` method, and a string holding its contents, and it gets written to the test -folder right before starting.. +directory right before starting.. This simple technique can be seen throughout most of the end-to-end tests as it was the original technique provided to test developers, -but it is definitely not the preferred way to write a new test. +but it is no longer the preferred way to write a new test. To develop this way, you first need to create the necessary files and get them to work, then convert them to an embedded string form, which may involve lots of extra escaping. These embedded files are then tricky to maintain. As a test grows multiple steps, it becomes less easy to -read, since the embedded strings aren't quite the final files, and -the volume of test code obscures the flow of the testing steps. -Additionally, as SCons moves more to the use of code checkers and -formatters to detect problems and keep a standard coding style for -better readability, note that these techniques don't look inside -strings, so they're either left out or lots of manual work has to be done. +read, since many if the embedded strings aren't quite the final files, +and the volume of test code obscures the flow of the testing steps. +Additionally, as SCons moves more to the use of automated code checkers +and formatters to detect problems and keep a standard coding style for +better readability, note that such tools don't look inside strings +for code, so the effect is lost on them. In testing parlance, a fixture is a repeatable test setup. The SCons test harness allows the use of saved files or directories to be used @@ -335,16 +334,17 @@ Directory Fixtures ################## The test harness method ``dir_fixture(srcdir, [dstdir])`` -copies the contents of the specified folder ``srcdir`` from +copies the contents of the specified directory ``srcdir`` from the directory of the called test script to the current temporary test directory. The ``srcdir`` name may be a list, in which case the elements -are concatenated with the ``os.path.join()`` method. The ``dstdir`` +are concatenated into a path first. The ``dstdir`` is assumed to be under the temporary working directory, it gets created automatically, if it does not already exist. -If ``srcdir`` represents an absolute path, it is used as-is. Otherwise, -if the harness was invoked with the environment variable ``FIXTURE_DIRS`` -set, the test instance will present that list of directories to search +If ``srcdir`` represents an absolute path, it is used as-is. +Otherwise, if the harness was invoked with the environment variable +``FIXTURE_DIRS`` set (which ``runtest.py`` does by default), +the test instance will present that list of directories to search as ``self.fixture_dirs``, each of these are additionally searched for a directory with the name of ``srcdir``. @@ -354,7 +354,7 @@ A short syntax example:: test.dir_fixture('image') test.run() -would copy all files and subfolders from the local ``image`` folder +would copy all files and subdirectories from the local ``image`` directory to the temporary directory for the current test, then run it. To see a real example for this in action, refer to the test named @@ -365,14 +365,16 @@ File Fixtures Similarly, the method ``file_fixture(srcfile, [dstfile])`` copies the file ``srcfile`` from the directory of the called script, -to the temporary test directory. The ``dstfile`` is assumed to be -under the temporary working directory, unless it is an absolute path -name. If ``dstfile`` is specified, its target directory gets created +to the temporary test directory. The ``srcfile`` name may be a list, +in which case the elements are concatenated into a path first. +The ``dstfile`` is assumed to be under the temporary working directory, +unless it is an absolute path name. +If ``dstfile`` is specified, its target directory gets created automatically if it doesn't already exist. If ``srcfile`` represents an absolute path, it is used as-is. Otherwise, any passed in fixture directories are used as additional places to -search, as for the ``dir_fixture`` case. +search for the fixture file, as for the ``dir_fixture`` case. With the following code:: @@ -384,7 +386,7 @@ With the following code:: The files ``SConstruct`` and ``src/main.cpp`` are copied to the temporary test directory. Notice the second ``file_fixture`` line preserves the path of the original, otherwise ``main.cpp`` -would have landed in the top level of the test directory. +would have been placed in the top level of the test directory. Again, a reference example can be found in the current revision of SCons, it is ``test/packaging/sandbox-test/sandbox-test.py``. @@ -400,11 +402,12 @@ How to Convert Old Tests to Use Fixures Tests using the inline ``TestSCons.write()`` method can fairly easily be converted to the fixture based approach. For this, we need to get at the -files as they are written to each temporary test folder. +files as they are written to each temporary test directory, +which we can do by taking advantage of a debugging aid: ``runtest.py`` checks for the existence of an environment variable named ``PRESERVE``. If it is set to a non-zero value, the testing -framework preserves the test folder instead of deleting it, and prints +framework preserves the test directory instead of deleting it, and prints its name to the screen. So, you should be able to give the commands:: @@ -421,8 +424,8 @@ The output will then look something like this:: PASSED Preserved directory /tmp/testcmd.4060.twlYNI -You can now copy the files from that folder to your new -*fixture* folder. Then, in the test script you simply remove all the +You can now copy the files from that directory to your new +*fixture* directory. Then, in the test script you simply remove all the tedious ``TestSCons.write()`` statements and replace them by a single ``TestSCons.dir_fixture()``. @@ -493,7 +496,7 @@ Still, there are some techniques to help debugging. The first step should be to run the tests so the harness emits more information, without forcing more information into -the test stdout/stderr which will confuse result evaulation. +the test stdout/stderr which will confuse result evaluation. ``runtest.py`` has several verbose levels which can be used for this purpose: @@ -532,7 +535,7 @@ in ``testing/framework``. Start in to create files (``test.write()``) and run commands (``test.run()``). Use ``TestSCons`` for the end-to-end tests in ``test``, but use -``TestCmd`` for the unit tests in the ``src`` folder. +``TestCmd`` for the unit tests in the ``src`` directory. The match functions work like this: -- cgit v0.12 From b742241e46d3894749e14607b5435f125db77431 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 27 Feb 2020 09:19:06 -0700 Subject: Add comments on skipping tests to framework doc [ci skip] Build up the skipping tests section by adding a bit of discussion on why you'd want to skip, and on not skipping too much by mocking parts of tests and organizing test code. Signed-off-by: Mats Wichmann --- testing/framework/test-framework.rst | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/testing/framework/test-framework.rst b/testing/framework/test-framework.rst index ce2e3c9..dad09d3 100644 --- a/testing/framework/test-framework.rst +++ b/testing/framework/test-framework.rst @@ -9,7 +9,7 @@ any surprise changes in behavior. In general, no change goes into SCons unless it has one or more new or modified tests that demonstrably exercise the bug being fixed or the feature being added. There are exceptions to this guideline, but -they should be just that, ''exceptions''. When in doubt, make sure +they should be just that, *exceptions*. When in doubt, make sure it's tested. Test Organization @@ -500,7 +500,7 @@ the test stdout/stderr which will confuse result evaluation. ``runtest.py`` has several verbose levels which can be used for this purpose: - python runtest.py --verbose=2 test/foo.py + $ python runtest.py --verbose=2 test/foo.py You can also use the internal ``SCons.Debug.Trace()`` function, which prints output to @@ -529,7 +529,7 @@ Test Infrastructure The main test API in the ``TestSCons.py`` class. ``TestSCons`` is a subclass of ``TestCommon``, which is a subclass of ``TestCmd``. -All those classes are defined in python files of the same name +All those classes are defined in Python files of the same name in ``testing/framework``. Start in ``testing/framework/TestCmd.py`` for the base API definitions, like how to create files (``test.write()``) and run commands (``test.run()``). @@ -553,7 +553,7 @@ The match functions work like this: * Joins the lines with newline (unless already a string) * joins the REs with newline (unless it's a string) and puts ``^..$`` around the whole thing - * then whole thing must match with python re.DOTALL. + * then whole thing must match with Python re.DOTALL. Use them in a test like this:: @@ -566,6 +566,12 @@ or:: Avoiding Tests Based on Tool Existence ====================================== +For many tests, if the tool being tested is backed by an external program +which is not installed on the machine under test, it may not be worth +proceeding with the test. For example, it's hard to test complilng code with +a C compiler if no C compiler exists. In this case, the test should be +skipped. + Here's a simple example:: #!python @@ -581,3 +587,20 @@ The ``where_is`` method can be used to look for programs that are do not have tool specifications. The existing test code will have many samples of using either or both of these to detect if it is worth even proceeding with a test. + +Note that it is usually possible to test at least part of the operation of +a tool without the underlying program. Tools are responsible for setting up +construction variables and having the right builders, scanners and emitters +plumbed into the environment. These things can be tested by mocking the +behavior of the executable. Many examples of this can be found in the +``test`` directory. *TODO: point to one example*. + +This leads to a suggestion for test organization: keep tool tests which +don't need the underlying program in separate files from ones which do - +it is clearer what is going on if we can see in the test results that the +plumbing tests worked but the ones using the underlying program were skipped +rather than seeing all the tests for a tool passing or being skipped. +The framework doesn't have a way to indicate a partial skip - if you executed +200 lines of test, then found a condition which caused you to skip the +last 20 lines, the whole test is marked as a skip; +it also doesn't have a way to indicate a partial pass. -- cgit v0.12 From 1a890d631d128867affc73bcf608d376c8b9c6f6 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 29 Feb 2020 08:20:51 -0700 Subject: [PR #3571] more work on testing document [ci skip] Apply some of the ideas of the Python documentation style guide - headings, indents, etc. Fixed some lingering format problems. Add a table of contents (this will work on the wiki as long as we save the copy there in rest format instead of markdown - that is currently the case). Signed-off-by: Mats Wichmann --- testing/framework/test-framework.rst | 349 +++++++++++++++++------------------ 1 file changed, 174 insertions(+), 175 deletions(-) diff --git a/testing/framework/test-framework.rst b/testing/framework/test-framework.rst index dad09d3..16d3734 100644 --- a/testing/framework/test-framework.rst +++ b/testing/framework/test-framework.rst @@ -1,6 +1,11 @@ -======================= +*********************** SCons Testing Framework -======================= +*********************** +.. contents:: + :local: + +Introduction +============ SCons uses extensive automated tests to ensure quality. The primary goal is that users be able to upgrade from version to version without @@ -12,36 +17,36 @@ the feature being added. There are exceptions to this guideline, but they should be just that, *exceptions*. When in doubt, make sure it's tested. -Test Organization +Test organization ================= There are three types of SCons tests: *End-to-End Tests* - End-to-end tests of SCons are Python scripts (``*.py``) underneath the - ``test/`` subdirectory. They use the test infrastructure modules in - the ``testing/framework`` subdirectory. They build set up complete - projects and call scons to execute them, checking that the behavior is - as expected. + End-to-end tests of SCons are Python scripts (``*.py``) underneath the + ``test/`` subdirectory. They use the test infrastructure modules in + the ``testing/framework`` subdirectory. They build set up complete + projects and call scons to execute them, checking that the behavior is + as expected. *Unit Tests* - Unit tests for individual SCons modules live underneath the - ``src/engine/`` subdirectory and are the same base name as the module - to be tests, with ``Tests`` appended before the ``.py``. For example, - the unit tests for the ``Builder.py`` module are in the - ``BuilderTests.py`` script. Unit tests tend to be based on assertions. + Unit tests for individual SCons modules live underneath the + ``src/engine/`` subdirectory and are the same base name as the module + to be tests, with ``Tests`` appended before the ``.py``. For example, + the unit tests for the ``Builder.py`` module are in the + ``BuilderTests.py`` script. Unit tests tend to be based on assertions. *External Tests* - For the support of external Tools (in the form of packages, preferably), - the testing framework is extended so it can run in standalone mode. - You can start it from the top-level directory of your Tool's source tree, - where it then finds all Python scripts (``*.py``) underneath the local - ``test/`` directory. This implies that Tool tests have to be kept in - a directory named ``test``, like for the SCons core. + For the support of external Tools (in the form of packages, preferably), + the testing framework is extended so it can run in standalone mode. + You can start it from the top-level directory of your Tool's source tree, + where it then finds all Python scripts (``*.py``) underneath the local + ``test/`` directory. This implies that Tool tests have to be kept in + a directory named ``test``, like for the SCons core. -Contrasting End-to-End and Unit Tests -##################################### +Contrasting end-to-end and unit tests +------------------------------------- In general, functionality with end-to-end tests should be considered a hardened part of the public interface (that is, @@ -59,38 +64,33 @@ scripts by using the ``runtest.py --pdb`` option, but the end-to-end tests treat an SCons invocation as a "black box" and just look for external effects; simple methods like inserting ``print`` statements in the SCons code itself can disrupt those external effects. -See `Debugging End-to-End Tests`_ for some more thoughts. +See `Debugging end-to-end tests`_ for some more thoughts. -Naming Conventions -################## +Naming conventions +------------------ The end-to-end tests, more or less, stick to the following naming conventions: #. All tests end with a .py suffix. - #. In the *General* form we use ``Feature.py`` - for the test of a specified feature; try to keep this description - reasonably short - + for the test of a specified feature; try to keep this description + reasonably short ``Feature-x.py`` - for the test of a specified feature using option ``x`` + for the test of a specified feature using option ``x`` #. The *command line option* tests take the form ``option-x.py`` - for a lower-case single-letter option - + for a lower-case single-letter option ``option--X.py`` - upper-case single-letter option (with an extra hyphen, so the - file names will be unique on case-insensitive systems) - + upper-case single-letter option (with an extra hyphen, so the + file names will be unique on case-insensitive systems) ``option--lo.py`` - long option; abbreviate the long option name to a few characters + long option; abbreviate the long option name to a few characters - -Running Tests +Running tests ============= The standard set of SCons tests are run from the top-level source @@ -98,11 +98,11 @@ directory by the ``runtest.py`` script. Help is available through the ``-h`` option:: - $ python runtest.py -h + $ python runtest.py -h To simply run all the tests, use the ``-a`` option:: - $ python runtest.py -a + $ python runtest.py -a By default, ``runtest.py`` prints a count and percentage message for each test case, along with the name of the test file. If you need the output @@ -110,22 +110,22 @@ to be more silent, have a look at the ``-q``, ``-s`` and ``-k`` options. You may specifically list one or more tests to be run:: - $ python runtest.py src/engine/SCons/BuilderTests.py - $ python runtest.py test/option-j.py test/Program.py + $ python runtest.py src/engine/SCons/BuilderTests.py + $ python runtest.py test/option-j.py test/Program.py Folder names are allowed in the test list as well, so you can do:: - $ python runtest.py test/SWIG + $ python runtest.py test/SWIG to run all SWIG tests only. You can also use the ``-f`` option to execute just the tests listed in a test list file:: - $ cat testlist.txt - test/option-j.py - test/Program.py - $ python runtest.py -f testlist.txt + $ cat testlist.txt + test/option-j.py + test/Program.py + $ python runtest.py -f testlist.txt One test must be listed per line, and any lines that begin with '#' will be ignored (the intent being to allow you, for example, to comment @@ -143,10 +143,10 @@ effects in place. The ``runtest.py`` script supports additional options to run tests against unpacked packages in the ``build/test-*/`` subdirectories. -If you are testing a separate Tool outside of the SCons source tree, you -have to call the ``runtest.py`` script in *external* (stand-alone) mode:: +If you are testing a separate Tool outside of the SCons source tree, +call the ``runtest.py`` script in *external* (stand-alone) mode:: - $ python ~/scons/runtest.py -e -a + $ python ~/scons/runtest.py -e -a This ensures that the testing framework doesn't try to access SCons classes needed for some of the *internal* test cases. @@ -163,42 +163,42 @@ that path-component in the testing directory. The use of an ephemeral test directory means that you can't simply change into a directory to "debug things" after a test has gone wrong. For a way around this, check out the ``PRESERVE`` environment variable. -It can be seen in action in `How to Convert Old Tests to Use Fixures`_ below. +It can be seen in action in `How to convert old tests to use fixures`_ below. -Not Running Tests +Not running tests ================= If you simply want to check which tests would get executed, you can call the ``runtest.py`` script with the ``-l`` option combined with whichever -test finding options (see below) you intend to use. Example: +test finding options (see below) you intend to use. Example:: - $ python runtest.py -l test/scons-time + $ python runtest.py -l test/scons-time ``runtest.py`` also has a ``-n`` option, which prints the command line for each test which would have been run, but doesn't actually run them:: - $ python runtest.py -n -a + $ python runtest.py -n -a Finding Tests ============= When started in *standard* mode:: - $ python runtest.py -a + $ python runtest.py -a ``runtest.py`` assumes that it is run from the SCons top-level source directory. It then dives into the ``src`` and ``test`` directories, where it tries to find filenames ``*Test.py`` - for the ``src`` directory (unit tests) + for the ``src`` directory (unit tests) ``*.py`` - for the ``test`` directory (end-to-end tests) + for the ``test`` directory (end-to-end tests) -When using fixtures, you may quickly end up in a position where you have +When using fixtures, you may end up in a situation where you have supporting Python script files in a subdirectory which shouldn't be -picked up as test scripts. In this case you have two options: +picked up as test scripts. There are two options here: #. Add a file with the name ``sconstest.skip`` to your subdirectory. This tells ``runtest.py`` to skip the contents of the directory completely. @@ -215,91 +215,89 @@ Example End-to-End Test Script To illustrate how the end-to-end test scripts work, let's walk through a simple "Hello, world!" example:: - #!python - import TestSCons + #!python + import TestSCons - test = TestSCons.TestSCons() + test = TestSCons.TestSCons() - test.write('SConstruct', """\ - Program('hello.c') - """) + test.write('SConstruct', """\ + Program('hello.c') + """) - test.write('hello.c', """\ - #include + test.write('hello.c', """\ + #include - int - main(int argc, char *argv[]) - { + int + main(int argc, char *argv[]) + { printf("Hello, world!\\n"); exit (0); - } - """) + } + """) - test.run() + test.run() - test.run(program='./hello', stdout="Hello, world!\n") + test.run(program='./hello', stdout="Hello, world!\n") - test.pass_test() + test.pass_test() ``import TestSCons`` - Imports the main infrastructure for writing SCons tests. This is - normally the only part of the infrastructure that needs importing. - Sometimes other Python modules are necessary or helpful, and get - imported before this line. + Imports the main infrastructure for writing SCons tests. This is + normally the only part of the infrastructure that needs importing. + Sometimes other Python modules are necessary or helpful, and get + imported before this line. ``test = TestSCons.TestSCons()`` - This initializes an object for testing. A fair amount happens under - the covers when the object is created, including: - - * A temporary directory is created for all the in-line files that will - get created. + This initializes an object for testing. A fair amount happens under + the covers when the object is created, including: - * The temporary directory's removal is arranged for when - the test is finished. - - * The test does ``os.chdir()`` to the temporary directory. + * A temporary directory is created for all the in-line files that will + get created. + * The temporary directory's removal is arranged for when + the test is finished. + * The test does ``os.chdir()`` to the temporary directory. ``test.write('SConstruct', ...)`` - This line creates an ``SConstruct`` file in the temporary directory, - to be used as input to the ``scons`` run(s) that we're testing. - Note the use of the Python triple-quoted string for the contents - of the ``SConstruct`` file (and see the next section for an - alternative approach). + This line creates an ``SConstruct`` file in the temporary directory, + to be used as input to the ``scons`` run(s) that we're testing. + Note the use of the Python triple-quoted string for the contents + of the ``SConstruct`` file (and see the next section for an + alternative approach). ``test.write('hello.c', ...)`` - This line creates an ``hello.c`` file in the temporary directory. - Note that we have to escape the newline in the - ``"Hello, world!\\n"`` string so that it ends up as a single - backslash in the ``hello.c`` file on disk. + This line creates an ``hello.c`` file in the temporary directory. + Note that we have to escape the newline in the + ``"Hello, world!\\n"`` string so that it ends up as a single + backslash in the ``hello.c`` file on disk. ``test.run()`` - This actually runs SCons. Like the object initialization, things - happen under the covers: + This actually runs SCons. Like the object initialization, things + happen under the covers: - * The exit status is verified; the test exits with a failure if - the exit status is not zero. - * The error output is examined, and the test exits with a failure - if there is any. + * The exit status is verified; the test exits with a failure if + the exit status is not zero. + * The error output is examined, and the test exits with a failure + if there is any. ``test.run(program='./hello', stdout="Hello, world!\n")`` - This shows use of the ``TestSCons.run()`` method to execute a program - other than ``scons``, in this case the ``hello`` program we just - built. The ``stdout=`` keyword argument also tells the - ``TestSCons.run()`` method to fail if the program output does not - match the expected string ``"Hello, world!\n"``. Like the previous - ``test.run()`` line, it will also fail the test if the exit status is - non-zero, or there is any error output. + This shows use of the ``TestSCons.run()`` method to execute a program + other than ``scons``, in this case the ``hello`` program we just + built. The ``stdout=`` keyword argument also tells the + ``TestSCons.run()`` method to fail if the program output does not + match the expected string ``"Hello, world!\n"``. Like the previous + ``test.run()`` line, it will also fail the test if the exit status is + non-zero, or there is any error output. ``test.pass_test()`` - This is always the last line in a test script. If we get to - this line, it means we haven't bailed out on a failure or skip, - so the result was good. It prints ``PASSED`` - on the screen and makes sure we exit with a ``0`` status to indicate - the test passed. As a side effect of destroying the ``test`` object, - the created temporary directory will be removed. - -Working with Fixtures + This is always the last line in a test script. If we get to + this line, it means we haven't bailed out on a failure or skip, + so the result was good. It prints ``PASSED`` + on the screen and makes sure we exit with a ``0`` status to indicate + the test passed. As a side effect of destroying the ``test`` object, + the created temporary directory will be removed. + +Working with fixtures ===================== In the simple example above, the files to set up the test are created @@ -330,8 +328,8 @@ reusable across multiple tests, the *fixture* terminology applies well. Note: fixtures must not be treated by SCons as runnable tests. To exclude them, see instructions in the above section named "Finding Tests". -Directory Fixtures -################## +Directory fixtures +------------------ The test harness method ``dir_fixture(srcdir, [dstdir])`` copies the contents of the specified directory ``srcdir`` from @@ -350,9 +348,9 @@ a directory with the name of ``srcdir``. A short syntax example:: - test = TestSCons.TestSCons() - test.dir_fixture('image') - test.run() + test = TestSCons.TestSCons() + test.dir_fixture('image') + test.run() would copy all files and subdirectories from the local ``image`` directory to the temporary directory for the current test, then run it. @@ -360,8 +358,8 @@ to the temporary directory for the current test, then run it. To see a real example for this in action, refer to the test named ``test/packaging/convenience-functions/convenience-functions.py``. -File Fixtures -############# +File fixtures +------------- Similarly, the method ``file_fixture(srcfile, [dstfile])`` copies the file ``srcfile`` from the directory of the called script, @@ -378,10 +376,10 @@ search for the fixture file, as for the ``dir_fixture`` case. With the following code:: - test = TestSCons.TestSCons() - test.file_fixture('SConstruct') - test.file_fixture(['src','main.cpp'],['src','main.cpp']) - test.run() + test = TestSCons.TestSCons() + test.file_fixture('SConstruct') + test.file_fixture(['src','main.cpp'],['src','main.cpp']) + test.run() The files ``SConstruct`` and ``src/main.cpp`` are copied to the temporary test directory. Notice the second ``file_fixture`` line @@ -397,8 +395,8 @@ https://bitbucket.org/dirkbaechle/scons_qt4. Also visit the SCons Tools Index at https://github.com/SCons/scons/wiki/ToolsIndex for a complete list of available Tools, though not all may have tests yet. -How to Convert Old Tests to Use Fixures -####################################### +How to convert old tests to use fixures +--------------------------------------- Tests using the inline ``TestSCons.write()`` method can fairly easily be converted to the fixture based approach. For this, we need to get at the @@ -412,7 +410,7 @@ its name to the screen. So, you should be able to give the commands:: - $ PRESERVE=1 python runtest.py test/packaging/sandbox-test.py + $ PRESERVE=1 python runtest.py test/packaging/sandbox-test.py assuming Linux and a bash-like shell. For a Windows ``cmd`` shell, use ``set PRESERVE=1`` (that will leave it set for the duration of the @@ -420,9 +418,9 @@ assuming Linux and a bash-like shell. For a Windows ``cmd`` shell, use The output will then look something like this:: - 1/1 (100.00%) /usr/bin/python -tt test/packaging/sandbox-test.py - PASSED - Preserved directory /tmp/testcmd.4060.twlYNI + 1/1 (100.00%) /usr/bin/python -tt test/packaging/sandbox-test.py + pASSED + preserved directory /tmp/testcmd.4060.twlYNI You can now copy the files from that directory to your new *fixture* directory. Then, in the test script you simply remove all the @@ -439,14 +437,14 @@ files write multiple ``SConstruct`` files across the full run. These files can be given different names - perhaps using a sufffix - and then sucessively copied to the final name as needed:: - test.file_fixture('fixture/SConstruct.part1', 'SConstruct') - # more setup, then run test - test.file_fixture('fixture/SConstruct.part2', 'SConstruct') - # etc. + test.file_fixture('fixture/SConstruct.part1', 'SConstruct') + # more setup, then run test + test.file_fixture('fixture/SConstruct.part2', 'SConstruct') + # etc. -When Not to Use a Fixture -######################### +When not to use a fixture +------------------------- Note that some files are not appropriate for use in a fixture as-is: fixture files should be static. If the creation of the file involves @@ -454,18 +452,18 @@ interpolating data discovered during the run of the test script, that process should stay in the script. Here is an example of this kind of usage that does not lend itself to a fixture:: - import TestSCons - _python_ = TestSCons._python_ + import TestSCons + _python_ = TestSCons._python_ - test.write('SConstruct', """ - cc = Environment().Dictionary('CC') - env = Environment(LINK=r'%(_python_)s mylink.py', - LINKFLAGS=[], - CC=r'%(_python_)s mycc.py', - CXX=cc, - CXXFLAGS=[]) - env.Program(target='test1', source='test1.c') - """ % locals()) + test.write('SConstruct', """ + cc = Environment().Dictionary('CC') + env = Environment(LINK=r'%(_python_)s mylink.py', + LINKFLAGS=[], + CC=r'%(_python_)s mycc.py', + CXX=cc, + CXXFLAGS=[]) + env.Program(target='test1', source='test1.c') + """ % locals()) Here the value of ``_python_`` is picked out of the script's ``locals`` dictionary - which works because we've set it above - @@ -476,7 +474,7 @@ here because we don't know the value of `_python_` until runtime. The other files created in this test may still be candidates for use as fixture files, however. -Debugging End-to-End Tests +Debugging end-to-end tests ========================== Most of the end to end tests have expectations for standard output @@ -498,9 +496,9 @@ The first step should be to run the tests so the harness emits more information, without forcing more information into the test stdout/stderr which will confuse result evaluation. ``runtest.py`` has several verbose levels which can be used -for this purpose: +for this purpose:: - $ python runtest.py --verbose=2 test/foo.py + $ python runtest.py --verbose=2 test/foo.py You can also use the internal ``SCons.Debug.Trace()`` function, which prints output to @@ -515,7 +513,7 @@ Part of the technique discussed in the section `How to Convert Old Tests to Use Fixures`_ can also be helpful for debugging purposes. If you have a failing test, try:: - $ PRESERVE=1 python runtest.py test/failing-test.py + $ PRESERVE=1 python runtest.py test/failing-test.py You can now go to the save directory reported from this run and invoke the test manually to see what it is doing, without @@ -524,7 +522,7 @@ the presence of the test infrastructure which would otherwise adding debug prints may be more useful. -Test Infrastructure +Test infrastructure =================== The main test API in the ``TestSCons.py`` class. ``TestSCons`` @@ -540,30 +538,31 @@ Use ``TestSCons`` for the end-to-end tests in ``test``, but use The match functions work like this: ``TestSCons.match_re`` - match each line with a RE + match each line with a RE - * Splits the lines into a list (unless they already are) - * splits the REs at newlines (unless already a list) and puts ^..$ around each - * then each RE must match each line. This means there must be as many - REs as lines. + * Splits the lines into a list (unless they already are) + * splits the REs at newlines (unless already a list) + and puts ``^..$`` around each + * then each RE must match each line. This means there must be as many + REs as lines. ``TestSCons.match_re_dotall`` - match all the lines against a single RE - - * Joins the lines with newline (unless already a string) - * joins the REs with newline (unless it's a string) and puts ``^..$`` - around the whole thing - * then whole thing must match with Python re.DOTALL. + match all the lines against a single RE + + * Joins the lines with newline (unless already a string) + * joins the REs with newline (unless it's a string) and puts ``^..$`` + around the whole thing + * then whole thing must match with Python re.DOTALL. Use them in a test like this:: - test.run(..., match=TestSCons.match_re, ...) + test.run(..., match=TestSCons.match_re, ...) or:: - test.must_match(..., match=TestSCons.match_re, ...) + test.must_match(..., match=TestSCons.match_re, ...) -Avoiding Tests Based on Tool Existence +Avoiding tests based on tool existence ====================================== For many tests, if the tool being tested is backed by an external program @@ -574,10 +573,10 @@ skipped. Here's a simple example:: - #!python - intelc = test.detect_tool('intelc', prog='icpc') - if not intelc: - test.skip_test("Could not load 'intelc' Tool; skipping test(s).\n") + #!python + intelc = test.detect_tool('intelc', prog='icpc') + if not intelc: + test.skip_test("Could not load 'intelc' Tool; skipping test(s).\n") See ``testing/framework/TestSCons.py`` for the ``detect_tool`` method. It calls the tool's ``generate()`` method, and then looks for the given -- cgit v0.12