The new fetch_data_files.py downloads all the input data files

used by encoding tests.  Fiddled the Windows buildbot helper
scripts to invoke this if needed.  Note that this isn't needed
on the trunk (the encoding tests download input files automatically
in 2.5).

4 files changed, 80 insertions, 3 deletions

diff --git a/Tools/buildbot/README.txt b/Tools/buildbot/README.txt
new file mode 100644
index 0000000..7c6e05a
--- /dev/null
+++ b/Tools/buildbot/README.txt
@@ -0,0 +1,14 @@
+Helpers used by buildbot-driven core Python testing.
+
+external.bat
+build.bat
+test.bat
+clean.bat
+    On Windows, these scripts are executed by the code sent
+    from the buildbot master to the slaves.
+
+fetch_data_files.py
+    Download all the input files various encoding tests want.  This is
+    used by build.bat on Windows (but could be used on any platform).
+    Note that in Python >= 2.5, the encoding tests download input files
+    automatically.
diff --git a/Tools/buildbot/build.bat b/Tools/buildbot/build.bat
index e3b77be..ef9b092 100644
--- a/Tools/buildbot/build.bat
+++ b/Tools/buildbot/build.bat
@@ -1,4 +1,7 @@
 @rem Used by the buildbot "compile" step.
 cmd /c Tools\buildbot\external.bat
 call "%VS71COMNTOOLS%vsvars32.bat"
-devenv.com /useenv /build Debug PCbuild\pcbuild.sln
+cd PCbuild
+devenv.com /useenv /build Debug pcbuild.sln
+@rem Fetch encoding test files.  Note that python_d needs to be built first.
+if not exist BIG5.TXT python_d.exe ..\Tools\buildbot\fetch_data_files.py
\ No newline at end of file
diff --git a/Tools/buildbot/external.bat b/Tools/buildbot/external.bat
index 5dd1114..fff0af2 100644
--- a/Tools/buildbot/external.bat
+++ b/Tools/buildbot/external.bat
@@ -4,5 +4,4 @@
 cd ..

 

 @rem bzip

-if not exist bzip2-1.0.3 svn export http://svn.python.org/projects/external/bzip2-1.0.3

-

+if not exist bzip2-1.0.3 svn export http://svn.python.org/projects/external/bzip2-1.0.3
\ No newline at end of file
diff --git a/Tools/buildbot/fetch_data_files.py b/Tools/buildbot/fetch_data_files.py
new file mode 100644
index 0000000..f4b6096
--- /dev/null
+++ b/Tools/buildbot/fetch_data_files.py
@@ -0,0 +1,61 @@
+"""A helper to download input files needed by assorted encoding tests.
+
+fetch_data_files.py [directory]
+
+Files are downloaded to directory `directory`.  If a directory isn't given,
+it defaults to the current directory (.).
+"""
+
+DATA_URLS = """
+    http://people.freebsd.org/~perky/i18n/BIG5HKSCS.TXT
+    http://people.freebsd.org/~perky/i18n/EUC-CN.TXT
+    http://people.freebsd.org/~perky/i18n/EUC-JISX0213.TXT
+    http://people.freebsd.org/~perky/i18n/EUC-JP.TXT
+    http://people.freebsd.org/~perky/i18n/EUC-KR.TXT
+    http://people.freebsd.org/~perky/i18n/SHIFT_JISX0213.TXT
+
+    http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP932.TXT
+    http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP936.TXT
+    http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP949.TXT
+    http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP950.TXT
+
+    http://www.unicode.org/Public/3.2-Update/NormalizationTest-3.2.0.txt
+
+    http://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/JIS/SHIFTJIS.TXT
+    http://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/KSC/JOHAB.TXT
+    http://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/OTHER/BIG5.TXT
+"""
+
+# Adapted from test_support.open_urlresource() in Python 2.5.
+# Fetch the file give by `url` off the web, and store it in directory
+# `directory`.  The file name is extracted from the last URL component.
+# If the file already exists, it's not fetched again.
+def fetch_file_from_url(url, directory):
+    import urllib, urlparse
+    import os.path
+
+    filename = urlparse.urlparse(url)[2].split('/')[-1] # '/': it's a URL!
+    target = os.path.join(directory, filename)
+    if os.path.exists(target):
+        print "\tskipping %r -- already exists" % target
+    else:
+        print "\tfetching %s ..." % url
+        urllib.urlretrieve(url, target)
+
+def main(urls, directory):
+    print "Downloading data files to %r" % directory
+    for url in urls.split():
+        fetch_file_from_url(url, directory)
+
+if __name__ == "__main__":
+    import sys
+
+    n = len(sys.argv)
+    if n == 1:
+        directory = "."
+    elif n == 2:
+        directory = sys.argv[1]
+    else:
+        raise ValueError("no more than one argument allowed")
+
+    main(DATA_URLS, directory)