Fix problem where API doc gen misses some files

In doc/SConscript, the function in bootstrap.py to parse a
manifest was called with an scons path (#src/engine) but bootstrap
does not run in an scons context, it is run via subprocess.Popen.
Preprocess the path. Along the way, change the function to open
the file itself instead of being passed a list already read
from the file, update the comment, and clean some whitespace issues.

Signed-off-by: Mats Wichmann <mats@linux.com>

3 files changed, 39 insertions, 30 deletions

diff --git a/SConstruct b/SConstruct
index 3d93d1e..8f7ceaf 100644
--- a/SConstruct
+++ b/SConstruct
@@ -335,7 +335,7 @@ python_scons = {
         'debian_deps'   : [
                             'debian/changelog',
                             'debian/compat',
-                            'debian/control',	    
+                            'debian/control',
                             'debian/copyright',
                             'debian/dirs',
                             'debian/docs',
@@ -499,8 +499,7 @@ for p in [ scons ]:
     # destination files.
     #
     manifest_in = File(os.path.join(src, 'MANIFEST.in')).rstr()
-    manifest_in_lines = open(manifest_in).readlines()
-    src_files = bootstrap.parseManifestLines(src, manifest_in_lines)
+    src_files = bootstrap.parseManifestLines(src, manifest_in)
     raw_files = src_files[:]
     dst_files = src_files[:]
 
@@ -520,12 +519,11 @@ for p in [ scons ]:
 
             MANIFEST_in = File(os.path.join(src, ssubdir, 'MANIFEST.in')).rstr()
             MANIFEST_in_list.append(MANIFEST_in)
-            files = bootstrap.parseManifestLines(os.path.join(src, ssubdir), open(MANIFEST_in).readlines())
+            files = bootstrap.parseManifestLines(os.path.join(src, ssubdir), MANIFEST_in)
 
             raw_files.extend(files)
             src_files.extend([os.path.join(ssubdir, x) for x in files])
 
-               
             files = [os.path.join(isubdir, x) for x in files]
             dst_files.extend(files)
             for k, f in sp['filemap'].items():
diff --git a/bootstrap.py b/bootstrap.py
index ea7e0c8..4ade361 100755
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -21,6 +21,7 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
+from __future__ import print_function
 
 import os
 import os.path
@@ -74,15 +75,24 @@ the following SCons options:
         "eaten" by the bootstrap.py script.
 """
 
-def parseManifestLines(basedir, lines):
-    """ Scans the single lines of a MANIFEST file,
-        and returns the list of source files.
-        Has basic support for recursive globs '**',
-        filename wildcards of the form '*.xml' and
-        comment lines, starting with a '#'.
+def parseManifestLines(basedir, manifest):
+    """
+    Scans a MANIFEST file, and returns the list of source files.
+
+    Has basic support for recursive globs '**',
+    filename wildcards of the form '*.xml' and
+    comment lines, starting with a '#'.
+
+    :param basedir: base path to find files in. Note this does not
+                    run in an SCons context so path must not need
+                    further processing (e.g. no '#' signs)
+    :param manifest: path to manifest file
+    :returns: list of files
     """
     sources = []
     basewd = os.path.abspath(basedir)
+    with open(manifest) as m:
+        lines = m.readlines()
     for l in lines:
         if l.startswith('#'):
             # Skip comments
@@ -107,16 +117,16 @@ def parseManifestLines(basedir, lines):
 
 def main():
     script_dir = os.path.abspath(os.path.dirname(__file__))
-    
+
     bootstrap_dir = os.path.join(script_dir, 'bootstrap')
-    
+
     pass_through_args = []
     update_only = None
-    
+
     requires_an_argument = 'bootstrap.py:  %s requires an argument\n'
-    
+
     search = [script_dir]
-    
+
     def find(filename, search=search):
         for dir_name in search:
             filepath = os.path.join(dir_name, filename)
@@ -125,20 +135,20 @@ def main():
         sys.stderr.write("could not find `%s' in search path:\n" % filename)
         sys.stderr.write("\t" + "\n\t".join(search) + "\n")
         sys.exit(2)
-    
+
     def must_copy(dst, src):
         if not os.path.exists(dst):
             return 1
         return not filecmp.cmp(dst,src)
-    
+
     # Note:  We don't use the getopt module to process the command-line
     # arguments because we'd have to teach it about all of the SCons options.
-    
+
     command_line_args = sys.argv[1:]
-    
+
     while command_line_args:
         arg = command_line_args.pop(0)
-    
+
         if arg == '--bootstrap_dir':
             try:
                 bootstrap_dir = command_line_args.pop(0)
@@ -180,7 +190,7 @@ def main():
     MANIFEST_in = find(os.path.join(src_engine, 'MANIFEST.in'))
     manifest_files = [os.path.join(src_engine, x)
                       for x in parseManifestLines(os.path.join(script_dir, src_engine),
-                                                  open(MANIFEST_in).readlines())]
+                                                  MANIFEST_in)]
 
     files = [scons_py] + manifest_files
 
diff --git a/doc/SConscript b/doc/SConscript
index ad3eb9b..51ef2db 100644
--- a/doc/SConscript
+++ b/doc/SConscript
@@ -326,7 +326,7 @@ else:
     # get included by the document XML files in the subdirectories.
     #
     manifest = File('MANIFEST').rstr()
-    src_files = bootstrap.parseManifestLines('.', open(manifest).readlines())
+    src_files = bootstrap.parseManifestLines('.', manifest)
     for s in src_files:
         if not s:
             continue
@@ -351,7 +351,7 @@ else:
         if not os.path.exists(os.path.join(build, doc, 'titlepage')):
             env.Execute(Mkdir(os.path.join(build, doc, 'titlepage')))
         manifest = File(os.path.join(doc, 'MANIFEST')).rstr()
-        src_files = bootstrap.parseManifestLines(doc, open(manifest).readlines())
+        src_files = bootstrap.parseManifestLines(doc, manifest)
         for s in src_files:
             if not s:
                 continue
@@ -571,14 +571,15 @@ if not epydoc_cli and not epydoc:
 else:
     # XXX Should be in common with reading the same thing in
     # the SConstruct file.
-    e = os.path.join('#src', 'engine')
-    manifest_in = File(os.path.join(e, 'MANIFEST.in')).rstr()
-    sources = bootstrap.parseManifestLines(e, open(manifest_in).readlines())
-    
-    # Don't omit this as we need Platform.virtualenv for the examples to be run
+    # bootstrap.py runs outside of SCons, so need to process the path
+    e = Dir(os.path.join('#src', 'engine')).rstr()
+    sources = bootstrap.parseManifestLines(e, os.path.join(e, 'MANIFEST.in'))
+
+    # Omit some files:
+    #
+    # Don't omit Platform as we need Platform.virtualenv for the examples to be run
     # sources = [x for x in sources if x.find('Platform') == -1]
     sources = [x for x in sources if x.find('Tool') == -1]
-    # XXX
     sources = [x for x in sources if x.find('Options') == -1]
 
     e = os.path.join(build, '..', 'scons', 'engine')