Add Fortran 90/95 support. (Chris Murray)

2 files changed, 374 insertions, 16 deletions

diff --git a/src/engine/SCons/Scanner/Fortran.py b/src/engine/SCons/Scanner/Fortran.py
index 9e9a990..786d4ab 100644
--- a/src/engine/SCons/Scanner/Fortran.py
+++ b/src/engine/SCons/Scanner/Fortran.py
@@ -1,6 +1,6 @@
 """SCons.Scanner.Fortran
 
-This module implements the dependency scanner for Fortran code. 
+This module implements the dependency scanner for Fortran code.
 
 """
 
@@ -29,18 +29,266 @@ This module implements the dependency scanner for Fortran code.
 
 __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
+import re
+import string
+
 import SCons.Node
 import SCons.Node.FS
 import SCons.Scanner
 import SCons.Util
 import SCons.Warnings
 
-def FortranScan(fs = SCons.Node.FS.default_fs):
+class F90Scanner(SCons.Scanner.Classic):
+    """
+    A Classic Scanner subclass for Fortran source files which takes
+    into account both USE and INCLUDE statements.  This scanner will
+    work for both F77 and F90 (and beyond) compilers.
+
+    Currently, this scanner assumes that the include files do not contain
+    USE statements.  To enable the ability to deal with USE statements
+    in include files, add logic right after the module names are found
+    to loop over each include file, search for and locate each USE
+    statement, and append each module name to the list of dependencies.
+    Caching the search results in a common dictionary somewhere so that
+    the same include file is not searched multiple times would be a
+    smart thing to do.
+    """
+
+    def __init__(self, name, suffixes, path_variable, use_regex,
+                 incl_regex, fs=SCons.Node.FS.default_fs, *args, **kw):
+
+        self.cre_use = re.compile(use_regex, re.M)
+        self.cre_incl = re.compile(incl_regex, re.M)
+        self.fs = fs
+
+        def _scan(node, env, path, self=self, fs=fs):
+            return self.scan(node, env, path)
+
+        kw['function'] = _scan
+        kw['path_function'] = SCons.Scanner.FindPathDirs(path_variable, fs)
+        kw['recursive'] = 1
+        kw['skeys'] = suffixes
+        kw['name'] = name
+
+        apply(SCons.Scanner.Current.__init__, (self,) + args, kw)
+
+    def scan(self, node, env, path=()):
+        node = node.rfile()
+
+        if not node.exists():
+            return []
+
+        # cache the includes list in node so we only scan it once:
+        if node.includes != None:
+            mods_and_includes = node.includes
+        else:
+            # retrieve all included filenames
+            includes = self.cre_incl.findall(node.get_contents())
+            # retrieve all USE'd module names
+            modules = self.cre_use.findall(node.get_contents())
+
+            # Convert module name to a .mod filename
+            suffix = env.subst('$FORTRANMODSUFFIX')
+            modules = map(lambda x, s=suffix: string.lower(x) + s, modules)
+            # Remove unique items from the list
+            mods_and_includes = SCons.Util.unique(includes+modules)
+            node.includes = mods_and_includes
+
+        nodes = []
+        source_dir = node.get_dir()
+        for dep in mods_and_includes:
+            n, i = self.find_include(dep, source_dir, path)
+
+            if not n is None:
+                nodes.append(n)
+            else:
+                SCons.Warnings.warn(SCons.Warnings.DependencyWarning,
+                                    "No dependency generated for file: %s (referenced by: %s) -- file not found" % (i, node))
+
+        #  Sort the list of dependencies
+
+        # Schwartzian transform from the Python FAQ Wizard
+        def st(List, Metric):
+            def pairing(element, M = Metric):
+                return (M(element), element)
+            def stripit(pair):
+                return pair[1]
+            paired = map(pairing, List)
+            paired.sort()
+            return map(stripit, paired)
+
+        def normalize(node):
+            # We don't want the order of includes to be
+            # modified by case changes on case insensitive OSes, so
+            # normalize the case of the filename here:
+            # (see test/win32pathmadness.py for a test of this)
+            return SCons.Node.FS._my_normcase(str(node))
+
+        # Apply a Schwartzian transform to return the list of
+        # dependencies, sorted according to their normalized names
+        transformed = st(nodes, normalize)
+#        print "ClassicF90: " + str(node) + " => " + str(map(lambda x: str(x),list(transformed)))
+        return transformed
+
+
+def FortranScan(path_variable="FORTRANPATH", fs=SCons.Node.FS.default_fs):
     """Return a prototype Scanner instance for scanning source files
-    for Fortran INCLUDE statements"""
-    scanner = SCons.Scanner.Classic("FortranScan",
-                                    "$FORTRANSUFFIXES",
-                                    "F77PATH",
-                                    "(?i)INCLUDE[ \t]+'([\\w./\\\\]+)'",
-                                    fs = fs)
+    for Fortran USE & INCLUDE statements"""
+
+#   The USE statement regex matches the following:
+#
+#   USE module_name
+#   USE :: module_name
+#   USE, INTRINSIC :: module_name
+#   USE, NON_INTRINSIC :: module_name
+#
+#   Limitations
+#
+#   --  While the regex can handle multiple USE statements on one line,
+#       it cannot properly handle them if they are commented out.
+#       In either of the following cases:
+#
+#            !  USE mod_a ; USE mod_b         [entire line is commented out]
+#               USE mod_a ! ; USE mod_b       [in-line comment of second USE statement]
+#
+#       the second module name (mod_b) will be picked up as a dependency
+#       even though it should be ignored.  The only way I can see
+#       to rectify this would be to modify the scanner to eliminate
+#       the call to re.findall, read in the contents of the file,
+#       treating the comment character as an end-of-line character
+#       in addition to the normal linefeed, loop over each line,
+#       weeding out the comments, and looking for the USE statements.
+#       One advantage to this is that the regex passed to the scanner
+#       would no longer need to match a semicolon.
+#
+#   --  I question whether or not we need to detect dependencies to
+#       INTRINSIC modules because these are built-in to the compiler.
+#       If we consider them a dependency, will SCons look for them, not
+#       find them, and kill the build?  Or will we there be standard
+#       compiler-specific directories we will need to point to so the
+#       compiler and SCons can locate the proper object and mod files?
+
+#   Here is a breakdown of the regex:
+#
+#   (?i)               : regex is case insensitive
+#   ^                  : start of line
+#   (?:                : group a collection of regex symbols without saving the match as a "group"
+#      ^|;             : matches either the start of the line or a semicolon - semicolon
+#   )                  : end the unsaved grouping
+#   \s*                : any amount of white space
+#   USE                : match the string USE, case insensitive
+#   (?:                : group a collection of regex symbols without saving the match as a "group"
+#      \s+|            : match one or more whitespace OR ....  (the next entire grouped set of regex symbols)
+#      (?:             : group a collection of regex symbols without saving the match as a "group"
+#         (?:          : establish another unsaved grouping of regex symbols
+#            \s*          : any amount of white space
+#            ,         : match a comma
+#            \s*       : any amount of white space
+#            (?:NON_)? : optionally match the prefix NON_, case insensitive
+#            INTRINSIC : match the string INTRINSIC, case insensitive
+#         )?           : optionally match the ", INTRINSIC/NON_INTRINSIC" grouped expression
+#         \s*          : any amount of white space
+#         ::           : match a double colon that must appear after the INTRINSIC/NON_INTRINSIC attribute
+#      )               : end the unsaved grouping
+#   )                  : end the unsaved grouping
+#   \s*                : match any amount of white space
+#   (\w+)              : match the module name that is being USE'd
+#
+#
+    use_regex = "(?i)(?:^|;)\s*USE(?:\s+|(?:(?:\s*,\s*(?:NON_)?INTRINSIC)?\s*::))\s*(\w+)"
+
+
+#   The INCLUDE statement regex matches the following:
+#
+#   INCLUDE 'some_Text'
+#   INCLUDE "some_Text"
+#   INCLUDE "some_Text" ; INCLUDE "some_Text"
+#   INCLUDE kind_"some_Text"
+#   INCLUDE kind_'some_Text"
+#
+#   where some_Text can include any alphanumeric and/or special character
+#   as defined by the Fortran 2003 standard.
+#
+#   Limitations:
+#
+#   --  The Fortran standard dictates that a " or ' in the INCLUDE'd
+#       string must be represented as a "" or '', if the quotes that wrap
+#       the entire string are either a ' or ", respectively.   While the
+#       regular expression below can detect the ' or " characters just fine,
+#       the scanning logic, presently is unable to detect them and reduce
+#       them to a single instance.  This probably isn't an issue since,
+#       in practice, ' or " are not generally used in filenames.
+#
+#   --  This regex will not properly deal with multiple INCLUDE statements
+#       when the entire line has been commented out, ala
+#
+#           ! INCLUDE 'some_file' ; INCLUDE 'some_file'
+#
+#       In such cases, it will properly ignore the first INCLUDE file,
+#       but will actually still pick up the second.  Interestingly enough,
+#       the regex will properly deal with these cases:
+#
+#             INCLUDE 'some_file'
+#             INCLUDE 'some_file' !; INCLUDE 'some_file'
+#
+#       To get around the above limitation, the FORTRAN programmer could
+#       simply comment each INCLUDE statement separately, like this
+#
+#           ! INCLUDE 'some_file' !; INCLUDE 'some_file'
+#
+#       The way I see it, the only way to get around this limitation would
+#       be to modify the scanning logic to replace the calls to re.findall
+#       with a custom loop that processes each line separately, throwing
+#       away fully commented out lines before attempting to match against
+#       the INCLUDE syntax.
+#
+#   Here is a breakdown of the regex:
+#
+#   (?i)               : regex is case insensitive
+#   (?:                : begin a non-saving group that matches the following:
+#      ^               :    either the start of the line
+#      |               :                or
+#      ['">]\s*;       :    a semicolon that follows a single quote,
+#                           double quote or greater than symbol (with any
+#                           amount of whitespace in between).  This will
+#                           allow the regex to match multiple INCLUDE
+#                           statements per line (although it also requires
+#                           the positive lookahead assertion that is
+#                           used below).  It will even properly deal with
+#                           (i.e. ignore) cases in which the additional
+#                           INCLUDES are part of an in-line comment, ala
+#                                           "  INCLUDE 'someFile' ! ; INCLUDE 'someFile2' "
+#   )                  : end of non-saving group
+#   \s*                : any amount of white space
+#   INCLUDE            : match the string INCLUDE, case insensitive
+#   \s+                : match one or more white space characters
+#   (?\w+_)?           : match the optional "kind-param _" prefix allowed by the standard
+#   [<"']              : match the include delimiter - an apostrophe, double quote, or less than symbol
+#   (.+?)              : match one or more characters that make up
+#                        the included path and file name and save it
+#                        in a group.  The Fortran standard allows for
+#                        any non-control character to be used.  The dot
+#                        operator will pick up any character, including
+#                        control codes, but I can't conceive of anyone
+#                        putting control codes in their file names.
+#                        The question mark indicates it is non-greedy so
+#                        that regex will match only up to the next quote,
+#                        double quote, or greater than symbol
+#   (?=["'>])          : positive lookahead assertion to match the include
+#                        delimiter - an apostrophe, double quote, or
+#                        greater than symbol.  This level of complexity
+#                        is required so that the include delimiter is
+#                        not consumed by the match, thus allowing the
+#                        sub-regex discussed above to uniquely match a
+#                        set of semicolon-separated INCLUDE statements
+#                        (as allowed by the F2003 standard)
+
+    include_regex = """(?i)(?:^|['">]\s*;)\s*INCLUDE\s+(?:\w+_)?[<"'](.+?)(?=["'>])"""
+
+    scanner = F90Scanner("FortranScan",
+                         "$FORTRANSUFFIXES",
+                         path_variable,
+                         use_regex,
+                         include_regex,
+                         fs = fs)
     return scanner
diff --git a/src/engine/SCons/Scanner/FortranTests.py b/src/engine/SCons/Scanner/FortranTests.py
index 766f0ee..b7e527c 100644
--- a/src/engine/SCons/Scanner/FortranTests.py
+++ b/src/engine/SCons/Scanner/FortranTests.py
@@ -134,19 +134,90 @@ test.write([ 'repository', 'src', 'ccc.f'], """
 
 test.write([ 'repository', 'src', 'ddd.f'], "\n")
 
+
+test.write('fff90a.f90',"""
+      PROGRAM FOO
+
+!  Test comments - these includes should NOT be picked up
+C     INCLUDE 'fi.f'
+#     INCLUDE 'fi.f'
+  !   INCLUDE 'fi.f'
+
+      INCLUDE 'f1.f'  ! in-line comments are valid syntax
+      INCLUDE"fi.f"   ! space is significant - this should be ignored
+      INCLUDE  <f2.f>  ! Absoft compiler allows greater than/less than delimiters
+!
+!  Allow kind type parameters
+      INCLUDE kindType_"f3.f"
+      INCLUDE kind_Type_"f4.f"
+!
+!  Test multiple statements per line - use various spacings between semicolons
+      incLUDE 'f5.f';include "f6.f"  ;  include <f7.f>; include 'f8.f' ;include kindType_'f9.f'
+!
+!  Test various USE statement syntaxes
+!
+      USE Mod01
+      use mod02
+      use use
+      USE mOD03, ONLY : someVar
+      USE MOD04 ,only:someVar
+      USE Mod05 , ONLY: someVar ! in-line comment
+      USE Mod06,ONLY :someVar,someOtherVar
+
+      USE  mod07;USE  mod08; USE mod09 ;USE mod10 ; USE mod11  ! Test various semicolon placements
+      use mod12 ;use mod13! Test comment at end of line
+
+!     USE modi
+!     USE modia ; use modib    ! Scanner regexp will only ignore the first - this is a deficiency in the regexp
+    ! USE modic ; ! use modid  ! Scanner regexp should ignore both modules
+      USE mod14 !; USE modi    ! Only ignore the second
+      USE mod15!;USE modi
+      USE mod16  !  ;  USE  modi
+
+!  Test semicolon syntax - use various spacings
+      USE :: mod17
+      USE::mod18
+      USE ::mod19 ; USE:: mod20
+
+      use, non_intrinsic :: mod21, ONLY : someVar ; use,intrinsic:: mod22
+      USE, NON_INTRINSIC::mod23 ; USE ,INTRINSIC ::mod24
+
+USE mod25  ! Test USE statement at the beginning of line
+
+
+; USE modi   ! Scanner should ignore this since it isn't valid syntax
+      USEmodi   ! No space in between USE and module name - ignore it
+      USE mod01   ! This one is a duplicate - there should only be one dependency to it.
+
+      STOP
+      END
+""")
+
+modules = ['mod01.mod', 'mod02.mod', 'mod03.mod', 'mod04.mod', 'mod05.mod',
+           'mod06.mod', 'mod07.mod', 'mod08.mod', 'mod09.mod', 'mod10.mod',
+           'mod11.mod', 'mod12.mod', 'mod13.mod', 'mod14.mod', 'mod15.mod',
+           'mod16.mod', 'mod17.mod', 'mod18.mod', 'mod19.mod', 'mod20.mod',
+           'mod21.mod', 'mod22.mod', 'mod23.mod', 'mod24.mod', 'mod25.mod']
+
+for m in modules:
+    test.write(m, "\n")
+
+test.subdir('modules')
+test.write(['modules', 'use.mod'], "\n")
+
 # define some helpers:
 
 class DummyEnvironment:
     def __init__(self, listCppPath):
         self.path = listCppPath
-        
+
     def Dictionary(self, *args):
         if not args:
-            return { 'F77PATH': self.path }
-        elif len(args) == 1 and args[0] == 'F77PATH':
+            return { 'FORTRANPATH': self.path, 'FORTRANMODSUFFIX' : ".mod" }
+        elif len(args) == 1 and args[0] == 'FORTRANPATH':
             return self.path
         else:
-            raise KeyError, "Dummy environment only has F77PATH attribute."
+            raise KeyError, "Dummy environment only has FORTRANPATH attribute."
 
     def has_key(self, key):
         return self.Dictionary().has_key(key)
@@ -161,6 +232,8 @@ class DummyEnvironment:
         del self.Dictionary()[key]
 
     def subst(self, arg):
+        if arg[0] == '$':
+            return self[arg[1:]]
         return arg
 
     def subst_path(self, path):
@@ -274,7 +347,7 @@ class FortranScannerTestCase8(unittest.TestCase):
         headers =  ['d1/d2/f2.f', 'd1/f2.f', 'f2.f']
         deps_match(self, deps, map(test.workpath, headers))
         test.unlink('f2.f')
-        
+
 class FortranScannerTestCase9(unittest.TestCase):
     def runTest(self):
         test.write('f3.f', "\n")
@@ -290,11 +363,11 @@ class FortranScannerTestCase9(unittest.TestCase):
         setattr(n, 'rexists', my_rexists)
 
         deps = s(n, env, path)
-        
+
         # Make sure rexists() got called on the file node being
         # scanned, essential for cooperation with BuildDir functionality.
         assert n.rexists_called
-        
+
         headers =  ['d1/f3.f', 'f3.f']
         deps_match(self, deps, map(test.workpath, headers))
         test.unlink('f3.f')
@@ -334,7 +407,7 @@ class FortranScannerTestCase11(unittest.TestCase):
 
         # Did we catch the warning from not finding not_there.f?
         assert to.out
-        
+
         deps_match(self, deps, [ 'f5.f' ])
 
 class FortranScannerTestCase12(unittest.TestCase):
@@ -402,6 +475,42 @@ class FortranScannerTestCase15(unittest.TestCase):
         deps_match(self, deps, map(test.workpath, headers))
         test.write(['d1', 'f2.f'], "\n")
 
+class FortranScannerTestCase16(unittest.TestCase):
+    def runTest(self):
+        test.write('f1.f', "\n")
+        test.write('f2.f', "\n")
+        test.write('f3.f', "\n")
+        test.write('f4.f', "\n")
+        test.write('f5.f', "\n")
+        test.write('f6.f', "\n")
+        test.write('f7.f', "\n")
+        test.write('f8.f', "\n")
+        test.write('f9.f', "\n")
+        test.write('f10.f', "\n")
+        env = DummyEnvironment([test.workpath('modules')])
+        s = SCons.Scanner.Fortran.FortranScan()
+        path = s.path(env)
+        fs = SCons.Node.FS.FS(original)
+        deps = s(make_node('fff90a.f90', fs), env, path)
+        headers = ['f1.f', 'f2.f', 'f3.f', 'f4.f', 'f5.f', 'f6.f', 'f7.f', 'f8.f', 'f9.f']
+        modules = ['mod01.mod', 'mod02.mod', 'mod03.mod', 'mod04.mod', 'mod05.mod',
+                   'mod06.mod', 'mod07.mod', 'mod08.mod', 'mod09.mod', 'mod10.mod',
+                   'mod11.mod', 'mod12.mod', 'mod13.mod', 'mod14.mod', 'mod15.mod',
+                   'mod16.mod', 'mod17.mod', 'mod18.mod', 'mod19.mod', 'mod20.mod',
+                   'mod21.mod', 'mod22.mod', 'mod23.mod', 'mod24.mod', 'mod25.mod', 'modules/use.mod']
+        deps_expected = headers + modules
+        deps_match(self, deps, map(test.workpath, deps_expected))
+        test.unlink('f1.f')
+        test.unlink('f2.f')
+        test.unlink('f3.f')
+        test.unlink('f4.f')
+        test.unlink('f5.f')
+        test.unlink('f6.f')
+        test.unlink('f7.f')
+        test.unlink('f8.f')
+        test.unlink('f9.f')
+        test.unlink('f10.f')
+
 def suite():
     suite = unittest.TestSuite()
     suite.addTest(FortranScannerTestCase1())
@@ -419,6 +528,7 @@ def suite():
     suite.addTest(FortranScannerTestCase13())
     suite.addTest(FortranScannerTestCase14())
     suite.addTest(FortranScannerTestCase15())
+    suite.addTest(FortranScannerTestCase16())
     return suite
 
 if __name__ == "__main__":