Merge branch 'master' into master

8 files changed, 216 insertions, 26 deletions

diff --git a/CHANGES.txt b/CHANGES.txt
index 90da03c..42d20d7 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -65,6 +65,15 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
     - Added ninja mingw support and improved ninja CommandGeneratorAction support.
     - Update ninja file generation to only create response files for build commands 
       which exceed MAXLINELENGTH
+    - Ninja: Added NINJA_GENERATED_SOURCE_ALIAS_NAME which allows user to specify an
+      Alias() which the ninja tool can use to determine which files are generated sources.
+      If this is not set by the user then the ninja tool will still dynamically determine
+      which files are generated sources based on NINJA_GENERATED_SOURCE_SUFFIXES, and create
+      a phony target _ninja_generated_sources. Generated sources will be built first by 
+      ninja. This is needed because ninja cannot determine which generated sources are
+      required by other build targets. Code contributed by MongoDB
+      The downstream commit is here:
+      https://github.com/mongodb/mongo/commit/2fef432fa6e7cf3fd4f22ba3b193222c2887f14f
     - Added special case for ninja scons daemon to work in win32 python3.6 environments.
       This particular environment does a bad job managing popen standard file handles, so 
       some special workarounds are needed.
diff --git a/RELEASE.txt b/RELEASE.txt
index 2ed4a8b..6bb1546 100755
--- a/RELEASE.txt
+++ b/RELEASE.txt
@@ -69,6 +69,13 @@ FIXES
   default compiler to MSVC which wasn't installed, yielding broken build.
   Updated mingw tool so that the generate and exists methods use the same mingw search paths
   (issue #4134).
+- Ninja: Added NINJA_GENERATED_SOURCE_ALIAS_NAME which allows user to specify an
+  Alias() which the ninja tool can use to determine which files are generated sources.
+  If this is not set by the user then the ninja tool will still dynamically determine
+  which files are generated sources based on NINJA_GENERATED_SOURCE_SUFFIXES, and create
+  a phony target _ninja_generated_sources. Generated sources will be built first by 
+  ninja. This is needed because ninja cannot determine which generated sources are
+  required by other build targets.  Code contributed by MongoDB.
 - Added special case for ninja scons daemon to work in win32 python3.6 environments.
   This particular environment does a bad job managing popen standard file handles, so 
   some special workarounds are needed.
diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py
index 28ee576..63ea3a1 100644
--- a/SCons/Tool/ninja/NinjaState.py
+++ b/SCons/Tool/ninja/NinjaState.py
@@ -395,26 +395,58 @@ class NinjaState:
                 kwargs['pool'] = 'local_pool'
             ninja.rule(rule, **kwargs)
 
-        generated_source_files = sorted({
-            output
-            # First find builds which have header files in their outputs.
-            for build in self.builds.values()
-            if self.has_generated_sources(build["outputs"])
-            for output in build["outputs"]
-            # Collect only the header files from the builds with them
-            # in their output. We do this because is_generated_source
-            # returns True if it finds a header in any of the outputs,
-            # here we need to filter so we only have the headers and
-            # not the other outputs.
-            if self.is_generated_source(output)
-        })
+        # If the user supplied an alias to determine generated sources, use that, otherwise
+        # determine what the generated sources are dynamically.
+        generated_sources_alias = self.env.get('NINJA_GENERATED_SOURCE_ALIAS_NAME')
+        generated_sources_build = None
+
+        if generated_sources_alias:
+            generated_sources_build = self.builds.get(generated_sources_alias)
+            if generated_sources_build is None or generated_sources_build["rule"] != 'phony':
+                raise Exception(
+                    "ERROR: 'NINJA_GENERATED_SOURCE_ALIAS_NAME' set, but no matching Alias object found."
+                )
 
-        if generated_source_files:
-            ninja.build(
-                outputs="_generated_sources",
-                rule="phony",
-                implicit=generated_source_files
+        if generated_sources_alias and generated_sources_build:
+            generated_source_files = sorted(
+                [] if not generated_sources_build else generated_sources_build['implicit']
             )
+            
+            def check_generated_source_deps(build):
+                return (
+                    build != generated_sources_build
+                    and set(build["outputs"]).isdisjoint(generated_source_files)
+                )
+        else:
+            generated_sources_build = None
+            generated_source_files = sorted({
+                output
+                # First find builds which have header files in their outputs.
+                for build in self.builds.values()
+                if self.has_generated_sources(build["outputs"])
+                for output in build["outputs"]
+                # Collect only the header files from the builds with them
+                # in their output. We do this because is_generated_source
+                # returns True if it finds a header in any of the outputs,
+                # here we need to filter so we only have the headers and
+                # not the other outputs.
+                if self.is_generated_source(output)
+            })
+
+            if generated_source_files:
+                generated_sources_alias = "_ninja_generated_sources"
+                ninja.build(
+                    outputs=generated_sources_alias,
+                    rule="phony",
+                    implicit=generated_source_files
+                )
+                
+                def check_generated_source_deps(build):
+                    return (
+                        not build["rule"] == "INSTALL"
+                        and set(build["outputs"]).isdisjoint(generated_source_files)
+                        and set(build.get("implicit", [])).isdisjoint(generated_source_files)
+                    )
 
         template_builders = []
         scons_compiledb = False
@@ -437,9 +469,7 @@ class NinjaState:
             # cycle.
             if (
                 generated_source_files
-                and not build["rule"] == "INSTALL"
-                and set(build["outputs"]).isdisjoint(generated_source_files)
-                and set(build.get("implicit", [])).isdisjoint(generated_source_files)
+                and check_generated_source_deps(build)
             ):
                 # Make all non-generated source targets depend on
                 # _generated_sources. We use order_only for generated
@@ -448,7 +478,7 @@ class NinjaState:
                 # sure that all of these sources are generated before
                 # other builds.
                 order_only = build.get("order_only", [])
-                order_only.append("_generated_sources")
+                order_only.append(generated_sources_alias)
                 build["order_only"] = order_only
             if "order_only" in build:
                 build["order_only"].sort()
diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py
index 4820ee1..526db8a 100644
--- a/SCons/Tool/ninja/__init__.py
+++ b/SCons/Tool/ninja/__init__.py
@@ -276,8 +276,8 @@ def generate(env):
         provider = gen_get_response_file_command(env, rule, tool)
         env.NinjaRuleMapping("${" + var + "}", provider)
 
-        # some of these construction vars could be generators, e.g. 
-        # CommandGeneratorAction, so if the var is not a string, we 
+        # some of these construction vars could be generators, e.g.
+        # CommandGeneratorAction, so if the var is not a string, we
         # can't parse the generated string.
         if isinstance(env.get(var), str):
             env.NinjaRuleMapping(env.get(var, None), provider)
@@ -297,7 +297,7 @@ def generate(env):
     # requires that all generated sources are added as order_only
     # dependencies to any builds that *might* use them.
     # TODO: switch to using SCons to help determine this (Github Issue #3624)
-    env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"]
+    env["NINJA_GENERATED_SOURCE_SUFFIXES"] = env.get('NINJA_GENERATED_SOURCE_SUFFIXES', [".h", ".hpp"])
 
     # Force ARCOM so use 's' flag on ar instead of separately running ranlib
     ninja_hack_arcom(env)
diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml
index b875922..6b247d0 100644
--- a/SCons/Tool/ninja/ninja.xml
+++ b/SCons/Tool/ninja/ninja.xml
@@ -65,6 +65,7 @@ See its __doc__ string for a discussion of the format.
             <item>NINJA_ENV_VAR_CACHE</item>
             <item>NINJA_FILE_NAME</item>
             <item>NINJA_GENERATED_SOURCE_SUFFIXES</item>
+            <item>NINJA_GENERATED_SOURCE_ALIAS_NAME</item>
             <item>NINJA_MSVC_DEPS_PREFIX</item>
             <item>NINJA_DEPFILE_PARSE_FORMAT</item>
             <item>NINJA_POOL</item>
@@ -199,6 +200,17 @@ python -m pip install ninja
         </summary>
     </cvar>
 
+    <cvar name="NINJA_GENERATED_SOURCE_ALIAS_NAME">
+        <summary>
+            <para>
+                A string matching the name of a user defined alias which represents a list of all generated sources.
+                This will prevent the auto-detection of generated sources from &cv-NINJA_GENERATED_SOURCE_SUFFIXES;.
+                Then all other source files will be made to depend on this in the &ninja; build file, forcing the
+                generated sources to be built first.
+            </para>
+        </summary>
+    </cvar>
+
     <cvar name="NINJA_MSVC_DEPS_PREFIX">
         <summary>
             <para>
@@ -274,7 +286,7 @@ python -m pip install ninja
             </para>
 
             <para>
-                If not explicitly set, &SCons; will generate this dynamically from the 
+                If not explicitly set, &SCons; will generate this dynamically from the
                 execution environment stored in the current &consenv;
                 (e.g. <literal>env['ENV']</literal>)
                 where those values differ from the existing shell..
diff --git a/test/ninja/generated_sources_alias.py b/test/ninja/generated_sources_alias.py
new file mode 100644
index 0000000..2c4ed36
--- /dev/null
+++ b/test/ninja/generated_sources_alias.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+#
+# Copyright The SCons Foundation
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+import os
+
+import TestSCons
+from TestCmd import IS_WINDOWS
+
+test = TestSCons.TestSCons()
+
+try:
+    import ninja
+except ImportError:
+    test.skip_test("Could not find module in python")
+
+_python_ = TestSCons._python_
+_exe = TestSCons._exe
+
+ninja_bin = os.path.abspath(os.path.join(
+    ninja.BIN_DIR,
+    'ninja' + _exe))
+
+test.dir_fixture('ninja-fixture')
+
+test.file_fixture('ninja_test_sconscripts/sconstruct_generated_sources_alias', 'SConstruct')
+
+for alias_value in [0, 1]:
+
+    # Gen with source alias
+    test.run(arguments=['--disable-execute-ninja', f'USE_GEN_SOURCE_ALIAS={alias_value}'], stdout=None)
+    test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja'])
+    test.must_not_exist(test.workpath('gen_source' + _exe))
+
+    # run ninja independently
+    program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin
+    test.run(program=program, stdout=None)
+    test.run(program=test.workpath('gen_source' + _exe), stdout="4")
+
+    # generate simple build
+    test.run(arguments=[f'USE_GEN_SOURCE_ALIAS={alias_value}'], stdout=None)
+    test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja'])
+    test.must_contain_all(test.stdout(), 'Executing:')
+    test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals())
+    test.run(program=test.workpath('gen_source' + _exe), stdout="4")
+
+    # clean build and ninja files
+    test.run(arguments=[f'USE_GEN_SOURCE_ALIAS={alias_value}', '-c'], stdout=None)
+    test.must_contain_all_lines(test.stdout(), [
+        'Removed generated_header1.htest',
+        'Removed generated_header2.htest',
+        'Removed generated_header2.c',
+        'Removed build.ninja'])
+
+
+test.pass_test()
+
+# Local Variables:
+# tab-width:4
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=4 shiftwidth=4:
diff --git a/test/ninja/ninja-fixture/gen_source.c b/test/ninja/ninja-fixture/gen_source.c
new file mode 100644
index 0000000..38ac3fe
--- /dev/null
+++ b/test/ninja/ninja-fixture/gen_source.c
@@ -0,0 +1,9 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "generated_header2.htest"
+
+int main(int argc, char *argv[])
+{
+    printf("%d", func2());
+}
diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generated_sources_alias b/test/ninja/ninja_test_sconscripts/sconstruct_generated_sources_alias
new file mode 100644
index 0000000..3661bcc
--- /dev/null
+++ b/test/ninja/ninja_test_sconscripts/sconstruct_generated_sources_alias
@@ -0,0 +1,41 @@
+import SCons
+
+SetOption('experimental','ninja')
+DefaultEnvironment(tools=[])
+
+vars = Variables()
+vars.Add("USE_GEN_SOURCE_ALIAS")
+
+env = Environment(variables=vars)
+my_gen_alias = 'my_ninja_gen_sources'
+
+
+env['SCANNERS'][0].add_scanner('.htest', SCons.Tool.CScanner)
+env.Command('out.txt', [], f'echo USE_GEN_SOURCE_ALIAS={env["USE_GEN_SOURCE_ALIAS"]}')
+
+if env['USE_GEN_SOURCE_ALIAS'] == "1":
+    env['NINJA_GENERATED_SOURCE_ALIAS_NAME'] = my_gen_alias
+else:
+    env['NINJA_GENERATED_SOURCE_SUFFIXES'] = ['.htest']
+
+env.Tool('ninja')
+
+env.Alias(my_gen_alias, env.Textfile('generated_header1.htest', [
+    '#pragma once',
+    'int func1(){return 4;};'
+]))
+alias = env.Alias(my_gen_alias, env.Textfile('generated_header2.htest', [
+    '#pragma once',
+    '',
+    'int func2();'
+]))
+env.Depends(alias, 'out.txt')
+
+my_gen_alias, env.Textfile('generated_header2.c', [
+    '#include "generated_header1.htest"',
+    '#include "generated_header2.htest"',
+    '',
+    'int func2(){return func1();}'
+])
+
+env.Program(target='gen_source', source=['gen_source.c', 'generated_header2.c'])
\ No newline at end of file