From 7ddeb166c05a68ebea9b4976373694337614c5ae Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 23 Mar 2023 09:28:54 -0600 Subject: yacc tool updates The yacc tool now recognizes the bison syntax of using --header, --defines and --graph options in YACCFLAGS without supplying an option argument. Note that byacc doesn't allow leaving off the option argument, and would error out in such cases. Fixes #4326 The yacc tool now recognizes -H as a partial synonym for --header. It only works in the no-option-argument form - giving an option argument will likely break things. Fixes #4327 Tests are expanded to cover these. The graph file name (-g) is now generated relative to the requested target file name, not to the source name, to match actual current behavior. This is rare case - only kicks in if target explicitly requested with a different base name than source - in this case SCons would emit a different name than bison produces. Unlikely to affect any real usage - you would have to set up the condition described, and additionally use the graph file as a source for some other build target. The default file suffix for graph files is set to .gv, which is current for Bison as of 3.8. The previous suffix for Bison, dating back to 2.4, was .dot, which is still the suffix byacc uses. Docs updated to better describe when you need to set this. The former default, .vcg, has not been used since 2006. Documentation updated for the two header suffix construction variables to better describe (I hope) what these mean and when to set them. Other doc updates as well. Signed-off-by: Mats Wichmann --- CHANGES.txt | 9 +++ SCons/Tool/Tool.xml | 26 +++++-- SCons/Tool/yacc.py | 35 ++++++---- SCons/Tool/yacc.xml | 92 +++++++++++++------------ test/YACC/BISONFLAGS.py | 49 ++++++++++++- test/YACC/YACCFLAGS-fixture/myyacc.py | 125 ++++++++++++++++++++++------------ test/YACC/YACCFLAGS.py | 2 +- 7 files changed, 230 insertions(+), 108 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d4ad973..2977fa8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -23,6 +23,15 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER calls dunder method __call__. Invoke instance directly." - Python 3.9 dropped the alias base64.decodestring, deprecated since 3.1. Only used in msvs.py. Use base64.decodebytes instead. + - The yacc tool now understands the bison behavior of --header, + --defines and --graph being called without option-argument as being + synonyms for -d (first two) and -g. -H also recognized as a synonym + for -d. Default value for $YACCVCGFILESUFFIX changed to '.gv' + to match current bison default (since bison 3.8). The graph file name + (-g) is now generated relative to the requested target file name, + not to the source file name, to match actual current behavior (only + affects if target explicitly requested with a different base name + than source). Docs updated. Fixes #4326 and #4327. RELEASE 4.5.2 - Sun, 21 Mar 2023 14:08:29 -0700 diff --git a/SCons/Tool/Tool.xml b/SCons/Tool/Tool.xml index de71efb..d25d7a5 100644 --- a/SCons/Tool/Tool.xml +++ b/SCons/Tool/Tool.xml @@ -37,10 +37,19 @@ Example: # builds foo.c -env.CFile(target = 'foo.c', source = 'foo.l') +env.CFile(target='foo.c', source='foo.l') + # builds bar.c -env.CFile(target = 'bar', source = 'bar.y') +env.CFile(target='bar', source='bar.y') + + +Note that for yacc files, +the output file name is derived from target, +or from the source file name if a target is not specified; +the traditional yacc default name +y.tab.c is not used. + @@ -59,10 +68,19 @@ Example: # builds foo.cc -env.CXXFile(target = 'foo.cc', source = 'foo.ll') +env.CXXFile(target='foo.cc', source='foo.ll') + # builds bar.cc -env.CXXFile(target = 'bar', source = 'bar.yy') +env.CXXFile(target='bar', source='bar.yy') + + +Note that for yacc files, +the output file name is derived from target, +or from the source file name if a target is not specified; +the traditional yacc default name +y.tab.cc is not used. + diff --git a/SCons/Tool/yacc.py b/SCons/Tool/yacc.py index 67ecd86..d408e62 100644 --- a/SCons/Tool/yacc.py +++ b/SCons/Tool/yacc.py @@ -23,8 +23,11 @@ """Tool-specific initialization for yacc. -This tool should support multiple yacc implementations, -but is in actuality biased towards GNU Bison. +This tool should support multiple yacc implementations, but is in actuality +biased towards GNU Bison. In particular, it forces the output file name (thus +avoiding the default convention of y.tab.c or foo.tab.c), so the tool *must* +support the -o option, which pure POSIX yacc does not. byacc should be okay +as an alternative to bison. There normally shouldn't be any need to import this module directly. It will usually be imported through the generic SCons.Tool.Tool() @@ -64,15 +67,17 @@ def _yaccEmitter(target, source, env, ysuf, hsuf) -> tuple: target = [targetBase + ".m"] # the extension is ".m". # If -d is specified on the command line, yacc will emit a .h - # or .hpp file with the same name as the .c or .cpp output file. - if '-d' in flags: + # or .hpp file with the same base name as the .c or .cpp output file. + # if '-d' in flags: # add bison options -H, --header, --defines (obsolete) + if "-d" in flags or "-H" in flags or "--header" in flags or "--defines" in flags: target.append(targetBase + env.subst(hsuf, target=target, source=source)) - # If -g is specified on the command line, yacc will emit a .vcg - # file with the same base name as the .y, .yacc, .ym or .yy file. - if "-g" in flags: - base, ext = os.path.splitext(to_String(source[0])) - target.append(base + env.subst("$YACCVCGFILESUFFIX")) + # If -g is specified on the command line, yacc will emit a graph + # file with the same base name as the .c or .cpp output file. + # TODO: should this be handled like -v? i.e. a side effect, not target + # if "-g" in flags: # add bison option --graph + if "-g" in flags or "--graph" in flags: + target.append(targetBase + env.subst("$YACCVCGFILESUFFIX")) # If -v is specified yacc will create the output debug file # which is not really source for any process, but should @@ -82,11 +87,13 @@ def _yaccEmitter(target, source, env, ysuf, hsuf) -> tuple: env.Clean(target[0], targetBase + '.output') # With --defines and --graph, the file to write is defined by the option - # argument. Extract this and include in the list of targets. - # NOTE: a filename passed to the command this way is not modified by SCons, - # and so will be interpreted relative to the project top directory at - # execution time, while the name added to the target list will be + # argument, if present (the no-option-argument cases were caught above). + # Extract this and include in the list of targets. + # NOTE: a filename passed to the command this way is not modified by + # SCons, and so will be interpreted relative to the project top directory + # at execution time, while the name added to the target list will be # interpreted relative to the SConscript directory - a possible mismatch. + # Better to use YACC_HEADER_FILE and YACC_GRAPH_FILE to pass these. # # These are GNU bison-only options. # Since bison 3.8, --header is the preferred name over --defines @@ -185,7 +192,7 @@ def generate(env) -> None: env['YACCCOM'] = '$YACC $YACCFLAGS $_YACC_HEADER $_YACC_GRAPH -o $TARGET $SOURCES' env['YACCHFILESUFFIX'] = '.h' env['YACCHXXFILESUFFIX'] = '.hpp' - env['YACCVCGFILESUFFIX'] = '.vcg' + env['YACCVCGFILESUFFIX'] = '.gv' env['_YACC_HEADER'] = '${YACC_HEADER_FILE and "--header=" + str(YACC_HEADER_FILE)}' env['_YACC_GRAPH'] = '${YACC_GRAPH_FILE and "--graph=" + str(YACC_GRAPH_FILE)}' diff --git a/SCons/Tool/yacc.xml b/SCons/Tool/yacc.xml index 9ccc4e6..cf466a2 100644 --- a/SCons/Tool/yacc.xml +++ b/SCons/Tool/yacc.xml @@ -115,6 +115,7 @@ with the suffix defined by &cv-link-YACCHFILESUFFIX; if the yacc source file ends in a .y suffix, or a file with the suffix defined by &cv-link-YACCHXXFILESUFFIX; if the yacc source file ends in a .yy suffix. +The header will have the same base name as the requested target. @@ -131,22 +132,31 @@ with the suffix .output. Also recognized are GNU &bison; options - and its deprecated synonym -, + +(and its deprecated synonym ), which is similar to -but the output filename is named by the option argument; -and , +but gives the option to explicitly name the output header file +through an option argument; +and , which is similar to -but the output filename is named by the option argument. +but gives the option to explicitly name the output graph file +through an option argument. +The file suffixes described for + and above +are not applied if these are used in the option=argument form. Note that files specified by and may not be properly handled -by &SCons; in all situations. Consider using -&cv-link-YACC_HEADER_FILE; and &cv-link-YACC_GRAPH_FILE; instead. +by &SCons; in all situations, and using those in &cv-YACCFLAGS; +should be considered legacy support only. +Consider using &cv-link-YACC_HEADER_FILE; +and &cv-link-YACC_GRAPH_FILE; instead +if the files need to be explicitly named +(new in version 4.4.0). @@ -159,6 +169,7 @@ Will be emitted as a command-line option. Use this in preference to including in &cv-link-YACCFLAGS; directly. +New in version 4.4.0. @@ -171,6 +182,7 @@ Will be emitted as a command-line option. Use this in preference to including in &cv-link-YACCFLAGS; directly. +New in version 4.4.0. @@ -179,14 +191,13 @@ command-line option. Use this in preference to including The suffix of the C header file generated by the parser generator -when the - -option is used. -Note that setting this variable does not cause -the parser generator to generate a header -file with the specified suffix, -it exists to allow you to specify -what suffix the parser generator will use of its own accord. +when the option +(or without an option-argument) +is used in &cv-link-YACCFLAGS;. +Note that setting this variable informs &SCons; +how to construct the header filename for tracking purposes, +it does not affect the actual generated filename. +Set this to match what your parser generator produces. The default value is .h. @@ -198,22 +209,14 @@ The default value is The suffix of the C++ header file generated by the parser generator -when the - -option is used. -Note that setting this variable does not cause -the parser generator to generate a header -file with the specified suffix, -it exists to allow you to specify -what suffix the parser generator will use of its own accord. -The default value is -.hpp, -except on Mac OS X, -where the default is -${TARGET.suffix}.h. -because the default &bison; parser generator just -appends .h -to the name of the generated C++ file. +when the option +(or without an option-argument) +is used in &cv-link-YACCFLAGS;. +Note that setting this variable informs &SCons; +how to construct the header filename for tracking purposes, +it does not affect the actual generated filename. +Set this to match what your parser generator produces. +The default value is .hpp. @@ -222,17 +225,22 @@ to the name of the generated C++ file. The suffix of the file -containing the VCG grammar automaton definition -when the - -option is used. -Note that setting this variable does not cause -the parser generator to generate a VCG -file with the specified suffix, -it exists to allow you to specify -what suffix the parser generator will use of its own accord. -The default value is -.vcg. +containing a graph of the grammar automaton +when the option +(or without an option-argument) +is used in &cv-link-YACCFLAGS;. +Note that setting this variable informs &SCons; +how to construct the graph filename for tracking purposes, +it does not affect the actual generated filename. +Various yacc tools have emitted various formats +at different times. +Set this to match what your parser generator produces. +The default value is .gv. + + +Changed in version 4.X.Y: the default value +changed from .vcg (&bison; stopped generating +.vcg output with version 2.4, in 2006). diff --git a/test/YACC/BISONFLAGS.py b/test/YACC/BISONFLAGS.py index cae673d..9b8cf03 100644 --- a/test/YACC/BISONFLAGS.py +++ b/test/YACC/BISONFLAGS.py @@ -47,12 +47,13 @@ test = TestSCons.TestSCons() test.subdir('sub1') test.subdir('sub2') +test.subdir('sub3') test.dir_fixture('YACCFLAGS-fixture') test.write('SConstruct', """\ DefaultEnvironment(tools=[]) -SConscript(dirs=['sub1', 'sub2']) +SConscript(dirs=['sub1', 'sub2', 'sub3']) """) # this SConscript is for the options-in-flags version @@ -90,8 +91,50 @@ env.CFile( """ % locals()) test.write(['sub2', 'aaa.y'], "aaa.y\nYACCFLAGS\n") +# this SConscript is to try various other flag combos +test.write(['sub3', 'SConscript'], """\ +import sys + +env = Environment( + YACC=r'%(_python_)s myyacc.py', + YACCFLAGS='-x --header=header.h --graph=graph.g', + tools=['yacc', '%(linker)s', '%(compiler)s'], +) + +def check(targets, expected): + t = [str(target) for target in targets] + assert t == expected, t + +targs1 = env.CFile('trg1', source='aaa.y', YACCFLAGS='-d') +check(targs1, ['trg1.c', 'trg1.h']) + +targs2 = env.CXXFile('trg2', source='aaa.yy', YACCFLAGS='-d') +check(targs2, ['trg2.cc', 'trg2.hpp']) + +targs3 = env.CFile('trg3', source='aaa.y', YACCFLAGS='--defines=zot.q') +check(targs3, ['trg3.c', 'zot.q']) + +targs4 = env.CFile('trg4', source='aaa.y', YACCFLAGS='--header') +check(targs4, ['trg4.c', 'trg4.h']) + +targs5 = env.CFile('trg5', source='aaa.y', YACCFLAGS='-H') +check(targs5, ['trg5.c', 'trg5.h']) + +targs6 = env.CFile('trg6', source='aaa.y', YACCFLAGS='-g') +check(targs6, ['trg6.c', 'trg6.gv']) + +targs7 = env.CFile('trg7', source='aaa.y', YACCFLAGS='-g -H') +check(targs7, ['trg7.c', 'trg7.h', 'trg7.gv']) + +targs8 = env.CFile('trg8', source='aaa.y', YACCFLAGS='--graph --header') +check(targs8, ['trg8.c', 'trg8.h', 'trg8.gv']) +""" % locals()) + +test.write(['sub3', 'aaa.y'], "aaa.y\nYACCFLAGS\n") +test.write(['sub3', 'aaa.yy'], "aaa.yy\nYACCFLAGS\n") + test.run('.', stderr=None) -test.must_match(['sub1', 'aaa.c'], "aaa.y\n -x --header=header.h --graph=graph.g\n") +test.must_match(['sub1', 'aaa.c'], "aaa.y\n-x --header=header.h --graph=graph.g\n") # NOTE: this behavior is "wrong" but we're keeping it for compat: # the generated files should go into 'sub1', not the topdir. @@ -109,7 +152,7 @@ test.must_match(['graph.g'], 'yacc graph\n') sub2 = Path('sub2') headerfile = sub2 / 'header.h' graphfile = sub2 / 'graph.g' -yaccflags = f"aaa.y\n -x --header={headerfile} --graph={graphfile}\n" +yaccflags = f"aaa.y\n-x --header={headerfile} --graph={graphfile}\n" test.must_match(['sub2', 'aaa.c'], yaccflags) test.must_match(['sub2', 'header.h'], 'yacc header\n') test.must_match(['sub2', 'graph.g'], 'yacc graph\n') diff --git a/test/YACC/YACCFLAGS-fixture/myyacc.py b/test/YACC/YACCFLAGS-fixture/myyacc.py index 3bc1375..8b793a1 100644 --- a/test/YACC/YACCFLAGS-fixture/myyacc.py +++ b/test/YACC/YACCFLAGS-fixture/myyacc.py @@ -1,65 +1,102 @@ -import getopt +#!/usr/bin/python +# +# SPDX-License-Identifier: MIT +# +# Copyright The SCons Foundation + +""" +Mock yacc/bison command for testing yacc tool with options +""" + +import argparse +import os import sys from pathlib import Path -def make_side_effect(path, text): +# Sentinels to make sure we pick the defaults +HEADER_DEFAULT = "_header_compute" +GRAPH_DEFAULT = "_graph_compute" +REPORT_DEFAULT = "_report_compute" + + +def make_side_effect(path, text, outfile): + if path == HEADER_DEFAULT: + file, ext = os.path.splitext(outfile) + if ext in [".y", ".yacc"]: + path = file + ".h" + elif ext in [".ym"]: + path = file + ".h.m" + elif ext in [".yy"]: + path = file + ".hpp" + else: # just guess + path = file + ".h" + if path == GRAPH_DEFAULT: + path = os.path.splitext(outfile)[0] + ".gv" + if path == REPORT_DEFAULT: + path = os.path.splitext(outfile)[0] + ".output" + p = Path(path) - if str(p.parent) != '.': + if str(p.parent) != ".": p.parent.mkdir(parents=True, exist_ok=True) with p.open(mode="wb") as f: f.write(text) def fake_yacc(): - make_header = None - make_graph = None + parser = argparse.ArgumentParser(allow_abbrev=False) + parser.add_argument("-g", "--graph", nargs="?", const=GRAPH_DEFAULT) + parser.add_argument("-d", dest="graph", action="store_const", const=HEADER_DEFAULT) + parser.add_argument("-v", "--verbose", action="store_const", const=REPORT_DEFAULT) + parser.add_argument("-x", action="store_true") # accept, do nothing + parser.add_argument("-H", "--header", "--defines", nargs="?", const=HEADER_DEFAULT) + parser.add_argument("-o", "--output", dest="out", required=True) + parser.add_argument("-I", dest="i_arguments", default=[], action="append") + parser.add_argument("files", nargs="+") + args = parser.parse_args() + # print(f"DEBUG: {args}") - longopts = ["defines=", "header=", "graph="] - cmd_opts, args = getopt.getopt(sys.argv[1:], 'o:I:x', longopts) - opt_string = '' - i_arguments = '' - - for opt, arg in cmd_opts: - if opt == '-o': - out = arg - elif opt == '-I': - i_arguments = f'{i_arguments} {arg}' - elif opt in ('--defines', '--header'): - make_header = arg - opt_string = f'{opt_string} {opt}={arg}' - elif opt == '--graph': - make_graph = arg - opt_string = f'{opt_string} {opt}={arg}' + # Synthesize "opt_string", which, when this script used getopt, was + # collected in the arg processing loop - from some arguments. The tests + # expect this to be subbed in for YACCFLAGS in making the output file. + opt_list = [] + skip = False + for arg in sys.argv[1:]: + if skip: + skip = False + elif arg == "-o": + skip = True + elif arg.startswith("-I"): + pass else: - opt_string = f'{opt_string} {opt}' + opt_list.append(arg) + # The original didn't use the file argument(s) so we have to get rid of. + for file in args.files: + if file in opt_list: + opt_list.remove(file) + opt_string = " ".join(opt_list) + + # Now we need to do something similar for i_arguments, which is easier + i_arguments = " ".join(args.i_arguments) - with open(out, 'wb') as ofp: - for a in args: - with open(a, 'rb') as ifp: + with open(args.out, "wb") as ofp: + for file in args.files: + with open(file, "rb") as ifp: contents = ifp.read() - contents = contents.replace(b'YACCFLAGS', opt_string.encode()) - contents = contents.replace(b'YACC', b'myyacc.py') - contents = contents.replace(b'I_ARGS', i_arguments.encode()) + contents = contents.replace(b"YACCFLAGS", opt_string.encode()) + contents = contents.replace(b"YACC", b"myyacc.py") + contents = contents.replace(b"I_ARGS", i_arguments.encode()) ofp.write(contents) - # Extra bits: - if make_header: - make_side_effect(make_header, b"yacc header\n") - if make_graph: - make_side_effect(make_graph, b"yacc graph\n") + # Make extra output files + if args.header: + make_side_effect(args.header, b"yacc header\n", args.out) + if args.graph: + make_side_effect(args.graph, b"yacc graph\n", args.out) + if args.verbose: + make_side_effect(args.verbose, b"yacc debug\n", args.out) -if __name__ == '__main__': +if __name__ == "__main__": fake_yacc() sys.exit(0) - -# If -d is specified on the command line, yacc will emit a .h -# or .hpp file with the same name as the .c or .cpp output file. - -# If -g is specified on the command line, yacc will emit a .vcg -# file with the same base name as the .y, .yacc, .ym or .yy file. - -# If -v is specified yacc will create the output debug file -# which is not really source for any process, but should -# be noted and also be cleaned (issue #2558) diff --git a/test/YACC/YACCFLAGS.py b/test/YACC/YACCFLAGS.py index 1f3ad78..7a07072 100644 --- a/test/YACC/YACCFLAGS.py +++ b/test/YACC/YACCFLAGS.py @@ -56,7 +56,7 @@ env.CFile(target='out/aaa', source='in/aaa.y') test.write(['in', 'aaa.y'], "aaa.y\nYACCFLAGS\nI_ARGS\n") test.run('.', stderr=None) -test.must_match(['out', 'aaa.c'], "aaa.y\n -x\n out in\n") +test.must_match(['out', 'aaa.c'], "aaa.y\n-x\nout in\n") test.pass_test() -- cgit v0.12