Revise the markup related to the grammar productions to increase the

level of predictability.  This is not really "good" markup, but is arguably
better than we had before.
This closes SF bug #523117.

7 files changed, 95 insertions, 98 deletions

diff --git a/Doc/perl/python.perl b/Doc/perl/python.perl
index 332f518..92cf655 100644
--- a/Doc/perl/python.perl
+++ b/Doc/perl/python.perl
@@ -807,6 +807,18 @@ sub do_cmd_production{
             . $_);
 }
 
+sub do_cmd_productioncont{
+    local($_) = @_;
+    my $defn = next_argument();
+    return ("<tr valign=\"baseline\">\n"
+            . "    <td>&nbsp;</td>\n"
+            . "    <td>&nbsp;</td>\n"
+            . "    <td><code>"
+            . translate_commands($defn)
+            . "</code></td></tr>"
+            . $_);
+}
+
 sub process_grammar_files{
     my $lang;
     my $filename;
@@ -847,6 +859,7 @@ sub process_grammar_files{
 
 sub strip_grammar_markup{
     local($_) = @_;
+    s/\\productioncont/              /g;
     s/\\production(<<\d+>>)(.+)\1/\n\2 ::= /g;
     s/\\token(<<\d+>>)(.+)\1/\2/g;
     s/\\e([^a-zA-Z])/\\\1/g;
diff --git a/Doc/ref/ref2.tex b/Doc/ref/ref2.tex
index 74aaae4..1956e02 100644
--- a/Doc/ref/ref2.tex
+++ b/Doc/ref/ref2.tex
@@ -318,8 +318,8 @@ String literals are described by the following lexical definitions:
              {"'" \token{shortstringitem}* "'"
               | '"' \token{shortstringitem}* '"'}
   \production{longstring}
-             {"'''" \token{longstringitem}* "'''"
-              | '"""' \token{longstringitem}* '"""'}
+             {"'''" \token{longstringitem}* "'''"}
+  \productioncont{| '"""' \token{longstringitem}* '"""'}
   \production{shortstringitem}
              {\token{shortstringchar} | \token{escapeseq}}
   \production{longstringitem}
diff --git a/Doc/ref/ref5.tex b/Doc/ref/ref5.tex
index 9ed9337..6005f51 100644
--- a/Doc/ref/ref5.tex
+++ b/Doc/ref/ref5.tex
@@ -54,8 +54,8 @@ categorized syntactically as atoms.  The syntax for atoms is:
   \production{atom}
              {\token{identifier} | \token{literal} | \token{enclosure}}
   \production{enclosure}
-             {\token{parenth_form} | \token{list_display}
-              | \token{dict_display} | \token{string_conversion}}
+             {\token{parenth_form} | \token{list_display}}
+  \productioncont{| \token{dict_display} | \token{string_conversion}}
 \end{productionlist}
 
 
@@ -112,9 +112,8 @@ Python supports string literals and various numeric literals:
 
 \begin{productionlist}
   \production{literal}
-             {\token{stringliteral} | \token{integer}
-              | \token{longinteger} | \token{floatnumber}
-              | \token{imagnumber}}
+             {\token{stringliteral} | \token{integer} | \token{longinteger}}
+  \productioncont{| \token{floatnumber} | \token{imagnumber}}
 \end{productionlist}
 
 Evaluation of a literal yields an object of the given type (string,
@@ -439,13 +438,12 @@ series of arguments:
   \production{call}
              {\token{primary} "(" [\token{argument_list} [","]] ")"}
   \production{argument_list}
-             {\token{positional_arguments} ["," \token{keyword_arguments} 
-		 ["," "*" \token{expression} ["," "**" \token{expression}]]]
-              | \token{keyword_arguments} ["," "*" \token{expression} 
-		["," "**" \token{expression}]]
-	      | "*" \token{expression} ["," "**" \token{expression}]
-	      | "**" \token{expression}
-	     }
+             {\token{positional_arguments} ["," \token{keyword_arguments}}
+  \productioncont{   ["," "*" \token{expression} ["," "**" \token{expression}]]]}
+  \productioncont{| \token{keyword_arguments} ["," "*" \token{expression}}
+  \productioncont{                             ["," "**" \token{expression}]]}
+  \productioncont{| "*" \token{expression} ["," "**" \token{expression}]}
+  \productioncont{| "**" \token{expression}}
   \production{positional_arguments}
              {\token{expression} ("," \token{expression})*}
   \production{keyword_arguments}
@@ -666,8 +664,8 @@ operators:
 \begin{productionlist}
   \production{m_expr}
              {\token{u_expr} | \token{m_expr} "*" \token{u_expr}
-              | \token{m_expr} "/" \token{u_expr}
-              | \token{m_expr} "\%" \token{u_expr}}
+              | \token{m_expr} "/" \token{u_expr}}
+  \productioncont{| \token{m_expr} "\%" \token{u_expr}}
   \production{a_expr}
              {\token{m_expr} | \token{aexpr} "+" \token{m_expr}
               \token{aexpr} "-" \token{m_expr}}
@@ -801,8 +799,8 @@ interpretation that is conventional in mathematics:
   \production{comparison}
              {\token{or_expr} ( \token{comp_operator} \token{or_expr} )*}
   \production{comp_operator}
-             {"<" | ">" | "==" | ">=" | "<=" | "<>" | "!="
-              | "is" ["not"] | ["not"] "in"}
+             {"<" | ">" | "==" | ">=" | "<=" | "<>" | "!="}
+  \productioncont{| "is" ["not"] | ["not"] "in"}
 \end{productionlist}
 
 Comparisons yield integer values: \code{1} for true, \code{0} for false.
diff --git a/Doc/ref/ref6.tex b/Doc/ref/ref6.tex
index 122f499..7710472 100644
--- a/Doc/ref/ref6.tex
+++ b/Doc/ref/ref6.tex
@@ -6,22 +6,21 @@ Several simple statements may occur on a single line separated
 by semicolons.  The syntax for simple statements is:
 
 \begin{productionlist}
-  \production{simple_stmt}
-             {\token{expression_stmt}
-              | \token{assert_stmt}
-              | \token{assignment_stmt}
-              | \token{augmented_assignment_stmt}
-              | \token{pass_stmt}
-              | \token{del_stmt}
-              | \token{print_stmt}
-              | \token{return_stmt}
-              | \token{yield_stmt}
-              | \token{raise_stmt}
-              | \token{break_stmt}
-              | \token{continue_stmt}
-              | \token{import_stmt}
-              | \token{global_stmt}
-              | \token{exec_stmt}}
+  \production{simple_stmt}{\token{expression_stmt}}
+  \productioncont{| \token{assert_stmt}}
+  \productioncont{| \token{assignment_stmt}}
+  \productioncont{| \token{augmented_assignment_stmt}}
+  \productioncont{| \token{pass_stmt}}
+  \productioncont{| \token{del_stmt}}
+  \productioncont{| \token{print_stmt}}
+  \productioncont{| \token{return_stmt}}
+  \productioncont{| \token{yield_stmt}}
+  \productioncont{| \token{raise_stmt}}
+  \productioncont{| \token{break_stmt}}
+  \productioncont{| \token{continue_stmt}}
+  \productioncont{| \token{import_stmt}}
+  \productioncont{| \token{global_stmt}}
+  \productioncont{| \token{exec_stmt}}
 \end{productionlist}
 
 
@@ -112,12 +111,12 @@ objects:
   \production{target_list}
              {\token{target} ("," \token{target})* [","]}
   \production{target}
-             {\token{identifier}
-              | "(" \token{target_list} ")"
-              | "[" \token{target_list} "]"
-              | \token{attributeref}
-              | \token{subscription}
-              | \token{slicing}}
+             {\token{identifier}}
+  \productioncont{| "(" \token{target_list} ")"}
+  \productioncont{| "[" \token{target_list} "]"}
+  \productioncont{| \token{attributeref}}
+  \productioncont{| \token{subscription}}
+  \productioncont{| \token{slicing}}
 \end{productionlist}
 
 (See section~\ref{primaries} for the syntax definitions for the last
@@ -264,7 +263,7 @@ print x
 \end{verbatim}
 
 
-\subsection{Augmented Assignment statements \label{augassign}}
+\subsection{Augmented assignment statements \label{augassign}}
 
 Augmented assignment is the combination, in a single statement, of a binary
 operation and an assignment statement:
@@ -275,15 +274,8 @@ operation and an assignment statement:
   \production{augmented_assignment_stmt}
              {\token{target} \token{augop} \token{expression_list}}
   \production{augop}
-             {"+=" | "-=" | "*=" | "/=" | "\%=" | "**="
-              | ">>=" | "<<=" | "\&=" | "\textasciicircum=" | "|="}
-  \production{target}
-             {\token{identifier}
-              | "(" \token{target_list} ")"
-              | "[" \token{target_list} "]"
-              | \token{attributeref}
-              | \token{subscription}
-              | \token{slicing}}
+             {"+=" | "-=" | "*=" | "/=" | "\%=" | "**="}
+  \productioncont{| ">>=" | "<<=" | "\&=" | "\textasciicircum=" | "|="}
 \end{productionlist}
 
 (See section~\ref{primaries} for the syntax definitions for the last
@@ -364,9 +356,9 @@ right type (but even this is determined by the sliced object).
 
 \begin{productionlist}
   \production{print_stmt}
-             {"print" ( \optional{\token{expression} ("," \token{expression})* \optional{","}}
-                      | ">\code{>}" \token{expression}
-                        \optional{("," \token{expression})+ \optional{","}})}
+             {"print" ( \optional{\token{expression} ("," \token{expression})* \optional{","}}}
+  \productioncont{| ">\code{>}" \token{expression}
+                  \optional{("," \token{expression})+ \optional{","}} )}
 \end{productionlist}
 
 \keyword{print} evaluates each expression in turn and writes the
@@ -600,11 +592,11 @@ It continues with the next cycle of the nearest enclosing loop.
 \begin{productionlist}
   \production{import_stmt}
              {"import" \token{module} ["as" \token{name}]
-                ( "," \token{module} ["as" \token{name}] )*
-              | "from" \token{module} "import" \token{identifier}
-                ["as" \token{name}]
-                ( "," \token{identifier} ["as" \token{name}] )*
-              | "from" \token{module} "import" "*"}
+                ( "," \token{module} ["as" \token{name}] )*}
+  \productioncont{| "from" \token{module} "import" \token{identifier}
+                    ["as" \token{name}]}
+  \productioncont{  ( "," \token{identifier} ["as" \token{name}] )*}
+  \productioncont{| "from" \token{module} "import" "*"}
   \production{module}
              {(\token{identifier} ".")* \token{identifier}}
 \end{productionlist}
diff --git a/Doc/ref/ref7.tex b/Doc/ref/ref7.tex
index f22be71..eeff1aa 100644
--- a/Doc/ref/ref7.tex
+++ b/Doc/ref/ref7.tex
@@ -42,8 +42,12 @@ Summarizing:
 
 \begin{productionlist}
   \production{compound_stmt}
-             {\token{if_stmt} | \token{while_stmt} | \token{for_stmt}
-              | \token{try_stmt} | \token{funcdef} | \token{classdef}}
+             {\token{if_stmt}}
+  \productioncont{| \token{while_stmt}}
+  \productioncont{| \token{for_stmt}}
+  \productioncont{| \token{try_stmt}}
+  \productioncont{| \token{funcdef}}
+  \productioncont{| \token{classdef}}
   \production{suite}
              {\token{stmt_list} NEWLINE
               | NEWLINE INDENT \token{statement}+ DEDENT}
@@ -73,9 +77,9 @@ The \keyword{if} statement is used for conditional execution:
 
 \begin{productionlist}
   \production{if_stmt}
-             {"if" \token{expression} ":" \token{suite}
-              ( "elif" \token{expression} ":" \token{suite} )*
-              ["else" ":" \token{suite}]}
+             {"if" \token{expression} ":" \token{suite}}
+  \productioncont{( "elif" \token{expression} ":" \token{suite} )*}
+  \productioncont{["else" ":" \token{suite}]}
 \end{productionlist}
 
 It selects exactly one of the suites by evaluating the expressions one
@@ -97,8 +101,8 @@ as an expression is true:
 
 \begin{productionlist}
   \production{while_stmt}
-             {"while" \token{expression} ":" \token{suite}
-               ["else" ":" \token{suite}]}
+             {"while" \token{expression} ":" \token{suite}}
+  \productioncont{["else" ":" \token{suite}]}
 \end{productionlist}
 
 This repeatedly tests the expression and, if it is true, executes the
@@ -126,8 +130,8 @@ sequence (such as a string, tuple or list) or other iterable object:
 \begin{productionlist}
   \production{for_stmt}
              {"for" \token{target_list} "in" \token{expression_list}
-              ":" \token{suite}
-              ["else" ":" \token{suite}]}
+              ":" \token{suite}}
+  \productioncont{["else" ":" \token{suite}]}
 \end{productionlist}
 
 The expression list is evaluated once; it should yield a sequence.  The
@@ -192,10 +196,10 @@ code for a group of statements:
   \production{try_stmt}
              {\token{try_exc_stmt} | \token{try_fin_stmt}}
   \production{try_exc_stmt}
-             {"try" ":" \token{suite}
-              ("except" [\token{expression} ["," \token{target}]] ":"
-              \token{suite})+
-              ["else" ":" \token{suite}]}
+             {"try" ":" \token{suite}}
+  \productioncont{("except" [\token{expression}
+                             ["," \token{target}]] ":" \token{suite})+}
+  \productioncont{["else" ":" \token{suite}]}
   \production{try_fin_stmt}
              {"try" ":" \token{suite}
               "finally" ":" \token{suite}}
@@ -310,10 +314,10 @@ section \ref{types}):
              {"def" \token{funcname} "(" [\token{parameter_list}] ")"
               ":" \token{suite}}
   \production{parameter_list}
-             {(\token{defparameter} ",")*
-              ("*" \token{identifier} [, "**" \token{identifier}]
-               | "**" \token{identifier}
-               | \token{defparameter} [","])}
+             {(\token{defparameter} ",")*}
+  \productioncont{("*" \token{identifier} [, "**" \token{identifier}]}
+  \productioncont{| "**" \token{identifier}
+                  | \token{defparameter} [","])}
   \production{defparameter}
              {\token{parameter} ["=" \token{expression}]}
   \production{sublist}
diff --git a/Doc/ref/refa1.tex b/Doc/ref/refa1.tex
index 4cd10d5..bf40c2d 100644
--- a/Doc/ref/refa1.tex
+++ b/Doc/ref/refa1.tex
@@ -20,13 +20,13 @@ versions of Python that introduce incompatible changes to the
 language.  It allows use of the new features on a per-module basis
 before the release in which the feature becomes standard.
 
-\begin{verbatim}
-future_statement: "from" "__future__" "import" feature ["as" name]
-                 ("," feature ["as" name])*
-
-feature: identifier
-name: identifier
-\end{verbatim}
+\begin{productionlist}[*]
+  \production{future_statement}
+             {"from" "__future__" "import" feature ["as" name]}
+  \productioncont{("," feature ["as" name])*}
+  \production{feature}{identifier}
+  \production{name}{identifier}
+\end{productionlist}
 
 A future statement must appear near the top of the module.  The only
 lines that can appear before a future statement are:
diff --git a/Doc/texinputs/python.sty b/Doc/texinputs/python.sty
index 021ba5c..9fa6d4c 100644
--- a/Doc/texinputs/python.sty
+++ b/Doc/texinputs/python.sty
@@ -53,10 +53,10 @@
   % the same for each link, and avoids having lots of garbage all over 
   % this style file.
   \newcommand{\py@linkToName}[2]{%
-    \pdfannotlink attr{/Border [0 0 0]} goto name{#1}%
+    \pdfstartlink attr{/Border [0 0 0]} goto name{#1}%
       \py@LinkColor#2\py@NormalColor%
     \pdfendlink%
-  }    
+  }
   % Compute the padded page number separately since we end up with a pair of
   % \relax tokens; this gets the right string computed and works.
   \renewcommand{\contentsline}[3]{%
@@ -788,7 +788,7 @@
 % but only if we actually used hyperref:
 \ifpdf
   \newcommand{\url}[1]{{%
-    \pdfannotlink attr{/Border [0 0 0]} user{/S /URI /URI (#1)}%
+    \pdfstartlink attr{/Border [0 0 0]} user{/S /URI /URI (#1)}%
     \py@LinkColor%                              color of the link text
     \small\sf #1%
     \py@NormalColor%                    Turn it back off; these are declarative
@@ -864,9 +864,9 @@
 % \ulink{link text}{URL}
 \ifpdf
   % The \noindent here is a hack -- we're forcing pdfTeX into
-  % horizontal mode since \pdfannotlink requires that.
+  % horizontal mode since \pdfstartlink requires that.
   \newcommand{\ulink}[2]{\noindent{%
-    \pdfannotlink attr{/Border [0 0 0]} user{/S /URI /URI (#2)}%
+    \pdfstartlink attr{/Border [0 0 0]} user{/S /URI /URI (#2)}%
     \py@LinkColor%                              color of the link text
     #1%
     \py@NormalColor%                    Turn it back off; these are declarative
@@ -899,17 +899,7 @@
 \newenvironment{productionlist}[1][\py@badkey]{
   \def\optional##1{{\Large[}##1{\Large]}}
   \def\production##1##2{\code{##1}&::=&\code{##2}\\}
-  \def\orgroup##1{{\def\oritem{\textbar\ }##1}}
-  \def\orgroup*##1{{
-      \def\oritem{\\ \textbar&}
-      % This uses math mode's ``negative thin space'' to avoid a weird
-      % indentation that I've not been able to figure out, but
-      % probably relates to nesting tabular environments.
-      $\!\!\!\!\!\!\!\!\!\!$%
-      \begin{tabular}[t]{ll}
-        \ & ##1
-      \end{tabular}
-    }}
+  \def\productioncont##1{& &\code{##1}\\}
   \def\token##1{##1}
   \let\grammartoken=\token
   \parindent=2em