bpo-36310: Allow pygettext.py to detect calls to gettext in f-strings. (GH-19875)

Adds support to Tools/i18n/pygettext.py for gettext calls in f-strings. This process is done by parsing the f-strings, processing each value, and flagging the ones which contain a gettext call.

Co-authored-by: Batuhan Taskaya <batuhanosmantaskaya@gmail.com>

4 files changed, 126 insertions, 0 deletions

diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py
index 8da6579..12f778d 100644
--- a/Lib/test/test_tools/test_i18n.py
+++ b/Lib/test/test_tools/test_i18n.py
@@ -220,6 +220,76 @@ class Test_pygettext(unittest.TestCase):
         '''))
         self.assertIn('doc', msgids)
 
+    def test_calls_in_fstrings(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_('foo bar')}"
+        '''))
+        self.assertIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_raw(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        rf"{_('foo bar')}"
+        '''))
+        self.assertIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_nested(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"""{f'{_("foo bar")}'}"""
+        '''))
+        self.assertIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_attribute(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{obj._('foo bar')}"
+        '''))
+        self.assertIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_with_call_on_call(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{type(str)('foo bar')}"
+        '''))
+        self.assertNotIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_with_format(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_('foo {bar}').format(bar='baz')}"
+        '''))
+        self.assertIn('foo {bar}', msgids)
+
+    def test_calls_in_fstrings_with_wrong_input_1(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_(f'foo {bar}')}"
+        '''))
+        self.assertFalse([msgid for msgid in msgids if 'foo {bar}' in msgid])
+
+    def test_calls_in_fstrings_with_wrong_input_2(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_(1)}"
+        '''))
+        self.assertNotIn(1, msgids)
+
+    def test_calls_in_fstring_with_multiple_args(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_('foo', 'bar')}"
+        '''))
+        self.assertNotIn('foo', msgids)
+        self.assertNotIn('bar', msgids)
+
+    def test_calls_in_fstring_with_keyword_args(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_('foo', bar='baz')}"
+        '''))
+        self.assertNotIn('foo', msgids)
+        self.assertNotIn('bar', msgids)
+        self.assertNotIn('baz', msgids)
+
+    def test_calls_in_fstring_with_partially_wrong_expression(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_(f'foo') + _('bar')}"
+        '''))
+        self.assertNotIn('foo', msgids)
+        self.assertIn('bar', msgids)
+
     def test_files_list(self):
         """Make sure the directories are inspected for source files
            bpo-31920
diff --git a/Misc/ACKS b/Misc/ACKS
index 5285693a..35a87ae 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -949,6 +949,7 @@ Ivan Krstić
 Anselm Kruis
 Steven Kryskalla
 Andrew Kuchling
+Jakub Kuczys
 Dave Kuhlman
 Jon Kuhn
 Ilya Kulakov
diff --git a/Misc/NEWS.d/next/Tools-Demos/2020-05-03-01-30-46.bpo-36310.xDxxwY.rst b/Misc/NEWS.d/next/Tools-Demos/2020-05-03-01-30-46.bpo-36310.xDxxwY.rst
new file mode 100644
index 0000000..16749a8
--- /dev/null
+++ b/Misc/NEWS.d/next/Tools-Demos/2020-05-03-01-30-46.bpo-36310.xDxxwY.rst
@@ -0,0 +1,2 @@
+Allow :file:`Tools/i18n/pygettext.py` to detect calls to ``gettext`` in
+f-strings.
\ No newline at end of file
diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py
index b1d281d..9dd65c2 100755
--- a/Tools/i18n/pygettext.py
+++ b/Tools/i18n/pygettext.py
@@ -162,6 +162,7 @@ import sys
 import glob
 import time
 import getopt
+import ast
 import token
 import tokenize
 
@@ -343,6 +344,58 @@ class TokenEater:
                 return
         if ttype == tokenize.NAME and tstring in opts.keywords:
             self.__state = self.__keywordseen
+            return
+        if ttype == tokenize.STRING:
+            maybe_fstring = ast.parse(tstring, mode='eval').body
+            if not isinstance(maybe_fstring, ast.JoinedStr):
+                return
+            for value in filter(lambda node: isinstance(node, ast.FormattedValue),
+                                maybe_fstring.values):
+                for call in filter(lambda node: isinstance(node, ast.Call),
+                                   ast.walk(value)):
+                    func = call.func
+                    if isinstance(func, ast.Name):
+                        func_name = func.id
+                    elif isinstance(func, ast.Attribute):
+                        func_name = func.attr
+                    else:
+                        continue
+
+                    if func_name not in opts.keywords:
+                        continue
+                    if len(call.args) != 1:
+                        print(_(
+                            '*** %(file)s:%(lineno)s: Seen unexpected amount of'
+                            ' positional arguments in gettext call: %(source_segment)s'
+                            ) % {
+                            'source_segment': ast.get_source_segment(tstring, call) or tstring,
+                            'file': self.__curfile,
+                            'lineno': lineno
+                            }, file=sys.stderr)
+                        continue
+                    if call.keywords:
+                        print(_(
+                            '*** %(file)s:%(lineno)s: Seen unexpected keyword arguments'
+                            ' in gettext call: %(source_segment)s'
+                            ) % {
+                            'source_segment': ast.get_source_segment(tstring, call) or tstring,
+                            'file': self.__curfile,
+                            'lineno': lineno
+                            }, file=sys.stderr)
+                        continue
+                    arg = call.args[0]
+                    if not isinstance(arg, ast.Constant):
+                        print(_(
+                            '*** %(file)s:%(lineno)s: Seen unexpected argument type'
+                            ' in gettext call: %(source_segment)s'
+                            ) % {
+                            'source_segment': ast.get_source_segment(tstring, call) or tstring,
+                            'file': self.__curfile,
+                            'lineno': lineno
+                            }, file=sys.stderr)
+                        continue
+                    if isinstance(arg.value, str):
+                        self.__addentry(arg.value, lineno)
 
     def __suiteseen(self, ttype, tstring, lineno):
         # skip over any enclosure pairs until we see the colon