closes bpo-31650: PEP 552 (Deterministic pycs) implementation (#4575)

Python now supports checking bytecode cache up-to-dateness with a hash of the
source contents rather than volatile source metadata. See the PEP for details.

While a fairly straightforward idea, quite a lot of code had to be modified due
to the pervasiveness of pyc implementation details in the codebase. Changes in
this commit include:

- The core changes to importlib to understand how to read, validate, and
  regenerate hash-based pycs.

- Support for generating hash-based pycs in py_compile and compileall.

- Modifications to our siphash implementation to support passing a custom
  key. We then expose it to importlib through _imp.

- Updates to all places in the interpreter, standard library, and tests that
  manually generate or parse pyc files to grok the new format.

- Support in the interpreter command line code for long options like
  --check-hash-based-pycs.

- Tests and documentation for all of the above.

1 files changed, 36 insertions, 1 deletions

diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h
index ddd694f..1955e50 100644
--- a/Python/clinic/import.c.h
+++ b/Python/clinic/import.c.h
@@ -354,6 +354,41 @@ exit:
     return return_value;
 }
 
+PyDoc_STRVAR(_imp_source_hash__doc__,
+"source_hash($module, /, key, source)\n"
+"--\n"
+"\n");
+
+#define _IMP_SOURCE_HASH_METHODDEF    \
+    {"source_hash", (PyCFunction)_imp_source_hash, METH_FASTCALL|METH_KEYWORDS, _imp_source_hash__doc__},
+
+static PyObject *
+_imp_source_hash_impl(PyObject *module, long key, Py_buffer *source);
+
+static PyObject *
+_imp_source_hash(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"key", "source", NULL};
+    static _PyArg_Parser _parser = {"ly*:source_hash", _keywords, 0};
+    long key;
+    Py_buffer source = {NULL, NULL};
+
+    if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser,
+        &key, &source)) {
+        goto exit;
+    }
+    return_value = _imp_source_hash_impl(module, key, &source);
+
+exit:
+    /* Cleanup for source */
+    if (source.obj) {
+       PyBuffer_Release(&source);
+    }
+
+    return return_value;
+}
+
 #ifndef _IMP_CREATE_DYNAMIC_METHODDEF
     #define _IMP_CREATE_DYNAMIC_METHODDEF
 #endif /* !defined(_IMP_CREATE_DYNAMIC_METHODDEF) */
@@ -361,4 +396,4 @@ exit:
 #ifndef _IMP_EXEC_DYNAMIC_METHODDEF
     #define _IMP_EXEC_DYNAMIC_METHODDEF
 #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */
-/*[clinic end generated code: output=d068dd493e513604 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=e8b2c0b0d0a75da8 input=a9049054013a1b77]*/