diff options
| -rw-r--r-- | Misc/sbom.spdx.json | 624 | ||||
| -rw-r--r-- | Tools/build/generate_sbom.py | 263 |
2 files changed, 837 insertions, 50 deletions
diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index 5b3cd04..9456677 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -1696,6 +1696,510 @@ "versionInfo": "2.5.1" }, { + "SPDXID": "SPDXRef-PACKAGE-cachecontrol", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "95dedbec849f46dda3137866dc28b9d133fc9af55f5b805ab1291833e4457aa4" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/1d/e3/a22348e6226dcd585d5a4b5f0175b3a16dabfd3912cbeb02f321d00e56c7/cachecontrol-0.13.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/cachecontrol@0.13.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "cachecontrol", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.13.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-colorama", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/colorama@0.4.6", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "colorama", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.4.6" + }, + { + "SPDXID": "SPDXRef-PACKAGE-distlib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/76/cb/6bbd2b10170ed991cf64e8c8b85e01f2fb38f95d1bc77617569e0b0b26ac/distlib-0.3.6-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/distlib@0.3.6", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "distlib", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.3.6" + }, + { + "SPDXID": "SPDXRef-PACKAGE-distro", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "99522ca3e365cac527b44bde033f64c6945d90eb9f769703caaec52b09bbd3ff" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/f4/2c/c90a3adaf0ddb70afe193f5ebfb539612af57cffe677c3126be533df3098/distro-1.8.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/distro@1.8.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "distro", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.8.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-msgpack", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/9f/4a/36d936e54cf71e23ad276564465f6a54fb129e3d61520b76e13e0bb29167/msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/msgpack@1.0.5", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "msgpack", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.5" + }, + { + "SPDXID": "SPDXRef-PACKAGE-packaging", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/05/8e/8de486cbd03baba4deef4142bd643a3e7bbe954a784dc1bb17142572d127/packaging-21.3-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/packaging@21.3", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "packaging", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "21.3" + }, + { + "SPDXID": "SPDXRef-PACKAGE-platformdirs", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/9e/d8/563a9fc17153c588c8c2042d2f0f84a89057cdb1c30270f589c88b42d62c/platformdirs-3.8.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/platformdirs@3.8.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "platformdirs", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.8.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-pyparsing", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d554a96d1a7d3ddaf7183104485bc19fd80543ad6ac5bdb6426719d766fb06c1" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/a4/24/6ae4c9c45cf99d96b06b5d99e25526c060303171fb0aea9da2bfd7dbde93/pyparsing-3.1.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/pyparsing@3.1.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "pyparsing", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.1.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-pyproject-hooks", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/pyproject-hooks@1.0.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "pyproject-hooks", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-requests", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/requests@2.31.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "requests", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.31.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-certifi", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/certifi@2023.7.22", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "certifi", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2023.7.22" + }, + { + "SPDXID": "SPDXRef-PACKAGE-chardet", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/74/8f/8fc49109009e8d2169d94d72e6b1f4cd45c13d147ba7d6170fb41f22b08f/chardet-5.1.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/chardet@5.1.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "chardet", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "5.1.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-idna", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/idna@3.4", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "idna", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "3.4" + }, + { + "SPDXID": "SPDXRef-PACKAGE-rich", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/fc/1e/482e5eec0b89b593e81d78f819a9412849814e22225842b598908e7ac560/rich-13.4.2-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/rich@13.4.2", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "rich", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "13.4.2" + }, + { + "SPDXID": "SPDXRef-PACKAGE-pygments", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/34/a7/37c8d68532ba71549db4212cb036dbd6161b40e463aba336770e80c72f84/Pygments-2.15.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/pygments@2.15.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "pygments", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.15.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-typing-extensions", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/ec/6b/63cc3df74987c36fe26157ee12e09e8f9db4de771e0f3404263117e75b95/typing_extensions-4.7.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/typing_extensions@4.7.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "typing_extensions", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "4.7.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-resolvelib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d2da45d1a8dfee81bdd591647783e340ef3bcb104b54c383f70d422ef5cc7dbf" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/d2/fc/e9ccf0521607bcd244aa0b3fbd574f71b65e9ce6a112c83af988bbbe2e23/resolvelib-1.0.1-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/resolvelib@1.0.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "resolvelib", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.0.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-setuptools", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/c7/42/be1c7bbdd83e1bfb160c94b9cafd8e25efc7400346cf7ccdbdb452c467fa/setuptools-68.0.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/setuptools@68.0.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "setuptools", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "68.0.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-six", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/six@1.16.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "six", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.16.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tenacity", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/e7/b0/c23bd61e1b32c9b96fbca996c87784e196a812da8d621d8d04851f6c8181/tenacity-8.2.2-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/tenacity@8.2.2", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "tenacity", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "8.2.2" + }, + { + "SPDXID": "SPDXRef-PACKAGE-tomli", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/tomli@2.0.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "tomli", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.0.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-truststore", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "e37a5642ae9fc48caa8f120b6283d77225d600d224965a672c9e8ef49ce4bb4c" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/20/56/7811d5439b6a56374f274a8672d8f18b4deadadeb3a9f0c86424b98b6f96/truststore-0.8.0-py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/truststore@0.8.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "truststore", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.8.0" + }, + { + "SPDXID": "SPDXRef-PACKAGE-webencodings", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/webencodings@0.5.1", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "webencodings", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "0.5.1" + }, + { + "SPDXID": "SPDXRef-PACKAGE-urllib3", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b" + } + ], + "downloadLocation": "https://files.pythonhosted.org/packages/48/fe/a5c6cc46e9fe9171d7ecf0f33ee7aae14642f8d74baa7af4d7840f9358be/urllib3-1.26.17-py2.py3-none-any.whl", + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": "pkg:pypi/urllib3@1.26.17", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "urllib3", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "1.26.17" + }, + { "SPDXID": "SPDXRef-PACKAGE-pip", "checksums": [ { @@ -1725,6 +2229,126 @@ ], "relationships": [ { + "relatedSpdxElement": "SPDXRef-PACKAGE-cachecontrol", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-certifi", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-chardet", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-colorama", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-distlib", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-distro", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-idna", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-msgpack", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-packaging", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-platformdirs", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-pygments", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-pyparsing", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-pyproject-hooks", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-requests", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-resolvelib", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-rich", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-setuptools", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-six", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-tenacity", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-tomli", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-truststore", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-typing-extensions", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-urllib3", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { + "relatedSpdxElement": "SPDXRef-PACKAGE-webencodings", + "relationshipType": "DEPENDS_ON", + "spdxElementId": "SPDXRef-PACKAGE-pip" + }, + { "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 317d48f..aceb13f 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -8,6 +8,7 @@ import pathlib import subprocess import sys import typing +import zipfile from urllib.request import urlopen CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent @@ -16,10 +17,16 @@ CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent # the license expression is a valid SPDX license expression: # See: https://spdx.org/licenses ALLOWED_LICENSE_EXPRESSIONS = { - "MIT", - "CC0-1.0", "Apache-2.0", + "Apache-2.0 OR BSD-2-Clause", "BSD-2-Clause", + "BSD-3-Clause", + "CC0-1.0", + "ISC", + "LGPL-2.1-only", + "MIT", + "MPL-2.0", + "Python-2.0.1", } # Properties which are required for our purposes. @@ -31,14 +38,13 @@ REQUIRED_PROPERTIES_PACKAGE = frozenset([ "checksums", "licenseConcluded", "externalRefs", - "originator", "primaryPackagePurpose", ]) class PackageFiles(typing.NamedTuple): """Structure for describing the files of a package""" - include: list[str] + include: list[str] | None exclude: list[str] | None = None @@ -118,62 +124,209 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")]) +def fetch_package_metadata_from_pypi(project: str, version: str, filename: str | None = None) -> tuple[str, str] | None: + """ + Fetches the SHA256 checksum and download location from PyPI. + If we're given a filename then we match with that, otherwise we use wheels. + """ + # Get pip's download location from PyPI. Check that the checksum is correct too. + try: + raw_text = urlopen(f"https://pypi.org/pypi/{project}/{version}/json").read() + release_metadata = json.loads(raw_text) + url: dict[str, typing.Any] + + # Look for a matching artifact filename and then check + # its remote checksum to the local one. + for url in release_metadata["urls"]: + # pip can only use Python-only dependencies, so there's + # no risk of picking the 'incorrect' wheel here. + if ( + (filename is None and url["packagetype"] == "bdist_wheel") + or (filename is not None and url["filename"] == filename) + ): + break + else: + raise ValueError(f"No matching filename on PyPI for '{filename}'") + + # Successfully found the download URL for the matching artifact. + download_url = url["url"] + checksum_sha256 = url["digests"]["sha256"] + return download_url, checksum_sha256 + + except (OSError, ValueError) as e: + # Fail if we're running in CI where we should have an internet connection. + error_if( + "CI" in os.environ, + f"Couldn't fetch metadata for project '{project}' from PyPI: {e}" + ) + return None + + +def find_ensurepip_pip_wheel() -> pathlib.Path | None: + """Try to find the pip wheel bundled in ensurepip. If missing return None""" + + ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled" + + pip_wheels = [] + try: + for wheel_filename in os.listdir(ensurepip_bundled_dir): + if wheel_filename.startswith("pip-"): + pip_wheels.append(wheel_filename) + else: + print(f"Unexpected wheel in ensurepip: '{wheel_filename}'") + sys.exit(1) + + # Ignore this error, likely caused by downstream distributors + # deleting the 'ensurepip/_bundled' directory. + except FileNotFoundError: + pass + + if len(pip_wheels) == 0: + return None + elif len(pip_wheels) > 1: + print("Multiple pip wheels detected in 'Lib/ensurepip/_bundled'") + sys.exit(1) + # Otherwise return the one pip wheel. + return ensurepip_bundled_dir / pip_wheels[0] + + +def maybe_remove_pip_and_deps_from_sbom(sbom_data: dict[str, typing.Any]) -> None: + """ + Removes pip and its dependencies from the SBOM data + if the pip wheel is removed from ensurepip. This is done + by redistributors of Python and pip. + """ + + # If there's a wheel we don't remove anything. + if find_ensurepip_pip_wheel() is not None: + return + + # Otherwise we traverse the relationships + # to find dependent packages to remove. + sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip") + sbom_spdx_ids_to_remove = {sbom_pip_spdx_id} + + # Find all package SPDXIDs that pip depends on. + for sbom_relationship in sbom_data["relationships"]: + if ( + sbom_relationship["relationshipType"] == "DEPENDS_ON" + and sbom_relationship["spdxElementId"] == sbom_pip_spdx_id + ): + sbom_spdx_ids_to_remove.add(sbom_relationship["relatedSpdxElement"]) + + # Remove all the packages and relationships. + sbom_data["packages"] = [ + sbom_package for sbom_package in sbom_data["packages"] + if sbom_package["SPDXID"] not in sbom_spdx_ids_to_remove + ] + sbom_data["relationships"] = [ + sbom_relationship for sbom_relationship in sbom_data["relationships"] + if sbom_relationship["relatedSpdxElement"] not in sbom_spdx_ids_to_remove + ] + + def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None: """pip is a part of a packaging ecosystem (Python, surprise!) so it's actually automatable to discover the metadata we need like the version and checksums - so let's do that on behalf of our friends at the PyPA. + so let's do that on behalf of our friends at the PyPA. This function also + discovers vendored packages within pip and fetches their metadata. """ global PACKAGE_TO_FILES - ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled" - pip_wheels = [] - - # Find the hopefully one pip wheel in the bundled directory. - for wheel_filename in os.listdir(ensurepip_bundled_dir): - if wheel_filename.startswith("pip-"): - pip_wheels.append(wheel_filename) - if len(pip_wheels) != 1: - print("Zero or multiple pip wheels detected in 'Lib/ensurepip/_bundled'") - sys.exit(1) - pip_wheel_filename = pip_wheels[0] + pip_wheel_filepath = find_ensurepip_pip_wheel() + if pip_wheel_filepath is None: + return # There's no pip wheel, nothing to discover. # Add the wheel filename to the list of files so the SBOM file # and relationship generator can work its magic on the wheel too. PACKAGE_TO_FILES["pip"] = PackageFiles( - include=[f"Lib/ensurepip/_bundled/{pip_wheel_filename}"] + include=[str(pip_wheel_filepath.relative_to(CPYTHON_ROOT_DIR))] ) # Wheel filename format puts the version right after the project name. - pip_version = pip_wheel_filename.split("-")[1] + pip_version = pip_wheel_filepath.name.split("-")[1] pip_checksum_sha256 = hashlib.sha256( - (ensurepip_bundled_dir / pip_wheel_filename).read_bytes() + pip_wheel_filepath.read_bytes() ).hexdigest() - # Get pip's download location from PyPI. Check that the checksum is correct too. - try: - raw_text = urlopen(f"https://pypi.org/pypi/pip/{pip_version}/json").read() - pip_release_metadata = json.loads(raw_text) - url: dict[str, typing.Any] + pip_metadata = fetch_package_metadata_from_pypi( + project="pip", + version=pip_version, + filename=pip_wheel_filepath.name, + ) + # We couldn't fetch any metadata from PyPI, + # so we give up on verifying if we're not in CI. + if pip_metadata is None: + return + + pip_download_url, pip_actual_sha256 = pip_metadata + if pip_actual_sha256 != pip_checksum_sha256: + raise ValueError("Unexpected") + + # Parse 'pip/_vendor/vendor.txt' from the wheel for sub-dependencies. + with zipfile.ZipFile(pip_wheel_filepath) as whl: + vendor_txt_data = whl.read("pip/_vendor/vendor.txt").decode() + + # With this version regex we're assuming that pip isn't using pre-releases. + # If any version doesn't match we get a failure below, so we're safe doing this. + version_pin_re = re.compile(r"^([a-zA-Z0-9_.-]+)==([0-9.]*[0-9])$") + sbom_pip_dependency_spdx_ids = set() + for line in vendor_txt_data.splitlines(): + line = line.partition("#")[0].strip() # Strip comments and whitespace. + if not line: # Skip empty lines. + continue + + # Non-empty lines we must be able to match. + match = version_pin_re.match(line) + error_if(match is None, f"Couldn't parse line from pip vendor.txt: '{line}'") + assert match is not None # Make mypy happy. + + # Parse out and normalize the project name. + project_name, project_version = match.groups() + project_name = project_name.lower() + + # At this point if pip's metadata fetch succeeded we should + # expect this request to also succeed. + project_metadata = ( + fetch_package_metadata_from_pypi(project_name, project_version) + ) + assert project_metadata is not None + project_download_url, project_checksum_sha256 = project_metadata + + # Update our SBOM data with what we received from PyPI. + # Don't overwrite any existing values. + sbom_project_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{project_name}") + sbom_pip_dependency_spdx_ids.add(sbom_project_spdx_id) + for package in sbom_data["packages"]: + if package["SPDXID"] != sbom_project_spdx_id: + continue - # Look for a matching artifact filename and then check - # its remote checksum to the local one. - for url in pip_release_metadata["urls"]: - if url["filename"] == pip_wheel_filename: + # Only thing missing from this blob is the `licenseConcluded`, + # that needs to be triaged by human maintainers if the list changes. + package.update({ + "SPDXID": sbom_project_spdx_id, + "name": project_name, + "versionInfo": project_version, + "downloadLocation": project_download_url, + "checksums": [ + {"algorithm": "SHA256", "checksumValue": project_checksum_sha256} + ], + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": f"pkg:pypi/{project_name}@{project_version}", + "referenceType": "purl", + }, + ], + "primaryPackagePurpose": "SOURCE" + }) break - else: - raise ValueError(f"No matching filename on PyPI for '{pip_wheel_filename}'") - if url["digests"]["sha256"] != pip_checksum_sha256: - raise ValueError(f"Local pip checksum doesn't match artifact on PyPI") - - # Successfully found the download URL for the matching artifact. - pip_download_url = url["url"] - except (OSError, ValueError) as e: - print(f"Couldn't fetch pip's metadata from PyPI: {e}") - sys.exit(1) + PACKAGE_TO_FILES[project_name] = PackageFiles(include=None) # Remove pip from the existing SBOM packages if it's there # and then overwrite its entry with our own generated one. + sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip") sbom_data["packages"] = [ sbom_package for sbom_package in sbom_data["packages"] @@ -181,7 +334,7 @@ def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None: ] sbom_data["packages"].append( { - "SPDXID": spdx_id("SPDXRef-PACKAGE-pip"), + "SPDXID": sbom_pip_spdx_id, "name": "pip", "versionInfo": pip_version, "originator": "Organization: Python Packaging Authority", @@ -205,12 +358,27 @@ def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None: "primaryPackagePurpose": "SOURCE", } ) + for sbom_dep_spdx_id in sorted(sbom_pip_dependency_spdx_ids): + sbom_data["relationships"].append({ + "spdxElementId": sbom_pip_spdx_id, + "relatedSpdxElement": sbom_dep_spdx_id, + "relationshipType": "DEPENDS_ON" + }) def main() -> None: sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json" sbom_data = json.loads(sbom_path.read_bytes()) + # Check if pip should be removed if the wheel is missing. + # We can't reset the SBOM relationship data until checking this. + maybe_remove_pip_and_deps_from_sbom(sbom_data) + + # We regenerate all of this information. Package information + # should be preserved though since that is edited by humans. + sbom_data["files"] = [] + sbom_data["relationships"] = [] + # Insert pip's SBOM metadata from the wheel. discover_pip_sbom_package(sbom_data) @@ -227,9 +395,10 @@ def main() -> None: "name" not in package, "Package is missing the 'name' field" ) + missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys()) error_if( - set(package.keys()) != REQUIRED_PROPERTIES_PACKAGE, - f"Package '{package['name']}' is missing required fields", + bool(missing_required_keys), + f"Package '{package['name']}' is missing required fields: {missing_required_keys}", ) error_if( package["SPDXID"] != spdx_id(f"SPDXRef-PACKAGE-{package['name']}"), @@ -257,15 +426,11 @@ def main() -> None: f"License identifier '{license_concluded}' not in SBOM tool allowlist" ) - # Regenerate file information from current data. - sbom_files = [] - sbom_relationships = [] - # We call 'sorted()' here a lot to avoid filesystem scan order issues. for name, files in sorted(PACKAGE_TO_FILES.items()): package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}") exclude = files.exclude or () - for include in sorted(files.include): + for include in sorted(files.include or ()): # Find all the paths and then filter them through .gitignore. paths = glob.glob(include, root_dir=CPYTHON_ROOT_DIR, recursive=True) paths = filter_gitignored_paths(paths) @@ -285,7 +450,7 @@ def main() -> None: checksum_sha256 = hashlib.sha256(data).hexdigest() file_spdx_id = spdx_id(f"SPDXRef-FILE-{path}") - sbom_files.append({ + sbom_data["files"].append({ "SPDXID": file_spdx_id, "fileName": path, "checksums": [ @@ -295,15 +460,13 @@ def main() -> None: }) # Tie each file back to its respective package. - sbom_relationships.append({ + sbom_data["relationships"].append({ "spdxElementId": package_spdx_id, "relatedSpdxElement": file_spdx_id, "relationshipType": "CONTAINS", }) # Update the SBOM on disk - sbom_data["files"] = sbom_files - sbom_data["relationships"] = sbom_relationships sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True)) |