From cb1f9ff5aaa2db02d7eb26b11d8e281859ec81f9 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 31 Mar 2022 11:00:43 -0600 Subject: Experimental: build a scons-local zipapp [skip appveyor] To use, set do_zipapp to True in site_scons/scons_local_package.py Will produce a build/dist/scons-local-${VERSION}.pyz which can be directly executed - copy to a work dir, and run as python scons-local-4.3.1.pyz or even ./scons-local-4.3.1.pyz Windows should have a file association for .pyz files, otherwise it works there too by calling it as an argument of the interpreter. Signed-off-by: Mats Wichmann --- site_scons/scons_local_package.py | 97 ++++++++++++++++---------- site_scons/zip_utils.py | 141 +++++++++++++++++++++++++------------- 2 files changed, 153 insertions(+), 85 deletions(-) diff --git a/site_scons/scons_local_package.py b/site_scons/scons_local_package.py index aaf058a..0a5d008 100644 --- a/site_scons/scons_local_package.py +++ b/site_scons/scons_local_package.py @@ -23,7 +23,7 @@ from glob import glob import os.path -from zip_utils import zipit +from zip_utils import zipit, zipappit from Utilities import is_windows @@ -36,9 +36,11 @@ def get_local_package_file_list(): # import pdb; pdb.set_trace() non_test = [f for f in s_files if "Tests.py" not in f] - non_test_non_doc = [f for f in non_test if '.xml' not in f or "SCons/Tool/docbook" in f] + non_test_non_doc = [ + f for f in non_test if '.xml' not in f or "SCons/Tool/docbook" in f + ] filtered_list = [f for f in non_test_non_doc if 'pyc' not in f] - filtered_list = [f for f in filtered_list if '__pycache__' not in f ] + filtered_list = [f for f in filtered_list if '__pycache__' not in f] filtered_list = [f for f in filtered_list if not os.path.isdir(f)] return filtered_list @@ -50,47 +52,71 @@ def install_local_package_files(env): files = get_local_package_file_list() target_dir = '#/build/scons-local/scons-local-$VERSION' for f in files: - all_local_installed.extend(env.Install(os.path.join(target_dir, os.path.dirname(f)), - f)) - - basedir_files = ['scripts/scons.bat', - 'scripts/scons.py', - 'scripts/scons-configure-cache.py', - 'scripts/sconsign.py', - 'bin/scons-time.py'] + all_local_installed.extend( + env.Install(os.path.join(target_dir, os.path.dirname(f)), f) + ) + + basedir_files = [ + 'scripts/scons.bat', + 'scripts/scons.py', + 'scripts/scons-configure-cache.py', + 'scripts/sconsign.py', + 'bin/scons-time.py', + ] for bf in basedir_files: fn = os.path.basename(bf) - all_local_installed.append(env.SCons_revision('#/build/scons-local/%s'%fn, bf)) + all_local_installed.append( + env.SCons_revision(f'#/build/scons-local/{fn}', bf) + ) # Now copy manpages into scons-local package - built_manpage_files = env.Glob('build/doc/man/*.1') + built_manpage_files = env.Glob('build/doc/man/*.1') for bmp in built_manpage_files: fn = os.path.basename(str(bmp)) - all_local_installed.append(env.SCons_revision('#/build/scons-local/%s'%fn, bmp)) - - rename_files = [('scons-${VERSION}.bat', 'scripts/scons.bat'), - ('scons-README', 'README-local'), - ('scons-LICENSE', 'LICENSE-local')] + all_local_installed.append( + env.SCons_revision(f'#/build/scons-local/{fn}', bmp) + ) + + rename_files = [ + ('scons-${VERSION}.bat', 'scripts/scons.bat'), + ('scons-README', 'README-local'), + ('scons-LICENSE', 'LICENSE-local'), + ] for t, f in rename_files: - target_file = "#/build/scons-local/%s"%t + target_file = f"#/build/scons-local/{t}" all_local_installed.append(env.SCons_revision(target_file, f)) return all_local_installed def create_local_packages(env): - # Add SubstFile builder - env.Tool('textfile') [env.Tool(x) for x in ['packaging', 'filesystem', 'zip']] installed_files = install_local_package_files(env) build_local_dir = 'build/scons-local' - package = env.Command('#build/dist/scons-local-${VERSION}.zip', - installed_files, - zipit, - CD=build_local_dir, - PSV='.', - ) + package = env.Command( + '#build/dist/scons-local-${VERSION}.zip', + installed_files, + zipit, + CD=build_local_dir, + PSV='.', + ) + + + do_zipapp = False + if do_zipapp: + # We need to descend into the versioned directory for zipapp, + # but we don't know the version. env.Glob lets us expand that. + # The action isn't going to use the sources here, but including + # them makes sure the deps work out right. + app_dir = env.Glob(f"{build_local_dir}/scons-local-*")[0] + zipapp = env.Command( + target='#build/dist/scons-local-${VERSION}.pyz', + source=installed_files, + action=zipappit, + CD=app_dir, + PSV='SCons', + ) if is_windows(): # avoid problem with tar interpreting c:/ as a remote machine @@ -98,13 +124,12 @@ def create_local_packages(env): else: tar_cargs = '-czf' - env.Command('#build/dist/scons-local-${VERSION}.tar.gz', - installed_files, - "cd %s && tar %s $( ${TARGET.abspath} $) *" % (build_local_dir, tar_cargs)) - - print("Package:%s"%package) - - - - + env.Command( + '#build/dist/scons-local-${VERSION}.tar.gz', + installed_files, + "cd %s && tar %s $( ${TARGET.abspath} $) *" % (build_local_dir, tar_cargs), + ) + print(f"Package:{package}") + if do_zipapp: + print(f"Zipapp:{zipapp}") diff --git a/site_scons/zip_utils.py b/site_scons/zip_utils.py index 1a0f843..d9f6a9b 100644 --- a/site_scons/zip_utils.py +++ b/site_scons/zip_utils.py @@ -1,54 +1,97 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Actions to zip and unzip, for working with SCons release bundles +Action for creating a zipapp +""" + import os.path +import zipfile +import zipapp -zcat = 'gzip -d -c' +def zipit(env, target, source): + """ + zip *source* into *target*, using some values from *env* -# -# Figure out if we can handle .zip files. -# -zipit = None -unzipit = None -try: - import zipfile - - def zipit(env, target, source): - print("Zipping %s:" % str(target[0])) - def visit(arg, dirname, filenames): - for filename in filenames: - path = os.path.join(dirname, filename) - if os.path.isfile(path): - arg.write(path) - # default ZipFile compression is ZIP_STORED - zf = zipfile.ZipFile(str(target[0]), 'w', compression=zipfile.ZIP_DEFLATED) - olddir = os.getcwd() - os.chdir(env.Dir(env['CD']).abspath) + *env* values: `CD` is the directory to work in, + `PSV` is the directory name to walk down to find the files + """ + print(f"Zipping {target[0]}:") + + def visit(arg, dirname, filenames): + for filename in filenames: + path = os.path.join(dirname, filename) + if os.path.isfile(path): + arg.write(path) + + # default ZipFile compression is ZIP_STORED + zf = zipfile.ZipFile(str(target[0]), 'w', compression=zipfile.ZIP_DEFLATED) + olddir = os.getcwd() + os.chdir(env.Dir(env['CD']).abspath) + try: + for dirname, dirnames, filenames in os.walk(env['PSV']): + visit(zf, dirname, filenames) + finally: + os.chdir(olddir) + zf.close() + + +def unzipit(env, target, source): + print(f"Unzipping {source[0]}:") + zf = zipfile.ZipFile(str(source[0]), 'r') + for name in zf.namelist(): + dest = os.path.join(env['UNPACK_ZIP_DIR'], name) + dir = os.path.dirname(dest) try: - for dirname, dirnames, filenames in os.walk(env['PSV']): - visit(zf, dirname, filenames) - finally: - os.chdir(olddir) - zf.close() - - def unzipit(env, target, source): - print("Unzipping %s:" % str(source[0])) - zf = zipfile.ZipFile(str(source[0]), 'r') - for name in zf.namelist(): - dest = os.path.join(env['UNPACK_ZIP_DIR'], name) - dir = os.path.dirname(dest) - try: - os.makedirs(dir) - except: - pass - print(dest,name) - # if the file exists, then delete it before writing - # to it so that we don't end up trying to write to a symlink: - if os.path.isfile(dest) or os.path.islink(dest): - os.unlink(dest) - if not os.path.isdir(dest): - with open(dest, 'wb') as fp: - fp.write(zf.read(name)) - -except ImportError: - if unzip and zip: - zipit = "cd $CD && $ZIP $ZIPFLAGS $( ${TARGET.abspath} $) $PSV" - unzipit = "$UNZIP $UNZIPFLAGS $SOURCES" + os.makedirs(dir) + except: + pass + print(dest, name) + # if the file exists, then delete it before writing + # to it so that we don't end up trying to write to a symlink: + if os.path.isfile(dest) or os.path.islink(dest): + os.unlink(dest) + if not os.path.isdir(dest): + with open(dest, 'wb') as fp: + fp.write(zf.read(name)) + + +def zipappit(env, target, source): + """ + Create a zipapp *target* from specified directory. + + *env* values: ``"CD"`` is the Dir node for the place we want to work. + ``"PSV"`` is the directory name to point :meth:`zipapp.create_archive` at + (note *source* is unused here in favor of PSV) + """ + print(f"Creating zipapp {target[0]}:") + dest = target[0].abspath + olddir = os.getcwd() + #os.chdir(env.Dir(env['CD']).abspath) + os.chdir(env['CD'].abspath) + try: + zipapp.create_archive(env['PSV'], dest, "/usr/bin/env python") + finally: + os.chdir(olddir) -- cgit v0.12 From 982dbaceb447b275b56ffc2e993845592e4bc01f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 31 Mar 2022 12:00:16 -0600 Subject: Fix sider complaint Signed-off-by: Mats Wichmann --- site_scons/zip_utils.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/site_scons/zip_utils.py b/site_scons/zip_utils.py index d9f6a9b..754029c 100644 --- a/site_scons/zip_utils.py +++ b/site_scons/zip_utils.py @@ -64,10 +64,7 @@ def unzipit(env, target, source): for name in zf.namelist(): dest = os.path.join(env['UNPACK_ZIP_DIR'], name) dir = os.path.dirname(dest) - try: - os.makedirs(dir) - except: - pass + os.makedirs(dir, exist_ok=True) print(dest, name) # if the file exists, then delete it before writing # to it so that we don't end up trying to write to a symlink: @@ -89,7 +86,6 @@ def zipappit(env, target, source): print(f"Creating zipapp {target[0]}:") dest = target[0].abspath olddir = os.getcwd() - #os.chdir(env.Dir(env['CD']).abspath) os.chdir(env['CD'].abspath) try: zipapp.create_archive(env['PSV'], dest, "/usr/bin/env python") -- cgit v0.12 From 4938de49dc7681b6ff35916805d31c21035e2523 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 1 Apr 2022 10:59:00 -0600 Subject: Get the zipapp support working correctly [ci skip] Needed to pass a different source and also pass an entry point to the zipapp.create_archive method. Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 ++ RELEASE.txt | 5 +++-- site_scons/scons_local_package.py | 15 +++++++-------- site_scons/zip_utils.py | 28 ++++++++++++++++++---------- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1512700..ecc82a4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -96,6 +96,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - The single-file Util module was split into a package with a few functional areas getting their own files - Util.py had grown to over 2100 lines. + - Add a zipapp package of scons-local: can use SCons from a local + file which does not need unpacking. diff --git a/RELEASE.txt b/RELEASE.txt index ecf23c3..e9f2d02 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -90,8 +90,9 @@ PACKAGING requirements_pkg.txt are the requirements to do a full build (including docs build) with an intent to create the packages. - Moved rpm and debian directories under packaging -- Added logic to help packagers enable reproducible builds into packaging/etc/. Please - read packaging/etc/README.txt if you are interested. +- Added logic to help packagers enable reproducible builds into packaging/etc/. + Please read packaging/etc/README.txt if you are interested. +- A zipapp of scons-local is now also built. DOCUMENTATION diff --git a/site_scons/scons_local_package.py b/site_scons/scons_local_package.py index 0a5d008..8eca758 100644 --- a/site_scons/scons_local_package.py +++ b/site_scons/scons_local_package.py @@ -28,9 +28,8 @@ from Utilities import is_windows def get_local_package_file_list(): - """ - Get list of all files which should be included in scons-local package - """ + """Get list of all files which should be included in scons-local package""" + s_files = glob("SCons/**", recursive=True) # import pdb; pdb.set_trace() @@ -102,20 +101,20 @@ def create_local_packages(env): PSV='.', ) - - do_zipapp = False + do_zipapp = True # Q: maybe an external way to specify whether to build? if do_zipapp: # We need to descend into the versioned directory for zipapp, # but we don't know the version. env.Glob lets us expand that. - # The action isn't going to use the sources here, but including - # them makes sure the deps work out right. + # The action isn't going to use the sources, but including + # them makes sure SCons has populated the dir we're going to zip. app_dir = env.Glob(f"{build_local_dir}/scons-local-*")[0] zipapp = env.Command( target='#build/dist/scons-local-${VERSION}.pyz', source=installed_files, action=zipappit, CD=app_dir, - PSV='SCons', + PSV='.', + entry='SCons.Script.Main:main', ) if is_windows(): diff --git a/site_scons/zip_utils.py b/site_scons/zip_utils.py index 754029c..a38a68f 100644 --- a/site_scons/zip_utils.py +++ b/site_scons/zip_utils.py @@ -32,11 +32,11 @@ import zipapp def zipit(env, target, source): - """ - zip *source* into *target*, using some values from *env* + """Action function to zip *source* into *target* - *env* values: `CD` is the directory to work in, - `PSV` is the directory name to walk down to find the files + Values extracted from *env*: + *CD*: the directory to work in + *PSV*: the directory name to walk down to find the files """ print(f"Zipping {target[0]}:") @@ -59,6 +59,8 @@ def zipit(env, target, source): def unzipit(env, target, source): + """Action function to unzip *source*""" + print(f"Unzipping {source[0]}:") zf = zipfile.ZipFile(str(source[0]), 'r') for name in zf.namelist(): @@ -76,18 +78,24 @@ def unzipit(env, target, source): def zipappit(env, target, source): - """ - Create a zipapp *target* from specified directory. + """Action function to Create a zipapp *target* from specified directory. - *env* values: ``"CD"`` is the Dir node for the place we want to work. - ``"PSV"`` is the directory name to point :meth:`zipapp.create_archive` at - (note *source* is unused here in favor of PSV) + Values extracted from *env*: + *CD*: the Dir node for the place we want to work. + *PSV*: the directory name to point :meth:`zipapp.create_archive` at + (note *source* is unused here in favor of PSV) + *main*: the entry point for the zipapp """ print(f"Creating zipapp {target[0]}:") dest = target[0].abspath olddir = os.getcwd() os.chdir(env['CD'].abspath) try: - zipapp.create_archive(env['PSV'], dest, "/usr/bin/env python") + zipapp.create_archive( + source=env['PSV'], + target=dest, + main=env['entry'], + interpreter="/usr/bin/env python", + ) finally: os.chdir(olddir) -- cgit v0.12