diff options
| -rwxr-xr-x | Tools/freeze/freeze.py | 576 |
1 files changed, 576 insertions, 0 deletions
diff --git a/Tools/freeze/freeze.py b/Tools/freeze/freeze.py new file mode 100755 index 0000000..52285a6 --- /dev/null +++ b/Tools/freeze/freeze.py @@ -0,0 +1,576 @@ +#! /usr/local/bin/python + +# Given a Python script, create a binary that runs the script. +# The binary is 100% independent of Python libraries and binaries. +# It will not contain any Python source code -- only "compiled" Python +# (as initialized static variables containing marshalled code objects). +# It even does the right thing for dynamically loaded modules! +# The module search path of the binary is set to the current directory. +# +# Some problems remain: +# - You need to have the Python source tree lying around as well as +# the various libraries used to generate the Python binary. +# - For scripts that use many modules it generates absurdly large +# files (frozen.c and config.o as well as the final binary), +# and is consequently rather slow. +# +# Caveats: +# - The search for modules sometimes finds modules that are never +# actually imported since the code importing them is never executed. +# - If an imported module isn't found, you get a warning but the +# process of freezing continues. The binary will fail if it +# actually tries to import one of these modules. +# - This often happens with the module 'mac', which module 'os' tries +# to import (to determine whether it is running on a Macintosh). +# You can ignore the warning about this. +# - If the program dynamically reads or generates Python code and +# executes it, this code may reference built-in or library modules +# that aren't present in the frozen binary, and this will fail. +# - Your program may be using external data files, e.g. compiled +# forms definitions (*.fd). These aren't incorporated. By default, +# the sys.path in the resulting binary is only '.' (but you can override +# that with the -P option). +# +# Usage hints: +# - If you have a bunch of scripts that you want to freeze, instead +# of freezing each of them separately, you might consider writing +# a tiny main script that looks at sys.argv[0] and then imports +# the corresponding module. You can then make links to the +# frozen binary named after the various scripts you support. +# Pass the additional scripts as arguments after the main script. +# A minimal script to do this is the following. +# import sys, posixpath +# exec('import ' + posixpath.basename(sys.argv[0]) + '\n') +# +# Mods by Jack, August 94: +# - Removed all static configuration stuff. Now, Setup and Makefile files +# are parsed to obtain the linking info for the libraries. You have to +# supply the -B option, though. +# - Added -P (set sys.path) and -I/-D/-L/-l options (passed on to cc and +# ld). + +import os +import sys +import regex +import getopt +import regsub +import string +import marshal + +# Exception used when scanfile fails +NoSuchFile = 'NoSuchFile' + +# Global options +builddir = '' # -B dir +quiet = 0 # -q +verbose = 0 # -v +noexec = 0 # -n +nowrite = 0 # -N +ofile = 'a.out' # -o file +path = '\'"."\'' # -P path + +cc_options = [] # Collects cc options +ld_options = [] # Collects ld options +module_libraries = {} # ld options for each module +global_libraries = [] # Libraries we always need +include_path = '' # Include path, from Makefile +lib_path = '' # and lib path, ditto +compiler = 'cc' # and compiler + +# Main program -- argument parsing etc. +def main(): + global quiet, verbose, noexec, nowrite, ofile, builddir, path + try: + opts, args = getopt.getopt(sys.argv[1:], 'B:nNo:P:qvI:D:L:l:') + except getopt.error, msg: + usage(str(msg)) + sys.exit(2) + for o, a in opts: + if o == '-B': builddir = a + if o == '-n': noexec = 1 + if o == '-N': nowrite = 1 + if o == '-o': ofile = a + if o == '-P': + if '"' in a: + usage('sorry, cannot have " in -P option') + sys.exit(2) + path = `'"' + a + '"'` + if o == '-q': verbose = 0; quiet = 1 + if o == '-v': verbose = verbose + 1; quiet = 0 + if o in ('-I', '-D'): cc_options.append(o+a) + if o in ('-L', '-l'): ld_options.append(o+a) + if not builddir: + usage('sorry, you have to pass a -B option') + sys.exit(2) + if len(args) < 1: + usage('please pass at least one file argument') + sys.exit(2) + process(args[0], args[1:]) + +# Print usage message to stderr +def usage(*msgs): + sys.stdout = sys.stderr + for msg in msgs: print msg + print 'Usage: freeze [options] scriptfile [modulefile ...]' + print '-B dir : name of python build dir (no default)' + print '-n : generate the files but don\'t compile and link' + print '-N : don\'t write frozen.c (do compile unless -n given)' + print '-o file : binary output file (default a.out)' + print '-P path : set sys.path for program (default ".")' + print '-q : quiet (no messages at all except errors)' + print '-v : verbose (lots of extra messages)' + print '-D and -I options are passed to cc, -L and -l to ld' + +# Process the script file +def process(filename, addmodules): + global noexec + # + if not quiet: print 'Computing needed modules ...' + todo = {} + todo['__main__'] = filename + for name in addmodules: + mod = os.path.basename(name) + if mod[-3:] == '.py': mod = mod[:-3] + todo[mod] = name + try: + dict = closure(todo) + except NoSuchFile, filename: + sys.stderr.write('Can\'t open file %s\n' % filename) + sys.exit(1) + # + mods = dict.keys() + mods.sort() + # + if verbose: + print '%-15s %s' % ('Module', 'Filename') + for mod in mods: + print '%-15s %s' % (`mod`, dict[mod]) + # + if not quiet: print 'Looking for dynamically linked modules ...' + dlmodules = [] + objs = [] + libs = [] + for mod in mods: + if dict[mod][-2:] == '.o': + if verbose: print 'Found', mod, dict[mod] + dlmodules.append(mod) + objs.append(dict[mod]) + libsname = dict[mod][:-2] + '.libs' + try: + f = open(libsname, 'r') + except IOError: + f = None + if f: + libtext = f.read() + f.close() + for lib in string.split(libtext): + if lib in libs: libs.remove(lib) + libs.append(lib) + # + if not nowrite: + if not quiet: print 'Writing frozen.c ...' + writefrozen('frozen.c', dict) + else: + if not quiet: print 'NOT writing frozen.c ...' + # + if not quiet: + print 'Deducing compile/link options from', builddir + # + # Parse the config info + # + parse(builddir) + CONFIG_IN = lib_path + '/config.c.in' + FMAIN = lib_path + '/frozenmain.c' + CC = compiler + # +## if not dlmodules: + if 0: + config = CONFIG + if not quiet: print 'Using existing', config, '...' + else: + config = 'tmpconfig.c' + if nowrite: + if not quiet: print 'NOT writing config.c ...' + else: + if not quiet: + print 'Writing config.c with dl modules ...' + f = open(CONFIG_IN, 'r') + g = open(config, 'w') + m1 = regex.compile('-- ADDMODULE MARKER 1 --') + m2 = regex.compile('-- ADDMODULE MARKER 2 --') + builtinmodules = [] + stdmodules = ('sys', '__main__', '__builtin__', + 'marshal') + for mod in dict.keys(): + if dict[mod] == '<builtin>' and \ + mod not in stdmodules: + builtinmodules.append(mod) + todomodules = builtinmodules + dlmodules + while 1: + line = f.readline() + if not line: break + g.write(line) + if m1.search(line) >= 0: + if verbose: print 'Marker 1 ...' + for mod in todomodules: + g.write('extern void init' + \ + mod + '();\n') + if m2.search(line) >= 0: + if verbose: print 'Marker 2 ...' + for mod in todomodules: + g.write('{"' + mod + \ + '", init' + mod + '},\n') + g.close() + # + if not quiet: + if noexec: print 'Generating compilation commands ...' + else: print 'Starting compilation ...' + defs = ['-DNO_MAIN', '-DUSE_FROZEN'] + defs.append('-DPYTHONPATH='+path) + # + incs = ['-I.', '-I' + include_path] +# if dict.has_key('stdwin'): +# incs.append('-I' + j(STDWIN, 'H')) + # + srcs = [config, FMAIN] + # + modlibs = module_libraries + + for mod in dict.keys(): + if modlibs.has_key(mod): + libs = libs + modlibs[mod] + + libs = libs + global_libraries + # + # remove dups: + # XXXX Not well tested... + nskip = 0 + newlibs = [] + while libs: + l = libs[0] + del libs[0] + if l[:2] == '-L' and l in newlibs: + nskip = nskip + 1 + continue + if (l[:2] == '-l' or l[-2:] == '.a') and l in libs: + nskip = nskip + 1 + continue + newlibs.append(l) + libs = newlibs + if nskip and not quiet: + print 'Removed %d duplicate libraries'%nskip + # + sts = 0 + # + cmd = CC + ' -c' + if cc_options: + cmd = cmd + ' ' + string.join(cc_options) + cmd = cmd + ' ' + string.join(defs) + cmd = cmd + ' ' + string.join(incs) + cmd = cmd + ' ' + string.join(srcs) + print cmd + # + if not noexec: + sts = os.system(cmd) + if sts: + print 'Exit status', sts, '-- turning on -n' + noexec = 1 + # + for s in srcs: + s = os.path.basename(s) + if s[-2:] == '.c': s = s[:-2] + o = s + '.o' + objs.insert(0, o) + # + cmd = CC + cmd = cmd + ' ' + string.join(objs) + cmd = cmd + ' ' + string.join(libs) + if ld_options: + cmd = cmd + ' ' + string.join(ld_options) + cmd = cmd + ' -o ' + ofile + print cmd + # + if not noexec: + sts = os.system(cmd) + if sts: + print 'Exit status', sts + else: + print 'Done.' + # + if not quiet and not noexec and sts == 0: + print 'Note: consider this:'; print '\tstrip', ofile + # + sys.exit(sts) + + +# Generate code for a given module +def makecode(filename): + if filename[-2:] == '.o': + return None + try: + f = open(filename, 'r') + except IOError: + return None + if verbose: print 'Making code from', filename, '...' + text = f.read() + code = compile(text, filename, 'exec') + f.close() + return marshal.dumps(code) + + +# Write the C source file containing the frozen Python code +def writefrozen(filename, dict): + f = open(filename, 'w') + codelist = [] + for mod in dict.keys(): + codestring = makecode(dict[mod]) + if codestring is not None: + codelist.append((mod, codestring)) + write = sys.stdout.write + save_stdout = sys.stdout + try: + sys.stdout = f + for mod, codestring in codelist: + if verbose: + write('Writing initializer for %s\n'%mod) + print 'static unsigned char M_' + mod + '[' + \ + str(len(codestring)) + '+1] = {' + for i in range(0, len(codestring), 16): + for c in codestring[i:i+16]: + print str(ord(c)) + ',', + print + print '};' + print 'struct frozen {' + print ' char *name;' + print ' unsigned char *code;' + print ' int size;' + print '} frozen_modules[] = {' + for mod, codestring in codelist: + print ' {"' + mod + '",', + print 'M_' + mod + ',', + print str(len(codestring)) + '},' + print ' {0, 0, 0} /* sentinel */' + print '};' + finally: + sys.stdout = save_stdout + f.close() + + +# Determine the names and filenames of the modules imported by the +# script, recursively. This is done by scanning for lines containing +# import statements. (The scanning has only superficial knowledge of +# Python syntax and no knowledge of semantics, so in theory the result +# may be incorrect -- however this is quite unlikely if you don't +# intentionally obscure your Python code.) + +# Compute the closure of scanfile() -- special first file because of script +def closure(todo): + done = {} + while todo: + newtodo = {} + for modname in todo.keys(): + if not done.has_key(modname): + filename = todo[modname] + if filename is None: + filename = findmodule(modname) + done[modname] = filename + if filename in ('<builtin>', '<unknown>'): + continue + modules = scanfile(filename) + for m in modules: + if not done.has_key(m): + newtodo[m] = None + todo = newtodo + return done + +# Scan a file looking for import statements +importstr = '\(^\|:\)[ \t]*import[ \t]+\([a-zA-Z0-9_, \t]+\)' +fromstr = '\(^\|:\)[ \t]*from[ \t]+\([a-zA-Z0-9_]+\)[ \t]+import[ \t]+' +isimport = regex.compile(importstr) +isfrom = regex.compile(fromstr) +def scanfile(filename): + allmodules = {} + try: + f = open(filename, 'r') + except IOError, msg: + raise NoSuchFile, filename + while 1: + line = f.readline() + if not line: break # EOF + while line[-2:] == '\\\n': # Continuation line + line = line[:-2] + ' ' + line = line + f.readline() + if isimport.search(line) >= 0: + rawmodules = isimport.group(2) + modules = string.splitfields(rawmodules, ',') + for i in range(len(modules)): + modules[i] = string.strip(modules[i]) + elif isfrom.search(line) >= 0: + modules = [isfrom.group(2)] + else: + continue + for mod in modules: + allmodules[mod] = None + f.close() + return allmodules.keys() + +# Find the file containing a module, given its name; None if not found +builtins = sys.builtin_module_names + ['sys'] +def findmodule(modname): + if modname in builtins: return '<builtin>' + for dirname in sys.path: + dlfullname = os.path.join(dirname, modname + 'module.o') + try: + f = open(dlfullname, 'r') + except IOError: + f = None + if f: + f.close() + return dlfullname + fullname = os.path.join(dirname, modname + '.py') + try: + f = open(fullname, 'r') + except IOError: + continue + f.close() + return fullname + if not quiet: + sys.stderr.write('Warning: module %s not found\n' % modname) + return '<unknown>' +# +# Parse a setup file. Returns two dictionaries, one containing variables +# defined with their values and one containing module definitions +# +def parse_setup(fp): + modules = {} + variables = {} + for line in fp.readlines(): + if '#' in line: # Strip comments + line = string.splitfields(line, '#')[0] + line = string.strip(line[:-1]) # Strip whitespace + if not line: + continue + words = string.split(line) + if '=' in words[0]: + # + # equal sign before first space. Definition + # + pos = string.index(line, '=') + name = line[:pos] + value = string.strip(line[pos+1:]) + variables[name] = value + else: + modules[words[0]] = words[1:] + return modules, variables +# +# Parse a makefile. Returns a list of the variables defined. +# +def parse_makefile(fp): + variables = {} + for line in fp.readlines(): + if '#' in line: # Strip comments + line = string.splitfields(line, '#')[0] + if not line: + continue + if line[0] in string.whitespace: + continue + line = string.strip(line[:-1]) # Strip whitespace + if not line: + continue + if '=' in string.splitfields(line, ':')[0]: + # + # equal sign before first colon. Definition + # + pos = string.index(line, '=') + name = line[:pos] + value = string.strip(line[pos+1:]) + variables[name] = value + return variables + +# +# Recursively add loader options from Setup files in extension +# directories. +# +def add_extension_directory(name, isinstalldir): + if verbose: + print 'Adding extension directory', name + fp = open(name + '/Setup', 'r') + modules, variables = parse_setup(fp) + # + # Locate all new modules and remember the ld flags needed for them + # + for m in modules.keys(): + if module_libraries.has_key(m): + continue + options = modules[m] + if isinstalldir: + ld_options = [] + else: + ld_options = [name + '/lib.a'] + for o in options: + # ld options are all capital except DUIC and l + if o[:-2] == '.a': + ld_options.append(o) + elif o[0] == '-': + if o[1] == 'l': + ld_options.append(o) + elif o[1] in string.uppercase and not o[1] in 'DUIC': + ld_options.append(o) + module_libraries[m] = ld_options + # + # See if we have to bother with base setups + # + if variables.has_key('BASESETUP'): + if isinstalldir: + raise 'installdir has base setup' + setupfiles = string.split(variables['BASESETUP']) + for s in setupfiles: + if s[-6:] <> '/Setup': + raise 'Incorrect BASESETUP', s + s = s[:-6] + if s[0] <> '/': + s = name + '/' + s + s = os.path.normpath(s) + add_extension_directory(s, 0) +# +# Main routine for this module: given a build directory, get all +# information needed for the linker. +# +def parse(dir): + global include_path + global lib_path + global compiler + + fp = open(dir + '/Makefile', 'r') + # + # First find the global libraries and the base python + # + vars = parse_makefile(fp) + if vars.has_key('CC'): + compiler = vars['CC'] + if not vars.has_key('installdir'): + raise 'No $installdir in Makefile' + include_path = vars['installdir'] + '/include/Py' + lib_path = vars['installdir'] + '/lib/python/lib' + global_libraries.append('-L' + lib_path) + global_libraries.append('-lPython') + global_libraries.append('-lParser') + global_libraries.append('-lObjects') + global_libraries.append('-lModules') + for name in ('LIBS', 'LIBM', 'LIBC'): + if not vars.has_key(name): + raise 'Missing required def in Makefile', name + for lib in string.split(vars[name]): + global_libraries.append(lib) + # + # Next, parse the modules from the base python + # + add_extension_directory(lib_path, 1) + # + # Finally, parse the modules from the extension python + # + if dir <> lib_path: + add_extension_directory(dir, 0) + +# Call the main program +main() |