From 57454e57f83b407dd2653cbfcead7c9801beeff0 Mon Sep 17 00:00:00 2001 From: Skip Montanaro Date: Fri, 14 Jun 2002 20:30:31 +0000 Subject: This introduces stricter library/header file checking for the Berkeley DB library. Since multiple versions can be installed simultaneously, it's crucial that you only select libraries and header files which are compatible with each other. Version checking is done from highest version to lowest. Building using version 1 of Berkeley DB is disabled by default because of the hash file bugs people keep rediscovering. It can be enabled by uncommenting a few lines in setup.py. Closes patch 553108. --- Misc/NEWS | 5 ++ Modules/dbmmodule.c | 3 -- README | 40 ++++++++++++++ configure | 9 ++-- configure.in | 4 +- pyconfig.h.in | 9 ---- setup.py | 152 ++++++++++++++++++++++++++++++++++++++-------------- 7 files changed, 162 insertions(+), 60 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 4a949f4..758a347 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -276,6 +276,11 @@ Build - The tools used to build the documentation now work under Cygwin as well as Unix. +- The bsddb and dbm module builds have been changed to try and avoid version + skew problems and disable linkage with Berkeley DB 1.85 unless the + installer knows what s/he's doing. See the section on building these + modules in the README file for details. + C API - Added new macro PySequence_ITEM(o, i) that directly calls diff --git a/Modules/dbmmodule.c b/Modules/dbmmodule.c index 056af49..bbc70a8 100644 --- a/Modules/dbmmodule.c +++ b/Modules/dbmmodule.c @@ -18,9 +18,6 @@ static char *which_dbm = "ndbm"; #else static char *which_dbm = "GNU gdbm"; /* EMX port of GDBM */ #endif -#elif defined(HAVE_DB1_NDBM_H) -#include -static char *which_dbm = "BSD db"; #elif defined(HAVE_GDBM_NDBM_H) #include static char *which_dbm = "GNU gdbm"; diff --git a/README b/README index 83cdf3c..0553d1c 100644 --- a/README +++ b/README @@ -503,6 +503,46 @@ Cygwin: With recent (relative to the time of writing, 2001-12-19) News regarding these platforms with more recent Cygwin versions would be appreciated! +Configuring the bsddb and dbm modules +------------------------------------- + +Configuring the bsddb module can sometimes be a bit tricky. This module +provides a Python interface to the Berkeley DB library. As of this writing +several versions of the underlying library are in common use (versions 1.85, +2.x, 3.x, and 4.x). The file formats across the various versions tend to be +incompatible. Some Linux distributions install multiple versions by +default. It is important that compatible versions of header files and +libraries are used when building bsddb. To make matters worse, version 1.85 +of Berkeley DB has known bugs in its hash file implementation, but is still +the most widely available version of the library. Many people build bsddb +with version 1.85 but aren't aware of the bugs. This affects people using +the anydbm and dbhash modules because they are both use Berkeley DB's hash +file format as a side effect of calling bsddb.hashopen. + +To try and remedy this problem, beginning with Python version 2.3 a number +of changes to the bsddb build process were made. First, and most important, +the bsddb module will not be built with version 1.85 unless the relevant +lines in setup.py are uncommented first and no other higher-numbered +versions are found. Second, matching versions of the library and include +files must be found. Third, searching is performed in order, starting from +version 4 and proceeding to version 2 (or version 1 if it is enabled). +Version-independent libraries and header files (e.g. /usr/lib/libdb.a and +/usr/include/db.h) are never considered. They must be in version-specific +directories or have version-specific filenames (e.g. /usr/lib/libdb-3.2.so +and /usr/include/db3/db_185.h). + +Since the bsddb module is programmed using the Berkeley DB version 1 API, +the underlying library must be configured with the --enable-compat185 flag. +Most vendor-provided distributions are so-configured. This is generally +only an issue if you build Berkeley DB from source. + +All this affects the dbm module as well. There are several dbm-compliant +APIs provided by different libraries, including ndbm, gdbm and Berkeley DB. +The build process for dbm would previously use the version 1.85 library, +thus extending the potential hash file corruption to the dbm module as well. +The dbm module will use the library and include files found for the bsddb +module if neither ndbm nor gdbm libraries are found. + Configuring threads ------------------- diff --git a/configure b/configure index d35b169..4c7a436 100755 --- a/configure +++ b/configure @@ -1,5 +1,5 @@ #! /bin/sh -# From configure.in Revision: 1.322 . +# From configure.in Revision: 1.323 . # Guess values for system-dependent variables and create Makefiles. # Generated by GNU Autoconf 2.53. # @@ -3865,16 +3865,13 @@ fi - - - for ac_header in dlfcn.h fcntl.h grp.h limits.h langinfo.h \ libintl.h locale.h ncurses.h poll.h pthread.h \ signal.h stdarg.h stddef.h stdlib.h thread.h unistd.h utime.h termios.h \ -sys/audioio.h sys/file.h sys/lock.h sys/mkdev.h sys/modem.h db_185.h db.h \ +sys/audioio.h sys/file.h sys/lock.h sys/mkdev.h sys/modem.h \ sys/param.h sys/poll.h sys/select.h sys/socket.h sys/time.h sys/times.h \ sys/un.h sys/utsname.h sys/wait.h pty.h term.h libutil.h \ -ndbm.h db1/ndbm.h gdbm/ndbm.h sys/resource.h netpacket/packet.h +ndbm.h gdbm/ndbm.h sys/resource.h netpacket/packet.h do as_ac_Header=`echo "ac_cv_header_$ac_header" | $as_tr_sh` if eval "test \"\${$as_ac_Header+set}\" = set"; then diff --git a/configure.in b/configure.in index 35b890d..5a53aab 100644 --- a/configure.in +++ b/configure.in @@ -596,10 +596,10 @@ AC_HEADER_STDC AC_CHECK_HEADERS(dlfcn.h fcntl.h grp.h limits.h langinfo.h \ libintl.h locale.h ncurses.h poll.h pthread.h \ signal.h stdarg.h stddef.h stdlib.h thread.h unistd.h utime.h termios.h \ -sys/audioio.h sys/file.h sys/lock.h sys/mkdev.h sys/modem.h db_185.h db.h \ +sys/audioio.h sys/file.h sys/lock.h sys/mkdev.h sys/modem.h \ sys/param.h sys/poll.h sys/select.h sys/socket.h sys/time.h sys/times.h \ sys/un.h sys/utsname.h sys/wait.h pty.h term.h libutil.h \ -ndbm.h db1/ndbm.h gdbm/ndbm.h sys/resource.h netpacket/packet.h) +ndbm.h gdbm/ndbm.h sys/resource.h netpacket/packet.h) AC_HEADER_DIRENT # checks for typedefs diff --git a/pyconfig.h.in b/pyconfig.h.in index 0606962..fd90a39 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -61,15 +61,6 @@ /* Define to 1 if you have the `ctermid_r' function. */ #undef HAVE_CTERMID_R -/* Define to 1 if you have the header file. */ -#undef HAVE_DB1_NDBM_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_DB_185_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_DB_H - /* Define to 1 if you have the header file, and it defines `DIR'. */ #undef HAVE_DIRENT_H diff --git a/setup.py b/setup.py index a204acf..5b99b05 100644 --- a/setup.py +++ b/setup.py @@ -418,14 +418,124 @@ class PyBuildExt(build_ext): # implementation independent wrapper for these; dumbdbm.py provides # similar functionality (but slower of course) implemented in Python. + # Berkeley DB interface. + # + # This requires the Berkeley DB code, see + # ftp://ftp.cs.berkeley.edu/pub/4bsd/db.1.85.tar.gz + # + # (See http://pybsddb.sourceforge.net/ for an interface to + # Berkeley DB 3.x.) + + # when sorted in reverse order, keys for this dict must appear in the + # order you wish to search - e.g., search for db3 before db2, db2 + # before db1 + db_try_this = { + 'db4': {'libs': ('db-4.3', 'db-4.2', 'db-4.1', 'db-4.0'), + 'libdirs': ('/usr/local/BerkeleyDB.4.3/lib', + '/usr/local/BerkeleyDB.4.2/lib', + '/usr/local/BerkeleyDB.4.1/lib', + '/usr/local/BerkeleyDB.4.0/lib', + '/usr/lib', + '/opt/sfw', + '/sw/lib', + '/lib', + ), + 'incdirs': ('/usr/local/BerkeleyDB.4.3/include', + '/usr/local/BerkeleyDB.4.2/include', + '/usr/local/BerkeleyDB.4.1/include', + '/usr/local/BerkeleyDB.4.0/include', + '/usr/include/db3', + '/opt/sfw/include/db3', + '/sw/include/db3', + '/usr/local/include/db3', + ), + 'incs': ('db_185.h',)}, + 'db3': {'libs': ('db-3.3', 'db-3.2', 'db-3.1', 'db-3.0'), + 'libdirs': ('/usr/local/BerkeleyDB.3.3/lib', + '/usr/local/BerkeleyDB.3.2/lib', + '/usr/local/BerkeleyDB.3.1/lib', + '/usr/local/BerkeleyDB.3.0/lib', + '/usr/lib', + '/opt/sfw', + '/sw/lib', + '/lib', + ), + 'incdirs': ('/usr/local/BerkeleyDB.3.3/include', + '/usr/local/BerkeleyDB.3.2/include', + '/usr/local/BerkeleyDB.3.1/include', + '/usr/local/BerkeleyDB.3.0/include', + '/usr/include/db3', + '/opt/sfw/include/db3', + '/sw/include/db3', + '/usr/local/include/db3', + ), + 'incs': ('db_185.h',)}, + 'db2': {'libs': ('db2',), + 'libdirs': ('/usr/lib', '/sw/lib', '/lib'), + 'incdirs': ('/usr/include/db2', + '/usr/local/include/db2', '/sw/include/db2'), + 'incs': ('db_185.h',)}, + # if you are willing to risk hash db file corruption you can + # uncomment the lines below for db1. Note that this will affect + # not only the bsddb module, but the dbhash and anydbm modules + # as well. you have been warned!!! + ##'db1': {'libs': ('db1', 'db'), + ## 'libdirs': ('/usr/lib', '/sw/lib', '/lib'), + ## 'incdirs': ('/usr/include/db1', '/usr/local/include/db1', + ## '/usr/include', '/usr/local/include'), + ## 'incs': ('db.h',)}, + } + + # override this list to affect the library version search order + # for example, if you want to force version 2 to be used: + # db_search_order = ["db2"] + db_search_order = db_try_this.keys() + db_search_order.sort() + db_search_order.reverse() + + find_lib_file = self.compiler.find_library_file + class found(Exception): pass + try: + for dbkey in db_search_order: + dbd = db_try_this[dbkey] + for dblib in dbd['libs']: + for dbinc in dbd['incs']: + db_incs = find_file(dbinc, [], dbd['incdirs']) + dblib_dir = find_lib_file(dbd['libdirs'], dblib) + if db_incs and dblib_dir: + dblib_dir = os.path.dirname(dblib_dir) + dblibs = [dblib] + raise found + except found: + if dbinc == 'db_185.h': + exts.append(Extension('bsddb', ['bsddbmodule.c'], + library_dirs=[dblib_dir], + include_dirs=db_incs, + define_macros=[('HAVE_DB_185_H',1)], + libraries=[dblib])) + else: + exts.append(Extension('bsddb', ['bsddbmodule.c'], + library_dirs=[dblib_dir], + include_dirs=db_incs, + libraries=[dblib])) + else: + db_incs = None + dblibs = [] + dblib_dir = None + # The standard Unix dbm module: if platform not in ['cygwin']: if (self.compiler.find_library_file(lib_dirs, 'ndbm')): exts.append( Extension('dbm', ['dbmmodule.c'], libraries = ['ndbm'] ) ) - elif self.compiler.find_library_file(lib_dirs, 'db1'): + elif self.compiler.find_library_file(lib_dirs, 'gdbm'): exts.append( Extension('dbm', ['dbmmodule.c'], - libraries = ['db1'] ) ) + libraries = ['gdbm'] ) ) + elif db_incs is not None: + exts.append( Extension('dbm', ['dbmmodule.c'], + library_dirs=dblib_dir, + include_dirs=db_incs, + libraries=dblibs)) else: exts.append( Extension('dbm', ['dbmmodule.c']) ) @@ -434,44 +544,6 @@ class PyBuildExt(build_ext): exts.append( Extension('gdbm', ['gdbmmodule.c'], libraries = ['gdbm'] ) ) - # Berkeley DB interface. - # - # This requires the Berkeley DB code, see - # ftp://ftp.cs.berkeley.edu/pub/4bsd/db.1.85.tar.gz - # - # Edit the variables DB and DBPORT to point to the db top directory - # and the subdirectory of PORT where you built it. - # - # (See http://pybsddb.sourceforge.net/ for an interface to - # Berkeley DB 3.x.) - - dblib = [] - if self.compiler.find_library_file(lib_dirs, 'db-3.2'): - dblib = ['db-3.2'] - elif self.compiler.find_library_file(lib_dirs, 'db-3.1'): - dblib = ['db-3.1'] - elif self.compiler.find_library_file(lib_dirs, 'db3'): - dblib = ['db3'] - elif self.compiler.find_library_file(lib_dirs, 'db2'): - dblib = ['db2'] - elif self.compiler.find_library_file(lib_dirs, 'db1'): - dblib = ['db1'] - elif self.compiler.find_library_file(lib_dirs, 'db'): - dblib = ['db'] - - db185_incs = find_file('db_185.h', inc_dirs, - ['/usr/include/db3', '/usr/include/db2']) - db_inc = find_file('db.h', inc_dirs, ['/usr/include/db1']) - if db185_incs is not None: - exts.append( Extension('bsddb', ['bsddbmodule.c'], - include_dirs = db185_incs, - define_macros=[('HAVE_DB_185_H',1)], - libraries = dblib ) ) - elif db_inc is not None: - exts.append( Extension('bsddb', ['bsddbmodule.c'], - include_dirs = db_inc, - libraries = dblib) ) - # The mpz module interfaces to the GNU Multiple Precision library. # You need to ftp the GNU MP library. # This was originally written and tested against GMP 1.2 and 1.3.2. -- cgit v0.12