diff options
25 files changed, 1213 insertions, 714 deletions
diff --git a/Lib/bsddb/__init__.py b/Lib/bsddb/__init__.py index 7eb625c..44cfdfb 100644 --- a/Lib/bsddb/__init__.py +++ b/Lib/bsddb/__init__.py @@ -33,10 +33,10 @@ #---------------------------------------------------------------------- -"""Support for BerkeleyDB 3.3 through 4.4 with a simple interface. +"""Support for Berkeley DB 3.3 through 4.6 with a simple interface. For the full featured object oriented interface use the bsddb.db module -instead. It mirrors the Sleepycat BerkeleyDB C API. +instead. It mirrors the Oracle Berkeley DB C API. """ try: @@ -188,7 +188,7 @@ class _DBWithCursor(_iter_mixin): self.saved_dbc_key = None # This method is needed for all non-cursor DB calls to avoid - # BerkeleyDB deadlocks (due to being opened with DB_INIT_LOCK + # Berkeley DB deadlocks (due to being opened with DB_INIT_LOCK # and DB_THREAD to be thread safe) when intermixing database # operations that use the cursor internally with those that don't. def _closeCursors(self, save=1): @@ -372,7 +372,7 @@ def _checkflag(flag, file): elif flag == 'n': flags = db.DB_CREATE #flags = db.DB_CREATE | db.DB_TRUNCATE - # we used db.DB_TRUNCATE flag for this before but BerkeleyDB + # we used db.DB_TRUNCATE flag for this before but Berkeley DB # 4.2.52 changed to disallowed truncate with txn environments. if file is not None and os.path.isfile(file): os.unlink(file) @@ -385,10 +385,10 @@ def _checkflag(flag, file): # This is a silly little hack that allows apps to continue to use the # DB_THREAD flag even on systems without threads without freaking out -# BerkeleyDB. +# Berkeley DB. # # This assumes that if Python was built with thread support then -# BerkeleyDB was too. +# Berkeley DB was too. try: import thread diff --git a/Lib/bsddb/dbshelve.py b/Lib/bsddb/dbshelve.py index 96f604a..6d7414e 100644 --- a/Lib/bsddb/dbshelve.py +++ b/Lib/bsddb/dbshelve.py @@ -37,9 +37,18 @@ import sys #DictMixin was added if sys.version_info[:3] >= (2, 3, 0): HIGHEST_PROTOCOL = cPickle.HIGHEST_PROTOCOL - def _dumps(object, protocol): - return cPickle.dumps(object, protocol=protocol) +# In python 2.3.*, "cPickle.dumps" accepts no +# named parameters. "pickle.dumps" accepts them, +# so this seems a bug. + if sys.version_info[:3] < (2, 4, 0): + def _dumps(object, protocol): + return cPickle.dumps(object, protocol) + else : + def _dumps(object, protocol): + return cPickle.dumps(object, protocol=protocol) + from UserDict import DictMixin + else: HIGHEST_PROTOCOL = None def _dumps(object, protocol): @@ -133,7 +142,7 @@ class DBShelf(DictMixin): def keys(self, txn=None): - if txn is not None: + if txn != None: return self.db.keys(txn) else: return self.db.keys() @@ -157,7 +166,7 @@ class DBShelf(DictMixin): def items(self, txn=None): - if txn is not None: + if txn != None: items = self.db.items(txn) else: items = self.db.items() @@ -168,7 +177,7 @@ class DBShelf(DictMixin): return newitems def values(self, txn=None): - if txn is not None: + if txn != None: values = self.db.values(txn) else: values = self.db.values() diff --git a/Lib/bsddb/test/test_1413192.py b/Lib/bsddb/test/test_1413192.py deleted file mode 100644 index c5d0036..0000000 --- a/Lib/bsddb/test/test_1413192.py +++ /dev/null @@ -1,48 +0,0 @@ -# http://bugs.python.org/issue1413192 -# -# See the bug report for details. -# The problem was that the env was deallocated prior to the txn. - -import shutil -import tempfile -from test.test_support import catch_warning -import warnings - -try: - # For Pythons w/distutils and add-on pybsddb - from bsddb3 import db -except ImportError: - # For Python >= 2.3 builtin bsddb distribution - from bsddb import db - -env_name = tempfile.mkdtemp() - -# Wrap test operation in a class so we can control destruction rather than -# waiting for the controlling Python executable to exit - -class Context: - - def __init__(self): - self.env = db.DBEnv() - self.env.open(env_name, - db.DB_CREATE | db.DB_INIT_TXN | db.DB_INIT_MPOOL) - self.the_txn = self.env.txn_begin() - - self.map = db.DB(self.env) - self.map.open('xxx.db', "p", - db.DB_HASH, db.DB_CREATE, 0666, txn=self.the_txn) - del self.env - del self.the_txn - - -with catch_warning(): - warnings.filterwarnings('ignore', 'DBTxn aborted in destructor') - context = Context() - del context - - -# try not to leave a turd -try: - shutil.rmtree(env_name) -except EnvironmentError: - pass diff --git a/Lib/bsddb/test/test_all.py b/Lib/bsddb/test/test_all.py index ad8b1e9..580bc7f 100644 --- a/Lib/bsddb/test/test_all.py +++ b/Lib/bsddb/test/test_all.py @@ -11,6 +11,11 @@ except ImportError: # For Python 2.3 from bsddb import db +try: + from bsddb3 import test_support +except ImportError: + from test import test_support + verbose = 0 if 'verbose' in sys.argv: verbose = 1 @@ -33,6 +38,53 @@ def print_versions(): print '-=' * 38 +def get_new_path(name) : + get_new_path.mutex.acquire() + try : + import os + path=os.path.join(get_new_path.prefix, + name+"_"+str(os.getpid())+"_"+str(get_new_path.num)) + get_new_path.num+=1 + finally : + get_new_path.mutex.release() + return path + +def get_new_environment_path() : + path=get_new_path("environment") + import os + try: + os.makedirs(path,mode=0700) + except os.error: + test_support.rmtree(path) + os.makedirs(path) + return path + +def get_new_database_path() : + path=get_new_path("database") + import os + if os.path.exists(path) : + os.remove(path) + return path + + +get_new_path.prefix="/tmp/z-Berkeley_DB" +get_new_path.num=0 + +try : + import threading + get_new_path.mutex=threading.Lock() + del threading +except ImportError: + class Lock(object) : + def acquire(self) : + pass + def release(self) : + pass + get_new_path.mutex=Lock() + del Lock + + + class PrintInfoFakeTest(unittest.TestCase): def testPrintVersions(self): print_versions() @@ -60,7 +112,9 @@ def suite(): 'test_dbobj', 'test_dbshelve', 'test_dbtables', - 'test_env_close', + 'test_early_close', + 'test_distributed_transactions', + 'test_replication', 'test_get_none', 'test_join', 'test_lock', diff --git a/Lib/bsddb/test/test_associate.py b/Lib/bsddb/test/test_associate.py index 80b1a8e..10907e3 100644 --- a/Lib/bsddb/test/test_associate.py +++ b/Lib/bsddb/test/test_associate.py @@ -3,7 +3,6 @@ TestCases for DB.associate. """ import sys, os, string -import tempfile import time from pprint import pprint @@ -14,7 +13,7 @@ except ImportError: have_threads = 0 import unittest -from test_all import verbose +from test_all import verbose, get_new_environment_path try: # For Pythons w/distutils pybsddb @@ -96,17 +95,9 @@ musicdata = { class AssociateErrorTestCase(unittest.TestCase): def setUp(self): self.filename = self.__class__.__name__ + '.db' - homeDir = os.path.join(tempfile.gettempdir(), 'db_home%d'%os.getpid()) - self.homeDir = homeDir - try: - os.mkdir(homeDir) - except os.error: - import glob - files = glob.glob(os.path.join(self.homeDir, '*')) - for file in files: - os.remove(file) + self.homeDir = get_new_environment_path() self.env = db.DBEnv() - self.env.open(homeDir, db.DB_CREATE | db.DB_INIT_MPOOL) + self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL) def tearDown(self): self.env.close() @@ -127,7 +118,7 @@ class AssociateErrorTestCase(unittest.TestCase): secDB.open(self.filename, "secondary", db.DB_BTREE, db.DB_CREATE) # dupDB has been configured to allow duplicates, it can't - # associate with a secondary. BerkeleyDB will return an error. + # associate with a secondary. Berkeley DB will return an error. try: def f(a,b): return a+b dupDB.associate(secDB, f) @@ -152,27 +143,16 @@ class AssociateTestCase(unittest.TestCase): def setUp(self): self.filename = self.__class__.__name__ + '.db' - homeDir = os.path.join(tempfile.gettempdir(), 'db_home%d'%os.getpid()) - self.homeDir = homeDir - try: - os.mkdir(homeDir) - except os.error: - import glob - files = glob.glob(os.path.join(self.homeDir, '*')) - for file in files: - os.remove(file) + self.homeDir = get_new_environment_path() self.env = db.DBEnv() - self.env.open(homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | + self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_THREAD | self.envFlags) def tearDown(self): self.closeDB() self.env.close() self.env = None - import glob - files = glob.glob(os.path.join(self.homeDir, '*')) - for file in files: - os.remove(file) + test_support.rmtree(self.homeDir) def addDataToDB(self, d, txn=None): for key, value in musicdata.items(): @@ -249,10 +229,10 @@ class AssociateTestCase(unittest.TestCase): def finish_test(self, secDB, txn=None): # 'Blues' should not be in the secondary database vals = secDB.pget('Blues', txn=txn) - assert vals == None, vals + self.assertEqual(vals, None, vals) vals = secDB.pget('Unknown', txn=txn) - assert vals[0] == 99 or vals[0] == '99', vals + self.assert_(vals[0] == 99 or vals[0] == '99', vals) vals[1].index('Unknown') vals[1].index('Unnamed') vals[1].index('unknown') @@ -264,14 +244,14 @@ class AssociateTestCase(unittest.TestCase): rec = self.cur.first() while rec is not None: if type(self.keytype) == type(''): - assert string.atoi(rec[0]) # for primary db, key is a number + self.assert_(string.atoi(rec[0])) # for primary db, key is a number else: - assert rec[0] and type(rec[0]) == type(0) + self.assert_(rec[0] and type(rec[0]) == type(0)) count = count + 1 if verbose: print rec rec = self.cur.next() - assert count == len(musicdata) # all items accounted for + self.assertEqual(count, len(musicdata)) # all items accounted for if verbose: @@ -281,29 +261,29 @@ class AssociateTestCase(unittest.TestCase): # test cursor pget vals = self.cur.pget('Unknown', flags=db.DB_LAST) - assert vals[1] == 99 or vals[1] == '99', vals - assert vals[0] == 'Unknown' + self.assert_(vals[1] == 99 or vals[1] == '99', vals) + self.assertEqual(vals[0], 'Unknown') vals[2].index('Unknown') vals[2].index('Unnamed') vals[2].index('unknown') vals = self.cur.pget('Unknown', data='wrong value', flags=db.DB_GET_BOTH) - assert vals == None, vals + self.assertEqual(vals, None, vals) rec = self.cur.first() - assert rec[0] == "Jazz" + self.assertEqual(rec[0], "Jazz") while rec is not None: count = count + 1 if verbose: print rec rec = self.cur.next() # all items accounted for EXCEPT for 1 with "Blues" genre - assert count == len(musicdata)-1 + self.assertEqual(count, len(musicdata)-1) self.cur = None def getGenre(self, priKey, priData): - assert type(priData) == type("") + self.assertEqual(type(priData), type("")) if verbose: print 'getGenre key: %r data: %r' % (priKey, priData) genre = string.split(priData, '|')[2] @@ -388,7 +368,7 @@ class ShelveAssociateTestCase(AssociateTestCase): def getGenre(self, priKey, priData): - assert type(priData) == type(()) + self.assertEqual(type(priData), type(())) if verbose: print 'getGenre key: %r data: %r' % (priKey, priData) genre = priData[2] @@ -419,6 +399,8 @@ class ThreadedAssociateTestCase(AssociateTestCase): t2 = Thread(target = self.writer2, args = (d, )) + t1.setDaemon(True) + t2.setDaemon(True) t1.start() t2.start() t1.join() diff --git a/Lib/bsddb/test/test_basics.py b/Lib/bsddb/test/test_basics.py index 9b38d57..a234d2a 100644 --- a/Lib/bsddb/test/test_basics.py +++ b/Lib/bsddb/test/test_basics.py @@ -6,7 +6,6 @@ various DB flags, etc. import os import errno import string -import tempfile from pprint import pprint import unittest import time @@ -23,7 +22,7 @@ try: except ImportError: from test import test_support -from test_all import verbose +from test_all import verbose, get_new_environment_path, get_new_database_path DASH = '-' @@ -38,8 +37,8 @@ class VersionTestCase(unittest.TestCase): print 'bsddb.db.version(): %s' % (info, ) print db.DB_VERSION_STRING print '-=' * 20 - assert info == (db.DB_VERSION_MAJOR, db.DB_VERSION_MINOR, - db.DB_VERSION_PATCH) + self.assertEqual(info, (db.DB_VERSION_MAJOR, db.DB_VERSION_MINOR, + db.DB_VERSION_PATCH)) #---------------------------------------------------------------------- @@ -57,27 +56,22 @@ class BasicTestCase(unittest.TestCase): def setUp(self): if self.useEnv: - homeDir = os.path.join(tempfile.gettempdir(), 'db_home%d'%os.getpid()) - self.homeDir = homeDir - test_support.rmtree(homeDir) - os.mkdir(homeDir) + self.homeDir=get_new_environment_path() try: self.env = db.DBEnv() self.env.set_lg_max(1024*1024) self.env.set_tx_max(30) self.env.set_tx_timestamp(int(time.time())) self.env.set_flags(self.envsetflags, 1) - self.env.open(homeDir, self.envflags | db.DB_CREATE) - tempfile.tempdir = homeDir - self.filename = os.path.split(tempfile.mktemp())[1] - tempfile.tempdir = None + self.env.open(self.homeDir, self.envflags | db.DB_CREATE) + self.filename = "test" # Yes, a bare except is intended, since we're re-raising the exc. except: - test_support.rmtree(homeDir) + test_support.rmtree(self.homeDir) raise else: self.env = None - self.filename = tempfile.mktemp() + self.filename = get_new_database_path() # create and open the DB self.d = db.DB(self.env) @@ -99,13 +93,6 @@ class BasicTestCase(unittest.TestCase): if self.env is not None: self.env.close() test_support.rmtree(self.homeDir) - ## XXX(nnorwitz): is this comment stil valid? - ## Make a new DBEnv to remove the env files from the home dir. - ## (It can't be done while the env is open, nor after it has been - ## closed, so we make a new one to do it.) - #e = db.DBEnv() - #e.remove(self.homeDir) - #os.remove(os.path.join(self.homeDir, self.filename)) else: os.remove(self.filename) @@ -153,44 +140,44 @@ class BasicTestCase(unittest.TestCase): if verbose: print data - assert d.get('0321') == '0321-0321-0321-0321-0321' + self.assertEqual(d.get('0321'), '0321-0321-0321-0321-0321') # By default non-existant keys return None... - assert d.get('abcd') == None + self.assertEqual(d.get('abcd'), None) # ...but they raise exceptions in other situations. Call # set_get_returns_none() to change it. try: d.delete('abcd') except db.DBNotFoundError, val: - assert val[0] == db.DB_NOTFOUND + self.assertEqual(val[0], db.DB_NOTFOUND) if verbose: print val else: self.fail("expected exception") d.put('abcd', 'a new record') - assert d.get('abcd') == 'a new record' + self.assertEqual(d.get('abcd'), 'a new record') d.put('abcd', 'same key') if self.dbsetflags & db.DB_DUP: - assert d.get('abcd') == 'a new record' + self.assertEqual(d.get('abcd'), 'a new record') else: - assert d.get('abcd') == 'same key' + self.assertEqual(d.get('abcd'), 'same key') try: d.put('abcd', 'this should fail', flags=db.DB_NOOVERWRITE) except db.DBKeyExistError, val: - assert val[0] == db.DB_KEYEXIST + self.assertEqual(val[0], db.DB_KEYEXIST) if verbose: print val else: self.fail("expected exception") if self.dbsetflags & db.DB_DUP: - assert d.get('abcd') == 'a new record' + self.assertEqual(d.get('abcd'), 'a new record') else: - assert d.get('abcd') == 'same key' + self.assertEqual(d.get('abcd'), 'same key') d.sync() @@ -204,28 +191,28 @@ class BasicTestCase(unittest.TestCase): self.d.open(self.filename) d = self.d - assert d.get('0321') == '0321-0321-0321-0321-0321' + self.assertEqual(d.get('0321'), '0321-0321-0321-0321-0321') if self.dbsetflags & db.DB_DUP: - assert d.get('abcd') == 'a new record' + self.assertEqual(d.get('abcd'), 'a new record') else: - assert d.get('abcd') == 'same key' + self.assertEqual(d.get('abcd'), 'same key') rec = d.get_both('0555', '0555-0555-0555-0555-0555') if verbose: print rec - assert d.get_both('0555', 'bad data') == None + self.assertEqual(d.get_both('0555', 'bad data'), None) # test default value data = d.get('bad key', 'bad data') - assert data == 'bad data' + self.assertEqual(data, 'bad data') # any object can pass through data = d.get('bad key', self) - assert data == self + self.assertEqual(data, self) s = d.stat() - assert type(s) == type({}) + self.assertEqual(type(s), type({})) if verbose: print 'd.stat() returned this dictionary:' pprint(s) @@ -243,47 +230,47 @@ class BasicTestCase(unittest.TestCase): for key in ['0002', '0101', '0401', '0701', '0998']: data = d[key] - assert data == self.makeData(key) + self.assertEqual(data, self.makeData(key)) if verbose: print data - assert len(d) == self._numKeys + self.assertEqual(len(d), self._numKeys) keys = d.keys() - assert len(keys) == self._numKeys - assert type(keys) == type([]) + self.assertEqual(len(keys), self._numKeys) + self.assertEqual(type(keys), type([])) d['new record'] = 'a new record' - assert len(d) == self._numKeys+1 + self.assertEqual(len(d), self._numKeys+1) keys = d.keys() - assert len(keys) == self._numKeys+1 + self.assertEqual(len(keys), self._numKeys+1) d['new record'] = 'a replacement record' - assert len(d) == self._numKeys+1 + self.assertEqual(len(d), self._numKeys+1) keys = d.keys() - assert len(keys) == self._numKeys+1 + self.assertEqual(len(keys), self._numKeys+1) if verbose: print "the first 10 keys are:" pprint(keys[:10]) - assert d['new record'] == 'a replacement record' + self.assertEqual(d['new record'], 'a replacement record') - assert d.has_key('0001') == 1 - assert d.has_key('spam') == 0 + self.assertEqual(d.has_key('0001'), 1) + self.assertEqual(d.has_key('spam'), 0) items = d.items() - assert len(items) == self._numKeys+1 - assert type(items) == type([]) - assert type(items[0]) == type(()) - assert len(items[0]) == 2 + self.assertEqual(len(items), self._numKeys+1) + self.assertEqual(type(items), type([])) + self.assertEqual(type(items[0]), type(())) + self.assertEqual(len(items[0]), 2) if verbose: print "the first 10 items are:" pprint(items[:10]) values = d.values() - assert len(values) == self._numKeys+1 - assert type(values) == type([]) + self.assertEqual(len(values), self._numKeys+1) + self.assertEqual(type(values), type([])) if verbose: print "the first 10 values are:" @@ -315,14 +302,15 @@ class BasicTestCase(unittest.TestCase): rec = c.next() except db.DBNotFoundError, val: if get_raises_error: - assert val[0] == db.DB_NOTFOUND + self.assertEqual(val[0], db.DB_NOTFOUND) if verbose: print val rec = None else: self.fail("unexpected DBNotFoundError") - assert c.get_current_size() == len(c.current()[1]), "%s != len(%r)" % (c.get_current_size(), c.current()[1]) + self.assertEqual(c.get_current_size(), len(c.current()[1]), + "%s != len(%r)" % (c.get_current_size(), c.current()[1])) - assert count == self._numKeys + self.assertEqual(count, self._numKeys) rec = c.last() @@ -335,49 +323,49 @@ class BasicTestCase(unittest.TestCase): rec = c.prev() except db.DBNotFoundError, val: if get_raises_error: - assert val[0] == db.DB_NOTFOUND + self.assertEqual(val[0], db.DB_NOTFOUND) if verbose: print val rec = None else: self.fail("unexpected DBNotFoundError") - assert count == self._numKeys + self.assertEqual(count, self._numKeys) rec = c.set('0505') rec2 = c.current() - assert rec == rec2 - assert rec[0] == '0505' - assert rec[1] == self.makeData('0505') - assert c.get_current_size() == len(rec[1]) + self.assertEqual(rec, rec2) + self.assertEqual(rec[0], '0505') + self.assertEqual(rec[1], self.makeData('0505')) + self.assertEqual(c.get_current_size(), len(rec[1])) # make sure we get empty values properly rec = c.set('empty value') - assert rec[1] == '' - assert c.get_current_size() == 0 + self.assertEqual(rec[1], '') + self.assertEqual(c.get_current_size(), 0) try: n = c.set('bad key') except db.DBNotFoundError, val: - assert val[0] == db.DB_NOTFOUND + self.assertEqual(val[0], db.DB_NOTFOUND) if verbose: print val else: if set_raises_error: self.fail("expected exception") - if n is not None: + if n != None: self.fail("expected None: %r" % (n,)) rec = c.get_both('0404', self.makeData('0404')) - assert rec == ('0404', self.makeData('0404')) + self.assertEqual(rec, ('0404', self.makeData('0404'))) try: n = c.get_both('0404', 'bad data') except db.DBNotFoundError, val: - assert val[0] == db.DB_NOTFOUND + self.assertEqual(val[0], db.DB_NOTFOUND) if verbose: print val else: if get_raises_error: self.fail("expected exception") - if n is not None: + if n != None: self.fail("expected None: %r" % (n,)) if self.d.get_type() == db.DB_BTREE: @@ -401,7 +389,7 @@ class BasicTestCase(unittest.TestCase): rec = c.current() except db.DBKeyEmptyError, val: if get_raises_error: - assert val[0] == db.DB_KEYEMPTY + self.assertEqual(val[0], db.DB_KEYEMPTY) if verbose: print val else: self.fail("unexpected DBKeyEmptyError") @@ -411,14 +399,14 @@ class BasicTestCase(unittest.TestCase): c.next() c2 = c.dup(db.DB_POSITION) - assert c.current() == c2.current() + self.assertEqual(c.current(), c2.current()) c2.put('', 'a new value', db.DB_CURRENT) - assert c.current() == c2.current() - assert c.current()[1] == 'a new value' + self.assertEqual(c.current(), c2.current()) + self.assertEqual(c.current()[1], 'a new value') c2.put('', 'er', db.DB_CURRENT, dlen=0, doff=5) - assert c2.current()[1] == 'a newer value' + self.assertEqual(c2.current()[1], 'a newer value') c.close() c2.close() @@ -446,7 +434,7 @@ class BasicTestCase(unittest.TestCase): # a bug may cause a NULL pointer dereference... apply(getattr(c, method), args) except db.DBError, val: - assert val[0] == 0 + self.assertEqual(val[0], 0) if verbose: print val else: self.fail("no exception raised when using a buggy cursor's" @@ -471,7 +459,7 @@ class BasicTestCase(unittest.TestCase): self.__class__.__name__ old = self.d.set_get_returns_none(0) - assert old == 2 + self.assertEqual(old, 2) self.test03_SimpleCursorStuff(get_raises_error=1, set_raises_error=1) def test03b_SimpleCursorWithGetReturnsNone1(self): @@ -493,9 +481,9 @@ class BasicTestCase(unittest.TestCase): self.__class__.__name__ old = self.d.set_get_returns_none(1) - assert old == 2 + self.assertEqual(old, 2) old = self.d.set_get_returns_none(2) - assert old == 1 + self.assertEqual(old, 1) self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=0) #---------------------------------------- @@ -510,23 +498,24 @@ class BasicTestCase(unittest.TestCase): key = "partialTest" data = "1" * 1000 + "2" * 1000 d.put(key, data) - assert d.get(key) == data - assert d.get(key, dlen=20, doff=990) == ("1" * 10) + ("2" * 10) + self.assertEqual(d.get(key), data) + self.assertEqual(d.get(key, dlen=20, doff=990), + ("1" * 10) + ("2" * 10)) d.put("partialtest2", ("1" * 30000) + "robin" ) - assert d.get("partialtest2", dlen=5, doff=30000) == "robin" + self.assertEqual(d.get("partialtest2", dlen=5, doff=30000), "robin") # There seems to be a bug in DB here... Commented out the test for # now. - ##assert d.get("partialtest2", dlen=5, doff=30010) == "" + ##self.assertEqual(d.get("partialtest2", dlen=5, doff=30010), "") if self.dbsetflags != db.DB_DUP: # Partial put with duplicate records requires a cursor d.put(key, "0000", dlen=2000, doff=0) - assert d.get(key) == "0000" + self.assertEqual(d.get(key), "0000") d.put(key, "1111", dlen=1, doff=2) - assert d.get(key) == "0011110" + self.assertEqual(d.get(key), "0011110") #---------------------------------------- @@ -541,14 +530,14 @@ class BasicTestCase(unittest.TestCase): #print "before ", i, d.put(key, "1" * i) #print "after", - assert d.get_size(key) == i + self.assertEqual(d.get_size(key), i) #print "done" #---------------------------------------- def test06_Truncate(self): if db.version() < (3,3): - # truncate is a feature of BerkeleyDB 3.3 and above + # truncate is a feature of Berkeley DB 3.3 and above return d = self.d @@ -558,9 +547,10 @@ class BasicTestCase(unittest.TestCase): d.put("abcde", "ABCDE"); num = d.truncate() - assert num >= 1, "truncate returned <= 0 on non-empty database" + self.assert_(num >= 1, "truncate returned <= 0 on non-empty database") num = d.truncate() - assert num == 0, "truncate on empty DB returned nonzero (%r)" % (num,) + self.assertEqual(num, 0, + "truncate on empty DB returned nonzero (%r)" % (num,)) #---------------------------------------- @@ -625,6 +615,11 @@ class BasicHashWithEnvTestCase(BasicWithEnvTestCase): #---------------------------------------------------------------------- class BasicTransactionTestCase(BasicTestCase): + import sys + if sys.version_info[:3] < (2, 4, 0): + def assertTrue(self, expr, msg=None): + self.failUnless(expr,msg=msg) + dbopenflags = db.DB_THREAD | db.DB_AUTO_COMMIT useEnv = 1 envflags = (db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | @@ -650,19 +645,21 @@ class BasicTransactionTestCase(BasicTestCase): print '\n', '-=' * 30 print "Running %s.test06_Transactions..." % self.__class__.__name__ - assert d.get('new rec', txn=self.txn) == None + self.assertEqual(d.get('new rec', txn=self.txn), None) d.put('new rec', 'this is a new record', self.txn) - assert d.get('new rec', txn=self.txn) == 'this is a new record' + self.assertEqual(d.get('new rec', txn=self.txn), + 'this is a new record') self.txn.abort() - assert d.get('new rec') == None + self.assertEqual(d.get('new rec'), None) self.txn = self.env.txn_begin() - assert d.get('new rec', txn=self.txn) == None + self.assertEqual(d.get('new rec', txn=self.txn), None) d.put('new rec', 'this is a new record', self.txn) - assert d.get('new rec', txn=self.txn) == 'this is a new record' + self.assertEqual(d.get('new rec', txn=self.txn), + 'this is a new record') self.txn.commit() - assert d.get('new rec') == 'this is a new record' + self.assertEqual(d.get('new rec'), 'this is a new record') self.txn = self.env.txn_begin() c = d.cursor(self.txn) @@ -673,7 +670,7 @@ class BasicTransactionTestCase(BasicTestCase): if verbose and count % 100 == 0: print rec rec = c.next() - assert count == self._numKeys+1 + self.assertEqual(count, self._numKeys+1) c.close() # Cursors *MUST* be closed before commit! self.txn.commit() @@ -686,20 +683,20 @@ class BasicTransactionTestCase(BasicTestCase): if db.version() >= (4,0): statDict = self.env.log_stat(0); - assert statDict.has_key('magic') - assert statDict.has_key('version') - assert statDict.has_key('cur_file') - assert statDict.has_key('region_nowait') + self.assert_(statDict.has_key('magic')) + self.assert_(statDict.has_key('version')) + self.assert_(statDict.has_key('cur_file')) + self.assert_(statDict.has_key('region_nowait')) # must have at least one log file present: logs = self.env.log_archive(db.DB_ARCH_ABS | db.DB_ARCH_LOG) - assert logs != None + self.assertNotEqual(logs, None) for log in logs: if verbose: print 'log file: ' + log if db.version() >= (4,2): logs = self.env.log_archive(db.DB_ARCH_REMOVE) - assert not logs + self.assertTrue(not logs) self.txn = self.env.txn_begin() @@ -707,7 +704,7 @@ class BasicTransactionTestCase(BasicTestCase): def test07_TxnTruncate(self): if db.version() < (3,3): - # truncate is a feature of BerkeleyDB 3.3 and above + # truncate is a feature of Berkeley DB 3.3 and above return d = self.d @@ -718,9 +715,10 @@ class BasicTransactionTestCase(BasicTestCase): d.put("abcde", "ABCDE"); txn = self.env.txn_begin() num = d.truncate(txn) - assert num >= 1, "truncate returned <= 0 on non-empty database" + self.assert_(num >= 1, "truncate returned <= 0 on non-empty database") num = d.truncate(txn) - assert num == 0, "truncate on empty DB returned nonzero (%r)" % (num,) + self.assertEqual(num, 0, + "truncate on empty DB returned nonzero (%r)" % (num,)) txn.commit() #---------------------------------------- @@ -766,20 +764,20 @@ class BTreeRecnoTestCase(BasicTestCase): print "Running %s.test07_RecnoInBTree..." % self.__class__.__name__ rec = d.get(200) - assert type(rec) == type(()) - assert len(rec) == 2 + self.assertEqual(type(rec), type(())) + self.assertEqual(len(rec), 2) if verbose: print "Record #200 is ", rec c = d.cursor() c.set('0200') num = c.get_recno() - assert type(num) == type(1) + self.assertEqual(type(num), type(1)) if verbose: print "recno of d['0200'] is ", num rec = c.current() - assert c.set_recno(num) == rec + self.assertEqual(c.set_recno(num), rec) c.close() @@ -806,23 +804,23 @@ class BasicDUPTestCase(BasicTestCase): d.put("dup2", "after") data = d.get("dup1") - assert data == "The" + self.assertEqual(data, "The") if verbose: print data c = d.cursor() rec = c.set("dup1") - assert rec == ('dup1', 'The') + self.assertEqual(rec, ('dup1', 'The')) next = c.next() - assert next == ('dup1', 'quick') + self.assertEqual(next, ('dup1', 'quick')) rec = c.set("dup1") count = c.count() - assert count == 9 + self.assertEqual(count, 9) next_dup = c.next_dup() - assert next_dup == ('dup1', 'quick') + self.assertEqual(next_dup, ('dup1', 'quick')) rec = c.set('dup1') while rec is not None: @@ -832,7 +830,7 @@ class BasicDUPTestCase(BasicTestCase): c.set('dup1') rec = c.next_nodup() - assert rec[0] != 'dup1' + self.assertNotEqual(rec[0], 'dup1') if verbose: print rec @@ -912,7 +910,7 @@ class BasicMultiDBTestCase(BasicTestCase): if verbose and (count % 50) == 0: print rec rec = c1.next() - assert count == self._numKeys + self.assertEqual(count, self._numKeys) count = 0 rec = c2.first() @@ -921,7 +919,7 @@ class BasicMultiDBTestCase(BasicTestCase): if verbose: print rec rec = c2.next() - assert count == 9 + self.assertEqual(count, 9) count = 0 rec = c3.first() @@ -930,7 +928,7 @@ class BasicMultiDBTestCase(BasicTestCase): if verbose: print rec rec = c3.next() - assert count == 52 + self.assertEqual(count, 52) c1.close() diff --git a/Lib/bsddb/test/test_compare.py b/Lib/bsddb/test/test_compare.py index ac9ee6b..940e56e 100644 --- a/Lib/bsddb/test/test_compare.py +++ b/Lib/bsddb/test/test_compare.py @@ -4,7 +4,6 @@ TestCases for python DB Btree key comparison function. import sys, os, re import test_all -import tempfile from cStringIO import StringIO import unittest @@ -15,6 +14,8 @@ except ImportError: # For Python 2.3 from bsddb import db, dbshelve +from test_all import get_new_environment_path, get_new_database_path + try: from bsddb3 import test_support except ImportError: @@ -57,23 +58,17 @@ class AbstractBtreeKeyCompareTestCase (unittest.TestCase): def setUp (self): self.filename = self.__class__.__name__ + '.db' - homeDir = os.path.join (tempfile.gettempdir(), 'db_home%d'%os.getpid()) - self.homeDir = homeDir - try: - os.mkdir (homeDir) - except os.error: - pass - - env = db.DBEnv () - env.open (homeDir, + self.homeDir = get_new_environment_path() + env = db.DBEnv() + env.open (self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_THREAD) self.env = env def tearDown (self): - self.closeDB () + self.closeDB() if self.env is not None: - self.env.close () + self.env.close() self.env = None test_support.rmtree(self.homeDir) @@ -236,7 +231,7 @@ class BtreeExceptionsTestCase (AbstractBtreeKeyCompareTestCase): self.createDB (my_compare) try: self.db.set_bt_compare (my_compare) - assert False, "this set should fail" + self.assert_(0, "this set should fail") except RuntimeError, msg: pass diff --git a/Lib/bsddb/test/test_compat.py b/Lib/bsddb/test/test_compat.py index bc165cf..f5c7873 100644 --- a/Lib/bsddb/test/test_compat.py +++ b/Lib/bsddb/test/test_compat.py @@ -5,9 +5,9 @@ regression test suite. import os, string import unittest -import tempfile -from test_all import verbose +from test_all import verbose, get_new_database_path + try: # For Pythons w/distutils pybsddb @@ -16,10 +16,9 @@ except ImportError: # For Python 2.3 from bsddb import db, hashopen, btopen, rnopen - class CompatibilityTestCase(unittest.TestCase): def setUp(self): - self.filename = tempfile.mktemp() + self.filename = get_new_database_path() def tearDown(self): try: @@ -47,7 +46,7 @@ class CompatibilityTestCase(unittest.TestCase): if verbose: print '%s %s %s' % getTest - assert getTest[1] == 'quick', 'data mismatch!' + self.assertEqual(getTest[1], 'quick', 'data mismatch!') rv = f.set_location(3) if rv != (3, 'brown'): @@ -120,13 +119,13 @@ class CompatibilityTestCase(unittest.TestCase): try: rec = f.next() except KeyError: - assert rec == f.last(), 'Error, last <> last!' + self.assertEqual(rec, f.last(), 'Error, last <> last!') f.previous() break if verbose: print rec - assert f.has_key('f'), 'Error, missing key!' + self.assert_(f.has_key('f'), 'Error, missing key!') # test that set_location() returns the next nearest key, value # on btree databases and raises KeyError on others. diff --git a/Lib/bsddb/test/test_cursor_pget_bug.py b/Lib/bsddb/test/test_cursor_pget_bug.py index 4868562..b1cfc79 100644 --- a/Lib/bsddb/test/test_cursor_pget_bug.py +++ b/Lib/bsddb/test/test_cursor_pget_bug.py @@ -1,5 +1,4 @@ import unittest -import tempfile import os, glob try: @@ -9,12 +8,13 @@ except ImportError: # For Python 2.3 from bsddb import db +from test_all import get_new_environment_path, get_new_database_path + try: from bsddb3 import test_support except ImportError: from test import test_support - #---------------------------------------------------------------------- class pget_bugTestCase(unittest.TestCase): @@ -22,11 +22,7 @@ class pget_bugTestCase(unittest.TestCase): db_name = 'test-cursor_pget.db' def setUp(self): - self.homeDir = os.path.join(tempfile.gettempdir(), 'db_home%d'%os.getpid()) - try: - os.mkdir(self.homeDir) - except os.error: - pass + self.homeDir = get_new_environment_path() self.env = db.DBEnv() self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL) self.primary_db = db.DB(self.env) diff --git a/Lib/bsddb/test/test_dbobj.py b/Lib/bsddb/test/test_dbobj.py index c15face..2af88ba 100644 --- a/Lib/bsddb/test/test_dbobj.py +++ b/Lib/bsddb/test/test_dbobj.py @@ -1,7 +1,6 @@ import os, string import unittest -import tempfile try: # For Pythons w/distutils pybsddb @@ -10,24 +9,21 @@ except ImportError: # For Python 2.3 from bsddb import db, dbobj +from test_all import get_new_environment_path, get_new_database_path + try: from bsddb3 import test_support except ImportError: from test import test_support - #---------------------------------------------------------------------- class dbobjTestCase(unittest.TestCase): """Verify that dbobj.DB and dbobj.DBEnv work properly""" - db_home = 'db_home' db_name = 'test-dbobj.db' def setUp(self): - homeDir = os.path.join(tempfile.gettempdir(), 'db_home%d'%os.getpid()) - self.homeDir = homeDir - try: os.mkdir(homeDir) - except os.error: pass + self.homeDir = get_new_environment_path() def tearDown(self): if hasattr(self, 'db'): @@ -48,10 +44,10 @@ class dbobjTestCase(unittest.TestCase): self.db = TestDB(self.env) self.db.open(self.db_name, db.DB_HASH, db.DB_CREATE) self.db.put('spam', 'eggs') - assert self.db.get('spam') == None, \ - "overridden dbobj.DB.put() method failed [1]" - assert self.db.get('SPAM') == 'eggs', \ - "overridden dbobj.DB.put() method failed [2]" + self.assertEqual(self.db.get('spam'), None, + "overridden dbobj.DB.put() method failed [1]") + self.assertEqual(self.db.get('SPAM'), 'eggs', + "overridden dbobj.DB.put() method failed [2]") self.db.close() self.env.close() @@ -63,12 +59,12 @@ class dbobjTestCase(unittest.TestCase): # __setitem__ self.db['spam'] = 'eggs' # __len__ - assert len(self.db) == 1 + self.assertEqual(len(self.db), 1) # __getitem__ - assert self.db['spam'] == 'eggs' + self.assertEqual(self.db['spam'], 'eggs') # __del__ del self.db['spam'] - assert self.db.get('spam') == None, "dbobj __del__ failed" + self.assertEqual(self.db.get('spam'), None, "dbobj __del__ failed") self.db.close() self.env.close() diff --git a/Lib/bsddb/test/test_dbshelve.py b/Lib/bsddb/test/test_dbshelve.py index 752ebc3..cbc58ca 100644 --- a/Lib/bsddb/test/test_dbshelve.py +++ b/Lib/bsddb/test/test_dbshelve.py @@ -3,7 +3,7 @@ TestCases for checking dbShelve objects. """ import os, string -import tempfile, random +import random from types import * import unittest @@ -19,7 +19,8 @@ try: except ImportError: from test import test_support -from test_all import verbose +from test_all import verbose, get_new_environment_path, get_new_database_path + #---------------------------------------------------------------------- @@ -35,7 +36,7 @@ class DataClass: class DBShelveTestCase(unittest.TestCase): def setUp(self): - self.filename = tempfile.mktemp() + self.filename = get_new_database_path() self.do_open() def tearDown(self): @@ -91,15 +92,15 @@ class DBShelveTestCase(unittest.TestCase): print "keys:", k print "stats:", s - assert 0 == d.has_key(self.mk('bad key')) - assert 1 == d.has_key(self.mk('IA')) - assert 1 == d.has_key(self.mk('OA')) + self.assertEqual(0, d.has_key(self.mk('bad key'))) + self.assertEqual(1, d.has_key(self.mk('IA'))) + self.assertEqual(1, d.has_key(self.mk('OA'))) d.delete(self.mk('IA')) del d[self.mk('OA')] - assert 0 == d.has_key(self.mk('IA')) - assert 0 == d.has_key(self.mk('OA')) - assert len(d) == l-2 + self.assertEqual(0, d.has_key(self.mk('IA'))) + self.assertEqual(0, d.has_key(self.mk('OA'))) + self.assertEqual(len(d), l-2) values = [] for key in d.keys(): @@ -110,29 +111,29 @@ class DBShelveTestCase(unittest.TestCase): self.checkrec(key, value) dbvalues = d.values() - assert len(dbvalues) == len(d.keys()) + self.assertEqual(len(dbvalues), len(d.keys())) values.sort() dbvalues.sort() - assert values == dbvalues + self.assertEqual(values, dbvalues) items = d.items() - assert len(items) == len(values) + self.assertEqual(len(items), len(values)) for key, value in items: self.checkrec(key, value) - assert d.get(self.mk('bad key')) == None - assert d.get(self.mk('bad key'), None) == None - assert d.get(self.mk('bad key'), 'a string') == 'a string' - assert d.get(self.mk('bad key'), [1, 2, 3]) == [1, 2, 3] + self.assertEqual(d.get(self.mk('bad key')), None) + self.assertEqual(d.get(self.mk('bad key'), None), None) + self.assertEqual(d.get(self.mk('bad key'), 'a string'), 'a string') + self.assertEqual(d.get(self.mk('bad key'), [1, 2, 3]), [1, 2, 3]) d.set_get_returns_none(0) self.assertRaises(db.DBNotFoundError, d.get, self.mk('bad key')) d.set_get_returns_none(1) d.put(self.mk('new key'), 'new data') - assert d.get(self.mk('new key')) == 'new data' - assert d[self.mk('new key')] == 'new data' + self.assertEqual(d.get(self.mk('new key')), 'new data') + self.assertEqual(d[self.mk('new key')], 'new data') @@ -156,7 +157,7 @@ class DBShelveTestCase(unittest.TestCase): rec = c.next() del c - assert count == len(d) + self.assertEqual(count, len(d)) count = 0 c = d.cursor() @@ -169,7 +170,7 @@ class DBShelveTestCase(unittest.TestCase): self.checkrec(key, value) rec = c.prev() - assert count == len(d) + self.assertEqual(count, len(d)) c.set(self.mk('SS')) key, value = c.current() @@ -191,25 +192,25 @@ class DBShelveTestCase(unittest.TestCase): # override this in a subclass if the key type is different x = key[1] if key[0] == 'S': - assert type(value) == StringType - assert value == 10 * x + self.assertEqual(type(value), StringType) + self.assertEqual(value, 10 * x) elif key[0] == 'I': - assert type(value) == IntType - assert value == ord(x) + self.assertEqual(type(value), IntType) + self.assertEqual(value, ord(x)) elif key[0] == 'L': - assert type(value) == ListType - assert value == [x] * 10 + self.assertEqual(type(value), ListType) + self.assertEqual(value, [x] * 10) elif key[0] == 'O': - assert type(value) == InstanceType - assert value.S == 10 * x - assert value.I == ord(x) - assert value.L == [x] * 10 + self.assertEqual(type(value), InstanceType) + self.assertEqual(value.S, 10 * x) + self.assertEqual(value.I, ord(x)) + self.assertEqual(value.L, [x] * 10) else: - raise AssertionError, 'Unknown key type, fix the test' + self.assert_(0, 'Unknown key type, fix the test') #---------------------------------------------------------------------- @@ -246,12 +247,9 @@ class ThreadHashShelveTestCase(BasicShelveTestCase): class BasicEnvShelveTestCase(DBShelveTestCase): def do_open(self): - self.homeDir = homeDir = os.path.join( - tempfile.gettempdir(), 'db_home%d'%os.getpid()) - try: os.mkdir(homeDir) - except os.error: pass self.env = db.DBEnv() - self.env.open(homeDir, self.envflags | db.DB_INIT_MPOOL | db.DB_CREATE) + self.env.open(self.homeDir, + self.envflags | db.DB_INIT_MPOOL | db.DB_CREATE) self.filename = os.path.split(self.filename)[1] self.d = dbshelve.DBShelf(self.env) @@ -263,6 +261,10 @@ class BasicEnvShelveTestCase(DBShelveTestCase): self.env.close() + def setUp(self) : + self.homeDir = get_new_environment_path() + DBShelveTestCase.setUp(self) + def tearDown(self): self.do_close() test_support.rmtree(self.homeDir) diff --git a/Lib/bsddb/test/test_dbtables.py b/Lib/bsddb/test/test_dbtables.py index faddd11..d750111 100644 --- a/Lib/bsddb/test/test_dbtables.py +++ b/Lib/bsddb/test/test_dbtables.py @@ -21,16 +21,14 @@ # $Id$ import os, re -import tempfile try: import cPickle pickle = cPickle except ImportError: import pickle -import tempfile import unittest -from test_all import verbose +from test_all import verbose, get_new_environment_path, get_new_database_path try: # For Pythons w/distutils pybsddb @@ -48,16 +46,12 @@ except ImportError: #---------------------------------------------------------------------- class TableDBTestCase(unittest.TestCase): - db_home = 'db_home' db_name = 'test-table.db' def setUp(self): - homeDir = tempfile.mkdtemp() - self.testHomeDir = homeDir - try: os.mkdir(homeDir) - except os.error: pass + self.testHomeDir = get_new_environment_path() self.tdb = dbtables.bsdTableDB( - filename='tabletest.db', dbhome=homeDir, create=1) + filename='tabletest.db', dbhome=self.testHomeDir, create=1) def tearDown(self): self.tdb.close() @@ -323,7 +317,7 @@ class TableDBTestCase(unittest.TestCase): self.tdb.Insert(tabname, {'Type': 'Unknown', 'Access': '0'}) def set_type(type): - if type is None: + if type == None: return 'MP3' return type diff --git a/Lib/bsddb/test/test_distributed_transactions.py b/Lib/bsddb/test/test_distributed_transactions.py new file mode 100644 index 0000000..82f36e9 --- /dev/null +++ b/Lib/bsddb/test/test_distributed_transactions.py @@ -0,0 +1,170 @@ +"""TestCases for distributed transactions. +""" + +import os +import unittest + +try: + # For Pythons w/distutils pybsddb + from bsddb3 import db +except ImportError: + # For Python 2.3 + from bsddb import db + +from test_all import get_new_environment_path, get_new_database_path + +try: + from bsddb3 import test_support +except ImportError: + from test import test_support + +try : + a=set() +except : # Python 2.3 + from sets import Set as set +else : + del a + +from test_all import verbose + +#---------------------------------------------------------------------- + +class DBTxn_distributed(unittest.TestCase): + num_txns=1234 + nosync=True + must_open_db=False + def _create_env(self, must_open_db) : + self.dbenv = db.DBEnv() + self.dbenv.set_tx_max(self.num_txns) + self.dbenv.set_lk_max_lockers(self.num_txns*2) + self.dbenv.set_lk_max_locks(self.num_txns*2) + self.dbenv.set_lk_max_objects(self.num_txns*2) + if self.nosync : + self.dbenv.set_flags(db.DB_TXN_NOSYNC,True) + self.dbenv.open(self.homeDir, db.DB_CREATE | db.DB_THREAD | + db.DB_RECOVER | + db.DB_INIT_TXN | db.DB_INIT_LOG | db.DB_INIT_MPOOL | + db.DB_INIT_LOCK, 0666) + self.db = db.DB(self.dbenv) + self.db.set_re_len(db.DB_XIDDATASIZE) + if must_open_db : + if db.version() > (4,1) : + txn=self.dbenv.txn_begin() + self.db.open(self.filename, + db.DB_QUEUE, db.DB_CREATE | db.DB_THREAD, 0666, + txn=txn) + txn.commit() + else : + self.db.open(self.filename, + db.DB_QUEUE, db.DB_CREATE | db.DB_THREAD, 0666) + + def setUp(self) : + self.homeDir = get_new_environment_path() + self.filename = "test" + return self._create_env(must_open_db=True) + + def _destroy_env(self): + if self.nosync or (db.version()[:2] == (4,6)): # Known bug + self.dbenv.log_flush() + self.db.close() + self.dbenv.close() + + def tearDown(self): + self._destroy_env() + test_support.rmtree(self.homeDir) + + def _recreate_env(self,must_open_db) : + self._destroy_env() + self._create_env(must_open_db) + + def test01_distributed_transactions(self) : + txns=set() + # Create transactions, "prepare" them, and + # let them be garbage collected. + for i in xrange(self.num_txns) : + txn=self.dbenv.txn_begin() + gid="%%%dd" %db.DB_XIDDATASIZE + gid=gid %i + self.db.put(i, gid, txn=txn, flags=db.DB_APPEND) + txns.add(gid) + txn.prepare(gid) + del txn + + self._recreate_env(self.must_open_db) + + # Get "to be recovered" transactions but + # let them be garbage collected. + recovered_txns=self.dbenv.txn_recover() + self.assertEquals(self.num_txns,len(recovered_txns)) + for gid,txn in recovered_txns : + self.assert_(gid in txns) + del txn + del recovered_txns + + self._recreate_env(self.must_open_db) + + # Get "to be recovered" transactions. Commit, abort and + # discard them. + recovered_txns=self.dbenv.txn_recover() + self.assertEquals(self.num_txns,len(recovered_txns)) + discard_txns=set() + committed_txns=set() + state=0 + for gid,txn in recovered_txns : + if state==0 or state==1: + committed_txns.add(gid) + txn.commit() + elif state==2 : + txn.abort() + elif state==3 : + txn.discard() + discard_txns.add(gid) + state=-1 + state+=1 + del txn + del recovered_txns + + self._recreate_env(self.must_open_db) + + # Verify the discarded transactions are still + # around, and dispose them. + recovered_txns=self.dbenv.txn_recover() + self.assertEquals(len(discard_txns),len(recovered_txns)) + for gid,txn in recovered_txns : + txn.abort() + del txn + del recovered_txns + + self._recreate_env(must_open_db=True) + + # Be sure there are not pending transactions. + # Check also database size. + recovered_txns=self.dbenv.txn_recover() + self.assert_(len(recovered_txns)==0) + self.assertEquals(len(committed_txns),self.db.stat()["nkeys"]) + +class DBTxn_distributedSYNC(DBTxn_distributed): + nosync=False + +class DBTxn_distributed_must_open_db(DBTxn_distributed): + must_open_db=True + +class DBTxn_distributedSYNC_must_open_db(DBTxn_distributed): + nosync=False + must_open_db=True + +#---------------------------------------------------------------------- + +def test_suite(): + suite = unittest.TestSuite() + if db.version() >= (4,5) : + suite.addTest(unittest.makeSuite(DBTxn_distributed)) + suite.addTest(unittest.makeSuite(DBTxn_distributedSYNC)) + if db.version() >= (4,6) : + suite.addTest(unittest.makeSuite(DBTxn_distributed_must_open_db)) + suite.addTest(unittest.makeSuite(DBTxn_distributedSYNC_must_open_db)) + return suite + + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/bsddb/test/test_early_close.py b/Lib/bsddb/test/test_early_close.py new file mode 100644 index 0000000..076b868 --- /dev/null +++ b/Lib/bsddb/test/test_early_close.py @@ -0,0 +1,207 @@ +"""TestCases for checking that it does not segfault when a DBEnv object +is closed before its DB objects. +""" + +import os +import unittest + +try: + # For Pythons w/distutils pybsddb + from bsddb3 import db +except ImportError: + # For Python 2.3 + from bsddb import db + +try: + from bsddb3 import test_support +except ImportError: + from test import test_support + +from test_all import verbose, get_new_environment_path, get_new_database_path + +# We're going to get warnings in this module about trying to close the db when +# its env is already closed. Let's just ignore those. +try: + import warnings +except ImportError: + pass +else: + warnings.filterwarnings('ignore', + message='DB could not be closed in', + category=RuntimeWarning) + + +#---------------------------------------------------------------------- + +class DBEnvClosedEarlyCrash(unittest.TestCase): + def setUp(self): + self.homeDir = get_new_environment_path() + self.filename = "test" + + def tearDown(self): + test_support.rmtree(self.homeDir) + + def test01_close_dbenv_before_db(self): + dbenv = db.DBEnv() + dbenv.open(self.homeDir, + db.DB_INIT_CDB| db.DB_CREATE |db.DB_THREAD|db.DB_INIT_MPOOL, + 0666) + + d = db.DB(dbenv) + d2 = db.DB(dbenv) + d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) + + self.assertRaises(db.DBNoSuchFileError, d2.open, + self.filename+"2", db.DB_BTREE, db.DB_THREAD, 0666) + + d.put("test","this is a test") + self.assertEqual(d.get("test"), "this is a test", "put!=get") + dbenv.close() # This "close" should close the child db handle also + self.assertRaises(db.DBError, d.get, "test") + + def test02_close_dbenv_before_dbcursor(self): + dbenv = db.DBEnv() + dbenv.open(self.homeDir, + db.DB_INIT_CDB| db.DB_CREATE |db.DB_THREAD|db.DB_INIT_MPOOL, + 0666) + + d = db.DB(dbenv) + d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) + + d.put("test","this is a test") + d.put("test2","another test") + d.put("test3","another one") + self.assertEqual(d.get("test"), "this is a test", "put!=get") + c=d.cursor() + c.first() + c.next() + d.close() # This "close" should close the child db handle also + # db.close should close the child cursor + self.assertRaises(db.DBError,c.next) + + d = db.DB(dbenv) + d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) + c=d.cursor() + c.first() + c.next() + dbenv.close() + # The "close" should close the child db handle also, with cursors + self.assertRaises(db.DBError, c.next) + + def test03_close_db_before_dbcursor_without_env(self): + import os.path + path=os.path.join(self.homeDir,self.filename) + d = db.DB() + d.open(path, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) + + d.put("test","this is a test") + d.put("test2","another test") + d.put("test3","another one") + self.assertEqual(d.get("test"), "this is a test", "put!=get") + c=d.cursor() + c.first() + c.next() + d.close() + # The "close" should close the child db handle also + self.assertRaises(db.DBError, c.next) + + def test04_close_massive(self): + dbenv = db.DBEnv() + dbenv.open(self.homeDir, + db.DB_INIT_CDB| db.DB_CREATE |db.DB_THREAD|db.DB_INIT_MPOOL, + 0666) + + dbs=[db.DB(dbenv) for i in xrange(16)] + cursors=[] + for i in dbs : + i.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) + + dbs[10].put("test","this is a test") + dbs[10].put("test2","another test") + dbs[10].put("test3","another one") + self.assertEqual(dbs[4].get("test"), "this is a test", "put!=get") + + for i in dbs : + cursors.extend([i.cursor() for j in xrange(32)]) + + for i in dbs[::3] : + i.close() + for i in cursors[::3] : + i.close() + + # Check for missing exception in DB! (after DB close) + self.assertRaises(db.DBError, dbs[9].get, "test") + + # Check for missing exception in DBCursor! (after DB close) + self.assertRaises(db.DBError, cursors[101].first) + + cursors[80].first() + cursors[80].next() + dbenv.close() # This "close" should close the child db handle also + # Check for missing exception! (after DBEnv close) + self.assertRaises(db.DBError, cursors[80].next) + + def test05_close_dbenv_delete_db_success(self): + dbenv = db.DBEnv() + dbenv.open(self.homeDir, + db.DB_INIT_CDB| db.DB_CREATE |db.DB_THREAD|db.DB_INIT_MPOOL, + 0666) + + d = db.DB(dbenv) + d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) + + dbenv.close() # This "close" should close the child db handle also + + del d + try: + import gc + except ImportError: + gc = None + if gc: + # force d.__del__ [DB_dealloc] to be called + gc.collect() + + def test06_close_txn_before_dup_cursor(self) : + dbenv = db.DBEnv() + dbenv.open(self.homeDir,db.DB_INIT_TXN | db.DB_INIT_MPOOL | + db.DB_INIT_LOG | db.DB_CREATE) + d = db.DB(dbenv) + txn = dbenv.txn_begin() + if db.version() < (4,1) : + d.open(self.filename, dbtype = db.DB_HASH, flags = db.DB_CREATE) + else : + d.open(self.filename, dbtype = db.DB_HASH, flags = db.DB_CREATE, + txn=txn) + d.put("XXX", "yyy", txn=txn) + txn.commit() + txn = dbenv.txn_begin() + c1 = d.cursor(txn) + c2 = c1.dup() + self.assertEquals(("XXX", "yyy"), c1.first()) + import warnings + # Not interested in warnings about implicit close. + warnings.simplefilter("ignore") + txn.commit() + warnings.resetwarnings() + self.assertRaises(db.DBCursorClosedError, c2.first) + + if db.version() > (4,3,0) : + def test07_close_db_before_sequence(self): + import os.path + path=os.path.join(self.homeDir,self.filename) + d = db.DB() + d.open(path, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) + dbs=db.DBSequence(d) + d.close() # This "close" should close the child DBSequence also + dbs.close() # If not closed, core dump (in Berkeley DB 4.6.*) + +#---------------------------------------------------------------------- + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(DBEnvClosedEarlyCrash)) + return suite + + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/bsddb/test/test_env_close.py b/Lib/bsddb/test/test_env_close.py deleted file mode 100644 index e07e472..0000000 --- a/Lib/bsddb/test/test_env_close.py +++ /dev/null @@ -1,107 +0,0 @@ -"""TestCases for checking that it does not segfault when a DBEnv object -is closed before its DB objects. -""" - -import os -import tempfile -import unittest - -try: - # For Pythons w/distutils pybsddb - from bsddb3 import db -except ImportError: - # For Python 2.3 - from bsddb import db - -try: - from bsddb3 import test_support -except ImportError: - from test import test_support - -from test_all import verbose - -# We're going to get warnings in this module about trying to close the db when -# its env is already closed. Let's just ignore those. -try: - import warnings -except ImportError: - pass -else: - warnings.filterwarnings('ignore', - message='DB could not be closed in', - category=RuntimeWarning) - - -#---------------------------------------------------------------------- - -class DBEnvClosedEarlyCrash(unittest.TestCase): - def setUp(self): - self.homeDir = os.path.join(tempfile.gettempdir(), 'db_home%d'%os.getpid()) - try: os.mkdir(self.homeDir) - except os.error: pass - tempfile.tempdir = self.homeDir - self.filename = os.path.split(tempfile.mktemp())[1] - tempfile.tempdir = None - - def tearDown(self): - test_support.rmtree(self.homeDir) - - def test01_close_dbenv_before_db(self): - dbenv = db.DBEnv() - dbenv.open(self.homeDir, - db.DB_INIT_CDB| db.DB_CREATE |db.DB_THREAD|db.DB_INIT_MPOOL, - 0666) - - d = db.DB(dbenv) - d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) - - try: - dbenv.close() - except db.DBError: - try: - d.close() - except db.DBError: - return - assert 0, \ - "DB close did not raise an exception about its "\ - "DBEnv being trashed" - - # XXX This may fail when using older versions of BerkeleyDB. - # E.g. 3.2.9 never raised the exception. - assert 0, "dbenv did not raise an exception about its DB being open" - - - def test02_close_dbenv_delete_db_success(self): - dbenv = db.DBEnv() - dbenv.open(self.homeDir, - db.DB_INIT_CDB| db.DB_CREATE |db.DB_THREAD|db.DB_INIT_MPOOL, - 0666) - - d = db.DB(dbenv) - d.open(self.filename, db.DB_BTREE, db.DB_CREATE | db.DB_THREAD, 0666) - - try: - dbenv.close() - except db.DBError: - pass # good, it should raise an exception - - del d - try: - import gc - except ImportError: - gc = None - if gc: - # force d.__del__ [DB_dealloc] to be called - gc.collect() - - -#---------------------------------------------------------------------- - -def test_suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(DBEnvClosedEarlyCrash)) - return suite - - -if __name__ == '__main__': - unittest.main(defaultTest='test_suite') diff --git a/Lib/bsddb/test/test_get_none.py b/Lib/bsddb/test/test_get_none.py index d6454cc..af6a49c 100644 --- a/Lib/bsddb/test/test_get_none.py +++ b/Lib/bsddb/test/test_get_none.py @@ -3,7 +3,6 @@ TestCases for checking set_get_returns_none. """ import os, string -import tempfile import unittest try: @@ -13,14 +12,14 @@ except ImportError: # For Python 2.3 from bsddb import db -from test_all import verbose +from test_all import verbose, get_new_database_path #---------------------------------------------------------------------- class GetReturnsNoneTestCase(unittest.TestCase): def setUp(self): - self.filename = tempfile.mktemp() + self.filename = get_new_database_path() def tearDown(self): try: @@ -38,10 +37,10 @@ class GetReturnsNoneTestCase(unittest.TestCase): d.put(x, x * 40) data = d.get('bad key') - assert data == None + self.assertEqual(data, None) data = d.get('a') - assert data == 'a'*40 + self.assertEqual(data, 'a'*40) count = 0 c = d.cursor() @@ -50,8 +49,8 @@ class GetReturnsNoneTestCase(unittest.TestCase): count = count + 1 rec = c.next() - assert rec == None - assert count == 52 + self.assertEqual(rec, None) + self.assertEqual(count, 52) c.close() d.close() @@ -69,7 +68,7 @@ class GetReturnsNoneTestCase(unittest.TestCase): self.assertRaises(KeyError, d.get, 'bad key') data = d.get('a') - assert data == 'a'*40 + self.assertEqual(data, 'a'*40) count = 0 exceptionHappened = 0 @@ -83,9 +82,9 @@ class GetReturnsNoneTestCase(unittest.TestCase): exceptionHappened = 1 break - assert rec != None - assert exceptionHappened - assert count == 52 + self.assertNotEqual(rec, None) + self.assert_(exceptionHappened) + self.assertEqual(count, 52) c.close() d.close() diff --git a/Lib/bsddb/test/test_join.py b/Lib/bsddb/test/test_join.py index c6c5c88..4c9ca8e 100644 --- a/Lib/bsddb/test/test_join.py +++ b/Lib/bsddb/test/test_join.py @@ -2,13 +2,6 @@ """ import os -import tempfile - -try: - from threading import Thread, currentThread - have_threads = 1 -except ImportError: - have_threads = 0 import unittest from test_all import verbose @@ -20,11 +13,14 @@ except ImportError: # For Python 2.3 from bsddb import db, dbshelve +from test_all import get_new_environment_path, get_new_database_path + try: from bsddb3 import test_support except ImportError: from test import test_support + #---------------------------------------------------------------------- ProductIndex = [ @@ -51,12 +47,9 @@ class JoinTestCase(unittest.TestCase): def setUp(self): self.filename = self.__class__.__name__ + '.db' - homeDir = os.path.join(tempfile.gettempdir(), 'db_home%d'%os.getpid()) - self.homeDir = homeDir - try: os.mkdir(homeDir) - except os.error: pass + self.homeDir = get_new_environment_path() self.env = db.DBEnv() - self.env.open(homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK ) + self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK ) def tearDown(self): self.env.close() @@ -87,7 +80,7 @@ class JoinTestCase(unittest.TestCase): # Don't do the .set() in an assert, or you can get a bogus failure # when running python -O tmp = sCursor.set('red') - assert tmp + self.assert_(tmp) # FIXME: jCursor doesn't properly hold a reference to its # cursors, if they are closed before jcursor is used it diff --git a/Lib/bsddb/test/test_lock.py b/Lib/bsddb/test/test_lock.py index 3b2f9e4..9b9ce7b 100644 --- a/Lib/bsddb/test/test_lock.py +++ b/Lib/bsddb/test/test_lock.py @@ -2,7 +2,6 @@ TestCases for testing the locking sub-system. """ -import tempfile import time try: @@ -13,7 +12,7 @@ except ImportError: import unittest -from test_all import verbose +from test_all import verbose, get_new_environment_path, get_new_database_path try: # For Pythons w/distutils pybsddb @@ -31,9 +30,14 @@ except ImportError: #---------------------------------------------------------------------- class LockingTestCase(unittest.TestCase): + import sys + if sys.version_info[:3] < (2, 4, 0): + def assertTrue(self, expr, msg=None): + self.failUnless(expr,msg=msg) + def setUp(self): - self.homeDir = tempfile.mkdtemp('.test_lock') + self.homeDir = get_new_environment_path() self.env = db.DBEnv() self.env.open(self.homeDir, db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_CREATE) @@ -55,7 +59,6 @@ class LockingTestCase(unittest.TestCase): lock = self.env.lock_get(anID, "some locked thing", db.DB_LOCK_WRITE) if verbose: print "Aquired lock: %s" % lock - time.sleep(1) self.env.lock_put(lock) if verbose: print "Released lock: %s" % lock @@ -70,38 +73,73 @@ class LockingTestCase(unittest.TestCase): threads = [] threads.append(Thread(target = self.theThread, - args=(5, db.DB_LOCK_WRITE))) + args=(db.DB_LOCK_WRITE,))) threads.append(Thread(target = self.theThread, - args=(1, db.DB_LOCK_READ))) + args=(db.DB_LOCK_READ,))) threads.append(Thread(target = self.theThread, - args=(1, db.DB_LOCK_READ))) + args=(db.DB_LOCK_READ,))) threads.append(Thread(target = self.theThread, - args=(1, db.DB_LOCK_WRITE))) + args=(db.DB_LOCK_WRITE,))) threads.append(Thread(target = self.theThread, - args=(1, db.DB_LOCK_READ))) + args=(db.DB_LOCK_READ,))) threads.append(Thread(target = self.theThread, - args=(1, db.DB_LOCK_READ))) + args=(db.DB_LOCK_READ,))) threads.append(Thread(target = self.theThread, - args=(1, db.DB_LOCK_WRITE))) + args=(db.DB_LOCK_WRITE,))) threads.append(Thread(target = self.theThread, - args=(1, db.DB_LOCK_WRITE))) + args=(db.DB_LOCK_WRITE,))) threads.append(Thread(target = self.theThread, - args=(1, db.DB_LOCK_WRITE))) + args=(db.DB_LOCK_WRITE,))) for t in threads: + t.setDaemon(True) t.start() for t in threads: t.join() - def test03_set_timeout(self): - # test that the set_timeout call works - if hasattr(self.env, 'set_timeout'): - self.env.set_timeout(0, db.DB_SET_LOCK_TIMEOUT) - self.env.set_timeout(0, db.DB_SET_TXN_TIMEOUT) - self.env.set_timeout(123456, db.DB_SET_LOCK_TIMEOUT) - self.env.set_timeout(7890123, db.DB_SET_TXN_TIMEOUT) + def test03_lock_timeout(self): + self.env.set_timeout(0, db.DB_SET_LOCK_TIMEOUT) + self.env.set_timeout(0, db.DB_SET_TXN_TIMEOUT) + self.env.set_timeout(123456, db.DB_SET_LOCK_TIMEOUT) + self.env.set_timeout(7890123, db.DB_SET_TXN_TIMEOUT) + + def deadlock_detection() : + while not deadlock_detection.end : + deadlock_detection.count = \ + self.env.lock_detect(db.DB_LOCK_EXPIRE) + if deadlock_detection.count : + while not deadlock_detection.end : + pass + break + time.sleep(0.01) + + deadlock_detection.end=False + deadlock_detection.count=0 + t=Thread(target=deadlock_detection) + t.setDaemon(True) + t.start() + self.env.set_timeout(100000, db.DB_SET_LOCK_TIMEOUT) + anID = self.env.lock_id() + anID2 = self.env.lock_id() + self.assertNotEqual(anID, anID2) + lock = self.env.lock_get(anID, "shared lock", db.DB_LOCK_WRITE) + start_time=time.time() + self.assertRaises(db.DBLockNotGrantedError, + self.env.lock_get,anID2, "shared lock", db.DB_LOCK_READ) + end_time=time.time() + deadlock_detection.end=True + self.assertTrue((end_time-start_time) >= 0.1) + self.env.lock_put(lock) + t.join() + + if db.version() >= (4,0): + self.env.lock_id_free(anID) + self.env.lock_id_free(anID2) + + if db.version() >= (4,6): + self.assertTrue(deadlock_detection.count>0) - def theThread(self, sleepTime, lockType): + def theThread(self, lockType): name = currentThread().getName() if lockType == db.DB_LOCK_WRITE: lt = "write" @@ -112,15 +150,15 @@ class LockingTestCase(unittest.TestCase): if verbose: print "%s: locker ID: %s" % (name, anID) - lock = self.env.lock_get(anID, "some locked thing", lockType) - if verbose: - print "%s: Aquired %s lock: %s" % (name, lt, lock) + for i in xrange(1000) : + lock = self.env.lock_get(anID, "some locked thing", lockType) + if verbose: + print "%s: Aquired %s lock: %s" % (name, lt, lock) - time.sleep(sleepTime) + self.env.lock_put(lock) + if verbose: + print "%s: Released %s lock: %s" % (name, lt, lock) - self.env.lock_put(lock) - if verbose: - print "%s: Released %s lock: %s" % (name, lt, lock) if db.version() >= (4,0): self.env.lock_id_free(anID) diff --git a/Lib/bsddb/test/test_misc.py b/Lib/bsddb/test/test_misc.py index 9a790e7..1da5830 100644 --- a/Lib/bsddb/test/test_misc.py +++ b/Lib/bsddb/test/test_misc.py @@ -3,7 +3,6 @@ import os import unittest -import tempfile try: # For Pythons w/distutils pybsddb @@ -12,6 +11,8 @@ except ImportError: # For Python 2.3 from bsddb import db, dbshelve, hashopen +from test_all import get_new_environment_path, get_new_database_path + try: from bsddb3 import test_support except ImportError: @@ -22,12 +23,7 @@ except ImportError: class MiscTestCase(unittest.TestCase): def setUp(self): self.filename = self.__class__.__name__ + '.db' - homeDir = os.path.join(tempfile.gettempdir(), 'db_home%d'%os.getpid()) - self.homeDir = homeDir - try: - os.mkdir(homeDir) - except OSError: - pass + self.homeDir = get_new_environment_path() def tearDown(self): test_support.unlink(self.filename) @@ -41,9 +37,9 @@ class MiscTestCase(unittest.TestCase): def test02_db_home(self): env = db.DBEnv() # check for crash fixed when db_home is used before open() - assert env.db_home is None + self.assert_(env.db_home is None) env.open(self.homeDir, db.DB_CREATE) - assert self.homeDir == env.db_home + self.assertEqual(self.homeDir, env.db_home) def test03_repr_closed_db(self): db = hashopen(self.filename) @@ -93,7 +89,7 @@ class MiscTestCase(unittest.TestCase): def test_DB_set_flags_persists(self): if db.version() < (4,2): # The get_flags API required for this to work is only available - # in BerkeleyDB >= 4.2 + # in Berkeley DB >= 4.2 return try: db1 = db.DB() diff --git a/Lib/bsddb/test/test_pickle.py b/Lib/bsddb/test/test_pickle.py index d691e37..b1d3805 100644 --- a/Lib/bsddb/test/test_pickle.py +++ b/Lib/bsddb/test/test_pickle.py @@ -6,7 +6,6 @@ try: except ImportError: cPickle = None import unittest -import tempfile try: # For Pythons w/distutils pybsddb @@ -15,6 +14,8 @@ except ImportError, e: # For Python 2.3 from bsddb import db +from test_all import get_new_environment_path, get_new_database_path + try: from bsddb3 import test_support except ImportError: @@ -25,14 +26,10 @@ except ImportError: class pickleTestCase(unittest.TestCase): """Verify that DBError can be pickled and unpickled""" - db_home = 'db_home' db_name = 'test-dbobj.db' def setUp(self): - homeDir = os.path.join(tempfile.gettempdir(), 'db_home%d'%os.getpid()) - self.homeDir = homeDir - try: os.mkdir(homeDir) - except os.error: pass + self.homeDir = get_new_environment_path() def tearDown(self): if hasattr(self, 'db'): @@ -47,7 +44,7 @@ class pickleTestCase(unittest.TestCase): self.db = db.DB(self.env) self.db.open(self.db_name, db.DB_HASH, db.DB_CREATE) self.db.put('spam', 'eggs') - assert self.db['spam'] == 'eggs' + self.assertEqual(self.db['spam'], 'eggs') try: self.db.put('spam', 'ham', flags=db.DB_NOOVERWRITE) except db.DBError, egg: diff --git a/Lib/bsddb/test/test_queue.py b/Lib/bsddb/test/test_queue.py index 1acf35a..c514bd1 100644 --- a/Lib/bsddb/test/test_queue.py +++ b/Lib/bsddb/test/test_queue.py @@ -3,7 +3,6 @@ TestCases for exercising a Queue DB. """ import os, string -import tempfile from pprint import pprint import unittest @@ -14,14 +13,14 @@ except ImportError: # For Python 2.3 from bsddb import db -from test_all import verbose +from test_all import verbose, get_new_database_path #---------------------------------------------------------------------- class SimpleQueueTestCase(unittest.TestCase): def setUp(self): - self.filename = tempfile.mktemp() + self.filename = get_new_database_path() def tearDown(self): try: @@ -48,14 +47,14 @@ class SimpleQueueTestCase(unittest.TestCase): for x in string.letters: d.append(x * 40) - assert len(d) == 52 + self.assertEqual(len(d), 52) d.put(100, "some more data") d.put(101, "and some more ") d.put(75, "out of order") d.put(1, "replacement data") - assert len(d) == 55 + self.assertEqual(len(d), 55) if verbose: print "before close" + '-' * 30 @@ -88,9 +87,9 @@ class SimpleQueueTestCase(unittest.TestCase): print "after consume loop" + '-' * 30 pprint(d.stat()) - assert len(d) == 0, \ + self.assertEqual(len(d), 0, \ "if you see this message then you need to rebuild " \ - "BerkeleyDB 3.1.17 with the patch in patches/qam_stat.diff" + "Berkeley DB 3.1.17 with the patch in patches/qam_stat.diff") d.close() @@ -120,14 +119,14 @@ class SimpleQueueTestCase(unittest.TestCase): for x in string.letters: d.append(x * 40) - assert len(d) == 52 + self.assertEqual(len(d), 52) d.put(100, "some more data") d.put(101, "and some more ") d.put(75, "out of order") d.put(1, "replacement data") - assert len(d) == 55 + self.assertEqual(len(d), 55) if verbose: print "before close" + '-' * 30 diff --git a/Lib/bsddb/test/test_recno.py b/Lib/bsddb/test/test_recno.py index 43cf176..d96dc86 100644 --- a/Lib/bsddb/test/test_recno.py +++ b/Lib/bsddb/test/test_recno.py @@ -3,11 +3,10 @@ import os import errno -import tempfile from pprint import pprint import unittest -from test_all import verbose +from test_all import verbose, get_new_environment_path, get_new_database_path try: # For Pythons w/distutils pybsddb @@ -27,8 +26,13 @@ letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' #---------------------------------------------------------------------- class SimpleRecnoTestCase(unittest.TestCase): + import sys + if sys.version_info[:3] < (2, 4, 0): + def assertFalse(self, expr, msg=None): + self.failIf(expr,msg=msg) + def setUp(self): - self.filename = tempfile.mktemp() + self.filename = get_new_database_path() self.homeDir = None def tearDown(self): @@ -46,8 +50,8 @@ class SimpleRecnoTestCase(unittest.TestCase): for x in letters: recno = d.append(x * 60) - assert type(recno) == type(0) - assert recno >= 1 + self.assertEqual(type(recno), type(0)) + self.assert_(recno >= 1) if verbose: print recno, @@ -62,13 +66,13 @@ class SimpleRecnoTestCase(unittest.TestCase): if verbose: print data - assert type(data) == type("") - assert data == d.get(recno) + self.assertEqual(type(data), type("")) + self.assertEqual(data, d.get(recno)) try: data = d[0] # This should raise a KeyError!?!?! except db.DBInvalidArgError, val: - assert val[0] == db.EINVAL + self.assertEqual(val[0], db.EINVAL) if verbose: print val else: self.fail("expected exception") @@ -94,35 +98,35 @@ class SimpleRecnoTestCase(unittest.TestCase): if get_returns_none: self.fail("unexpected exception") else: - assert data == None + self.assertEqual(data, None) keys = d.keys() if verbose: print keys - assert type(keys) == type([]) - assert type(keys[0]) == type(123) - assert len(keys) == len(d) + self.assertEqual(type(keys), type([])) + self.assertEqual(type(keys[0]), type(123)) + self.assertEqual(len(keys), len(d)) items = d.items() if verbose: pprint(items) - assert type(items) == type([]) - assert type(items[0]) == type(()) - assert len(items[0]) == 2 - assert type(items[0][0]) == type(123) - assert type(items[0][1]) == type("") - assert len(items) == len(d) + self.assertEqual(type(items), type([])) + self.assertEqual(type(items[0]), type(())) + self.assertEqual(len(items[0]), 2) + self.assertEqual(type(items[0][0]), type(123)) + self.assertEqual(type(items[0][1]), type("")) + self.assertEqual(len(items), len(d)) - assert d.has_key(25) + self.assert_(d.has_key(25)) del d[25] - assert not d.has_key(25) + self.assertFalse(d.has_key(25)) d.delete(13) - assert not d.has_key(13) + self.assertFalse(d.has_key(13)) data = d.get_both(26, "z" * 60) - assert data == "z" * 60, 'was %r' % data + self.assertEqual(data, "z" * 60, 'was %r' % data) if verbose: print data @@ -146,7 +150,7 @@ class SimpleRecnoTestCase(unittest.TestCase): c.set(50) rec = c.current() - assert rec == (50, "a replacement record") + self.assertEqual(rec, (50, "a replacement record")) if verbose: print rec @@ -157,7 +161,7 @@ class SimpleRecnoTestCase(unittest.TestCase): # test that non-existant key lookups work (and that # DBC_set_range doesn't have a memleak under valgrind) rec = c.set_range(999999) - assert rec == None + self.assertEqual(rec, None) if verbose: print rec @@ -170,7 +174,7 @@ class SimpleRecnoTestCase(unittest.TestCase): # put a record beyond the consecutive end of the recno's d[100] = "way out there" - assert d[100] == "way out there" + self.assertEqual(d[100], "way out there") try: data = d[99] @@ -185,7 +189,7 @@ class SimpleRecnoTestCase(unittest.TestCase): if get_returns_none: self.fail("unexpected DBKeyEmptyError exception") else: - assert val[0] == db.DB_KEYEMPTY + self.assertEqual(val[0], db.DB_KEYEMPTY) if verbose: print val else: if not get_returns_none: @@ -207,7 +211,7 @@ class SimpleRecnoTestCase(unittest.TestCase): just a line in the file, but you can set a different record delimiter if needed. """ - homeDir = os.path.join(tempfile.gettempdir(), 'db_home%d'%os.getpid()) + homeDir = get_new_environment_path() self.homeDir = homeDir source = os.path.join(homeDir, 'test_recno.txt') if not os.path.isdir(homeDir): @@ -236,7 +240,7 @@ class SimpleRecnoTestCase(unittest.TestCase): print data print text.split('\n') - assert text.split('\n') == data + self.assertEqual(text.split('\n'), data) # open as a DB again d = db.DB() @@ -255,8 +259,8 @@ class SimpleRecnoTestCase(unittest.TestCase): print text print text.split('\n') - assert text.split('\n') == \ - "The quick reddish-brown fox jumped over the comatose dog".split() + self.assertEqual(text.split('\n'), + "The quick reddish-brown fox jumped over the comatose dog".split()) def test03_FixedLength(self): d = db.DB() @@ -273,7 +277,7 @@ class SimpleRecnoTestCase(unittest.TestCase): try: # this one will fail d.append('bad' * 20) except db.DBInvalidArgError, val: - assert val[0] == db.EINVAL + self.assertEqual(val[0], db.EINVAL) if verbose: print val else: self.fail("expected exception") diff --git a/Lib/bsddb/test/test_replication.py b/Lib/bsddb/test/test_replication.py new file mode 100644 index 0000000..e4f626a --- /dev/null +++ b/Lib/bsddb/test/test_replication.py @@ -0,0 +1,199 @@ +"""TestCases for distributed transactions. +""" + +import os +import unittest + +try: + # For Pythons w/distutils pybsddb + from bsddb3 import db +except ImportError: + # For Python 2.3 + from bsddb import db + +from test_all import get_new_environment_path, get_new_database_path + +try: + from bsddb3 import test_support +except ImportError: + from test import test_support + +from test_all import verbose + +#---------------------------------------------------------------------- + +class DBReplicationManager(unittest.TestCase): + import sys + if sys.version_info[:3] < (2, 4, 0): + def assertTrue(self, expr, msg=None): + self.failUnless(expr,msg=msg) + + def setUp(self) : + self.homeDirMaster = get_new_environment_path() + self.homeDirClient = get_new_environment_path() + + self.dbenvMaster = db.DBEnv() + self.dbenvClient = db.DBEnv() + + # Must use "DB_THREAD" because the Replication Manager will + # be executed in other threads but will use the same environment. + # http://forums.oracle.com/forums/thread.jspa?threadID=645788&tstart=0 + self.dbenvMaster.open(self.homeDirMaster, db.DB_CREATE | db.DB_INIT_TXN + | db.DB_INIT_LOG | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | + db.DB_INIT_REP | db.DB_RECOVER | db.DB_THREAD, 0666) + self.dbenvClient.open(self.homeDirClient, db.DB_CREATE | db.DB_INIT_TXN + | db.DB_INIT_LOG | db.DB_INIT_MPOOL | db.DB_INIT_LOCK | + db.DB_INIT_REP | db.DB_RECOVER | db.DB_THREAD, 0666) + + self.confirmed_master=self.client_startupdone=False + def confirmed_master(a,b,c) : + if b==db.DB_EVENT_REP_MASTER : + self.confirmed_master=True + + def client_startupdone(a,b,c) : + if b==db.DB_EVENT_REP_STARTUPDONE : + self.client_startupdone=True + + self.dbenvMaster.set_event_notify(confirmed_master) + self.dbenvClient.set_event_notify(client_startupdone) + + self.dbenvMaster.repmgr_set_local_site("127.0.0.1",46117) + self.dbenvClient.repmgr_set_local_site("127.0.0.1",46118) + self.dbenvMaster.repmgr_add_remote_site("127.0.0.1",46118) + self.dbenvClient.repmgr_add_remote_site("127.0.0.1",46117) + self.dbenvMaster.rep_set_nsites(2) + self.dbenvClient.rep_set_nsites(2) + self.dbenvMaster.rep_set_priority(10) + self.dbenvClient.rep_set_priority(0) + + self.dbenvMaster.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) + self.dbenvClient.repmgr_set_ack_policy(db.DB_REPMGR_ACKS_ALL) + + self.dbenvMaster.repmgr_start(1, db.DB_REP_MASTER); + self.dbenvClient.repmgr_start(1, db.DB_REP_CLIENT); + + self.assertEquals(self.dbenvMaster.rep_get_nsites(),2) + self.assertEquals(self.dbenvClient.rep_get_nsites(),2) + self.assertEquals(self.dbenvMaster.rep_get_priority(),10) + self.assertEquals(self.dbenvClient.rep_get_priority(),0) + self.assertEquals(self.dbenvMaster.repmgr_get_ack_policy(), + db.DB_REPMGR_ACKS_ALL) + self.assertEquals(self.dbenvClient.repmgr_get_ack_policy(), + db.DB_REPMGR_ACKS_ALL) + + #self.dbenvMaster.set_verbose(db.DB_VERB_REPLICATION, True) + #self.dbenvMaster.set_verbose(db.DB_VERB_FILEOPS_ALL, True) + #self.dbenvClient.set_verbose(db.DB_VERB_REPLICATION, True) + #self.dbenvClient.set_verbose(db.DB_VERB_FILEOPS_ALL, True) + + self.dbMaster = self.dbClient = None + + # The timeout is necessary in BDB 4.5, since DB_EVENT_REP_STARTUPDONE + # is not generated if the master has no new transactions. + # This is solved in BDB 4.6 (#15542). + import time + timeout = time.time()+2 + while (time.time()<timeout) and not (self.confirmed_master and self.client_startupdone) : + time.sleep(0.001) + if db.version() >= (4,6) : + self.assertTrue(time.time()<timeout) + else : + self.assertTrue(time.time()>=timeout) + + d = self.dbenvMaster.repmgr_site_list() + self.assertEquals(len(d), 1) + self.assertEquals(d[0][0], "127.0.0.1") + self.assertEquals(d[0][1], 46118) + self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \ + (d[0][2]==db.DB_REPMGR_DISCONNECTED)) + + d = self.dbenvClient.repmgr_site_list() + self.assertEquals(len(d), 1) + self.assertEquals(d[0][0], "127.0.0.1") + self.assertEquals(d[0][1], 46117) + self.assertTrue((d[0][2]==db.DB_REPMGR_CONNECTED) or \ + (d[0][2]==db.DB_REPMGR_DISCONNECTED)) + + if db.version() >= (4,6) : + d = self.dbenvMaster.repmgr_stat(flags=db.DB_STAT_CLEAR); + self.assertTrue("msgs_queued" in d) + + def tearDown(self): + if self.dbClient : + self.dbClient.close() + if self.dbMaster : + self.dbMaster.close() + self.dbenvClient.close() + self.dbenvMaster.close() + test_support.rmtree(self.homeDirClient) + test_support.rmtree(self.homeDirMaster) + + def test01_basic_replication(self) : + self.dbMaster=db.DB(self.dbenvMaster) + txn=self.dbenvMaster.txn_begin() + self.dbMaster.open("test", db.DB_HASH, db.DB_CREATE, 0666, txn=txn) + txn.commit() + + import time,os.path + timeout=time.time()+10 + while (time.time()<timeout) and \ + not (os.path.exists(os.path.join(self.homeDirClient,"test"))) : + time.sleep(0.01) + + self.dbClient=db.DB(self.dbenvClient) + while True : + txn=self.dbenvClient.txn_begin() + try : + self.dbClient.open("test", db.DB_HASH, flags=db.DB_RDONLY, + mode=0666, txn=txn) + except db.DBRepHandleDeadError : + txn.abort() + self.dbClient.close() + self.dbClient=db.DB(self.dbenvClient) + continue + + txn.commit() + break + + txn=self.dbenvMaster.txn_begin() + self.dbMaster.put("ABC", "123", txn=txn) + txn.commit() + import time + timeout=time.time()+1 + v=None + while (time.time()<timeout) and (v==None) : + txn=self.dbenvClient.txn_begin() + v=self.dbClient.get("ABC", txn=txn) + txn.commit() + self.assertEquals("123", v) + + txn=self.dbenvMaster.txn_begin() + self.dbMaster.delete("ABC", txn=txn) + txn.commit() + timeout=time.time()+1 + while (time.time()<timeout) and (v!=None) : + txn=self.dbenvClient.txn_begin() + v=self.dbClient.get("ABC", txn=txn) + txn.commit() + self.assertEquals(None, v) + +#---------------------------------------------------------------------- + +def test_suite(): + suite = unittest.TestSuite() + if db.version() >= (4,5) : + dbenv = db.DBEnv() + try : + dbenv.repmgr_get_ack_policy() + ReplicationManager_available=True + except : + ReplicationManager_available=False + dbenv.close() + del dbenv + if ReplicationManager_available : + suite.addTest(unittest.makeSuite(DBReplicationManager)) + return suite + + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff --git a/Lib/bsddb/test/test_sequence.py b/Lib/bsddb/test/test_sequence.py index ff94b76..4ef65eb 100644 --- a/Lib/bsddb/test/test_sequence.py +++ b/Lib/bsddb/test/test_sequence.py @@ -1,6 +1,5 @@ import unittest import os -import tempfile try: # For Pythons w/distutils pybsddb @@ -13,18 +12,19 @@ try: except ImportError: from test import test_support +from test_all import get_new_environment_path, get_new_database_path + class DBSequenceTest(unittest.TestCase): + import sys + if sys.version_info[:3] < (2, 4, 0): + def assertTrue(self, expr, msg=None): + self.failUnless(expr,msg=msg) + def setUp(self): self.int_32_max = 0x100000000 - self.homeDir = os.path.join(tempfile.gettempdir(), 'db_home%d'%os.getpid()) - try: - os.mkdir(self.homeDir) - except os.error: - pass - tempfile.tempdir = self.homeDir - self.filename = os.path.split(tempfile.mktemp())[1] - tempfile.tempdir = None + self.homeDir = get_new_environment_path() + self.filename = "test" self.dbenv = db.DBEnv() self.dbenv.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL, 0666) @@ -100,6 +100,52 @@ class DBSequenceTest(unittest.TestCase): 'flags', 'cache_size', 'last_value', 'wait'): self.assertTrue(param in stat, "parameter %s isn't in stat info" % param) + if db.version() >= (4,7) : + # This code checks a crash solved in Berkeley DB 4.7 + def test_stat_crash(self) : + d=db.DB() + d.open(None,dbtype=db.DB_HASH,flags=db.DB_CREATE) # In RAM + seq = db.DBSequence(d, flags=0) + + self.assertRaises(db.DBNotFoundError, seq.open, + key='id', txn=None, flags=0) + + self.assertRaises(db.DBNotFoundError, seq.stat) + + d.close() + + def test_64bits(self) : + value_plus=(1L<<63)-1 + self.assertEquals(9223372036854775807L,value_plus) + value_minus=-1L<<63 # Two complement + self.assertEquals(-9223372036854775808L,value_minus) + if db.version() < (4,4): + # We don't use both extremes because it is + # problematic in Berkeley DB 4.3. + value_plus-=1 + value_minus+=1 + self.seq = db.DBSequence(self.d, flags=0) + self.assertEquals(None, self.seq.init_value(value_plus-1)) + self.assertEquals(None, self.seq.open(key='id', txn=None, + flags=db.DB_CREATE)) + self.assertEquals(value_plus-1, self.seq.get(1)) + self.assertEquals(value_plus, self.seq.get(1)) + + self.seq.remove(txn=None, flags=0) + + self.seq = db.DBSequence(self.d, flags=0) + self.assertEquals(None, self.seq.init_value(value_minus)) + self.assertEquals(None, self.seq.open(key='id', txn=None, + flags=db.DB_CREATE)) + self.assertEquals(value_minus, self.seq.get(1)) + self.assertEquals(value_minus+1, self.seq.get(1)) + + def test_multiple_close(self): + self.seq = db.DBSequence(self.d) + self.seq.close() # You can close a Sequence multiple times + self.seq.close() + self.seq.close() + def test_suite(): suite = unittest.TestSuite() if db.version() >= (4,3): diff --git a/Lib/bsddb/test/test_thread.py b/Lib/bsddb/test/test_thread.py index 0395f6d..5ac98ce 100644 --- a/Lib/bsddb/test/test_thread.py +++ b/Lib/bsddb/test/test_thread.py @@ -5,7 +5,6 @@ import os import sys import time import errno -import tempfile from random import random try: @@ -29,7 +28,8 @@ except NameError: pass import unittest -from test_all import verbose +from test_all import verbose, get_new_environment_path, get_new_database_path + try: # For Pythons w/distutils pybsddb @@ -52,19 +52,19 @@ class BaseThreadedTestCase(unittest.TestCase): dbsetflags = 0 envflags = 0 + import sys + if sys.version_info[:3] < (2, 4, 0): + def assertTrue(self, expr, msg=None): + self.failUnless(expr,msg=msg) + def setUp(self): if verbose: dbutils._deadlock_VerboseFile = sys.stdout - homeDir = os.path.join(tempfile.gettempdir(), 'db_home%d'%os.getpid()) - self.homeDir = homeDir - try: - os.mkdir(homeDir) - except OSError, e: - if e.errno != errno.EEXIST: raise + self.homeDir = get_new_environment_path() self.env = db.DBEnv() self.setEnvOpts() - self.env.open(homeDir, self.envflags | db.DB_CREATE) + self.env.open(self.homeDir, self.envflags | db.DB_CREATE) self.filename = self.__class__.__name__ + '.db' self.d = db.DB(self.env) @@ -100,63 +100,73 @@ class ConcurrentDataStoreBase(BaseThreadedTestCase): print "Running %s.test01_1WriterMultiReaders..." % \ self.__class__.__name__ - threads = [] - for x in range(self.writers): - wt = Thread(target = self.writerThread, - args = (self.d, self.records, x), - name = 'writer %d' % x, - )#verbose = verbose) - threads.append(wt) - - for x in range(self.readers): + keys=range(self.records) + import random + random.shuffle(keys) + records_per_writer=self.records/self.writers + readers_per_writer=self.readers/self.writers + self.assertEqual(self.records,self.writers*records_per_writer) + self.assertEqual(self.readers,self.writers*readers_per_writer) + self.assertTrue((records_per_writer%readers_per_writer)==0) + readers = [] + + for x in xrange(self.readers): rt = Thread(target = self.readerThread, args = (self.d, x), name = 'reader %d' % x, )#verbose = verbose) - threads.append(rt) + rt.setDaemon(True) + readers.append(rt) + + writers=[] + for x in xrange(self.writers): + a=keys[records_per_writer*x:records_per_writer*(x+1)] + a.sort() # Generate conflicts + b=readers[readers_per_writer*x:readers_per_writer*(x+1)] + wt = Thread(target = self.writerThread, + args = (self.d, a, b), + name = 'writer %d' % x, + )#verbose = verbose) + writers.append(wt) - for t in threads: + for t in writers: + t.setDaemon(True) t.start() - for t in threads: + + for t in writers: + t.join() + for t in readers: t.join() - def writerThread(self, d, howMany, writerNum): - #time.sleep(0.01 * writerNum + 0.01) + def writerThread(self, d, keys, readers): name = currentThread().getName() - start = howMany * writerNum - stop = howMany * (writerNum + 1) - 1 if verbose: print "%s: creating records %d - %d" % (name, start, stop) - for x in range(start, stop): + count=len(keys)/len(readers) + count2=count + for x in keys : key = '%04d' % x dbutils.DeadlockWrap(d.put, key, self.makeData(key), max_retries=12) if verbose and x % 100 == 0: print "%s: records %d - %d finished" % (name, start, x) + count2-=1 + if not count2 : + readers.pop().start() + count2=count + if verbose: print "%s: finished creating records" % name -## # Each write-cursor will be exclusive, the only one that can update the DB... -## if verbose: print "%s: deleting a few records" % name -## c = d.cursor(flags = db.DB_WRITECURSOR) -## for x in range(10): -## key = int(random() * howMany) + start -## key = '%04d' % key -## if d.has_key(key): -## c.set(key) -## c.delete() - -## c.close() if verbose: print "%s: thread finished" % name def readerThread(self, d, readerNum): - time.sleep(0.01 * readerNum) name = currentThread().getName() - for loop in range(5): + for i in xrange(5) : c = d.cursor() count = 0 rec = c.first() @@ -168,7 +178,6 @@ class ConcurrentDataStoreBase(BaseThreadedTestCase): if verbose: print "%s: found %d records" % (name, count) c.close() - time.sleep(0.05) if verbose: print "%s: thread finished" % name @@ -193,8 +202,8 @@ class HashConcurrentDataStore(ConcurrentDataStoreBase): class SimpleThreadedBase(BaseThreadedTestCase): dbopenflags = db.DB_THREAD envflags = db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK - readers = 5 - writers = 3 + readers = 10 + writers = 2 records = 1000 def setEnvOpts(self): @@ -205,34 +214,53 @@ class SimpleThreadedBase(BaseThreadedTestCase): print '\n', '-=' * 30 print "Running %s.test02_SimpleLocks..." % self.__class__.__name__ - threads = [] - for x in range(self.writers): - wt = Thread(target = self.writerThread, - args = (self.d, self.records, x), - name = 'writer %d' % x, - )#verbose = verbose) - threads.append(wt) - for x in range(self.readers): + + keys=range(self.records) + import random + random.shuffle(keys) + records_per_writer=self.records/self.writers + readers_per_writer=self.readers/self.writers + self.assertEqual(self.records,self.writers*records_per_writer) + self.assertEqual(self.readers,self.writers*readers_per_writer) + self.assertTrue((records_per_writer%readers_per_writer)==0) + + readers = [] + for x in xrange(self.readers): rt = Thread(target = self.readerThread, args = (self.d, x), name = 'reader %d' % x, )#verbose = verbose) - threads.append(rt) + rt.setDaemon(True) + readers.append(rt) + + writers = [] + for x in xrange(self.writers): + a=keys[records_per_writer*x:records_per_writer*(x+1)] + a.sort() # Generate conflicts + b=readers[readers_per_writer*x:readers_per_writer*(x+1)] + wt = Thread(target = self.writerThread, + args = (self.d, a, b), + name = 'writer %d' % x, + )#verbose = verbose) + writers.append(wt) - for t in threads: + for t in writers: + t.setDaemon(True) t.start() - for t in threads: + + for t in writers: + t.join() + for t in readers: t.join() - def writerThread(self, d, howMany, writerNum): + def writerThread(self, d, keys, readers): name = currentThread().getName() - start = howMany * writerNum - stop = howMany * (writerNum + 1) - 1 if verbose: print "%s: creating records %d - %d" % (name, start, stop) - # create a bunch of records - for x in xrange(start, stop): + count=len(keys)/len(readers) + count2=count + for x in keys : key = '%04d' % x dbutils.DeadlockWrap(d.put, key, self.makeData(key), max_retries=12) @@ -240,52 +268,28 @@ class SimpleThreadedBase(BaseThreadedTestCase): if verbose and x % 100 == 0: print "%s: records %d - %d finished" % (name, start, x) - # do a bit or reading too - if random() <= 0.05: - for y in xrange(start, x): - key = '%04d' % x - data = dbutils.DeadlockWrap(d.get, key, max_retries=12) - self.assertEqual(data, self.makeData(key)) - - # flush them - try: - dbutils.DeadlockWrap(d.sync, max_retries=12) - except db.DBIncompleteError, val: - if verbose: - print "could not complete sync()..." - - # read them back, deleting a few - for x in xrange(start, stop): - key = '%04d' % x - data = dbutils.DeadlockWrap(d.get, key, max_retries=12) - if verbose and x % 100 == 0: - print "%s: fetched record (%s, %s)" % (name, key, data) - self.assertEqual(data, self.makeData(key)) - if random() <= 0.10: - dbutils.DeadlockWrap(d.delete, key, max_retries=12) - if verbose: - print "%s: deleted record %s" % (name, key) + count2-=1 + if not count2 : + readers.pop().start() + count2=count if verbose: print "%s: thread finished" % name def readerThread(self, d, readerNum): - time.sleep(0.01 * readerNum) name = currentThread().getName() - for loop in range(5): - c = d.cursor() - count = 0 - rec = dbutils.DeadlockWrap(c.first, max_retries=10) - while rec: - count += 1 - key, data = rec - self.assertEqual(self.makeData(key), data) - rec = dbutils.DeadlockWrap(c.next, max_retries=10) - if verbose: - print "%s: found %d records" % (name, count) - c.close() - time.sleep(0.05) + c = d.cursor() + count = 0 + rec = dbutils.DeadlockWrap(c.first, max_retries=10) + while rec: + count += 1 + key, data = rec + self.assertEqual(self.makeData(key), data) + rec = dbutils.DeadlockWrap(c.next, max_retries=10) + if verbose: + print "%s: found %d records" % (name, count) + c.close() if verbose: print "%s: thread finished" % name @@ -325,120 +329,97 @@ class ThreadedTransactionsBase(BaseThreadedTestCase): print "Running %s.test03_ThreadedTransactions..." % \ self.__class__.__name__ - threads = [] - for x in range(self.writers): - wt = Thread(target = self.writerThread, - args = (self.d, self.records, x), - name = 'writer %d' % x, - )#verbose = verbose) - threads.append(wt) - - for x in range(self.readers): + keys=range(self.records) + import random + random.shuffle(keys) + records_per_writer=self.records/self.writers + readers_per_writer=self.readers/self.writers + self.assertEqual(self.records,self.writers*records_per_writer) + self.assertEqual(self.readers,self.writers*readers_per_writer) + self.assertTrue((records_per_writer%readers_per_writer)==0) + + readers=[] + for x in xrange(self.readers): rt = Thread(target = self.readerThread, args = (self.d, x), name = 'reader %d' % x, )#verbose = verbose) - threads.append(rt) + rt.setDaemon(True) + readers.append(rt) + + writers = [] + for x in xrange(self.writers): + a=keys[records_per_writer*x:records_per_writer*(x+1)] + b=readers[readers_per_writer*x:readers_per_writer*(x+1)] + wt = Thread(target = self.writerThread, + args = (self.d, a, b), + name = 'writer %d' % x, + )#verbose = verbose) + writers.append(wt) dt = Thread(target = self.deadlockThread) + dt.setDaemon(True) dt.start() - for t in threads: + for t in writers: + t.setDaemon(True) t.start() - for t in threads: + + for t in writers: + t.join() + for t in readers: t.join() self.doLockDetect = False dt.join() - def doWrite(self, d, name, start, stop): - finished = False - while not finished: + def writerThread(self, d, keys, readers): + name = currentThread().getName() + count=len(keys)/len(readers) + while len(keys): try: txn = self.env.txn_begin(None, self.txnFlag) - for x in range(start, stop): + keys2=keys[:count] + for x in keys2 : key = '%04d' % x d.put(key, self.makeData(key), txn) if verbose and x % 100 == 0: print "%s: records %d - %d finished" % (name, start, x) txn.commit() - finished = True + keys=keys[count:] + readers.pop().start() except (db.DBLockDeadlockError, db.DBLockNotGrantedError), val: if verbose: print "%s: Aborting transaction (%s)" % (name, val[1]) txn.abort() - time.sleep(0.05) - def writerThread(self, d, howMany, writerNum): - name = currentThread().getName() - start = howMany * writerNum - stop = howMany * (writerNum + 1) - 1 if verbose: - print "%s: creating records %d - %d" % (name, start, stop) - - step = 100 - for x in range(start, stop, step): - self.doWrite(d, name, x, min(stop, x+step)) + print "%s: thread finished" % name - if verbose: - print "%s: finished creating records" % name - if verbose: - print "%s: deleting a few records" % name + def readerThread(self, d, readerNum): + name = currentThread().getName() finished = False while not finished: try: - recs = [] txn = self.env.txn_begin(None, self.txnFlag) - for x in range(10): - key = int(random() * howMany) + start - key = '%04d' % key - data = d.get(key, None, txn, db.DB_RMW) - if data is not None: - d.delete(key, txn) - recs.append(key) + c = d.cursor(txn) + count = 0 + rec = c.first() + while rec: + count += 1 + key, data = rec + self.assertEqual(self.makeData(key), data) + rec = c.next() + if verbose: print "%s: found %d records" % (name, count) + c.close() txn.commit() finished = True - if verbose: - print "%s: deleted records %s" % (name, recs) except (db.DBLockDeadlockError, db.DBLockNotGrantedError), val: if verbose: print "%s: Aborting transaction (%s)" % (name, val[1]) + c.close() txn.abort() - time.sleep(0.05) - - if verbose: - print "%s: thread finished" % name - - def readerThread(self, d, readerNum): - time.sleep(0.01 * readerNum + 0.05) - name = currentThread().getName() - - for loop in range(5): - finished = False - while not finished: - try: - txn = self.env.txn_begin(None, self.txnFlag) - c = d.cursor(txn) - count = 0 - rec = c.first() - while rec: - count += 1 - key, data = rec - self.assertEqual(self.makeData(key), data) - rec = c.next() - if verbose: print "%s: found %d records" % (name, count) - c.close() - txn.commit() - finished = True - except (db.DBLockDeadlockError, db.DBLockNotGrantedError), val: - if verbose: - print "%s: Aborting transaction (%s)" % (name, val[1]) - c.close() - txn.abort() - time.sleep(0.05) - - time.sleep(0.05) if verbose: print "%s: thread finished" % name @@ -446,7 +427,7 @@ class ThreadedTransactionsBase(BaseThreadedTestCase): def deadlockThread(self): self.doLockDetect = True while self.doLockDetect: - time.sleep(0.5) + time.sleep(0.05) try: aborted = self.env.lock_detect( db.DB_LOCK_RANDOM, db.DB_LOCK_CONFLICT) @@ -459,28 +440,28 @@ class ThreadedTransactionsBase(BaseThreadedTestCase): class BTreeThreadedTransactions(ThreadedTransactionsBase): dbtype = db.DB_BTREE - writers = 3 - readers = 5 - records = 2000 + writers = 2 + readers = 10 + records = 1000 class HashThreadedTransactions(ThreadedTransactionsBase): dbtype = db.DB_HASH - writers = 1 - readers = 5 - records = 2000 + writers = 2 + readers = 10 + records = 1000 class BTreeThreadedNoWaitTransactions(ThreadedTransactionsBase): dbtype = db.DB_BTREE - writers = 3 - readers = 5 - records = 2000 + writers = 2 + readers = 10 + records = 1000 txnFlag = db.DB_TXN_NOWAIT class HashThreadedNoWaitTransactions(ThreadedTransactionsBase): dbtype = db.DB_HASH - writers = 1 - readers = 5 - records = 2000 + writers = 2 + readers = 10 + records = 1000 txnFlag = db.DB_TXN_NOWAIT |