From a3888a3b007e818f4178b053606c8d1b46f0fa1b Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Thu, 28 Aug 2003 11:02:21 -0500 Subject: [svn-r7426] Purpose: Bug fix Description: When datasets are deleted from a file, they are removed from the sieve buffer, but instead of invalidating only the part of the sieve buffer affected, the sieve buffer code would throw away the entire sieve buffer, potentially including other raw data in the buffer that hadn't been written to disk yet. Solution: Improve the sieve buffer clearing code to handle partial invalidations. Platforms tested: FreeBSD 4.8 (sleipnir) h5committest --- release_docs/RELEASE.txt | 3 + src/H5Dcontig.c | 2 +- src/H5Distore.c | 2 +- src/H5F.c | 55 ++++++++++- src/H5Fcontig.c | 2 +- src/H5Fistore.c | 2 +- src/H5Fpkg.h | 2 +- test/Makefile.in | 25 ++--- test/tmisc.c | 237 ++++++++++++++++++++++++++++++++++++++++++++++- 9 files changed, 303 insertions(+), 27 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index a3eff51..158e722 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -74,6 +74,9 @@ Bug Fixes since HDF5-1.6.0 release Library ------- + - Corrected bug in sieve buffer code which could cause loss of data + when a small dataset was created and deleted in quick succession. + QAK - 2003/08/27 - Corrected bug in H5Gget_objname_by_idx which was not allowing NULL for the name when just querying for the object name's length. QAK - 2003/08/25 diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c index 64a4377..5f0c72d 100644 --- a/src/H5Dcontig.c +++ b/src/H5Dcontig.c @@ -332,7 +332,7 @@ H5F_contig_delete(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout) size *= layout->dim[u]; /* Check for overlap with the sieve buffer and reset it */ - if (H5F_sieve_overlap_clear(f, layout->addr, size)<0) + if (H5F_sieve_overlap_clear(f, dxpl_id, layout->addr, size)<0) HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to clear sieve buffer"); /* Free the file space for the chunk */ diff --git a/src/H5Distore.c b/src/H5Distore.c index 929a3e0..909007a 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -2662,7 +2662,7 @@ H5F_istore_remove(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key /*in,out FUNC_ENTER_NOAPI(H5F_istore_remove,H5B_INS_ERROR); /* Check for overlap with the sieve buffer and reset it */ - if (H5F_sieve_overlap_clear(f, addr, (hsize_t)lt_key->nbytes)<0) + if (H5F_sieve_overlap_clear(f, dxpl_id, addr, (hsize_t)lt_key->nbytes)<0) HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, H5B_INS_ERROR, "unable to clear sieve buffer"); /* Remove raw data chunk from file */ diff --git a/src/H5F.c b/src/H5F.c index fc3273b..2122ae2 100644 --- a/src/H5F.c +++ b/src/H5F.c @@ -4196,7 +4196,7 @@ H5F_addr_decode(const H5F_t *f, const uint8_t **pp/*in,out*/, haddr_t *addr_p/*o *------------------------------------------------------------------------- */ herr_t -H5F_sieve_overlap_clear(const H5F_t *f, haddr_t addr, hsize_t size) +H5F_sieve_overlap_clear(const H5F_t *f, hid_t dxpl_id, haddr_t addr, hsize_t size) { herr_t ret_value=SUCCEED; /* Return value */ @@ -4207,10 +4207,55 @@ H5F_sieve_overlap_clear(const H5F_t *f, haddr_t addr, hsize_t size) /* Check for the address range overlapping with the sieve buffer */ if(H5F_addr_overlap(f->shared->sieve_loc,f->shared->sieve_size,addr,size)) { - /* Reset sieve information */ - f->shared->sieve_loc=HADDR_UNDEF; - f->shared->sieve_size=0; - f->shared->sieve_dirty=0; + /* Check if only part of the sieve buffer is being invalidated */ + if(sizeshared->sieve_size) { + /* Check if the portion to invalidate is at the end */ + if((f->shared->sieve_loc+f->shared->sieve_size)==(addr+size)) { + /* Just shorten the buffer */ + f->shared->sieve_size-=size; + } /* end if */ + /* Check if the portion to invalidate is at the beginning */ + else if(f->shared->sieve_loc==addr) { + /* Advance the start of the sieve buffer (on disk) and shorten the buffer */ + f->shared->sieve_loc+=size; + f->shared->sieve_size-=size; + + /* Move the retained information in the buffer down */ + HDmemcpy(f->shared->sieve_buf,f->shared->sieve_buf+size,f->shared->sieve_size); + } /* end elif */ + /* Portion to invalidate is in middle */ + else { + size_t invalid_size; /* Portion of sieve buffer to invalidate */ + + /* Write out portion at the beginning of the buffer, if buffer is dirty */ + if(f->shared->sieve_dirty) { + size_t start_size; /* Portion of sieve buffer to write */ + + /* Compute size of block at beginning of buffer */ + start_size=(addr-f->shared->sieve_loc); + + /* Write to file */ + if (H5F_block_write(f, H5FD_MEM_DRAW, f->shared->sieve_loc, start_size, dxpl_id, f->shared->sieve_buf)<0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); + } /* end if */ + + /* Compute size of block to invalidate */ + invalid_size=((addr+size)-f->shared->sieve_loc); + + /* Advance the start of the sieve buffer (on disk) and shorten the buffer */ + f->shared->sieve_loc+=invalid_size; + f->shared->sieve_size-=invalid_size; + + /* Move the retained information in the buffer down */ + HDmemcpy(f->shared->sieve_buf,f->shared->sieve_buf+invalid_size,f->shared->sieve_size); + } /* end else */ + } /* end if */ + else { + /* Reset sieve information */ + f->shared->sieve_loc=HADDR_UNDEF; + f->shared->sieve_size=0; + f->shared->sieve_dirty=0; + } /* end else */ } /* end if */ done: diff --git a/src/H5Fcontig.c b/src/H5Fcontig.c index 64a4377..5f0c72d 100644 --- a/src/H5Fcontig.c +++ b/src/H5Fcontig.c @@ -332,7 +332,7 @@ H5F_contig_delete(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout) size *= layout->dim[u]; /* Check for overlap with the sieve buffer and reset it */ - if (H5F_sieve_overlap_clear(f, layout->addr, size)<0) + if (H5F_sieve_overlap_clear(f, dxpl_id, layout->addr, size)<0) HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to clear sieve buffer"); /* Free the file space for the chunk */ diff --git a/src/H5Fistore.c b/src/H5Fistore.c index 929a3e0..909007a 100644 --- a/src/H5Fistore.c +++ b/src/H5Fistore.c @@ -2662,7 +2662,7 @@ H5F_istore_remove(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key /*in,out FUNC_ENTER_NOAPI(H5F_istore_remove,H5B_INS_ERROR); /* Check for overlap with the sieve buffer and reset it */ - if (H5F_sieve_overlap_clear(f, addr, (hsize_t)lt_key->nbytes)<0) + if (H5F_sieve_overlap_clear(f, dxpl_id, addr, (hsize_t)lt_key->nbytes)<0) HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, H5B_INS_ERROR, "unable to clear sieve buffer"); /* Remove raw data chunk from file */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 5599fa9..8ba9806 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -196,7 +196,7 @@ H5_DLLVAR hbool_t H5_mpiposix_1_metawrite_g; H5_DLL void H5F_encode_length_unusual(const H5F_t *f, uint8_t **p, uint8_t *l); #endif /* NOT_YET */ H5_DLL herr_t H5F_mountpoint(struct H5G_entry_t *find/*in,out*/); -H5_DLL herr_t H5F_sieve_overlap_clear(const H5F_t *f, haddr_t addr, hsize_t size); +H5_DLL herr_t H5F_sieve_overlap_clear(const H5F_t *f, hid_t dxpl_id, haddr_t addr, hsize_t size); /* Functions that operate on indexed storage */ H5_DLL herr_t H5F_istore_init (H5F_t *f); diff --git a/test/Makefile.in b/test/Makefile.in index 5504940..4a89e03 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -48,21 +48,16 @@ PUB_LIB= ## the temporary file name in ways that the makefile is not aware of. MOSTLYCLEAN=cmpd_dset.h5 compact_dataset.h5 dataset.h5 extend.h5 istore.h5 \ tfile1.h5 tfile2.h5 tfile3.h5 th5s1.h5 lheap.h5 ohdr.h5 stab1.h5 \ - stab2.h5 extern_1.h5 extern_2.h5 extern_3.h5 extern_1a.raw \ - extern_1b.raw extern_2a.raw extern_2b.raw extern_3a.raw \ - extern_3b.raw extern_4a.raw extern_4b.raw gheap0.h5 gheap1.h5 \ - gheap2.h5 gheap3.h5 gheap4.h5 links.h5 links[1-3].h5 big.data \ - big[0-9][0-9][0-9][0-9][0-9].h5 dtypes[1-3].h5 tattr.h5 \ - tselect.h5 mtime.h5 unlink.h5 fillval_[0-9].h5 fillval.raw \ - mount_[0-9].h5 testmeta.h5 ttime.h5 trefer[1-3].h5 tvltypes.h5 \ - tvlstr.h5 flush.h5 enum1.h5 titerate.h5 ttsafe.h5 tarray1.h5 \ - tgenprop.h5 tmisc.h5 tmisc2a.h5 tmisc2b.h5 tmisc3.h5 tmisc4a.h5 \ - tmisc4b.h5 tmisc5.h5 tmisc6.h5 tmisc7.h5 tmisc8.h5 tmisc9.h5 \ - tmisc10.h5 tmisc11.h5 tmisc12.h5 tmisc13a.h5 tmisc13b.h5 \ - set_extent_read.h5 set_extent_create.h5 getname.h5 getname1.h5 \ - getname2.h5 getname3.h5 sec2_file.h5 family_file000[0-3][0-9].h5 \ - multi_file-[rs].h5 core_file new_move_[ab].h5 ntypes.h5 dangle.h5 \ - errors.h5 + stab2.h5 extern_[1-3].h5 extern_[1-4][ab].raw gheap[0-4].h5 \ + links.h5 links[1-3].h5 big.data big[0-9][0-9][0-9][0-9][0-9].h5 \ + dtypes[1-3].h5 tattr.h5 tselect.h5 mtime.h5 unlink.h5 \ + fillval_[0-9].h5 fillval.raw mount_[0-9].h5 testmeta.h5 ttime.h5 \ + trefer[1-3].h5 tvltypes.h5 tvlstr.h5 flush.h5 enum1.h5 \ + titerate.h5 ttsafe.h5 tarray1.h5 tgenprop.h5 tmisc[0-9]*.h5 \ + set_extent_read.h5 set_extent_create.h5 getname.h5 \ + getname[1-3].h5 sec2_file.h5 family_file000[0-3][0-9].h5 \ + multi_file-[rs].h5 core_file new_move_[ab].h5 ntypes.h5 \ + dangle.h5 errors.h5 CLEAN=$(TIMINGS) diff --git a/test/tmisc.c b/test/tmisc.c index 8c7ae93..ec804b1 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -28,7 +28,7 @@ #include "testhdf5.h" /* Definitions for misc. test #1 */ -#define MISC1_FILE "tmisc.h5" +#define MISC1_FILE "tmisc1.h5" #define MISC1_VAL (13417386) /* 0xccbbaa */ #define MISC1_VAL2 (15654348) /* 0xeeddcc */ #define MISC1_DSET_NAME "/scalar_set" @@ -184,6 +184,13 @@ typedef struct unsigned m13_data[MISC13_DIM1][MISC13_DIM2]; /* Data to write to dataset */ unsigned m13_rdata[MISC13_DIM1][MISC13_DIM2]; /* Data read from dataset */ +/* Definitions for misc. test #14 */ +#define MISC14_FILE "tmisc14.h5" +#define MISC14_DSET1_NAME "Dataset1" +#define MISC14_DSET2_NAME "Dataset2" +#define MISC14_DSET3_NAME "Dataset3" +#define MISC14_METADATA_SIZE 4096 + /**************************************************************** ** ** test_misc1(): test unlinking a dataset from a group and immediately @@ -2333,7 +2340,231 @@ test_misc13(void) /* Verify file contents are still correct */ verify_file(MISC13_FILE_2,(hsize_t)MISC13_USERBLOCK_SIZE,1); -} +} /* end test_misc13() */ + +/**************************************************************** +** +** test_misc14(): Test that file contents can be "slid down" by +** inserting a user block in front of an existing file. +** +****************************************************************/ +static void +test_misc14(void) +{ + hid_t file_id; /* File ID */ + hid_t fapl; /* File access property list ID */ + hid_t DataSpace; /* Dataspace ID */ + hid_t Dataset1; /* Dataset ID #1 */ + hid_t Dataset2; /* Dataset ID #2 */ + hid_t Dataset3; /* Dataset ID #3 */ + double data1 = 5.0; /* Data to write for dataset #1 */ + double data2 = 10.0; /* Data to write for dataset #2 */ + double data3 = 15.0; /* Data to write for dataset #3 */ + double rdata; /* Data read in */ + herr_t ret; /* Generic return value */ + + /* Test creating two datasets and deleting the second */ + + /* Increase the metadata block size */ + /* (This makes certain that all the data blocks are allocated together) */ + fapl=H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + + ret=H5Pset_meta_block_size(fapl,(hsize_t)MISC14_METADATA_SIZE); + CHECK(ret, FAIL, "H5Pset_meta_block_size"); + + /* Create dataspace to use */ + DataSpace = H5Screate(H5S_SCALAR); + CHECK(DataSpace, FAIL, "H5Screate"); + + /* Open the file */ + file_id = H5Fcreate(MISC14_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + CHECK(file_id, FAIL, "H5Fcreate"); + + /* Create first dataset & write data */ + Dataset1 = H5Dcreate(file_id, MISC14_DSET1_NAME, H5T_NATIVE_DOUBLE, DataSpace, H5P_DEFAULT); + CHECK(Dataset1, FAIL, "H5Dcreate"); + + ret = H5Dwrite(Dataset1, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data1); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Create second dataset (to be unlinked). */ + Dataset2 = H5Dcreate(file_id, MISC14_DSET2_NAME, H5T_NATIVE_DOUBLE, DataSpace, H5P_DEFAULT); + CHECK(Dataset2, FAIL, "H5Dcreate"); + + ret = H5Dwrite(Dataset2, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data2); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Check data from first dataset */ + ret = H5Dread(Dataset1, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdata); + CHECK(ret, FAIL, "H5Dread"); + if(rdata!=data1) { + num_errs++; + printf("Error on line %d: data1!=rdata\n",__LINE__); + } /* end if */ + + /* Unlink second dataset */ + ret = H5Gunlink(file_id, MISC14_DSET2_NAME); + CHECK(ret, FAIL, "H5Gunlink"); + + /* Close second dataset */ + ret = H5Dclose(Dataset2); + CHECK(ret, FAIL, "H5Dclose"); + + /* Verify the data from dataset #1 */ + ret = H5Dread(Dataset1, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdata); + CHECK(ret, FAIL, "H5Dread"); + if(rdata!=data1) { + num_errs++; + printf("Error on line %d: data1!=rdata\n",__LINE__); + } /* end if */ + + /* Close first dataset */ + ret = H5Dclose(Dataset1); + CHECK(ret, FAIL, "H5Dclose"); + + /* Close the file */ + ret = H5Fclose (file_id); + CHECK(ret, FAIL, "H5Fclose"); + + /* Test creating two datasets and deleting the first */ + + /* Open the file */ + file_id = H5Fcreate(MISC14_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + CHECK(file_id, FAIL, "H5Fcreate"); + + /* Create first dataset & write data */ + Dataset1 = H5Dcreate(file_id, MISC14_DSET1_NAME, H5T_NATIVE_DOUBLE, DataSpace, H5P_DEFAULT); + CHECK(Dataset1, FAIL, "H5Dcreate"); + + ret = H5Dwrite(Dataset1, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data1); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Create second dataset */ + Dataset2 = H5Dcreate(file_id, MISC14_DSET2_NAME, H5T_NATIVE_DOUBLE, DataSpace, H5P_DEFAULT); + CHECK(Dataset2, FAIL, "H5Dcreate"); + + ret = H5Dwrite(Dataset2, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data2); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Check data from second dataset */ + ret = H5Dread(Dataset2, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdata); + CHECK(ret, FAIL, "H5Dread"); + if(rdata!=data2) { + num_errs++; + printf("Error on line %d: data2!=rdata\n",__LINE__); + } /* end if */ + + /* Unlink first dataset */ + ret = H5Gunlink(file_id, MISC14_DSET1_NAME); + CHECK(ret, FAIL, "H5Gunlink"); + + /* Close first dataset */ + ret = H5Dclose(Dataset1); + CHECK(ret, FAIL, "H5Dclose"); + + /* Verify the data from dataset #2 */ + ret = H5Dread(Dataset2, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdata); + CHECK(ret, FAIL, "H5Dread"); + if(rdata!=data2) { + num_errs++; + printf("Error on line %d: data2!=rdata\n",__LINE__); + } /* end if */ + + /* Close second dataset */ + ret = H5Dclose(Dataset2); + CHECK(ret, FAIL, "H5Dclose"); + + /* Close the file */ + ret = H5Fclose (file_id); + CHECK(ret, FAIL, "H5Fclose"); + + /* Test creating three datasets and deleting the second */ + + /* Open the file */ + file_id = H5Fcreate(MISC14_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + CHECK(file_id, FAIL, "H5Fcreate"); + + /* Create first dataset & write data */ + Dataset1 = H5Dcreate(file_id, MISC14_DSET1_NAME, H5T_NATIVE_DOUBLE, DataSpace, H5P_DEFAULT); + CHECK(Dataset1, FAIL, "H5Dcreate"); + + ret = H5Dwrite(Dataset1, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data1); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Create second dataset */ + Dataset2 = H5Dcreate(file_id, MISC14_DSET2_NAME, H5T_NATIVE_DOUBLE, DataSpace, H5P_DEFAULT); + CHECK(Dataset2, FAIL, "H5Dcreate"); + + ret = H5Dwrite(Dataset2, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data2); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Create third dataset */ + Dataset3 = H5Dcreate(file_id, MISC14_DSET3_NAME, H5T_NATIVE_DOUBLE, DataSpace, H5P_DEFAULT); + CHECK(Dataset2, FAIL, "H5Dcreate"); + + ret = H5Dwrite(Dataset3, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data3); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Check data from first dataset */ + ret = H5Dread(Dataset1, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdata); + CHECK(ret, FAIL, "H5Dread"); + if(rdata!=data1) { + num_errs++; + printf("Error on line %d: data1!=rdata\n",__LINE__); + } /* end if */ + + /* Check data from third dataset */ + ret = H5Dread(Dataset3, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdata); + CHECK(ret, FAIL, "H5Dread"); + if(rdata!=data3) { + num_errs++; + printf("Error on line %d: data3!=rdata\n",__LINE__); + } /* end if */ + + /* Unlink second dataset */ + ret = H5Gunlink(file_id, MISC14_DSET2_NAME); + CHECK(ret, FAIL, "H5Gunlink"); + + /* Close second dataset */ + ret = H5Dclose(Dataset2); + CHECK(ret, FAIL, "H5Dclose"); + + /* Verify the data from dataset #1 */ + ret = H5Dread(Dataset1, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdata); + CHECK(ret, FAIL, "H5Dread"); + if(rdata!=data1) { + num_errs++; + printf("Error on line %d: data1!=rdata\n",__LINE__); + } /* end if */ + + /* Verify the data from dataset #3 */ + ret = H5Dread(Dataset3, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdata); + CHECK(ret, FAIL, "H5Dread"); + if(rdata!=data3) { + num_errs++; + printf("Error on line %d: data3!=rdata\n",__LINE__); + } /* end if */ + + /* Close first dataset */ + ret = H5Dclose(Dataset1); + CHECK(ret, FAIL, "H5Dclose"); + + /* Close third dataset */ + ret = H5Dclose(Dataset3); + CHECK(ret, FAIL, "H5Dclose"); + + /* Close the file */ + ret = H5Fclose (file_id); + CHECK(ret, FAIL, "H5Fclose"); + + /* Close shared objects (dataspace & fapl) */ + ret = H5Sclose (DataSpace); + CHECK(ret, FAIL, "H5Sclose"); + ret = H5Pclose (fapl); + CHECK(ret, FAIL, "H5Pclose"); + +} /* end test_misc14() */ /**************************************************************** ** @@ -2359,6 +2590,7 @@ test_misc(void) test_misc11(); /* Test for all properties of a file creation property list being stored */ test_misc12(); /* Test VL-strings in chunked datasets operating correctly */ test_misc13(); /* Test that a user block can be insert in front of file contents */ + test_misc14(); /* Test that deleted dataset's data is removed from sieve buffer correctly */ } /* test_misc() */ @@ -2396,4 +2628,5 @@ cleanup_misc(void) HDremove(MISC12_FILE); HDremove(MISC13_FILE_1); HDremove(MISC13_FILE_2); + HDremove(MISC14_FILE); } -- cgit v0.12