/* * Copyright (C) 1997 Spizella Software * All rights reserved. * * Programmer: Robb Matzke * Monday, November 10, 1997 * * Purpose: Implements a family of files that acts as a single hdf5 * file. The purpose is to be able to split a huge file on a * 64-bit platform, transfer all the <2GB members to a 32-bit * platform, and then access the entire huge file on the 32-bit * platform. * * All family members are logically the same size although their * physical sizes may vary. The logical member size is * determined by looking at the physical size of the first * member and rounding that up to the next power of two. When * creating a file family, the first member is created with a * predefined physical size (actually, this happens when the * file family is flushed, and can be quite time consuming on * file systems that don't implement holes, like nfs). * */ #include #include #include #include #define PABLO_MASK H5F_family static hbool_t interface_initialize_g = FALSE; /* * Number of bits in the member address. This can be up to (but not * including) the number of bits in the `off_t' type, but be warned that some * operating systems are not able to write to the last possible address of a * file, so a safe maximum is two less than the number of bits in an `off_t'. * Smaller values result in files of a more manageable size (from a human * perspective) but also limit the total logical size of the hdf5 file. */ #define H5F_FAM_DFLT_NBITS 26u /*64MB*/ #define H5F_FAM_MASK(N) (((uint64)1<<(N))-1) #define H5F_FAM_OFFSET(ADDR,N) ((ADDR)->offset & H5F_FAM_MASK(N)) #define H5F_FAM_MEMBNO(ADDR,N) ((ADDR)->offset >> N) static hbool_t H5F_fam_access (const char *name, int mode, H5F_search_t *key); static H5F_low_t *H5F_fam_open (const char *name, uintn flags, H5F_search_t*); static herr_t H5F_fam_close (H5F_low_t *lf); static herr_t H5F_fam_read (H5F_low_t *lf, const haddr_t *addr, size_t size, uint8 *buf); static herr_t H5F_fam_write (H5F_low_t *lf, const haddr_t *addr, size_t size, const uint8 *buf); static herr_t H5F_fam_flush (H5F_low_t *lf); const H5F_low_class_t H5F_LOW_FAM[1] = {{ H5F_fam_access, /* access method */ H5F_fam_open, /* open method */ H5F_fam_close, /* close method */ H5F_fam_read, /* read method */ H5F_fam_write, /* write method */ H5F_fam_flush, /* flush method */ NULL, /* extend method */ }}; /*------------------------------------------------------------------------- * Function: H5F_fam_open * * Purpose: Opens a file family with the specified base name. The name * should contain a printf-style "%d" field which will be * expanded with a zero-origin family member number. * * Bugs: We don't check for overflow on the name, so keep it under * 4kb, please. Also, we don't actually check for the `%d' * field because we assume that the caller already did. Who * knows what happens when all the family member names are the * same! * * Return: Success: Low-level file pointer * * Failure: NULL * * Programmer: Robb Matzke * Monday, November 10, 1997 * * Modifications: * *------------------------------------------------------------------------- */ static H5F_low_t * H5F_fam_open (const char *name, uintn flags, H5F_search_t *key/*out*/) { H5F_low_t *ret_value = NULL, *lf=NULL; H5F_low_t *member = NULL; /*a family member */ char member_name[4096]; /*name of family member */ intn membno; /*member number (zero-origin) */ size_t nbits=H5F_FAM_DFLT_NBITS;/*num bits in an offset */ haddr_t tmp_addr; /*temporary address */ FUNC_ENTER (H5F_fam_open, NULL, NULL); /* * If we're truncating the file then delete all but the first family * member. Use the default number of bits for the offset. */ if ((flags & H5F_ACC_WRITE) && (flags & H5F_ACC_TRUNC)) { for (membno=1; /*void*/; membno++) { sprintf (member_name, name, membno); if (!H5F_low_access (H5F_LOW_DFLT, member_name, F_OK, NULL)) { break; } else if (unlink (member_name)<0) { HGOTO_ERROR (H5E_IO, H5E_CANTINIT, NULL);/*can't delete member*/ } } } /* Create the file descriptor */ lf = H5MM_xcalloc (1, sizeof(H5F_low_t)); lf->u.fam.name = H5MM_xstrdup (name); lf->u.fam.flags = (flags & ~H5F_ACC_CREAT); /* Open all existing members */ for (membno=0; /*void*/; membno++) { sprintf (member_name, name, membno); /* * Open the family member. After the first member is opened or created, * turn off the creation flag so we don't create a zillion family * members. */ member = H5F_low_open (H5F_LOW_DFLT, member_name, flags, 0==membno?key:NULL); if (!member) { if (0==membno) { /* Can't open first family member */ HGOTO_ERROR (H5E_IO, H5E_CANTOPENFILE, NULL); } break; } flags &= ~H5F_ACC_CREAT; /* Add the member to the family */ if (lf->u.fam.nmemb>=lf->u.fam.nalloc) { lf->u.fam.nalloc = MAX (100, 2*lf->u.fam.nalloc); lf->u.fam.memb = H5MM_xrealloc (lf->u.fam.memb, lf->u.fam.nalloc*sizeof(H5F_low_t*)); } lf->u.fam.memb[lf->u.fam.nmemb++] = member; member = NULL; } /* * If the first and second files exists then round the first file size up * to the next power of two and use that as the number of bits per family * member. */ if (lf->u.fam.nmemb>=2) { size_t size = H5F_low_size (lf->u.fam.memb[0], &tmp_addr); for (nbits=8*sizeof(size_t)-1; nbits>0; --nbits) { size_t mask = (size_t)1 << nbits; if (size & mask) { if (size != mask) { size++; #ifdef H5F_DEBUG fprintf (stderr, "HDF5-DIAG: family member size was rounded up " "to a power of 2"); #endif } break; } } } lf->u.fam.offset_bits = nbits; #ifdef H5F_DEBUG if (nbits>=30) { fprintf (stderr, "HDF5-DIAG: family members are %dGB\n", 1<<(nbits-30)); } else if (nbits>=20) { fprintf (stderr, "HDF5-DIAG: family members are %dMB\n", 1<<(nbits-20)); } else if (nbits>=10) { fprintf (stderr, "HDF5-DIAG: family members are %dkB\n", 1<<(nbits-10)); } else { fprintf (stderr, "HDF5-DIAG: family members are %d bytes\n", 1<u.fam.nmemb>=1); lf->eof.offset = (size_t)1 << lf->u.fam.offset_bits; lf->eof.offset *= (lf->u.fam.nmemb-1); lf->eof.offset += lf->u.fam.memb[lf->u.fam.nmemb-1]->eof.offset; HRETURN (lf); done: if (!ret_value) { if (lf) { H5F_fam_close (lf); H5MM_xfree (lf); } } FUNC_LEAVE (ret_value); } /*------------------------------------------------------------------------- * Function: H5F_fam_close * * Purpose: Closes all members of a file family and releases resources * used by the file descriptor. * * Return: Success: SUCCEED * * Failure: FAIL * * Programmer: Robb Matzke * Monday, November 10, 1997 * * Modifications: * *------------------------------------------------------------------------- */ static herr_t H5F_fam_close (H5F_low_t *lf) { intn membno; FUNC_ENTER (H5F_fam_close, NULL, FAIL); assert (lf); for (membno=0; membnou.fam.nmemb; membno++) { lf->u.fam.memb[membno] = H5F_low_close (lf->u.fam.memb[membno]); } H5MM_xfree (lf->u.fam.memb); H5MM_xfree (lf->u.fam.name); FUNC_LEAVE (SUCCEED); } /*------------------------------------------------------------------------- * Function: H5F_fam_read * * Purpose: Reads a chunk of contiguous data from the file family. * Reading past the physical end of a file returns zeros instead * of failing. We must insure that if the logical end of file is * before the physical end of file that we will read zeros there * also (the only time this can happen is if we create a family * and then close it before the first member is filled, since * flushing the file causes the first member to be physically * extended to it's maximum size). * * Return: Success: SUCCEED * * Failure: FAIL * * Programmer: Robb Matzke * Monday, November 10, 1997 * * Modifications: * *------------------------------------------------------------------------- */ static herr_t H5F_fam_read (H5F_low_t *lf, const haddr_t *addr, size_t size, uint8 *buf) { size_t nbytes; haddr_t cur_addr; uintn membno; off_t offset; size_t member_size; FUNC_ENTER (H5F_fam_read, NULL, FAIL); assert (lf); assert (addr && H5F_addr_defined (addr)); assert (buf); member_size = (size_t)1 << lf->u.fam.offset_bits; membno = H5F_FAM_MEMBNO (addr, lf->u.fam.offset_bits); offset = H5F_FAM_OFFSET (addr, lf->u.fam.offset_bits); cur_addr = *addr; while (size>0) { if (membno>=lf->u.fam.nmemb) { HDmemset (buf, 0, size); break; } else { nbytes = MIN (size, member_size-offset); cur_addr.offset = offset; if (H5F_low_read (lf->u.fam.memb[membno], &cur_addr, nbytes, buf)<0) { /* Can't read from family member */ HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL); } buf += nbytes; size -= nbytes; membno++; offset=0; } } FUNC_LEAVE (SUCCEED); } /*------------------------------------------------------------------------- * Function: H5F_fam_write * * Purpose: Writes BUF to the family of files. The superclass has * already insured that we aren't writing past the logical end * of file, so this function will extend the physical file to * accommodate the new data if necessary. * * Return: Success: SUCCEED * * Failure: FAIL * * Programmer: Robb Matzke * Monday, November 10, 1997 * * Modifications: * *------------------------------------------------------------------------- */ static herr_t H5F_fam_write (H5F_low_t *lf, const haddr_t *addr, size_t size, const uint8 *buf) { size_t nbytes; haddr_t cur_addr, max_addr; uintn membno; off_t offset; H5F_low_t *member = NULL; char member_name[4096]; intn i; size_t member_size; FUNC_ENTER (H5F_fam_write, NULL, FAIL); assert (lf); assert (addr && H5F_addr_defined (addr)); assert (buf); member_size = (size_t)1 << lf->u.fam.offset_bits; membno = H5F_FAM_MEMBNO (addr, lf->u.fam.offset_bits); offset = H5F_FAM_OFFSET (addr, lf->u.fam.offset_bits); cur_addr = *addr; while (size>0) { nbytes = MIN (size, member_size-offset); cur_addr.offset = offset; if (membno>=lf->u.fam.nmemb) { /* * We're writing past the end of the last family member--create the * new family member(s) */ for (i=lf->u.fam.nmemb; i<=membno; i++) { sprintf (member_name, lf->u.fam.name, i); member = H5F_low_open (H5F_LOW_DFLT, member_name, lf->u.fam.flags|H5F_ACC_CREAT, NULL); if (!member) { /* Can't create a new member */ HRETURN_ERROR (H5E_IO, H5E_CANTOPENFILE, FAIL); } /* * For members in the middle, set their logical eof to the * maximum possible value. */ if (iu.fam.nmemb>=lf->u.fam.nalloc) { lf->u.fam.nalloc *= 2; lf->u.fam.memb = H5MM_xrealloc (lf->u.fam.memb, (lf->u.fam.nalloc * sizeof(H5F_low_t*))); } lf->u.fam.memb[lf->u.fam.nmemb++] = member; } } /* * Make sure the logical eof is large enough to handle the request. */ max_addr = cur_addr; H5F_addr_inc (&max_addr, nbytes); H5F_low_seteof (lf->u.fam.memb[membno], &max_addr); /* Write the data to the member */ if (H5F_low_write (lf->u.fam.memb[membno], &cur_addr, nbytes, buf)<0) { /* Can't write to family member */ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL); } buf += nbytes; size -= nbytes; membno++; offset=0; } FUNC_LEAVE (SUCCEED); } /*------------------------------------------------------------------------- * Function: H5F_fam_flush * * Purpose: Flushes all data to disk and makes sure that the first member * is as large as a member can be so we can accurately detect * the member size if we open this file for read access at a * later date. * * Return: Success: SUCCEED * * Failure: FAIL * * Programmer: Robb Matzke * Monday, November 10, 1997 * * Modifications: * *------------------------------------------------------------------------- */ static herr_t H5F_fam_flush (H5F_low_t *lf) { int membno, nerrors=0; uint8 buf[1]; haddr_t addr1, addr2, addr3; size_t max_offset; FUNC_ENTER (H5F_fam_flush, NULL, FAIL); /* * Make sure that the first family member is the maximum size because * H5F_fam_open() looks at the size of the first member to determine the * number of bits to use for each family member offset. We do this by * reading the last possible byte from the member (which defaults to zero * if we're reading past the end of the member) and then writing it back. */ max_offset = H5F_FAM_MASK (lf->u.fam.offset_bits); H5F_addr_reset (&addr1); H5F_addr_inc (&addr1, max_offset); H5F_low_size (lf->u.fam.memb[0], &addr2); /*remember logical eof*/ addr3 = addr1; H5F_addr_inc (&addr3, (size_t)1); H5F_low_seteof (lf->u.fam.memb[0], &addr3); /*prevent a warning*/ if (H5F_low_read (lf->u.fam.memb[0], &addr1, 1, buf)<0) { /* Can't read from first family member */ HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL); } if (H5F_low_write (lf->u.fam.memb[0], &addr1, 1, buf)<0) { /* Can't write to first family member */ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL); } H5F_low_seteof (lf->u.fam.memb[0], &addr2); /*reset old eof*/ /* * Flush each member file. Don't return an error status until we've * flushed as much as possible. */ for (membno=0; membnou.fam.nmemb; membno++) { if (H5F_low_flush (lf->u.fam.memb[membno])<0) { nerrors++; } } if (nerrors) { /* Can't flush family member */ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL); } FUNC_LEAVE (SUCCEED); } /*------------------------------------------------------------------------- * Function: H5F_fam_access * * Purpose: Determines if all members of the file family can be accessed * and returns the key for the first member of the family. * * Return: Success: TRUE or FALSE * * Failure: FAIL * * Programmer: Robb Matzke * Monday, November 10, 1997 * * Modifications: * *------------------------------------------------------------------------- */ static hbool_t H5F_fam_access (const char *name, int mode, H5F_search_t *key/*out*/) { intn membno; char member_name[4096]; hbool_t status; FUNC_ENTER (H5F_fam_access, NULL, FAIL); for (membno=0; /*void*/; membno++) { sprintf (member_name, name, membno); status = H5F_low_access (H5F_LOW_DFLT, member_name, mode, 0==membno?key:NULL); if (!status) { if (F_OK==mode) { /* * If we didn't find a member then we must have gotten to the end * of the family. As long as we found the first member(s) the * family exists. */ HRETURN (membno>0); } else if (H5F_low_access (H5F_LOW_DFLT, member_name, F_OK, NULL)) { /* * The file exists but didn't have the write access permissions. */ HRETURN (FALSE); } else { /* * The file doesn't exist because we got to the end of the * family. */ HRETURN (TRUE); } } if (status<0) { /* Access method failed for a member file */ HRETURN_ERROR (H5E_IO, H5E_CANTOPENFILE, FAIL); } } FUNC_LEAVE (TRUE); }