path: root/src/H5Ffamily.c

/*
 * Copyright (C) 1997 Spizella Software
 *                    All rights reserved.
 *
 * Programmer: Robb Matzke <matzke@llnl.gov>
 *             Monday, November 10, 1997
 *
 * Purpose:	Implements a family of files that acts as a single hdf5
 *		file.  The purpose is to be able to split a huge file on a
 *		64-bit platform, transfer all the <2GB members to a 32-bit
 *		platform, and then access the entire huge file on the 32-bit
 *		platform.
 *
 *		All family members are logically the same size although their
 *		physical sizes may vary.  The logical member size is
 *		determined by looking at the physical size of the first
 *		member and rounding that up to the next power of two.  When
 *		creating a file family, the first member is created with a
 *		predefined physical size (actually, this happens when the
 *		file family is flushed, and can be quite time consuming on
 *		file systems that don't implement holes, like nfs).
 *		
 */
#include <H5private.h>
#include <H5Eprivate.h>
#include <H5Fprivate.h>
#include <H5MMprivate.h>

#define PABLO_MASK H5F_family
static hbool_t interface_initialize_g = FALSE;
#define INTERFACE_INIT NULL

/*
 * Number of bits in the member address.  This can be up to (but not
 * including) the number of bits in the `off_t' type, but be warned that some
 * operating systems are not able to write to the last possible address of a
 * file, so a safe maximum is two less than the number of bits in an `off_t'.
 * Smaller values result in files of a more manageable size (from a human
 * perspective) but also limit the total logical size of the hdf5 file.
 */
#define H5F_FAM_DFLT_NBITS	26u	/*64MB*/

#define H5F_FAM_MASK(N)		(((uint64)1<<(N))-1)
#define H5F_FAM_OFFSET(ADDR,N)	((ADDR)->offset & H5F_FAM_MASK(N))
#define H5F_FAM_MEMBNO(ADDR,N)	((ADDR)->offset >> N)

static hbool_t H5F_fam_access (const char *name, int mode, H5F_search_t *key);
static H5F_low_t *H5F_fam_open (const char *name, uintn flags, H5F_search_t*);
static herr_t H5F_fam_close (H5F_low_t *lf);
static herr_t H5F_fam_read (H5F_low_t *lf, const haddr_t *addr, size_t size,
			    uint8 *buf);
static herr_t H5F_fam_write (H5F_low_t *lf, const haddr_t *addr, size_t size,
			     const uint8 *buf);
static herr_t H5F_fam_flush (H5F_low_t *lf);

const H5F_low_class_t H5F_LOW_FAM[1] = {{
   H5F_fam_access,		/* access method			*/
   H5F_fam_open, 		/* open method				*/
   H5F_fam_close, 		/* close method				*/
   H5F_fam_read,		/* read method				*/
   H5F_fam_write, 		/* write method				*/
   H5F_fam_flush, 		/* flush method				*/
   NULL,			/* extend method			*/
}};


/*-------------------------------------------------------------------------
 * Function:	H5F_fam_open
 *
 * Purpose:	Opens a file family with the specified base name.  The name
 *		should contain a printf-style "%d" field which will be
 *		expanded with a zero-origin family member number.
 *
 * Bugs:	We don't check for overflow on the name, so keep it under
 *		4kb, please.  Also, we don't actually check for the `%d'
 *		field because we assume that the caller already did.  Who
 *		knows what happens when all the family member names are the
 *		same!
 *
 * Return:	Success:	Low-level file pointer
 *
 *		Failure:	NULL
 *
 * Programmer:	Robb Matzke
 *              Monday, November 10, 1997
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
static H5F_low_t *
H5F_fam_open (const char *name, uintn flags, H5F_search_t *key/*out*/)
{
   H5F_low_t	*ret_value = NULL, *lf=NULL;
   H5F_low_t	*member = NULL;		/*a family member		*/
   char		member_name[4096];	/*name of family member		*/
   intn		membno;			/*member number (zero-origin)	*/
   size_t	nbits=H5F_FAM_DFLT_NBITS;/*num bits in an offset	*/
   haddr_t	tmp_addr;		/*temporary address		*/

   FUNC_ENTER (H5F_fam_open, NULL);

   /*
    * If we're truncating the file then delete all but the first family
    * member.  Use the default number of bits for the offset.
    */
   if ((flags & H5F_ACC_WRITE) && (flags & H5F_ACC_TRUNC)) {
      for (membno=1; /*void*/; membno++) {
	 sprintf (member_name, name, membno);
	 if (!H5F_low_access (H5F_LOW_DFLT, member_name, F_OK, NULL)) {
	    break;
	 } else if (unlink (member_name)<0) {
	    HGOTO_ERROR (H5E_IO, H5E_CANTINIT, NULL, "can't delete member");
	 }
      }
   }

   /* Create the file descriptor */
   lf = H5MM_xcalloc (1, sizeof(H5F_low_t));
   lf->u.fam.name = H5MM_xstrdup (name);
   lf->u.fam.flags = (flags & ~H5F_ACC_CREAT);

   /* Open all existing members */
   for (membno=0; /*void*/; membno++) {
      sprintf (member_name, name, membno);

      /*
       * Open the family member.  After the first member is opened or created,
       * turn off the creation flag so we don't create a zillion family
       * members.
       */
      member = H5F_low_open (H5F_LOW_DFLT, member_name, flags,
			     0==membno?key:NULL);
      if (!member) {
	 if (0==membno) {
	    HGOTO_ERROR (H5E_IO, H5E_CANTOPENFILE, NULL,
			 "can't open first family member");
	 }
	 break;
      }
      flags &= ~H5F_ACC_CREAT;

      /* Add the member to the family */
      if (lf->u.fam.nmemb>=lf->u.fam.nalloc) {
	 lf->u.fam.nalloc = MAX (100, 2*lf->u.fam.nalloc);
	 lf->u.fam.memb = H5MM_xrealloc (lf->u.fam.memb,
					 lf->u.fam.nalloc*sizeof(H5F_low_t*));
      }
      lf->u.fam.memb[lf->u.fam.nmemb++] = member;
      member = NULL;
   }
   
   /*
    * If the first and second files exists then round the first file size up
    * to the next power of two and use that as the number of bits per family
    * member.
    */
   if (lf->u.fam.nmemb>=2) {
      size_t size = H5F_low_size (lf->u.fam.memb[0], &tmp_addr);
      for (nbits=8*sizeof(size_t)-1; nbits>0; --nbits) {
	 size_t mask = (size_t)1 << nbits;
	 if (size & mask) {
	    if (size != mask) {
	       size++;
#ifdef H5F_DEBUG
	       fprintf (stderr, "HDF5-DIAG: family member size was rounded up "
			"to a power of 2");
#endif
	    }
	    break;
	 }
      }
   }
   lf->u.fam.offset_bits = nbits;
   
#ifdef H5F_DEBUG
   if (nbits>=30) {
      fprintf (stderr, "HDF5-DIAG: family members are %dGB\n", 1<<(nbits-30));
   } else if (nbits>=20) {
      fprintf (stderr, "HDF5-DIAG: family members are %dMB\n", 1<<(nbits-20));
   } else if (nbits>=10) {
      fprintf (stderr, "HDF5-DIAG: family members are %dkB\n", 1<<(nbits-10));
   } else {
      fprintf (stderr, "HDF5-DIAG: family members are %d bytes\n", 1<<nbits);
   }
#endif

   /*
    * Get the total family size and store it in the max_addr field.
    */
   assert (lf->u.fam.nmemb>=1);
   lf->eof.offset = (size_t)1 << lf->u.fam.offset_bits;
   lf->eof.offset *= (lf->u.fam.nmemb-1);
   lf->eof.offset += lf->u.fam.memb[lf->u.fam.nmemb-1]->eof.offset;

   HRETURN (lf);

 done:
   if (!ret_value) {
      if (lf) {
	 H5F_fam_close (lf);
	 H5MM_xfree (lf);
      }
   }
   FUNC_LEAVE (ret_value);
}


/*-------------------------------------------------------------------------
 * Function:	H5F_fam_close
 *
 * Purpose:	Closes all members of a file family and releases resources
 *		used by the file descriptor.
 *
 * Return:	Success:	SUCCEED
 *
 *		Failure:	FAIL
 *
 * Programmer:	Robb Matzke
 *              Monday, November 10, 1997
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
static herr_t
H5F_fam_close (H5F_low_t *lf)
{
   intn		membno;
   
   FUNC_ENTER (H5F_fam_close, FAIL);

   assert (lf);
   
   for (membno=0; membno<lf->u.fam.nmemb; membno++) {
      lf->u.fam.memb[membno] = H5F_low_close (lf->u.fam.memb[membno]);
   }
   H5MM_xfree (lf->u.fam.memb);
   H5MM_xfree (lf->u.fam.name);

   FUNC_LEAVE (SUCCEED);
}


/*-------------------------------------------------------------------------
 * Function:	H5F_fam_read
 *
 * Purpose:	Reads a chunk of contiguous data from the file family.
 *		Reading past the physical end of a file returns zeros instead
 *		of failing. We must insure that if the logical end of file is
 *		before the physical end of file that we will read zeros there
 *		also (the only time this can happen is if we create a family
 *		and then close it before the first member is filled, since
 *		flushing the file causes the first member to be physically
 *		extended to it's maximum size).
 *
 * Return:	Success:	SUCCEED
 *
 *		Failure:	FAIL
 *
 * Programmer:	Robb Matzke
 *              Monday, November 10, 1997
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
static herr_t
H5F_fam_read (H5F_low_t *lf, const haddr_t *addr, size_t size, uint8 *buf)
{
   size_t	nbytes;
   haddr_t	cur_addr;
   uintn	membno;
   off_t	offset;
   size_t	member_size;
   
   FUNC_ENTER (H5F_fam_read, FAIL);

   assert (lf);
   assert (addr && H5F_addr_defined (addr));
   assert (buf);

   member_size = (size_t)1 << lf->u.fam.offset_bits;
   membno = H5F_FAM_MEMBNO (addr, lf->u.fam.offset_bits);
   offset = H5F_FAM_OFFSET (addr, lf->u.fam.offset_bits);
   cur_addr = *addr;

   while (size>0) {
      if (membno>=lf->u.fam.nmemb) {
	 HDmemset (buf, 0, size);
	 break;
      } else {
	 nbytes = MIN (size, member_size-offset);
	 cur_addr.offset = offset;
	 if (H5F_low_read (lf->u.fam.memb[membno], &cur_addr,
			   nbytes, buf)<0) {
	    HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL,
			   "can't read from family member");
	 }
	 buf += nbytes;
	 size -= nbytes;
	 membno++;
	 offset=0;
      }
   }

   FUNC_LEAVE (SUCCEED);
}


/*-------------------------------------------------------------------------
 * Function:	H5F_fam_write
 *
 * Purpose:	Writes BUF to the family of files.  The superclass has
 *		already insured that we aren't writing past the logical end
 *		of file, so this function will extend the physical file to
 *		accommodate the new data if necessary.
 *
 * Return:	Success:	SUCCEED
 *
 *		Failure:	FAIL
 *
 * Programmer:	Robb Matzke
 *              Monday, November 10, 1997
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
static herr_t
H5F_fam_write (H5F_low_t *lf, const haddr_t *addr, size_t size,
	       const uint8 *buf)
{
   size_t	nbytes;
   haddr_t	cur_addr, max_addr;
   uintn	membno;
   off_t	offset;
   H5F_low_t	*member = NULL;
   char		member_name[4096];
   intn		i;
   size_t	member_size;
   
   FUNC_ENTER (H5F_fam_write, FAIL);

   assert (lf);
   assert (addr && H5F_addr_defined (addr));
   assert (buf);

   member_size = (size_t)1 << lf->u.fam.offset_bits;
   membno = H5F_FAM_MEMBNO (addr, lf->u.fam.offset_bits);
   offset = H5F_FAM_OFFSET (addr, lf->u.fam.offset_bits);
   cur_addr = *addr;
   
   while (size>0) {
      nbytes = MIN (size, member_size-offset);
      cur_addr.offset = offset;
      
      if (membno>=lf->u.fam.nmemb) {
	 /*
	  * We're writing past the end of the last family member--create the
	  * new family member(s)
	  */
	 for (i=lf->u.fam.nmemb; i<=membno; i++) {
	    sprintf (member_name, lf->u.fam.name, i);
	    member = H5F_low_open (H5F_LOW_DFLT, member_name,
				   lf->u.fam.flags|H5F_ACC_CREAT,
				   NULL);
	    if (!member) {
	       HRETURN_ERROR (H5E_IO, H5E_CANTOPENFILE, FAIL,
			      "can't create a new member");
	    }

	    /*
	     * For members in the middle, set their logical eof to the
	     * maximum possible value.
	     */
	    if (i<membno) {
	       H5F_addr_reset (&max_addr);
	       H5F_addr_inc (&max_addr, member_size);
	       H5F_low_seteof (member, &max_addr);
	    }
	    
	    if (lf->u.fam.nmemb>=lf->u.fam.nalloc) {
	       lf->u.fam.nalloc *= 2;
	       lf->u.fam.memb = H5MM_xrealloc (lf->u.fam.memb,
					       (lf->u.fam.nalloc *
						sizeof(H5F_low_t*)));
	    }
	    lf->u.fam.memb[lf->u.fam.nmemb++] = member;
	 }
      }

      /*
       * Make sure the logical eof is large enough to handle the request.
       */
      max_addr = cur_addr;
      H5F_addr_inc (&max_addr, nbytes);
      H5F_low_seteof (lf->u.fam.memb[membno], &max_addr);


      /* Write the data to the member */
      if (H5F_low_write (lf->u.fam.memb[membno], &cur_addr,
			 nbytes, buf)<0) {
	 HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
			"can't write to family member");
      }
      buf += nbytes;
      size -= nbytes;
      membno++;
      offset=0;
   }

   FUNC_LEAVE (SUCCEED);
}


/*-------------------------------------------------------------------------
 * Function:	H5F_fam_flush
 *
 * Purpose:	Flushes all data to disk and makes sure that the first member
 *		is as large as a member can be so we can accurately detect
 *		the member size if we open this file for read access at a
 *		later date.
 *
 * Return:	Success:	SUCCEED
 *
 *		Failure:	FAIL
 *
 * Programmer:	Robb Matzke
 *              Monday, November 10, 1997
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
static herr_t
H5F_fam_flush (H5F_low_t *lf)
{
   int		membno, nerrors=0;
   uint8	buf[1];
   haddr_t	addr1, addr2, addr3;
   size_t	max_offset;
   
   FUNC_ENTER (H5F_fam_flush, FAIL);

   /*
    * Make sure that the first family member is the maximum size because
    * H5F_fam_open() looks at the size of the first member to determine the
    * number of bits to use for each family member offset.  We do this by
    * reading the last possible byte from the member (which defaults to zero
    * if we're reading past the end of the member) and then writing it back.
    */
   max_offset = H5F_FAM_MASK (lf->u.fam.offset_bits);
   H5F_addr_reset (&addr1);
   H5F_addr_inc (&addr1, max_offset);
   H5F_low_size (lf->u.fam.memb[0], &addr2); /*remember logical eof*/
   addr3 = addr1;
   H5F_addr_inc (&addr3, (size_t)1);
   H5F_low_seteof (lf->u.fam.memb[0], &addr3); /*prevent a warning*/
   if (H5F_low_read (lf->u.fam.memb[0], &addr1, 1, buf)<0) {
      HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL,
		     "can't read from first family member");
   }
   if (H5F_low_write (lf->u.fam.memb[0], &addr1, 1, buf)<0) {
      HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
		     "can't write to first family member");
   }
   H5F_low_seteof (lf->u.fam.memb[0], &addr2); /*reset old eof*/
   
   /*
    * Flush each member file.  Don't return an error status until we've
    * flushed as much as possible.
    */
   for (membno=0; membno<lf->u.fam.nmemb; membno++) {
      if (H5F_low_flush (lf->u.fam.memb[membno])<0) {
	 nerrors++;
      }
   }
   if (nerrors) {
      HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
		     "can't flush family member");
   }

   FUNC_LEAVE (SUCCEED);
}


/*-------------------------------------------------------------------------
 * Function:	H5F_fam_access
 *
 * Purpose:	Determines if all members of the file family can be accessed
 *		and returns the key for the first member of the family.
 *
 * Return:	Success:	TRUE or FALSE
 *
 *		Failure:	FAIL
 *
 * Programmer:	Robb Matzke
 *              Monday, November 10, 1997
 *
 * Modifications:
 *
 *-------------------------------------------------------------------------
 */
static hbool_t
H5F_fam_access (const char *name, int mode, H5F_search_t *key/*out*/)
{
   intn		membno;
   char		member_name[4096];
   hbool_t	status;
   
   FUNC_ENTER (H5F_fam_access, FAIL);

   for (membno=0; /*void*/; membno++) {
      sprintf (member_name, name, membno);
      status = H5F_low_access (H5F_LOW_DFLT, member_name, mode,
			       0==membno?key:NULL);

      if (!status) {
	 if (F_OK==mode) {
	    /*
	     * If we didn't find a member then we must have gotten to the end
	     * of the family.  As long as we found the first member(s) the
	     * family exists.
	     */
	    HRETURN (membno>0);
	 } else if (H5F_low_access (H5F_LOW_DFLT, member_name, F_OK, NULL)) {
	    /*
	     * The file exists but didn't have the write access permissions.
	     */
	    HRETURN (FALSE);
	 } else {
	    /*
	     * The file doesn't exist because we got to the end of the
	     * family.
	     */
	    HRETURN (TRUE);
	 }
      }
				    
      if (status<0) {
	 HRETURN_ERROR (H5E_IO, H5E_CANTOPENFILE, FAIL,
			"access method failed for a member file");
      }
   }

   FUNC_LEAVE (TRUE);
}