[svn-r4720]

Purpose:
    A bug fixed for unlimited dimension with zero size
    A new feature(using a parameter file to pass some parameters in special
                  conversion cases: unlimited dimension with the current
                  size of 0 and enormous huge SDS array)
Description:
    1.conversion of unlimited dimension data with the current dimensional size 0
    2. Use a parameter file to control some special cases:
        1) To subdivide the huge array into hyperslabs, a memory buffer size has i
           to be set.
        2) when current dimensional size is 0, a default chunk size can be set.

Solution:
    bug fixed    1. Old approach: the current dimensional size is set to H5S_UNLIMITED, which is
                  a huge number. The default chunk size is set as a *FIXED* default value.
    2. New approach: the current dimensional size is set to the current value 0.
                     Users can provide the chunk size through a parameter file.

Platforms tested:
RedHat zoot 6.2

2 files changed, 274 insertions, 24 deletions

diff --git a/tools/h4toh5/h4toh5sds.c b/tools/h4toh5/h4toh5sds.c
index 245da81..01b7e8f 100644
--- a/tools/h4toh5/h4toh5sds.c
+++ b/tools/h4toh5/h4toh5sds.c
@@ -29,6 +29,13 @@ Author:  Kent Yang(ymuqun@ncsa.uiuc.edu)
 
 #include "h4toh5main.h"
 
+static int convert_zerosdsunlimit(int32 file_id,
+			 int32 sds_id,
+			 hid_t h5_group,
+			 hid_t h5_dimgroup,
+			 int32 chunk_size,
+			 int h4_attr);
+
 /*-------------------------------------------------------------------------
  * Function:	Sds_h4_to_h5
  *
@@ -55,9 +62,9 @@ int Sds_h4_to_h5(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimgroup,int
   int32*  sds_stride;
   int32   count_sdsdata;
   int32   sds_ref;
-  intn   sds_empty;
+  int   sds_empty;
   int32   istat;
-  int     i;
+  int     i,j;
   int32   num_sdsattrs;
   void*   sds_data;
 
@@ -79,8 +86,8 @@ int Sds_h4_to_h5(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimgroup,int
   int32           access_id;
   uint16          sd_ref;
   int             gzip_level;
-  /* define varibles for hdf5. */
 
+  /* define variables for hdf5. */
   hid_t   h5dset;
   hid_t   h5d_sid;
   hid_t   h5ty_id;
@@ -89,10 +96,52 @@ int Sds_h4_to_h5(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimgroup,int
   hid_t   write_plist;
   hsize_t h5dims[MAX_VAR_DIMS];
   hsize_t max_h5dims[MAX_VAR_DIMS];
+#ifdef H5_WANT_H5_V1_4_COMPAT
   hsize_t bufsize;
+#else /* H5_WANT_H5_V1_4_COMPAT */
+  size_t bufsize;
+#endif /* H5_WANT_H5_V1_4_COMPAT */
   char*   h5csds_name;
   herr_t  ret;
 
+  /* define variables to handle transformation when the maximum memory
+     buffer is set by users. */
+
+  int NUM_HSLAB_PERD;
+  int32* h4slab_start;
+  int32* h4slab_stride;
+  int32* h4slab_stop;
+  int32* h4slab_dims;
+  int32* h4slab_edges;
+  hsize_t* h5slab_offset;
+  hsize_t* h5slab_count;
+  int h4slab_count,h4slab_index;
+  int32 slabsize;
+  int32 chunksize;
+  int32 count_slabdata;
+  hid_t  slabmemspace;
+
+  FILE *fp;
+  int  memopt;
+  int  sdsopt_flag = 1;
+
+  memopt   = 0;
+  slabsize = 0;
+  if((fp= fopen("parafile","r"))==NULL){/*ignore the parameter file */
+    sdsopt_flag = 0;
+    memopt = 0;
+    chunksize = HDF4_CHUNKSIZE;
+  }
+
+  if(sdsopt_flag !=0) {
+    fscanf(fp,"%d",&memopt);
+    while(fgetc(fp)!='\n');
+    fscanf(fp,"%d",&slabsize);
+    while(fgetc(fp)!='\n');
+    fscanf(fp,"%d",&chunksize);
+    fclose(fp);
+  }
+
   special_code = -1;
   /* zeroing out the memory for sdsname and sdslabel.*/
 
@@ -106,7 +155,6 @@ int Sds_h4_to_h5(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimgroup,int
   }
 
   /*check whether the sds is created with unlimited dimension. */
-  
 
   /*obtain name,rank,dimsizes,datatype and num of attributes of sds */
   if (SDgetinfo(sds_id,sdsname,&sds_rank,sds_dimsizes,&sds_dtype,
@@ -116,7 +164,7 @@ int Sds_h4_to_h5(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimgroup,int
   }
   if(sds_empty !=0) {
     if(sds_dimsizes[0]==0) {
-      if(convert_zerosdsunlimit(file_id,sds_id,h5_group,h5_dimgroup,h4_attr)==FAIL){
+      if(convert_zerosdsunlimit(file_id,sds_id,h5_group,h5_dimgroup,chunksize,h4_attr)==FAIL){
 	printf("cannot convert unlimited dimension SDS with 0.\n");
 	return FAIL;
       }
@@ -197,7 +245,7 @@ int Sds_h4_to_h5(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimgroup,int
     }
   }
  
-  sds_data = malloc(h4memsize*count_sdsdata);
+  /*  sds_data = malloc(h4memsize*count_sdsdata);
 
   if(sds_data == NULL) {
     printf("error in allocating memory. \n");
@@ -216,6 +264,7 @@ int Sds_h4_to_h5(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimgroup,int
     free(sds_data);
     return FAIL;
   }
+  */
 
   /* obtaining reference number and name of h5 dataset 
      corresponding to sds. */
@@ -436,9 +485,32 @@ int Sds_h4_to_h5(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimgroup,int
     H5Pclose(create_plist);
     return FAIL;							      
   }									      
-  /*  write_plist = H5Pcreate(H5P_DATASET_XFER);
+
+  if(count_sdsdata*h4memsize <= slabsize || memopt!= 1) {
+
+    sds_data = malloc(h4memsize*count_sdsdata);
+
+    if(sds_data == NULL) {
+      printf("error in allocating memory. \n");
+      free(sds_start);
+      free(sds_edge);
+      free(sds_stride);
+      return FAIL;
+    }
+  istat    = SDreaddata(sds_id, sds_start, sds_stride, sds_edge, 
+			(VOIDP)sds_data);
+  if (istat == FAIL)  { 
+    printf("unable to read data from h5dset. \n");
+    free(sds_start);
+    free(sds_edge);
+    free(sds_stride);
+    free(sds_data);
+    return FAIL;
+  }
+
+  write_plist = H5Pcreate(H5P_DATASET_XFER);
   bufsize = h4memsize;
-  for(i=1;i<sds_rank;i++)
+  for(i=0;i<sds_rank;i++)
     bufsize *= h5dims[i];
   if(H5Pset_buffer(write_plist,bufsize,NULL,NULL)<0) {
     printf("fail to create data transfer property list.\n");
@@ -449,12 +521,11 @@ int Sds_h4_to_h5(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimgroup,int
     free(chunk_dims);
     H5Sclose(h5d_sid);
     H5Pclose(create_plist);
+    H5Pclose(write_plist);
     return FAIL;		
   }
   if (H5Dwrite(h5dset,h5_memtype,h5d_sid,h5d_sid,write_plist,	    
-  (void *)sds_data)<0) {	*/
-  if (H5Dwrite(h5dset,h5_memtype,h5d_sid,h5d_sid,H5P_DEFAULT,	    
-  (void *)sds_data)<0) {		      
+  (void *)sds_data)<0) {	
     printf("failed to write data into hdf5 dataset");	
     printf(" converted from SDS.\n");
     H5Sclose(h5d_sid);
@@ -466,8 +537,137 @@ int Sds_h4_to_h5(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimgroup,int
     free(sds_data);	
     free(chunk_dims);
     return FAIL;							      
-  }							      
-									      
+  }		
+  free(sds_data);
+  }
+
+  else { 
+    /* obtain the number of hyperslab per dimension. */
+  NUM_HSLAB_PERD= get_numslab_perD(h4memsize*count_sdsdata,slabsize,sds_rank);
+  
+  h4slab_start  = calloc(sds_rank,sizeof(int32));
+  h4slab_stride = calloc(sds_rank,sizeof(int32));
+  h4slab_stop   = calloc(sds_rank,sizeof(int32));
+  h4slab_dims   = calloc(sds_rank,sizeof(int32));
+  h4slab_edges  = calloc(sds_rank,sizeof(int32));
+  h5slab_offset = calloc(sds_rank,sizeof(hsize_t));
+  h5slab_count  = calloc(sds_rank,sizeof(hsize_t));
+
+  /* Initialize the hyperslab parameter. */
+  for ( i =0;i<sds_rank;i++) {
+    h4slab_start[i] =0;
+    h4slab_stop[i] = 0;
+    h4slab_stride[i] =1;
+    h4slab_dims[i] = h5ceil(sds_dimsizes[i],NUM_HSLAB_PERD);
+  }
+
+  h4slab_count = -1;
+
+  /* total number of hyperslab is equal to pow(NUM_HSLAB_PERD,sds_rank)-1, 
+     At first, all other dimensions are fixed, only the first dimension
+     (the most frequently changing dimension) changes its starting point
+     and ending point.
+     After the first dimension moves to the end of its dimension, 
+     the second dimension starts to change. So we use "mod fuction" to check the which 
+     dimension we are heading to. */  
+
+  while(h4slab_count != pow(NUM_HSLAB_PERD,sds_rank)-1){
+    h4slab_count++;
+    h4slab_index = -1;
+    for (i=0;i<sds_rank;i++){
+      /* check the current location of the slab. */
+      if((h4slab_count%pow(NUM_HSLAB_PERD,(i+1)))==0){
+        h4slab_index = i;
+      }
+    }
+    if(h4slab_index != -1 && h4slab_index != (sds_rank-1)) {
+      for(j=0;j<=h4slab_index;j++){
+	h4slab_start[j] =0;
+      }
+      h4slab_start[j] = h4slab_start[j]+h4slab_dims[j];
+    }
+
+    for(i=0;i<sds_rank;i++){
+      h4slab_stop[i] = h4slab_start[i]+h4slab_dims[i];
+      if(h4slab_stop[i]>sds_dimsizes[i])
+        h4slab_stop[i] = sds_dimsizes[i];
+    }
+    count_slabdata = 1;
+    for(i=0;i<sds_rank;i++){
+      count_slabdata = count_slabdata*(h4slab_stop[i]-h4slab_start[i]);
+      h4slab_edges[i] = h4slab_stop[i]-h4slab_start[i];
+    }
+    count_slabdata = count_slabdata*h4memsize;
+  sds_data = malloc(count_slabdata);
+  if(sds_data == NULL) {
+    printf("error in allocating memory. \n");
+    free(sds_start);
+    free(sds_edge);
+    free(sds_stride);
+    return FAIL;
+  }
+  istat    = SDreaddata(sds_id, h4slab_start, h4slab_stride, h4slab_edges, 
+			(VOIDP)sds_data);
+  if (istat == FAIL)  { 
+    printf("unable to read SDS data. \n");
+    free(sds_start);
+    free(sds_edge);
+    free(sds_stride);
+    free(sds_data);
+    return FAIL;
+  }
+  write_plist = H5Pcreate(H5P_DATASET_XFER);
+  /*  bufsize = h4memsize;
+  for(i=0;i<sds_rank;i++)
+  bufsize *= h5dims[i];*/
+  bufsize = slabsize;
+  if(H5Pset_buffer(write_plist,bufsize,NULL,NULL)<0) {
+    printf("fail to create data transfer property list.\n");
+    free(sds_start);
+    free(sds_edge);
+    free(sds_stride);
+    free(sds_data);
+    free(chunk_dims);
+    H5Sclose(h5d_sid);
+    H5Pclose(create_plist);
+    H5Pclose(write_plist);
+    return FAIL;		
+  }
+
+  for (i=0;i<sds_rank;i++) {
+    h5slab_offset[i] = (hsize_t)(h4slab_start[i]);
+    h5slab_count[i] =(hsize_t)(h4slab_stop[i]-h4slab_start[i]);
+  }
+
+  slabmemspace = H5Screate_simple(sds_rank,h5slab_count,NULL);
+  H5Sselect_hyperslab(h5d_sid,H5S_SELECT_SET,h5slab_offset,NULL,h5slab_count,NULL);
+  if (H5Dwrite(h5dset,h5_memtype,slabmemspace,h5d_sid,write_plist,	    
+  (void *)sds_data)<0) {	
+    printf("failed to write data into hdf5 dataset");	
+    printf(" converted from SDS.\n");
+    H5Sclose(h5d_sid);
+
+    H5Dclose(h5dset);
+    H5Pclose(create_plist);
+    free(sds_start);
+    free(sds_edge);
+    free(sds_stride);
+    free(sds_data);	
+    free(chunk_dims);
+    return FAIL;							      
+  }			
+  free(sds_data);				      
+							
+  h4slab_start[0] = h4slab_start[0]+h4slab_dims[0];
+  }
+
+  free(h4slab_start);
+  free(h4slab_stop);
+  free(h4slab_dims);
+  free(h4slab_edges);
+  free(h5slab_offset);
+  free(h5slab_count);
+  }
   /* convert sds annotation into attribute of sds dataset.
      Since there is no routines to find the exact tag of sds object,
      we will check three possible object tags of sds objects, that is:
@@ -594,11 +794,12 @@ int Sds_h4_to_h5(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimgroup,int
   ret   = H5Pclose(create_plist);
   ret   = H5Sclose(h5d_sid);
   ret   = H5Dclose(h5dset);
-  free(sds_data);
+  /*  free(sds_data);*/
   free(sds_start);
   free(sds_edge);
   free(sds_stride);
   free(chunk_dims);
+  
   return SUCCEED;
 }
 
@@ -1305,7 +1506,11 @@ int convert_sdsfillvalue(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimg
   hid_t   write_plist;
   hsize_t h5dims[MAX_VAR_DIMS];
   hsize_t max_h5dims[MAX_VAR_DIMS];
+#ifdef H5_WANT_H5_V1_4_COMPAT
   hsize_t bufsize;
+#else /* H5_WANT_H5_V1_4_COMPAT */
+  size_t bufsize;
+#endif /* H5_WANT_H5_V1_4_COMPAT */
   char*   h5csds_name;
 
   if (SDgetinfo(sds_id,sdsname,&sds_rank,sds_dimsizes,&sds_dtype,
@@ -1471,7 +1676,7 @@ int convert_sdsfillvalue(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimg
    chunk_dims   = malloc(sizeof(hsize_t)*sds_rank);
 
   if(chunk_dims == NULL) {
-    H5Pclose(H5P_DATASET_CREATE);
+    H5Pclose(create_plist);
     free(sds_start);
     free(sds_edge);
     free(sds_stride);
@@ -1531,6 +1736,7 @@ int convert_sdsfillvalue(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimg
      free(chunk_dims);
     H5Sclose(h5d_sid);
     H5Pclose(create_plist);
+    H5Pclose(write_plist);
     return FAIL;		
   }
 
@@ -1665,6 +1871,7 @@ int convert_sdsfillvalue(int32 file_id,int32 sds_id,hid_t h5_group,hid_t h5_dimg
   H5Sclose(h5d_sid);
   H5Dclose(h5dset);
   H5Pclose(create_plist);
+/*  H5Pclose(write_plist); */
   return SUCCEED;
 }
 
@@ -1700,11 +1907,12 @@ uint16 get_SDref(int32 file_id,uint16 tag,int32 sds_ref){
   return sd_ref;
 }
 
-int convert_zerosdsunlimit(int32 file_id,
-			 int32 sds_id,
-			 hid_t h5_group,
-			 hid_t h5_dimgroup,
-			 int h4_attr) {
+static int convert_zerosdsunlimit(int32 file_id,
+				  int32 sds_id,
+				  hid_t h5_group,
+				  hid_t h5_dimgroup,
+				  int32 chunk_size,
+				  int h4_attr) {
 
 
   int32   sds_dtype;
@@ -1764,7 +1972,7 @@ int convert_zerosdsunlimit(int32 file_id,
 
   if(SDisrecord(sds_id)){
     max_h5dims[0] = H5S_UNLIMITED;
-    h5dims[0] = H5S_UNLIMITED;
+    /*  h5dims[0] = 0;*/
   }
 
   /* convert hdf4 data type to hdf5 data type. */
@@ -1890,7 +2098,7 @@ int convert_zerosdsunlimit(int32 file_id,
       for(i=0;i<sds_rank;i++){
 	chunk_dims[i] = (hsize_t)(sds_dimsizes[i]);
         if(sds_dimsizes[0] == 0)
-	  chunk_dims[0] = (hsize_t)DEF_CHUNKSIZE;
+	  chunk_dims[0] = (hsize_t)chunk_size;
       }
       if(H5Pset_chunk(create_plist, sds_rank, chunk_dims)<0) {
 	printf("failed to set up chunking information for ");
@@ -1941,7 +2149,7 @@ int convert_zerosdsunlimit(int32 file_id,
   }
    
   if(access_id == FAIL) {
-    chunk_dims[0] = (hsize_t)DEF_CHUNKSIZE;
+    chunk_dims[0] = (hsize_t)chunk_size;
     for(i=1;i<sds_rank;i++)
       chunk_dims[i] = h5dims[i];
     if(H5Pset_chunk(create_plist, sds_rank, chunk_dims)<0) {
@@ -2071,3 +2279,41 @@ int convert_zerosdsunlimit(int32 file_id,
   return SUCCEED;
 }
 
+
+int get_numslab_perD(int32 totalsize,int slabsize,int rank) {
+
+  int i,flag;
+  int32 tempsize;
+  int   numslab_perD;
+  numslab_perD =1;
+  flag = 0;
+
+  while(!flag) {
+    tempsize =1;
+    for(i=0;i<rank;i++)
+      tempsize = tempsize*numslab_perD;
+    if((totalsize/tempsize)<=slabsize){
+      flag = 1;
+    }
+    else 
+      numslab_perD++;
+  }
+  printf("numslab_perD %d\n",numslab_perD);
+  return numslab_perD;
+}
+
+
+int h5ceil(int32 m,int32 n){
+  if(m/n * n < m) return m/n +1;
+  else return m/n;
+}
+
+
+int pow(int num,int h4index) {
+  int i;
+  int total;
+  total = 1;
+  for( i= 1; i<=h4index;i++)
+    total = total * num;
+  return total;
+}
diff --git a/tools/h4toh5/h4toh5util.h b/tools/h4toh5/h4toh5util.h
index e04aa35..3143687 100644
--- a/tools/h4toh5/h4toh5util.h
+++ b/tools/h4toh5/h4toh5util.h
@@ -151,7 +151,7 @@ public constant for hdf lib, this constant can be released.*/
 
 /*13. define a default first dimension chunking size when the
   current unlimited dimensional size of SDS is set to 0. */
-#define DEF_CHUNKSIZE 20
+#define HDF4_CHUNKSIZE 20
 extern int32 estnum_vg;
 extern int32 estnum_vd;
 extern int32 num_sds;
@@ -241,6 +241,10 @@ char* make_objname_no(char*,char*,const char*);
 char* make_objname_yes(char*,char*);
 char* correct_name(char*);
 
+/* routines for handling memory allocation of huge SDS dataset */
+int get_numslab_perD(int32,int,int);
+int h5ceil(int32,int32);
+int pow(int,int);
 #endif