[svn-r6087]

Purpose:
   Adding internal shuffle filter
Description:
    With the combination of shuffling filter with general compression
    algorithm, the compression ratio may be improved without adding much
    encoding and decoding time for many real NASA datasets(especially
    floating data) and other application
    datasets(See techNotes).
Solution:
    SHuffle the bytes within the data to utilize the locality.
Platforms tested:
    arabica , eirene, modi4
Misc. update:
    Update MANIFEST if you add or remove any file.
    Update release_docs/RELEASE for bug fixes, new features, etc.
    Update applicable document files too.

4 files changed, 141 insertions, 0 deletions

diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c
index 7e995ef..d7f8809 100644
--- a/src/H5Pdcpl.c
+++ b/src/H5Pdcpl.c
@@ -751,6 +751,61 @@ done:
     FUNC_LEAVE(ret_value);
 }
 
+
+/*-------------------------------------------------------------------------
+ * Function:	H5Pset_shuffle
+ *
+ * Purpose:	Sets the shuffling method for a permanent 
+ *		filter
+ *	        to H5Z_FILTER_SHUFFLE
+ *		and bytes of the datatype of the array to be shuffled 
+ *              
+ * Return:	Non-negative on success/Negative on failure
+ *
+ * Programmer:	Robb Matzke
+ *              Wednesday, April 15, 1998
+ *
+ * Modifications:
+ *
+ *              Raymond Lu
+ *              Tuesday, October 2, 2001
+ *              Changed the way to check parameter and set property for 
+ *              generic property list. 
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_shuffle(hid_t plist_id, unsigned bytes_of_type)
+{
+    H5O_pline_t         pline;
+    H5P_genplist_t *plist;      /* Property list pointer */
+    herr_t ret_value=SUCCEED;   /* return value */
+    
+    FUNC_ENTER_API(H5Pset_shuffle, FAIL);
+    H5TRACE2("e","iIu",plist_id,bytes_of_type);
+
+    /* Check arguments */
+    if(TRUE != H5P_isa_class(plist_id, H5P_DATASET_CREATE))
+        HGOTO_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list");
+    if (bytes_of_type == 0)
+        HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "number of bytes of datatype cannot be 0");
+   
+    /* Get the plist structure */
+    if(NULL == (plist = H5I_object(plist_id)))
+        HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
+
+    /* Add the filter */
+    if(H5P_get(plist, H5D_CRT_DATA_PIPELINE_NAME, &pline) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get pipeline");
+    if (H5Z_append(&pline, H5Z_FILTER_SHUFFLE, H5Z_FLAG_OPTIONAL, 1, &bytes_of_type)<0)
+        HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to shuffle the data");
+    if(H5P_set(plist, H5D_CRT_DATA_PIPELINE_NAME, &pline) < 0)
+        HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to set pipeline");
+
+done:
+    FUNC_LEAVE(ret_value);
+}
+
 
 /*-------------------------------------------------------------------------
  * Function:	H5Pset_fill_value
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index b0b332e..e1a6cf1 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -210,6 +210,7 @@ H5_DLL H5Z_filter_t H5Pget_filter(hid_t plist_id, int filter,
        unsigned cd_values[]/*out*/,
        size_t namelen, char name[]);
 H5_DLL herr_t H5Pset_deflate(hid_t plist_id, unsigned aggression);
+H5_DLL herr_t H5Pset_shuffle(hid_t plist_id, unsigned bytespertype);
 #ifdef H5_WANT_H5_V1_4_COMPAT
 H5_DLL herr_t H5Pset_cache(hid_t plist_id, int mdc_nelmts, int rdcc_nelmts,
        size_t rdcc_nbytes, double rdcc_w0);
diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h
index 466fbb6..46f74a3 100644
--- a/src/H5Zpublic.h
+++ b/src/H5Zpublic.h
@@ -19,6 +19,7 @@ typedef int H5Z_filter_t;
 #define H5Z_FILTER_ERROR	(-1)	/*no filter			*/
 #define H5Z_FILTER_NONE		0	/*reserved indefinitely		*/
 #define H5Z_FILTER_DEFLATE	1 	/*deflation like gzip	     	*/
+#define H5Z_FILTER_SHUFFLE      2       /* shuffle the data             */
 #define H5Z_FILTER_MAX		65535	/*maximum filter id		*/
 
 /* Flags for filter definition */
@@ -59,6 +60,9 @@ size_t H5Z_filter_deflate(unsigned flags, size_t cd_nelmts,
 			  const unsigned cd_values[], size_t nbytes,
 			  size_t *buf_size, void **buf);
 
+size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts,
+			  const unsigned cd_values[], size_t nbytes,
+			  size_t *buf_size, void **buf);
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/H5Zshuffle.c b/src/H5Zshuffle.c
new file mode 100644
index 0000000..61dcc4f
--- /dev/null
+++ b/src/H5Zshuffle.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright © 1999-2001 NCSA
+ *                       All rights reserved.
+ *
+ * Programmer:  Robb Matzke <matzke@llnl.gov>
+ *              Friday, August 27, 1999
+ */
+#include "H5private.h"
+#include "H5Eprivate.h"
+#include "H5MMprivate.h"
+#include "H5Zprivate.h"
+#include <stdio.h>
+
+#ifdef H5_HAVE_FILTER_SHUFFLE
+
+size_t H5Z_filter_shuffle(unsigned flags,
+			  size_t cd_nelmts,
+			  const unsigned cd_values[],
+			  size_t nbytes,
+			  size_t *buf_size,void **buf) {
+  
+  size_t i;
+  size_t j;
+  size_t k;
+  size_t ret_value = 0;
+  size_t byte_pos;
+  size_t bytesoftype;
+  void* dest = NULL;
+  char* _src;
+  char* _des;
+  char* _dest;
+  size_t numofelements;
+   
+  bytesoftype=cd_values[0];
+  numofelements=nbytes/bytesoftype;
+  _src =(char*)(*buf);
+
+  dest = malloc((size_t)nbytes);
+  _dest =(char*)dest;
+
+  j = 0;
+  k = 0;
+  if(flags & H5Z_FLAG_REVERSE) {
+     for(i=0;i<nbytes;i++) {
+       byte_pos = 1 +j *numofelements; 
+       if(byte_pos > nbytes) {
+	 k++;
+	 j=0;
+	 byte_pos=1;
+       }
+       *(_dest+i)=*((char*)(_src+byte_pos-1+k));
+       j++;
+     }
+     free(*buf);
+     *buf = dest;
+     dest = NULL;
+     *buf_size=nbytes;
+     ret_value = nbytes;
+  }
+
+  else {
+    for (i=0;i<nbytes;i++){
+      byte_pos = 1+j * bytesoftype; 
+      if(byte_pos >nbytes) {
+	k++;
+	j=0;
+	byte_pos = 1;
+      }
+
+      *(_dest+i)=*(_src+byte_pos-1+k);
+      j++;
+    }
+    free(*buf);
+     *buf = dest;
+     dest = NULL;
+     *buf_size=nbytes;
+     ret_value = nbytes;
+  }
+
+}
+#endif /*H5_HAVE_FILTER_SHUFFLE */