From be511b5b6d107726bd21b238810e722bdd1909fd Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Tue, 29 Dec 2020 15:37:54 -0600 Subject: HDFFV-10865 - merge from dev, HDFArray perf fix. (#234) --- java/src/hdf/hdf5lib/HDFArray.java | 304 +++++++++++++++---------------------- release_docs/RELEASE.txt | 11 +- 2 files changed, 130 insertions(+), 185 deletions(-) diff --git a/java/src/hdf/hdf5lib/HDFArray.java b/java/src/hdf/hdf5lib/HDFArray.java index 30f0fc8..63e17e8 100644 --- a/java/src/hdf/hdf5lib/HDFArray.java +++ b/java/src/hdf/hdf5lib/HDFArray.java @@ -16,6 +16,7 @@ package hdf.hdf5lib; import hdf.hdf5lib.exceptions.HDF5Exception; import hdf.hdf5lib.exceptions.HDF5JavaException; +import java.util.Arrays; /** * This is a class for handling multidimensional arrays for HDF. @@ -394,6 +395,7 @@ public class HDFArray { throw (ex); } _barray = bytes; /* hope that the bytes are correct.... */ + if (ArrayDescriptor.dims == 1) { /* special case */ /* 2 data copies here! */ @@ -496,8 +498,62 @@ public class HDFArray { Object oo = _theArray; int n = 0; /* the current byte */ + int m = 0; /* the current array index */ int index = 0; int i; + Object flattenedArray = null; + switch (ArrayDescriptor.NT) { + case 'J': + flattenedArray = (Object) HDFNativeData.byteToLong(_barray); + break; + case 'S': + flattenedArray = (Object) HDFNativeData.byteToShort(_barray); + break; + case 'I': + flattenedArray = (Object) HDFNativeData.byteToInt(_barray); + break; + case 'F': + flattenedArray = (Object) HDFNativeData.byteToFloat(_barray); + break; + case 'D': + flattenedArray = (Object) HDFNativeData.byteToDouble(_barray); + break; + case 'B': + flattenedArray = (Object) _barray; + break; + case 'L': + switch (ArrayDescriptor.className) { + case "java.lang.Byte": + flattenedArray = (Object) ByteToByteObj(_barray); + break; + case "java.lang.Short": + flattenedArray = (Object) ByteToShort(_barray); + break; + case "java.lang.Integer": + flattenedArray = (Object) ByteToInteger(_barray); + break; + case "java.lang.Long": + flattenedArray = (Object) ByteToLongObj(_barray); + break; + case "java.lang.Float": + flattenedArray = (Object) ByteToFloatObj(_barray); + break; + case "java.lang.Double": + flattenedArray = (Object) ByteToDoubleObj(_barray); + break; + default: + HDF5JavaException ex = new HDF5JavaException( + "HDFArray: unsupported Object type: " + + ArrayDescriptor.NT); + throw (ex); + } // end of switch statement for arrays of boxed objects + default: + HDF5JavaException ex = new HDF5JavaException( + "HDFArray: unknown or unsupported type: " + + ArrayDescriptor.NT); + throw (ex); + } // end of switch statement for arrays of primitives + while (n < ArrayDescriptor.totalSize) { oo = ArrayDescriptor.objs[0]; index = n / ArrayDescriptor.bytetoindex[0]; @@ -524,172 +580,58 @@ public class HDFArray { /* array-ify */ try { - if (ArrayDescriptor.NT == 'J') { - long[] arow = HDFNativeData.byteToLong(n, - ArrayDescriptor.dimlen[ArrayDescriptor.dims], - _barray); - java.lang.reflect.Array - .set( - ArrayDescriptor.objs[ArrayDescriptor.dims - 2], - (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]), - arow); - n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; - ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++; - } - else if (ArrayDescriptor.NT == 'I') { - int[] arow = HDFNativeData.byteToInt(n, - ArrayDescriptor.dimlen[ArrayDescriptor.dims], - _barray); - java.lang.reflect.Array - .set( - ArrayDescriptor.objs[ArrayDescriptor.dims - 2], - (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]), - arow); - - n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; - ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++; - } - else if (ArrayDescriptor.NT == 'S') { - short[] arow = HDFNativeData.byteToShort(n, - ArrayDescriptor.dimlen[ArrayDescriptor.dims], - _barray); - java.lang.reflect.Array - .set( - ArrayDescriptor.objs[ArrayDescriptor.dims - 2], - (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]), - arow); - - n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; - ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++; - } - else if (ArrayDescriptor.NT == 'B') { - System.arraycopy(_barray, n, - ArrayDescriptor.objs[ArrayDescriptor.dims - 1], 0, - ArrayDescriptor.dimlen[ArrayDescriptor.dims]); - n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; - } - else if (ArrayDescriptor.NT == 'F') { - float arow[] = HDFNativeData.byteToFloat(n, - ArrayDescriptor.dimlen[ArrayDescriptor.dims], - _barray); - java.lang.reflect.Array - .set( - ArrayDescriptor.objs[ArrayDescriptor.dims - 2], - (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]), - arow); - - n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; - ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++; - } - else if (ArrayDescriptor.NT == 'D') { - double[] arow = HDFNativeData.byteToDouble(n, - ArrayDescriptor.dimlen[ArrayDescriptor.dims], - _barray); - java.lang.reflect.Array - .set( - ArrayDescriptor.objs[ArrayDescriptor.dims - 2], - (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]), - arow); - - n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; - ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++; - } - else if (ArrayDescriptor.NT == 'L') { - if (ArrayDescriptor.className.equals("java.lang.Byte")) { - Byte I[] = ByteToByteObj(n, - ArrayDescriptor.dimlen[ArrayDescriptor.dims], - _barray); - java.lang.reflect.Array - .set( - ArrayDescriptor.objs[ArrayDescriptor.dims - 2], - (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]), - I); - - n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; - ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++; - } - else if (ArrayDescriptor.className - .equals("java.lang.Integer")) { - Integer I[] = ByteToInteger(n, - ArrayDescriptor.dimlen[ArrayDescriptor.dims], - _barray); - java.lang.reflect.Array - .set( - ArrayDescriptor.objs[ArrayDescriptor.dims - 2], - (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]), - I); - - n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; - ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++; - } - else if (ArrayDescriptor.className - .equals("java.lang.Short")) { - Short I[] = ByteToShort(n, - ArrayDescriptor.dimlen[ArrayDescriptor.dims], - _barray); - java.lang.reflect.Array - .set( - ArrayDescriptor.objs[ArrayDescriptor.dims - 2], - (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]), - I); - - n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; - ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++; - } - else if (ArrayDescriptor.className - .equals("java.lang.Float")) { - Float I[] = ByteToFloatObj(n, - ArrayDescriptor.dimlen[ArrayDescriptor.dims], - _barray); - java.lang.reflect.Array - .set( - ArrayDescriptor.objs[ArrayDescriptor.dims - 2], - (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]), - I); - - n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; - ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++; - } - else if (ArrayDescriptor.className - .equals("java.lang.Double")) { - Double I[] = ByteToDoubleObj(n, - ArrayDescriptor.dimlen[ArrayDescriptor.dims], - _barray); - java.lang.reflect.Array - .set( - ArrayDescriptor.objs[ArrayDescriptor.dims - 2], - (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]), - I); - - n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; - ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++; - } - else if (ArrayDescriptor.className.equals("java.lang.Long")) { - Long I[] = ByteToLongObj(n, - ArrayDescriptor.dimlen[ArrayDescriptor.dims], - _barray); - java.lang.reflect.Array - .set( - ArrayDescriptor.objs[ArrayDescriptor.dims - 2], - (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]), - I); - - n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; - ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++; - } - else { - HDF5JavaException ex = new HDF5JavaException( - "HDFArray: unsupported Object type: " - + ArrayDescriptor.NT); - throw (ex); - } - } - else { - HDF5JavaException ex = new HDF5JavaException( - "HDFArray: unknown or unsupported type: " - + ArrayDescriptor.NT); - throw (ex); - } + + Object arow = null; + int mm = m + ArrayDescriptor.dimlen[ArrayDescriptor.dims]; + switch (ArrayDescriptor.NT) { + case 'B': + arow = (Object) Arrays.copyOfRange((byte[]) flattenedArray, m, mm); + break; + case 'S': + arow = (Object) Arrays.copyOfRange((short[]) flattenedArray, m, mm); + break; + case 'I': + arow = (Object) Arrays.copyOfRange((int[]) flattenedArray, m, mm); + break; + case 'J': + arow = (Object) Arrays.copyOfRange((long[]) flattenedArray, m, mm); + break; + case 'F': + arow = (Object) Arrays.copyOfRange((float[]) flattenedArray, m, mm); + break; + case 'D': + arow = (Object) Arrays.copyOfRange((double[]) flattenedArray, m, mm); + break; + case 'L': + switch (ArrayDescriptor.className) { + case "java.lang.Byte": + arow = (Object) Arrays.copyOfRange((Byte[])flattenedArray, m, mm); + break; + case "java.lang.Short": + arow = (Object) Arrays.copyOfRange((Short[])flattenedArray, m, mm); + break; + case "java.lang.Integer": + arow = (Object) Arrays.copyOfRange((Integer[])flattenedArray, m, mm); + break; + case "java.lang.Long": + arow = (Object) Arrays.copyOfRange((Long[])flattenedArray, m, mm); + break; + case "java.lang.Float": + arow = (Object) Arrays.copyOfRange((Float[])flattenedArray, m, mm); + break; + case "java.lang.Double": + arow = (Object) Arrays.copyOfRange((Double[])flattenedArray, m, mm); + break; + } // end of switch statement for arrays of boxed numerics + } // end of switch statement for arrays of primitives + + java.lang.reflect.Array.set( + ArrayDescriptor.objs[ArrayDescriptor.dims - 2], + (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]), + arow); + n += ArrayDescriptor.bytetoindex[ArrayDescriptor.dims - 1]; + ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1]++; + m = mm; } catch (OutOfMemoryError err) { HDF5JavaException ex = new HDF5JavaException( @@ -717,25 +659,15 @@ public class HDFArray { + (ArrayDescriptor.dimlen[i] - 1) + "?")); } } - if (ArrayDescriptor.NT != 'B') { - if (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1] != ArrayDescriptor.dimlen[ArrayDescriptor.dims - 1]) { - throw new java.lang.InternalError(new String( - "HDFArray::arrayify Panic didn't complete all data: currentindex[" - + i + "] = " + ArrayDescriptor.currentindex[i] - + " (should be " + (ArrayDescriptor.dimlen[i]) - + "?")); - } - } - else { - if (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1] != (ArrayDescriptor.dimlen[ArrayDescriptor.dims - 1] - 1)) { - throw new java.lang.InternalError(new String( - "HDFArray::arrayify Panic didn't complete all data: currentindex[" - + i + "] = " + ArrayDescriptor.currentindex[i] - + " (should be " - + (ArrayDescriptor.dimlen[i] - 1) + "?")); - } + if (ArrayDescriptor.currentindex[ArrayDescriptor.dims - 1] != ArrayDescriptor.dimlen[ArrayDescriptor.dims - 1]) { + throw new java.lang.InternalError(new String( + "HDFArray::arrayify Panic didn't complete all data: currentindex[" + + i + "] = " + ArrayDescriptor.currentindex[i] + + " (should be " + (ArrayDescriptor.dimlen[i]) + + "?")); } + return _theArray; } @@ -944,6 +876,7 @@ class ArrayDescriptor { static int[] currentindex = null; static int[] bytetoindex = null; static int totalSize = 0; + static int totalElements = 0; static Object[] objs = null; static char NT = ' '; /* must be B,S,I,L,F,D, else error */ static int NTsize = 0; @@ -1052,6 +985,7 @@ class ArrayDescriptor { dimlen[0] = 1; dimstart[0] = 0; currentindex[0] = 0; + int elements = 1; int i; for (i = 1; i <= dims; i++) { dimlen[i] = java.lang.reflect.Array.getLength((Object) o); @@ -1059,7 +993,9 @@ class ArrayDescriptor { objs[i] = o; dimstart[i] = 0; currentindex[i] = 0; + elements *= dimlen[i]; } + totalElements = elements; int j; int dd; @@ -1083,7 +1019,7 @@ class ArrayDescriptor { System.out.println("Class: " + theClass); System.out.println("NT: " + NT + " NTsize: " + NTsize); System.out.println("Array has " + dims + " dimensions (" + totalSize - + " bytes)"); + + " bytes, " + totalElements + " elements)"); int i; for (i = 0; i <= dims; i++) { Class tc = objs[i].getClass(); diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 14b7f44..8b26c70 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -170,7 +170,16 @@ Bug Fixes since HDF5-1.10.7 release Java Library: ---------------- - - + - The H5FArray.java class, in which virtually the entire execution time + is spent using the HDFNativeData method that converts from an array + of bytes to an array of the destination Java type. + + 1. Convert the entire byte array into a 1-d array of the desired type, + rather than performing 1 conversion per row; + 2. Use the Java Arrays method copyOfRange to grab the section of the + array from (1) that is desired to be inserted into the destination array. + + (PGT,ADB - 2020/12/29, HDFFV-10865) Configuration ------------- -- cgit v0.12