diff options
| author | Ruth Aydt <aydt@hdfgroup.org> | 2014-04-22 20:11:09 (GMT) |
|---|---|---|
| committer | Ruth Aydt <aydt@hdfgroup.org> | 2014-04-22 20:11:09 (GMT) |
| commit | b95a1cba6bbed30bcdaf6cdefd482f1636efd188 (patch) | |
| tree | 1a0c79a143aaab6d9dcc34331662ba943118a443 /examples | |
| parent | 9e4ba4e7d726f6c894a6dffb87fa6ba23366b0dd (diff) | |
| download | hdf5-b95a1cba6bbed30bcdaf6cdefd482f1636efd188.zip hdf5-b95a1cba6bbed30bcdaf6cdefd482f1636efd188.tar.gz hdf5-b95a1cba6bbed30bcdaf6cdefd482f1636efd188.tar.bz2 | |
[svn-r25078] Test program that accepts parameters to control size of Datasets and Maps in container
and times/reports writes / persists / evicts / prefetches / reads.
Reads are of form "rank reads data it wrote", "rank reads data written by next rank" and "rank reads all data"
Reads are done from logged data, prefetched data, or data on DAOS.
Designed to give insights comparing performance of Arrays & KVs
and accessing data in various layers of the storage stack.
Diffstat (limited to 'examples')
| -rw-r--r-- | examples/h5ff_client_timings.c | 1118 |
1 files changed, 1118 insertions, 0 deletions
diff --git a/examples/h5ff_client_timings.c b/examples/h5ff_client_timings.c new file mode 100644 index 0000000..15ef0c7 --- /dev/null +++ b/examples/h5ff_client_timings.c @@ -0,0 +1,1118 @@ +/*
+ * h5ff_timings.c: Do some timing of various write / persist / prefetch / read
+ */
+/*
+ *
+ * Creates 3 groups, each with a dataset & a map
+ * In /G-logged, objects will read.
+ * In /G-prefetched, objects will be evicted & prefetched & read & evicted
+ * In /G-stored, objects will be evicted & read
+ * D = dataset; 2d with uint64 datatypes. size is [ (rows_per_rank*#ranks), (cols_per_row) ]
+ * M = map; uint64 keys ranging from 0 to (rows_per_rank*#ranks*cols_per_row)-1
+ * uint64 values, set to same as dset entries
+ * each rank updates "its" dataset entries & map pairs, based on rows_per_rank variable
+ *
+ *
+ * Notes for possible future modifications:
+ * 1) Make Datasets 1D, with entries 2D vectors to mimic ACG Edge List representation
+ * 2) Make Maps have values that are vectors of uint64, to mimic ACG Adjacency list representation (step 1)
+ * 3) Make Maps have values that variable-length vectors of uint64, to mimic ACG Adjaceny list represenations (really)
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include <sys/time.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+/* define filename for this app, and max size after username prepended */
+#define FILENAME_APP "eff_timings.h5"
+#define FILENAME_SIZE 256
+
+/* define default sizing parameters - can be changed via command line arguments */
+#define DEFAULT_ROWS_PER_RANK 10
+#define DEFAULT_COLS_PER_ROW 5
+
+/* macros related to error reporting */
+#define STATUS (ret >= 0) ? " " : " - FAILED"
+#define ASSERT_RET assert( ret >= 0 )
+
+/* option flags */
+int cols_per_row = DEFAULT_COLS_PER_ROW;
+int rows_per_rank = DEFAULT_ROWS_PER_RANK;
+int num_iterations = 1; /* Number of times to perform the commit/persist/evict/prefetch/read/evict cycle */
+int verbose = 0; /* Verbose output - defaults to no */
+int time_reads = 0; /* Time reads after container re-open */
+
+/* global variables and macros used to make timing easier */
+struct timeval tv_start;
+struct timeval tv_end;
+#define START_TIME gettimeofday( &tv_start, NULL )
+#define END_TIME gettimeofday( &tv_end, NULL )
+#define ELAPSED_TIME (ulong)( (tv_end.tv_usec + 1000000*tv_end.tv_sec) - (tv_start.tv_usec + 1000000*tv_start.tv_sec) )
+
+/* function prototypes */
+void print_container_contents( hid_t, hid_t, const char*, int );
+int parse_options( int, char**, int, int );
+void usage( const char* );
+
+/**************************************************************************************************************/
+int main( int argc, char **argv ) {
+
+ /* MPI */
+ int my_rank, comm_size;
+ int provided;
+
+ /* Container */
+ char *user_name; /* We'll prepend username to make filename unique */
+ char file_name[FILENAME_SIZE]; /* Actual filename to be opened will be user_name + FILENAME_APP */
+ hid_t fapl_id;
+ hid_t file_id;
+
+ /* Groups */
+ hid_t grp_l_id;
+ hid_t grp_p_id;
+ hid_t grp_s_id;
+
+ /* Dataspaces */
+ hid_t space_l_id;
+ hid_t space_p_id;
+ hid_t space_s_id;
+
+ /* Datasets */
+ hid_t dset_l_id;
+ hid_t dset_p_id;
+ hid_t dset_s_id;
+ hsize_t dset_size[2];
+ hid_t space_id;
+
+ /* Maps */
+ hid_t map_l_id;
+ hid_t map_p_id;
+ hid_t map_s_id;
+
+ /* Replicas and related properties */
+ hrpl_t dset_p_replica;
+ hid_t dxpl_p_id;
+ hid_t dapl_p_id;
+ hrpl_t map_p_replica;
+ hid_t mxpl_p_id;
+ hid_t mapl_p_id;
+
+ /* Container Version & Read Contexts */
+ uint64_t version, versionH;
+ hid_t rc_id, rc_idH;
+
+ /* Transactions */
+ uint64_t tr_num;
+ hid_t tr_id;
+ hid_t trspl_id;
+ int num_tr_leaders;
+
+ /* Memory buffer - sized to hold complete DS or Map Values; sometimes partially used */
+ uint64_t *mbuf;
+
+ /* Sizing, dataspace, hyperslab parameters set per-rank */
+ hsize_t rank_mbuf_size[1];
+ hsize_t rank_offset[2], rank_count[2];
+ hid_t rank_space_id;
+
+ /* Variables used for reading data written by a neighbor rank */
+ int neighbor_rank;
+ hid_t neighbor_space_l_id;
+ hid_t neighbor_space_p_id;
+ hid_t neighbor_space_s_id;
+ hsize_t neighbor_offset[2], neighbor_count[2];
+
+ /* Misc */
+ herr_t ret;
+ uint64_t u, r, c;
+ uint64_t key;
+ int bytesPerCell;
+ int i;
+ int iteration;
+
+ /****
+ * Initialize and create container
+ ****/
+
+ /* Confirm needed MPI functionality is available */
+ MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
+ if ( MPI_THREAD_MULTIPLE != provided ) {
+ fprintf( stderr, "APP: ERROR: MPI does not have MPI_THREAD_MULTIPLE support\n" );
+ exit( 1 );
+ }
+
+ /* Find MPI rank & communicator size */
+ MPI_Comm_rank( MPI_COMM_WORLD, &my_rank );
+ MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
+ if ( my_rank == 0 ) {
+ fprintf( stderr, "APP-r%d: Number of MPI processes = %d\n", my_rank, comm_size );
+ }
+
+ /* Parse command-line options controlling behavior */
+ if ( parse_options( argc, argv, my_rank, comm_size ) != 0 ) {
+ exit( 1 );
+ }
+ if ( my_rank == 0 ) {
+ fprintf( stderr, "APP-r0: Number of MPI processes = %d\n", comm_size );
+ fprintf( stderr, "APP-r0: Datasets will have %d rows (%d per rank).\n", (rows_per_rank*comm_size), rows_per_rank );
+ fprintf( stderr, "APP-r0: Datasets will have %d columns.\n", cols_per_row );
+ fprintf( stderr, "APP-r0: There will be %d iterations.\n", num_iterations );
+ }
+
+ /* Initialize the EFF stack, starting FS client, registering HDF5 VOL calls, requesting IOD init */
+ fprintf( stderr, "APP-r%d: Initialize EFF stack\n", my_rank );
+ EFF_init( MPI_COMM_WORLD, MPI_INFO_NULL );
+
+ /* Specify the IOD VOL plugin should be used and create H5File (EFF container), with user name prepended */
+ user_name = getenv( "USER" );
+ snprintf( file_name, FILENAME_SIZE, "%s_%s", user_name, FILENAME_APP );
+ fprintf( stderr, "APP-r%d: Create %s\n", my_rank, file_name );
+ fapl_id = H5Pcreate( H5P_FILE_ACCESS ); assert( fapl_id >= 0 );
+ ret = H5Pset_fapl_iod( fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL ); ASSERT_RET;
+ file_id = H5Fcreate_ff( file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL ); assert( file_id >= 0 );
+
+ /* Create the memory buffer that will be used in many places - both for writing and reading */
+ bytesPerCell = sizeof( uint64_t );
+ mbuf = (uint64_t *) calloc( (rows_per_rank * comm_size * cols_per_row), bytesPerCell );
+
+ /****
+ * Transaction 2: Rank 0 creates H5Objects in the container
+ ****/
+
+ if ( my_rank == 0 ) {
+
+ /* Acquire read context for CV 1 */
+ version = 1;
+ rc_id = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET; assert( version == 1 );
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
+
+ /* Start a transaction with a single leader (the default) */
+ tr_num = 2;
+ fprintf( stderr, "APP-r%d: tr %lu - Start\n", my_rank, tr_num );
+ tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
+ ret = H5TRstart( tr_id, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Add updates to the transaction for Group creates */
+ fprintf( stderr, "APP-r%d: tr %d - Create /G-logged /G-prefetched /G-stored\n", my_rank, tr_num );
+
+ grp_l_id = H5Gcreate_ff( file_id, "G-logged", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ assert( grp_l_id >= 0 );
+
+ grp_p_id = H5Gcreate_ff( file_id, "G-prefetched", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ assert( grp_p_id >= 0 );
+
+ grp_s_id = H5Gcreate_ff( file_id, "G-stored", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ assert( grp_s_id >= 0 );
+
+ /* Create Dataspace for Datasets */
+ dset_size[0] = rows_per_rank * comm_size;
+ dset_size[1] = cols_per_row;
+ space_id = H5Screate_simple( 2, dset_size, dset_size ); assert( space_id >= 0 );
+
+ /* Add updates to the transaction for Dataset creates */
+ fprintf( stderr, "APP-r%d: tr %d - Create /G-logged/D /G-prefetched/D /G-stored/D\n", my_rank, tr_num );
+
+ dset_l_id = H5Dcreate_ff( grp_l_id, "D", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( dset_l_id >= 0 );
+
+ dset_p_id = H5Dcreate_ff( grp_p_id, "D", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( dset_p_id >= 0 );
+
+ dset_s_id = H5Dcreate_ff( grp_s_id, "D", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( dset_s_id >= 0 );
+
+ ret = H5Sclose( space_id ); ASSERT_RET;
+
+ /* Add updates to the transaction for Map creates */
+ fprintf( stderr, "APP-r%d: tr %d - Create /G-logged/M /G-prefetched/M /G-stored/M\n", my_rank, tr_num );
+
+ map_l_id = H5Mcreate_ff( grp_l_id, "M", H5T_NATIVE_UINT, H5T_NATIVE_UINT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( map_l_id >= 0 );
+
+ map_p_id = H5Mcreate_ff( grp_p_id, "M", H5T_NATIVE_UINT, H5T_NATIVE_UINT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( map_p_id >= 0 );
+
+ map_s_id = H5Mcreate_ff( grp_s_id, "M", H5T_NATIVE_UINT, H5T_NATIVE_UINT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( map_s_id >= 0 );
+
+ /* Finish and commit TR and get RC for it */
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, &rc_idH, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: tr %lu - Finished\n", my_rank, tr_num );
+
+ assert( rc_idH >= 0 );
+ versionH = tr_num;
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, versionH );
+
+ /* Release the read handle and close read context on earlier CV */
+ ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
+ ret = H5RCclose( rc_id ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc %lu - Released and Closed\n", my_rank, version );
+
+ /* The just-acquired RC is now the primary rc_id we'll work with (for awhile) */
+ rc_id = rc_idH;
+ version = versionH;
+ }
+
+ /* Rank 0 notifies other ranks that transaction 2 committed and RC acquired */
+ MPI_Bcast( &version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+
+ /****
+ * Read Context 2: Other ranks create RC and open HObjects created by rank 0.
+ ****/
+ if ( my_rank != 0 ) {
+ rc_id = H5RCcreate( file_id, version );
+ assert( rc_id >= 0 ); assert ( version == 2 );
+ fprintf( stderr, "APP-r%d: rc %lu - Created for rank\n", my_rank, version );
+
+ grp_l_id = H5Gopen_ff( file_id, "G-logged", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( grp_l_id >= 0 );
+ grp_p_id = H5Gopen_ff( file_id, "G-prefetched", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( grp_p_id >= 0 );
+ grp_s_id = H5Gopen_ff( file_id, "G-stored", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( grp_s_id >= 0 );
+
+ dset_l_id = H5Dopen_ff( grp_l_id, "D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_l_id >= 0 );
+ dset_p_id = H5Dopen_ff( grp_p_id, "D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_p_id >= 0 );
+ dset_s_id = H5Dopen_ff( grp_s_id, "D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_s_id >= 0 );
+
+ map_l_id = H5Mopen_ff( grp_l_id, "M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( map_l_id >= 0 );
+ map_p_id = H5Mopen_ff( grp_p_id, "M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( map_p_id >= 0 );
+ map_s_id = H5Mopen_ff( grp_s_id, "M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( map_s_id >= 0 );
+ } else {
+ if ( verbose ) print_container_contents( file_id, rc_id, "/", my_rank );
+ }
+
+ /* Keep track of my_rank's neighbor */
+ neighbor_rank = my_rank + 1;
+ if ( neighbor_rank == comm_size ) {
+ neighbor_rank = 0;
+ }
+
+ /* Get the dataspaces for the three datasets and make a copy of each */
+ space_l_id = H5Dget_space( dset_l_id ); assert ( space_l_id >= 0 );
+ space_p_id = H5Dget_space( dset_p_id ); assert ( space_p_id >= 0 );
+ space_s_id = H5Dget_space( dset_s_id ); assert ( space_s_id >= 0 );
+
+ neighbor_space_l_id = H5Scopy( space_l_id );
+ neighbor_space_p_id = H5Scopy( space_p_id );
+ neighbor_space_s_id = H5Scopy( space_s_id );
+
+ /* Select the hyperslab that this rank will update and the hyperslab for reading neighbor's data */
+ rank_offset[0] = my_rank * rows_per_rank;
+ rank_offset[1] = 0;
+ rank_count[0] = rows_per_rank;
+ rank_count[1] = cols_per_row;
+
+ neighbor_offset[0] = neighbor_rank * rows_per_rank;
+ neighbor_offset[1] = 0;
+ neighbor_count[0] = rows_per_rank;
+ neighbor_count[1] = cols_per_row;
+
+ ret = H5Sselect_hyperslab( space_l_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_p_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_s_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
+
+ ret = H5Sselect_hyperslab( neighbor_space_l_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( neighbor_space_p_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( neighbor_space_s_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL ); ASSERT_RET;
+
+ /* Create the memory dataspace for a rank - always first part of memory buffer */
+ rank_mbuf_size[0] = rows_per_rank * cols_per_row;
+ rank_space_id = H5Screate_simple( 1, rank_mbuf_size, rank_mbuf_size ); assert( space_id >= 0 );
+
+ /* Create property lists that will be used */
+ dxpl_p_id = H5Pcreate( H5P_DATASET_XFER );
+ mxpl_p_id = H5Pcreate( H5P_DATASET_XFER );
+ dapl_p_id = H5Pcreate( H5P_DATASET_ACCESS );
+ mapl_p_id = H5Pcreate( H5P_MAP_ACCESS );
+
+ for ( iteration = 0; iteration < num_iterations; iteration++ ) {
+
+ /****
+ * Cycle of:
+ * Tranasction N Start / Update / Commit: All ranks update Dataset and Map objects in the 3 Groups then commit
+ * CV N Persist: Rank 0
+ * Prefetch: Rank 0
+ *
+ ****/
+ tr_num = tr_num + 1;
+ num_tr_leaders = comm_size;
+
+ /* Create a transaction and start it. */
+ fprintf( stderr, "APP-r%d: iter %d tr %lu - Transaction Start with %d leaders\n",
+ my_rank, iteration, tr_num, num_tr_leaders );
+ tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
+ trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
+ ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
+ ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Set cell values (in memory) for this rank based on Row, Rank, Column, Transaction */
+ u = 0;
+ for ( r = 0; r < rows_per_rank; r++ ) {
+ for ( c = 0; c < cols_per_row; c++ ) {
+ mbuf[u] = (my_rank*rows_per_rank+r)*1000000 + c*100 + tr_num;
+ u++;
+ }
+ }
+
+ fprintf( stderr, "APP-r%d: iter %d tr %lu - Add updates to transaction.\n", my_rank, iteration, tr_num );
+
+ /* Add dataset updates to transaction */
+ START_TIME;
+ ret = H5Dwrite_ff( dset_l_id, H5T_NATIVE_UINT64, rank_space_id, space_l_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to add updates (%lu bytes) for /G-logged/D: %lu usec\n",
+ my_rank, iteration, u*bytesPerCell, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Dwrite_ff( dset_p_id, H5T_NATIVE_UINT64, rank_space_id, space_p_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to add updates (%lu bytes) for /G-prefetched/D: %lu usec\n",
+ my_rank, iteration, u*bytesPerCell, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Dwrite_ff( dset_s_id, H5T_NATIVE_UINT64, rank_space_id, space_s_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to add updates (%lu bytes) for /G-stored/D: %lu usec\n",
+ my_rank, iteration, u*bytesPerCell, ELAPSED_TIME );
+
+ /* Add map updates to transaction; do it map-by-map so we can do timing */
+ key = rows_per_rank * cols_per_row * my_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mset_ff( map_l_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to add updates ( %lu K/V pairs - %lu bytes) for /G-logged/M: %lu usec\n",
+ my_rank, iteration, u, u*bytesPerCell*2, ELAPSED_TIME );
+
+ key = rows_per_rank * cols_per_row * my_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mset_ff( map_p_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to add updates ( %lu K/V pairs - %lu bytes) for /G-prefetched/M: %lu usec\n",
+ my_rank, iteration, u, u*bytesPerCell*2, ELAPSED_TIME );
+
+ key = rows_per_rank * cols_per_row * my_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mset_ff( map_s_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to add updates ( %lu K/V pairs - %lu bytes) for /G-stored/M: %lu usec\n",
+ my_rank, iteration, u, u*bytesPerCell*2, ELAPSED_TIME );
+
+ /* Finish, (commit), and close transaction */
+ fprintf( stderr, "APP-r%d: iter %d tr %lu - Finish and Commit\n", my_rank, iteration, tr_num );
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ ret = H5Pclose( trspl_id ); ASSERT_RET;
+
+ /* After commit completes on all ranks, acquire RC on rank 0 and create local handle for RC on other ranks */
+ MPI_Barrier( MPI_COMM_WORLD );
+ versionH = tr_num;
+ if ( my_rank == 0 ) {
+ rc_idH = H5RCacquire( file_id, &versionH, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ assert( rc_idH >= 0 ); assert ( versionH == tr_num );
+ fprintf( stderr, "APP-r%d: iter %d rc %lu - Acquired\n", my_rank, iteration, versionH );
+ } else {
+ /* Won't wait here for rank 0 to acquire, because we'll wait later before rc_idH is used by other ranks */
+ rc_idH = H5RCcreate( file_id, versionH );
+ assert( rc_idH >= 0 ); assert ( versionH == tr_num );
+ fprintf( stderr, "APP-r%d: iter %d rc %lu - Created\n", my_rank, iteration, versionH );
+ }
+ /* Release previous Read Context */
+ if ( my_rank == 0 ) {
+ ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: iter %d rc %lu - Released\n", my_rank, iteration, version );
+ }
+ ret = H5RCclose( rc_id ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: iter %d rc %lu - Closed\n", my_rank, iteration, version );
+
+ /* The just-acquired RC is now the primary rc_id we'll work with (for awhile) */
+ rc_id = rc_idH;
+ version = versionH;
+
+ /* Rank 0:
+ - persists the container
+ - evicts D and M under /G-prefetched and /G-stored
+ - prefetches D and M under /G-prefetched
+ */
+ if ( my_rank == 0 ) {
+ if ( verbose ) print_container_contents( file_id, rc_id, "/", my_rank );
+
+ START_TIME;
+ ret = H5RCpersist( rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Persist container: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Devict_ff( dset_p_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Evict /G-prefetched/D: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Devict_ff( dset_s_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Evict /G-stored/D: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Mevict_ff( map_p_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Evict /G-prefetched/M (currently no-op for IOD): %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Mevict_ff( map_s_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Evict /G-stored/M: (currently no-op for IOD) %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Dprefetch_ff( dset_p_id, rc_id, &dset_p_replica, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Prefetch /G-prefetched/D: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Mprefetch_ff( map_p_id, rc_id, &map_p_replica, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Prefetch /G-prefetched/M: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
+ }
+
+ /* This Bcast will make sure no nodes go ahead to use the RC before rank 0 has acquired it */
+ MPI_Bcast( &dset_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+ ret = H5Pset_read_replica( dxpl_p_id, dset_p_replica ); ASSERT_RET;
+ ret = H5Pset_evict_replica( dapl_p_id, dset_p_replica ); ASSERT_RET;
+
+ MPI_Bcast( &map_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+ ret = H5Pset_read_replica( mxpl_p_id, map_p_replica ); ASSERT_RET;
+ ret = H5Pset_evict_replica( mapl_p_id, map_p_replica ); ASSERT_RET;
+
+ /*
+ * All ranks read the same values they updated in last transaction
+ */
+
+ /* Datasets */
+ START_TIME;
+ ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, rank_space_id, space_l_id, H5P_DEFAULT, mbuf, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-logged/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: my first updated value in /G-logged/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, rank_space_id, space_p_id, dxpl_p_id, mbuf, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-prefetched/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: my first updated value in /G-prefetched/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, rank_space_id, space_s_id, H5P_DEFAULT, mbuf, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-stored/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: my first updated value in /G-stored/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ /* Maps */
+ key = rows_per_rank * cols_per_row * my_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mget_ff( map_l_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-logged/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: my first updated value in /G-logged/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ key = rows_per_rank * cols_per_row * my_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mget_ff( map_p_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-prefetched/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose )
+ { fprintf( stderr, "APP-r%d: my first updated value in /G-prefetched/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ key = rows_per_rank * cols_per_row * my_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mget_ff( map_s_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-stored/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: my first updated value in /G-stored/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ /*
+ * All ranks read the values "next rank" neighbor updated in last transaction
+ */
+
+ /* Datasets */
+ START_TIME;
+ ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, rank_space_id, neighbor_space_l_id, H5P_DEFAULT, mbuf, rc_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-logged/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-logged/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, rank_space_id, neighbor_space_p_id, dxpl_p_id, mbuf, rc_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-prefetched/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-prefetched/D: %09lu\n", my_rank, mbuf[0]);
+ }
+
+ START_TIME;
+ ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, rank_space_id, neighbor_space_s_id, H5P_DEFAULT, mbuf, rc_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-stored/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-stored/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ /* Maps */
+ key = rows_per_rank * cols_per_row * neighbor_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mget_ff( map_l_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-logged/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-logged/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ key = rows_per_rank * cols_per_row * neighbor_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mget_ff( map_p_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-prefetched/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-prefetched/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ key = rows_per_rank * cols_per_row * neighbor_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mget_ff( map_s_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-stored/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-stored/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ /*
+ * All ranks read all the values
+ */
+
+ /* Datasets */
+ START_TIME;
+ ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, mbuf, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-logged/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: first value in /G-logged/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, dxpl_p_id, mbuf, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-prefetched/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: first value in /G-prefetched/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, mbuf, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-stored/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: first value in /G-stored/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ /* Maps */
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row * comm_size; u++ ) {
+ ret = H5Mget_ff( map_l_id, H5T_NATIVE_UINT64, &u, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-logged/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: first value in /G-logged/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row * comm_size; u++ ) {
+ ret = H5Mget_ff( map_p_id, H5T_NATIVE_UINT64, &u, H5T_NATIVE_UINT64, &mbuf[u], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-prefetched/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: first value in /G-prefetched/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row * comm_size; u++ ) {
+ ret = H5Mget_ff( map_s_id, H5T_NATIVE_UINT64, &u, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-stored/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: first value in /G-stored/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+
+ /* Rank 0 evicts the replicas */
+ if ( my_rank == 0 ) {
+ START_TIME;
+ ret = H5Devict_ff( dset_p_id, version, dapl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Evict Replica of /G-prefetched/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+
+ ret = H5Mevict_ff( map_p_id, version, mapl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Evict Replica of /G-prefetched/M (currently no-op for IOD): %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ }
+
+ }
+
+ /*
+ * Wrap-up
+ */
+
+ /* Print (if verbose) the final state */
+ if ( my_rank == 0 ) {
+ if ( verbose ) print_container_contents( file_id, rc_id, "/", my_rank );
+ }
+
+ /* Close the dataset and map property lists */
+ ret = H5Pclose( dxpl_p_id ); ASSERT_RET;
+ ret = H5Pclose( mxpl_p_id ); ASSERT_RET;
+ ret = H5Pclose( dapl_p_id ); ASSERT_RET;
+ ret = H5Pclose( mapl_p_id ); ASSERT_RET;
+
+ /* Close the memory dataspace for the rank */
+ ret = H5Sclose( rank_space_id ); ASSERT_RET;
+
+ fprintf( stderr, "APP-r%d: Closing H5Objects\n", my_rank );
+
+ ret = H5Dclose_ff( dset_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Dclose_ff( dset_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Dclose_ff( dset_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ ret = H5Sclose( space_l_id ); ASSERT_RET;
+ ret = H5Sclose( space_p_id ); ASSERT_RET;
+ ret = H5Sclose( space_s_id ); ASSERT_RET;
+
+ ret = H5Mclose_ff( map_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Mclose_ff( map_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Mclose_ff( map_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ ret = H5Gclose_ff( grp_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Gclose_ff( grp_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Gclose_ff( grp_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Release the read handle and close read context */
+ if ( my_rank == 0 ) {
+ ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc %lu - Released\n", my_rank, version );
+ }
+ ret = H5RCclose( rc_id ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc %lu - Closed\n", my_rank, version );
+
+ /* Close H5 Objects that are still open */
+ fprintf( stderr, "APP-r%d: Closing %s\n", my_rank, file_name );
+ MPI_Barrier( MPI_COMM_WORLD );
+ ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Pclose( fapl_id ); ASSERT_RET;
+
+ /* Perform wrap-up operations */
+ fprintf( stderr, "APP-r%d: Finalize EFF stack\n", my_rank );
+ MPI_Barrier( MPI_COMM_WORLD );
+ EFF_finalize();
+ MPI_Finalize();
+
+ return 0;
+}
+
+
+/*
+ * Helper function used to recursively read and print container contents
+ * for container identified by "file_id"
+ * in read context identified by "rc_id"
+ * with path to current level in "grp_path"
+ * and "my_rank" used to identify the process doing the reading / printing.
+ */
+void
+print_container_contents( hid_t file_id, hid_t rc_id, const char* grp_path, int my_rank )
+{
+ herr_t ret;
+ uint64_t cv;
+ hbool_t exists;
+ char path_to_object[1024];
+ char name[30];
+ int i;
+
+ static int lvl = 0; /* level in recursion - used to format printing */
+ char preface[128];
+
+ /* Get the container version for the read context */
+ ret = H5RCget_version( rc_id, &cv ); ASSERT_RET;
+
+ /* Set up the preface and adding version number */
+ sprintf( preface, "APP-r%d: cv %d: ", my_rank, (int)cv );
+
+ /* Start the printing */
+ if ( lvl == 0 ) {
+ fprintf( stderr, "%s ----- Container Contents ------------\n", preface );
+ }
+
+ /* Datasets */
+ sprintf( path_to_object, "%s%s", grp_path, "D" );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+
+ if ( exists ) {
+ hid_t dset_id;
+ hid_t space_id;
+ int nDims;
+ hsize_t current_size[2];
+ hsize_t max_size[2];
+ hsize_t totalSize;
+ uint64_t *data;
+ uint64_t u, e;
+
+ dset_id = H5Dopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( dset_id >= 0 );
+
+ space_id = H5Dget_space( dset_id ); assert ( space_id >= 0 );
+ nDims = H5Sget_simple_extent_dims( space_id, current_size, max_size );
+ assert( nDims == 2 );
+
+ totalSize = current_size[0] * current_size[1];
+ data = (uint64_t *)calloc( totalSize, sizeof(uint64_t) );
+
+ ret = H5Dread_ff( dset_id, H5T_NATIVE_UINT64, space_id, space_id, H5P_DEFAULT, data, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* note that by printing piece-by-piece, other output may be interspersed. trade-off since won't know line size */
+ fprintf( stderr, "%s %s data:\n", preface, path_to_object );
+ e = 0;
+ for ( i = 0; i < totalSize; i++ ) {
+ if ( e == 0 ) fprintf( stderr, "%s\t", preface );
+ fprintf( stderr, "%09lu ", data[i] );
+ if ( ++e == cols_per_row ) {
+ fprintf( stderr, "\n" );
+ e = 0;
+ }
+ }
+
+ free( data );
+ sleep( 1 ); /* give output a chance to be printed before diag messages from lower layers on close */
+ ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Sclose( space_id ); ASSERT_RET;
+ }
+
+ /* Maps */
+ sprintf( path_to_object, "%s%s", grp_path, "M" );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+
+ if ( exists ) {
+ hid_t map_id;
+ hsize_t totalCount;
+ uint64_t *value;
+ uint64_t u, e;
+
+ map_id = H5Mopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( map_id >= 0 );
+
+ ret = H5Mget_count_ff( map_id, &totalCount, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ value = (uint64_t *)calloc( totalCount, sizeof(uint64_t) );
+
+ for ( u = 0; u < totalCount; u++ ) {
+ ret = H5Mget_ff( map_id, H5T_NATIVE_UINT64, &u, H5T_NATIVE_UINT64, &value[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+
+ /* note that by printing piece-by-piece, other output may be interspersed. trade-off since won't know line size */
+ fprintf( stderr, "%s %s key/values:\n", preface, path_to_object );
+ e = 0;
+ for ( u = 0; u < totalCount; u++ ) {
+ if ( e == 0 ) fprintf( stderr, "%s\t", preface );
+ fprintf( stderr, "%lu/%09lu ", u, value[u] );
+ if ( ++e == cols_per_row ) {
+ fprintf( stderr, "\n" );
+ e = 0;
+ }
+ }
+
+ free( value );
+ sleep( 1 ); /* give output a chance to be printed before diag messages from lower layers on close */
+ ret = H5Mclose_ff( map_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ }
+
+ /* Groups - if found, descend */
+ for ( i = 1; i < 4; i++ ) {
+ if ( i == 1 ) {
+ strcpy( name, "G-logged" );
+ } else if ( i == 2 ) {
+ strcpy( name, "G-prefetched" );
+ } else if ( i == 3 ) {
+ strcpy( name, "G-stored" );
+ }
+
+ sprintf( path_to_object, "%s%s/", grp_path, name );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+ if ( exists ) {
+ fprintf( stderr, "%s %s\n", preface, path_to_object );
+ lvl++;
+ print_container_contents( file_id, rc_id, path_to_object, my_rank );
+ lvl--;
+ }
+ }
+
+ /* End printing */
+ if ( lvl == 0 ) {
+ fprintf( stderr, "%s -----------------\n", preface );
+ }
+
+ return;
+}
+
+/*
+ * parse_options - parse command line options
+ */
+int
+parse_options( int argc, char** argv, int my_rank, int comm_size ) {
+ int i, n;
+ char* app = argv[0];
+
+ while ( --argc ) {
+ if ( **(++argv) != '-' ) {
+ break;
+ } else {
+ switch( *(*argv+1) ) {
+ case 'c':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "Error: No number specified after -c option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ cols_per_row = atoi( *argv );
+ }
+ break;
+ case 'r':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "Error: No number specified after -r option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ rows_per_rank = atoi( *argv );
+ }
+ break;
+ case 'i':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "APP-r0: Error: No number specified after -i option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ num_iterations = atoi( *argv );
+ }
+ break;
+ case 'v':
+ verbose = 1;
+ break;
+ default:
+ if ( my_rank == 0 ) {
+ usage( app );
+ }
+ return( 1 );
+ }
+ }
+ }
+ return( 0 );
+}
+
+/*
+ * usage - display usage message
+ */
+void
+usage( const char* app ) {
+ printf( "Usage: %s [-c cols_per_row] [-r rows_per_rank] [-i num_iterations] [-v] [-w]\n", app );
+ printf( "\tc: number of columns in each row of created H5Datasets\n" );
+ printf( "\tr: number of rows each rank writes in created H5Datasets (total rows = this * # of ranks)\n" );
+ printf( "\ti: number of iterations to do update/commit/persist/evict/prefetch/read/evict\n" );
+ printf( "\tv: verbose output\n" );
+}
+
+
+
+
+#ifdef THIS_CODE_WONT_COMPILE_BUT_KEEP_IT_AROUND
+/*
+ * This codeblock needs work to reintegrate back into main() after the iterate loop of complete overwrites.
+ * Didn't want to loose it in case I decide to add back in later.
+ * It allows ranks to overwrite only part of the dataset & map obects in a transaction.
+ * With more work, can be used to demonstrate / time reading from 'layers' of updates & persisting only partial objects
+ * for a container (where previous layers alreay persisted -- Ruth
+ */
+
+ /****
+ * Transaction X - All ranks update parts of the Dataset and Map objects in the 3 Groups
+ ****/
+ mbuf_size[0] = cols_per_row;
+ space_id = H5Screate_simple( 1, mbuf_size, mbuf_size ); assert( space_id >= 0 );
+
+ offset[0] = my_rank * rows_per_rank;
+ offset[1] = 0;
+ count[0] = 1;
+ count[1] = cols_per_row;
+
+ ret = H5Sselect_hyperslab( space_l_id, H5S_SELECT_SET, offset, NULL, count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_p_id, H5S_SELECT_SET, offset, NULL, count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_s_id, H5S_SELECT_SET, offset, NULL, count, NULL ); ASSERT_RET;
+
+ tr_num = X;
+ num_tr_leaders = comm_size;
+
+ /* Create a local transaction for transaction number and start it. */
+ fprintf( stderr, "APP-r%d: tr %lu - Start with %d leaders\n", my_rank, tr_num, num_tr_leaders );
+ tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
+ trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
+ ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
+ ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Set cell values (in memory) based on Column, Row, Rank, Transaction */
+ u = 0;
+ for ( r = 0; r < 1; r++ ) {
+ for ( c = 0; c < cols_per_row; c++ ) {
+ mbuf[u] = (my_rank*rows_per_rank+r)*1000000 + c*100 + tr_num;
+ u++;
+ }
+ }
+
+ /* Add dataset updates to transaction */
+ ret = H5Dwrite_ff( dset_l_id, H5T_NATIVE_UINT64, space_id, space_l_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ ret = H5Dwrite_ff( dset_p_id, H5T_NATIVE_UINT64, space_id, space_p_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ ret = H5Dwrite_ff( dset_s_id, H5T_NATIVE_UINT64, space_id, space_s_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+
+ /* Add map updates to transaction */
+ /* Set cell values (in memory) based on Column, Row, Rank, Transaction */
+ key = rows_per_rank * cols_per_row * my_rank;
+ for ( u = 0; u < cols_per_row; u++ ) {
+ ret = H5Mset_ff( map_l_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ ret = H5Mset_ff( map_p_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ ret = H5Mset_ff( map_s_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+
+ ret = H5Sclose( space_id ); ASSERT_RET; // RUTH - likely keep this around
+ free( mbuf ); // RUTH - likely keep this around
+
+ /* Finish and close transaction */
+ fprintf( stderr, "APP-r%d: tr %lu - Finish\n", my_rank, tr_num );
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ ret = H5Pclose( trspl_id ); ASSERT_RET;
+
+ // RUTH _ FIX THIS, for now just want to see if data updated.
+ MPI_Barrier( MPI_COMM_WORLD );
+ if ( my_rank == 0 ) {
+ version = tr_num;
+ rc_idH = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ assert( rc_idH >= 0 ); assert ( version == tr_num );
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
+ if ( verbose ) print_container_contents( file_id, rc_idH, "/", my_rank );
+ ret = H5RCrelease( rc_idH, H5_EVENT_STACK_NULL); ASSERT_RET;
+ ret = H5RCclose( rc_idH ); ASSERT_RET;
+ version = 2;
+ }
+ MPI_Barrier( MPI_COMM_WORLD );
+
+#endif
|