summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--examples/h5ff_client_M6.2_demo.c2254
-rw-r--r--examples/h5ff_client_M7.2-pep_demo.c2576
-rw-r--r--examples/h5ff_client_do.c578
-rw-r--r--examples/h5ff_client_evict_deltas.c664
-rw-r--r--examples/h5ff_client_links.c628
-rw-r--r--examples/h5ff_client_map.c1136
-rw-r--r--examples/h5ff_client_multiple_cont.c876
-rw-r--r--examples/h5ff_client_obj.c626
-rw-r--r--examples/h5ff_client_old_api.c560
-rw-r--r--examples/h5ff_client_prefetch.c878
-rw-r--r--examples/h5ff_client_time_datasets.c2710
-rw-r--r--examples/h5ff_client_timings.c2232
-rw-r--r--examples/h5ff_client_trans.c380
-rw-r--r--examples/h5ff_server.c96
14 files changed, 8097 insertions, 8097 deletions
diff --git a/examples/h5ff_client_M6.2_demo.c b/examples/h5ff_client_M6.2_demo.c
index b5d6a74..b080c10 100644
--- a/examples/h5ff_client_M6.2_demo.c
+++ b/examples/h5ff_client_M6.2_demo.c
@@ -1,1127 +1,1127 @@
-/*
- * M6.2-demo.c: This is the demo program for Milestone 6.2 - HDF5 and IO Dispatcher Container Versioning Demonstration.
- * This program runs on one or more compute nodes and makes calls to the HDF5 API.
- * The Function Shipper server should be running before this demo program is started.
- *
- * March, 2014: Updated to start user-initated transactions at number 1, not 0, to reflect DAOS numbering requirements.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-
-/* define filename for this app, and max size after username prepended */
-#define FILENAME_APP "m6.2-eff_container.h5"
-#define FILENAME_SIZE 256
-
-/* max length of attributes */
-#define ATTR_STR_LEN 128
-
-/* macros related to error reporting */
-#define STATUS (ret >= 0) ? "OK" : "FAILED"
-#define ASSERT_RET assert( ret >= 0 )
-
-/* option flags */
-int verbose = 0; // Verbose (print CV contents between every transaction) defaults to no
-int last_cv_only = 0; // Only print contents of last container version when file re-opened - defaults to no
-int abort4 = 0; // Abort transaction 4 defaults to no
-
-/* prototypes for helper functions */
-void create_string_attribute( hid_t, const char*, hid_t, const char*, int, uint64_t );
-void create_group( hid_t, const char*, hid_t, const char*, int, uint64_t );
-void create_dataset( hid_t, const char*, hid_t, const char*, int, uint64_t, int );
-void create_dataset2( hid_t, const char*, hid_t, const char*, int, uint64_t );
-void create_committed_datatype( hid_t, const char*, hid_t, const char*, int, uint64_t );
-void update_dataset( hid_t, const char*, hid_t, hid_t, const char*, int, uint64_t, int );
-void append_dataset2( hid_t, const char*, hid_t, hid_t, const char*, int, uint64_t );
-void print_container_contents( hid_t, hid_t, const char*, int );
-int parse_options( int, char**, int );
-
-int main( int argc, char **argv ) {
-
- char *user_name; /* We'll prepend username to make filename unique */
- char file_name[FILENAME_SIZE]; /* Actual filename to be opened will be user_name + FILENAME_APP */
-
- int my_rank, comm_size;
- int provided;
-
- hid_t fapl_id;
- hid_t file_id;
- herr_t ret;
-
- uint64_t version;
- hid_t rc_id, rc_id1, rc_id2, rc_id3, rc_id4, rc_id5;
-
- /* Check for MPI multi-thread support */
- MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
- if ( MPI_THREAD_MULTIPLE != provided ) {
- fprintf( stderr, "M6.2: ERROR: MPI does not have MPI_THREAD_MULTIPLE support\n" );
- exit( 1 );
- }
-
- MPI_Comm_rank( MPI_COMM_WORLD, &my_rank );
- MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
-
- if(comm_size < 2) {
- fprintf(stderr, "Need at least 2 MPI ranks to run this test. Aborting..\n");
- MPI_Abort(MPI_COMM_WORLD, 1);
- }
-
- fprintf( stderr, "M6.2-r%d: Number of MPI processes = %d\n", my_rank, comm_size );
-
- /* Parse command-line options controlling behavior */
- if ( parse_options( argc, argv, my_rank ) != 0 ) {
- exit( 1 );
- }
-
- /* Initialize the EFF stack. */
- fprintf( stderr, "M6.2-r%d: Initialize EFF stack (Step 1)\n", my_rank );
- EFF_init( MPI_COMM_WORLD, MPI_INFO_NULL );
-
- /* Specify that the IOD VOL plugin should be used and create H5File (EFF container) */
- user_name = getenv( "USER" );
- snprintf( file_name, FILENAME_SIZE, "%s_%s", user_name, FILENAME_APP );
- fprintf( stderr, "M6.2-r%d: Create the container %s (Step 2)\n", my_rank, file_name );
- fapl_id = H5Pcreate( H5P_FILE_ACCESS ); assert( fapl_id >= 0 );
- ret = H5Pset_fapl_iod( fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL ); ASSERT_RET;
- file_id = H5Fcreate_ff( file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL ); assert( file_id >= 0 );
-
- /* Acquire a read handle for container version 1 and create a read context. */
- version = 1;
- if ( 0 == my_rank ) {
- fprintf( stderr, "M6.2-r%d: Acquire read context for container version %d (Step 3)\n", my_rank, (int)version );
- rc_id1 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- } else {
- rc_id1 = H5RCcreate( file_id, version);
- }
- assert( rc_id1 >= 0 ); assert( version == 1 );
- MPI_Barrier( MPI_COMM_WORLD );
-
- /* Print container contents at this point */
- fprintf( stderr, "M6.2-r%d: 1st call to print container contents (Step 4)\n", my_rank );
- if ( verbose ) print_container_contents( file_id, rc_id1, "/", my_rank );
-
- /*
- * Transaction 2 is created, started, updated, and committed by rank 0.
- * Root group attributes AA and AB are created and populated.
- * Groups /GA and /GB are created.
- * Datasets /DA and /DB are created and populated.
- * Dataset /DC (2-d) and attribute AC@/DC are created & initialized.
- * Committed datatypes /TA and /TB are created.
- */
- if ( my_rank == 0 ) {
-
- uint64_t tr_num = 2;
- hid_t tr_id;
-
- /* Create a local transaction for transaction number. */
- fprintf( stderr, "M6.2-r%d: Create tr %d (Step 5 - begin) \n", my_rank, (int)tr_num );
- tr_id = H5TRcreate( file_id, rc_id1, (uint64_t)tr_num ); assert( tr_id >= 0 );
-
- /* Start transaction 2 with single transaction leader (the default) */
- fprintf( stderr, "M6.2-r%d: Start tr %d\n", my_rank, (int)tr_num );
- ret = H5TRstart( tr_id, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Add updates to the transaction */
-
- /* 2 Attributes */
- create_string_attribute( file_id, "AA", tr_id, "/", my_rank, tr_num );
- create_string_attribute( file_id, "AB", tr_id, "/", my_rank, tr_num );
-
- /* 3 Datasets */
- create_dataset( file_id, "DA", tr_id, "/", my_rank, tr_num, 1 );
- create_dataset( file_id, "DB", tr_id, "/", my_rank, tr_num, 2 );
- create_dataset2( file_id, "DC", tr_id, "/", my_rank, tr_num );
-
- /* 2 committed datatypes */
- create_committed_datatype( file_id, "TA", tr_id, "/", my_rank, tr_num );
- create_committed_datatype( file_id, "TB", tr_id, "/", my_rank, tr_num );
-
- /* 2 Groups */
- create_group( file_id, "GA", tr_id, "/", my_rank, tr_num );
- create_group( file_id, "GB", tr_id, "/", my_rank, tr_num );
-
- /* Finish and commit transaction 2, causing the updates to appear in container version 2. */
- fprintf( stderr, "M6.2-r%d: Finish and commit tr %d (Step 5 - end)\n", my_rank, (int)tr_num );
- ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Close the local transaction. */
- ret = H5TRclose( tr_id ); ASSERT_RET;
- }
-
- /* Print container contents at this point */
- fprintf( stderr, "M6.2-r%d: 2nd call to print container contents (Step 6)\n", my_rank );
- if (verbose) print_container_contents( file_id, rc_id1, "/", my_rank );
-
-
- /* Acquire a read handle for container version 2 and create a read context. */
- version = 2;
- if ( 0 == my_rank ) {
- fprintf( stderr, "M6.2-r%d: Try to acquire read context for cv %d (Step 7)\n", my_rank, (int)version );
- H5E_BEGIN_TRY {
- rc_id2 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- while ( rc_id2 < 0 ) {
- fprintf( stderr, "M6.2-r%d: Failed to acquire read context for cv %d - sleep then retry\n", my_rank, (int)version );
- sleep( 1 );
- version = 2;
- rc_id2 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- }
- } H5E_END_TRY;
- } else {
- rc_id2 = H5RCcreate( file_id, version );
- }
- assert( rc_id2 >= 0 ); assert ( version == 2 );
- fprintf( stderr, "M6.2-r%d: Acquired read context for cv 2\n", my_rank );
- MPI_Barrier( MPI_COMM_WORLD );
-
- fprintf( stderr, "M6.2-r%d: 3rd call to print container contents (Step 8)\n", my_rank );
- if (verbose) print_container_contents( file_id, rc_id2, "/", my_rank );
-
- MPI_Barrier( MPI_COMM_WORLD );
- /* Release the read handle and close read context on cv 1 */
- fprintf( stderr, "M6.2-r%d: Release read handle on cv 1 (Step 9)\n", my_rank );
- if ( 0 == my_rank ) {
- ret = H5RCrelease( rc_id1, H5_EVENT_STACK_NULL); ASSERT_RET;
- }
- ret = H5RCclose( rc_id1 ); ASSERT_RET;
-
- /*
- * Transaction 3 is created, started, updated, and committed by ranks 0 & 1, if 2 or more MPI processes.
- * Group /GA attributes AA and AB are created and populated by rank 0.
- * Group /GB attributes AA and AB are created and populated by rank 1.
- * Groups /GA/GA and /GA/GB are created by rank 0.
- * Groups /GB/GA and /GB/GB are created by rank 1.
- * Datasets /GA/DA and /GA/DB are created and populated by rank 0.
- * Datasets /GB/DA and /GB/DB are created and populated by rank 1.
- * Committed Datatypes /GA/AA and /GA/TB are created by rank 0.
- * Committed Datatypes /GB/AA and /GB/TB are created by rank 1.
- * Dataset /DC is appended to by rank 0.
- */
- if ( (my_rank == 0) || (my_rank == 1) ) {
-
- uint64_t tr_num = 3;
- hid_t tr_id;
- hid_t trspl_id;
- hid_t gr_id;
- char gr_path[128];
-
- /* Create a local transaction for transaction number 3. */
- tr_id = H5TRcreate( file_id, rc_id2, tr_num ); assert( tr_id >= 0 );
-
- /* Start transaction 3 with two transaction leaders (unless there is only 1 MPI process) */
- fprintf( stderr, "M6.2-r%d: Start tr %d (Step 10 - begin)\n", my_rank, (int)tr_num );
- trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
- if ( comm_size >= 2 ) {
- ret = H5Pset_trspl_num_peers( trspl_id, 2 ); ASSERT_RET;
- } else {
- ret = H5Pset_trspl_num_peers( trspl_id, 1 ); ASSERT_RET;
- }
- ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Add updates to the transaction */
- if ( my_rank == 0 ) {
- gr_id = H5Gopen_ff( file_id, "GA", H5P_DEFAULT, rc_id2, H5_EVENT_STACK_NULL ); assert( gr_id >= 0 );
- sprintf( gr_path, "/GA/" );
- } else {
- gr_id = H5Gopen_ff( file_id, "GB", H5P_DEFAULT, rc_id2, H5_EVENT_STACK_NULL ); assert( gr_id >= 0 );
- sprintf( gr_path, "/GB/" );
- }
-
- /* 2 Attributes in each group */
- create_string_attribute( gr_id, "AA", tr_id, gr_path, my_rank, tr_num );
- create_string_attribute( gr_id, "AB", tr_id, gr_path, my_rank, tr_num );
-
- /* 2 1-D Datasets */
- create_dataset( gr_id, "DA", tr_id, gr_path, my_rank, tr_num, 1 );
- create_dataset( gr_id, "DB", tr_id, gr_path, my_rank, tr_num, 2 );
-
- /* 2 committed datatypes */
- create_committed_datatype( gr_id, "TA", tr_id, gr_path, my_rank, tr_num );
- create_committed_datatype( gr_id, "TB", tr_id, gr_path, my_rank, tr_num );
-
- /* 2 Groups */
- create_group( gr_id, "GA", tr_id, gr_path, my_rank, tr_num );
- create_group( gr_id, "GB", tr_id, gr_path, my_rank, tr_num );
-
- /* Append to dataset /DC */
- if ( my_rank == 0 ) {
- append_dataset2( file_id, "/DC", tr_id, rc_id2, "/", my_rank, tr_num );
- }
-
- /* Close the group */
- ret = H5Gclose_ff ( gr_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Finish and commit transaction 3, causing the updates to appear in container version 3. */
- fprintf( stderr, "M6.2-r%d: Finish and commit tr %d (Step 10 - end)\n", my_rank, (int)tr_num );
- ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Close the local transaction and transaction start property list. */
- ret = H5TRclose( tr_id ); ASSERT_RET;
- ret = H5Pclose( trspl_id ); ASSERT_RET;
- }
-
- fprintf( stderr, "M6.2-r%d: 4th call to print container contents (Step 11)\n", my_rank );
- if (verbose) print_container_contents( file_id, rc_id2, "/", my_rank );
-
- version = 3;
- if ( 0 == my_rank ) {
- fprintf( stderr, "M6.2-r%d: Try to acquire read context for cv %d (Step 12)\n", my_rank, (int)version );
- H5E_BEGIN_TRY {
- rc_id3 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- while ( rc_id3 < 0 ) {
- fprintf( stderr, "M6.2-r%d: Failed to acquire read context for cv 3 - sleep then retry\n", my_rank );
- sleep( 1 );
- version = 3;
- rc_id3 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- }
- } H5E_END_TRY;
- } else {
- rc_id3 = H5RCcreate( file_id, version );
- }
- assert( rc_id3 >= 0 ); assert( version == 3 );
- fprintf( stderr, "M6.2-r%d: Acquired read context for cv 3\n", my_rank );
- MPI_Barrier( MPI_COMM_WORLD );
-
- fprintf( stderr, "M6.2-r%d: 5th call to print container contents (Step 13)\n", my_rank );
- if (verbose) print_container_contents( file_id, rc_id3, "/", my_rank );
-
- MPI_Barrier( MPI_COMM_WORLD );
- /* Release the read handle and close read context on cv 2 */
- fprintf( stderr, "M6.2-r%d: Release read handle on cv 2 (Step 14)\n", my_rank );
- if ( 0 == my_rank) {
- ret = H5RCrelease( rc_id2, H5_EVENT_STACK_NULL); ASSERT_RET;
- }
- ret = H5RCclose( rc_id2 ); ASSERT_RET;
-
- /*
- * Transaction 5 is created and started by all MPI processes.
- * Transaction 4 is created and started by all MPI processes.
- * Updates are added to the transactions in an interleaved manner,
- * with the ranks progressing independent of each other, and
- * additional updates to transaction 5 after transaction 4
- * is finished and committed.
- * Transaction 4 may be aborted
- */
- { /* Bracket to localize variables, and to later make it easier to restrict ranks participating if desired. */
- uint64_t tr_num4 = 4;
- uint64_t tr_num5 = 5;
- hid_t tr_id4, tr_id5;
- hid_t trspl_id;
-
- /* Set up transaction start property list with the number of transaction leaders */
- trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
- ret = H5Pset_trspl_num_peers( trspl_id, comm_size ); ASSERT_RET;
-
- /* Create & start transaction 5, based on cv 3 */
- tr_id5 = H5TRcreate( file_id, rc_id3, tr_num5 ); assert( tr_id5 >= 0 );
- ret = H5TRstart( tr_id5, trspl_id, H5_EVENT_STACK_NULL );
- fprintf( stderr, "M6.2-r%d: Start tr %d (Step 15a - begin) - %s\n", my_rank, (int)tr_num5, STATUS );
-
- /* Create & start transaction 4, based on cv 3 */
- tr_id4 = H5TRcreate( file_id, rc_id3, tr_num4 ); assert( tr_id4 >= 0 );
- ret = H5TRstart( tr_id4, trspl_id, H5_EVENT_STACK_NULL );
- fprintf( stderr, "M6.2-r%d: Start tr %d (Step 15b - begin) - %s\n", my_rank, (int)tr_num4, STATUS );
-
- /* Close the transaction start property list. */
- ret = H5Pclose( trspl_id ); ASSERT_RET;
-
- /* Add updates to the transactions */
- /* 1) /GA/GA/GB added to Tr 5 by rank 0 */
- if ( my_rank == 0 ) {
- create_group( file_id, "/GA/GA/GB", tr_id5, "/", my_rank, tr_num5 );
- }
- /* 2) /GB/GB/GB added to Tr 5 by rank 1 */
- if ( my_rank == 1 ) {
- create_group( file_id, "/GB/GB/GB", tr_id5, "/", my_rank, tr_num5 );
- }
- /* 3) AA@/ deleted in Tr 4 by rank 0 */
- if ( my_rank == 0 ) {
- ret = H5Adelete_by_name_ff( file_id, ".", "AA", H5P_DEFAULT, tr_id4, H5_EVENT_STACK_NULL);
- fprintf( stderr, "M6.2-r%d: delete AA @ / in tr %d - %s\n", my_rank, (int)tr_num4, STATUS );
- }
- /* 4) /GA/DA deleted in Tr 4 by rank 1 */
- if ( my_rank == 1 ) {
- ret = H5Ldelete_ff( file_id, "/GA/DA", H5P_DEFAULT, tr_id4, H5_EVENT_STACK_NULL);
- fprintf( stderr, "M6.2-r%d: delete /GA/DA in tr %d - %s\n", my_rank, (int)tr_num4, STATUS);
- }
- /* 5) /GA/TB deleted in Tr 5 by rank 0 */
- if ( my_rank == 0 ) {
- ret = H5Ldelete_ff( file_id, "/GA/TB", H5P_DEFAULT, tr_id5, H5_EVENT_STACK_NULL);
- fprintf( stderr, "M6.2-r%d: delete /GA/TB in tr %d - %s\n", my_rank, (int)tr_num5, STATUS );
- }
- /* 6) /DB deleted in Tr 5 by rank 1 */
- if ( my_rank == 1 ) {
- ret = H5Ldelete_ff( file_id, "/DB", H5P_DEFAULT, tr_id5, H5_EVENT_STACK_NULL);
- fprintf( stderr, "M6.2-r%d: delete /DB in tr %d - %s \n", my_rank, (int)tr_num5, STATUS );
- }
- /* 7) /GB/GA deleted in Tr 4 by rank 0 */
- if ( my_rank == 0 ) {
- ret = H5Ldelete_ff( file_id, "/GB/GA", H5P_DEFAULT, tr_id4, H5_EVENT_STACK_NULL);
- fprintf( stderr, "M6.2-r%d: delete /GB/GA in tr %d - %s\n", my_rank, (int)tr_num4, STATUS );
- }
- /* 8) /GA/TA deleted in Tr 4 by rank 1 */
- if ( my_rank == 1 ) {
- ret = H5Ldelete_ff( file_id, "/GA/TA", H5P_DEFAULT, tr_id4, H5_EVENT_STACK_NULL);
- fprintf( stderr, "M6.2-r%d: delete /GA/TA in tr %d - %s\n", my_rank, (int)tr_num4, STATUS );
- }
- /* 9) /GA/GA/GA added in Tr 4 by rank 0 */
- if ( my_rank == 0 ) {
- create_group( file_id, "/GA/GA/GA", tr_id4, "/", my_rank, tr_num4 );
- }
- /* 10) /GB/TA deleted in Tr 4 by rank 1 */
- if ( my_rank == 1 ) {
- ret = H5Ldelete_ff( file_id, "/GB/TA", H5P_DEFAULT, tr_id4, H5_EVENT_STACK_NULL);
- fprintf( stderr, "M6.2-r%d: delete /GB/TA in tr %d - %s\n", my_rank, (int)tr_num4, STATUS );
- }
- /* 11) /GB/GB/DB added in Tr 4 by rank 1 */
- if ( my_rank == 1 ) {
- create_dataset( file_id, "/GB/GB/DB", tr_id4, "/", my_rank, tr_num4, 2 );
- }
- /* 12) /DA updated in Tr 4 by all ranks */
- update_dataset( file_id, "/DA", tr_id4, rc_id3, "/", my_rank, tr_num4, 1 );
-
- /* 13) /GB/DA updated in Tr 4 by rank 0 */
- if ( my_rank == 0 ) {
- update_dataset( file_id, "/GB/DA", tr_id4, rc_id3, "/", my_rank, tr_num4, 1 );
- }
-
- /* 14) Append to dataset /DC in Tr 4 by rank 0 */
- if ( my_rank == 0 ) {
- append_dataset2( file_id, "/DC", tr_id4, rc_id3, "/", my_rank, tr_num4 );
- }
-
- /* Abort - one or more processes should be able to abort the Transaction with the same effect. */
- if ( abort4 ) {
- /* All ranks call TRabort; the order of actual calls is not determined by the code.
- * The first abort to reach IOD is the one that aborts the transaction, and abort need not be called by all ranks. */
- ret = H5TRabort( tr_id4, H5_EVENT_STACK_NULL );
- fprintf( stderr, "M6.2-r%d: ABORT tr %d (Step 15b - abort) - %s\n", my_rank, (int)tr_num4, STATUS );
- }
- else {
- /* Finish and commit transaction 4, causing the updates to appear in container version 4. */
- ret = H5TRfinish( tr_id4, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL );
- fprintf( stderr, "M6.2-r%d: Finish and commit tr %d (Step 15b - end) - %s\n", my_rank, (int)tr_num4, STATUS );
- }
-
- ret = H5TRclose( tr_id4 ); ASSERT_RET;
-
- /* Get read context for CV 4, then print the contents of container */
- /* Limit the number of tries, because if it was aborted we'll never get it */
- int max_tries = 4;
- int current_try = 0;
- version = 4;
- if ( 0 == my_rank ) {
- fprintf( stderr, "M6.2-r%d: Try to acquire read context for cv %d (Step 15c)\n", my_rank, (int)version );
- H5E_BEGIN_TRY {
- rc_id4 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- while ( (rc_id4 < 0) && (current_try < max_tries) ) {
- fprintf( stderr, "M6.2-r%d: Failed to acquire read context for cv 4 - sleep then retry\n", my_rank );
- sleep( 1 );
- current_try++;
- version = 4;
- rc_id4 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- }
- } H5E_END_TRY;
- }
-
- MPI_Bcast( &rc_id4, 1, MPI_INT64_T, 0, MPI_COMM_WORLD );
-
- if ( rc_id4 >= 0 ) {
- if ( 0 != my_rank ) {
- rc_id4 = H5RCcreate( file_id, version );
- }
- assert( version == 4 );
- fprintf( stderr, "M6.2-r%d: Acquired read context for cv 4\n", my_rank );
-
- fprintf( stderr, "M6.2-r%d: 6th call to print container contents (Step 15d)\n", my_rank );
- if (verbose) print_container_contents( file_id, rc_id4, "/", my_rank ); ASSERT_RET;
-
- /* Release read handle & close read context for CV 4 */
- fprintf( stderr, "M6.2-r%d: Release read handle on cv 4 (Step 15e)\n", my_rank );
- MPI_Barrier( MPI_COMM_WORLD );
- if ( 0 == my_rank ) {
- ret = H5RCrelease( rc_id4, H5_EVENT_STACK_NULL ); ASSERT_RET;
- }
- ret = H5RCclose( rc_id4 ); ASSERT_RET;
- } else {
- if ( 0 == my_rank ) {
- fprintf( stderr, "M6.2-r%d: Failed %d times to aquire read context for cv 4 - continuing\n", my_rank, max_tries );
- }
- }
-
- /* 15) /GB/GB/GA added in Tr 5 by rank 1 */
- if ( my_rank == 1 ) {
- create_group( file_id, "/GB/GB/GA", tr_id5, "/", my_rank, tr_num5 );
- }
-
- /* 16) /DA updated in Tr 5 by all ranks */
- update_dataset( file_id, "/DA", tr_id5, rc_id3, "/", my_rank, tr_num5, 1 );
-
- /* 17) Append to dataset /DC in Tr 5 by rank 0*/
- if ( my_rank == 0 ) {
- append_dataset2( file_id, "/DC", tr_id5, rc_id3, "/", my_rank, tr_num5 );
- }
-
- /* Finish and commit transaction 5, causing the updates to appear in container version 5. */
- fprintf( stderr, "M6.2-r%d: Finish and commit tr %d (Step 15a - end)\n", my_rank, (int)tr_num5 );
- ret = H5TRfinish( tr_id5, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5TRclose( tr_id5 ); ASSERT_RET;
-
- /* Get read context for CV 5, then print the contents of the container */
- version = 5;
- if ( 0 == my_rank) {
- fprintf( stderr, "M6.2-r%d: Try to acquire read context for cv %d (Step 15f)\n", my_rank, (int)version );
- H5E_BEGIN_TRY {
- rc_id5 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- while ( rc_id5 < 0 ) {
- fprintf( stderr, "M6.2-r%d: Failed to acquire read context for cv 5 - sleep then retry\n", my_rank );
- sleep( 1 );
- version = 5;
- rc_id5 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- }
- } H5E_END_TRY;
- } else {
- rc_id5 = H5RCcreate(file_id, version);
- }
- assert( rc_id5 >= 0 ); assert( version == 5 );
- fprintf( stderr, "M6.2-r%d: Acquired read context for cv 5\n", my_rank );
- MPI_Barrier( MPI_COMM_WORLD );
-
- fprintf( stderr, "M6.2-r%d: 7th call to print container contents (Step 15g)\n", my_rank );
- if (verbose) print_container_contents( file_id, rc_id5, "/", my_rank ); ASSERT_RET;
-
-
- /*
- * Get read context for CV 4 from rank 0.
- * Should be there first time if Transaction 4 was finished & committed (because we know Transaction 5 is done)
- * If Transaction 4 was aborted, it will never be there.
- */
- version = 4;
- if ( 0 == my_rank) {
- fprintf( stderr, "M6.2-r%d: Once again, try to acquire read context for cv %d (Step 15h)\n", my_rank, (int)version );
- rc_id4 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- if ( rc_id4 < 0 ) {
- fprintf( stderr, "M6.2-r%d: Read context for cv 4 not available.\n", my_rank );
- } else {
- assert( rc_id4 >= 0 ); assert( version == 4 );
- fprintf( stderr, "M6.2-r%d: Acquired read context for cv 4\n", my_rank );
- fprintf( stderr, "M6.2-r%d: 8th call to print container contents (Step 15i)\n", my_rank );
- if (verbose) print_container_contents( file_id, rc_id4, "/", my_rank ); ASSERT_RET;
-
- /* Release read handle & close read context for CV 4 */
- fprintf( stderr, "M6.2-r%d: Release read handle on cv 4 (Step 15j)\n", my_rank );
- ret = H5RCrelease( rc_id4, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5RCclose( rc_id4 ); ASSERT_RET;
- }
- }
-
- /*
- * Note that rc_id5 was obtained in this bracketed code, but we aren't releasing it until later.
- * Inelegant but I want it again & don't want to have to acquire again.
- * Wanted to use it in print above (15f) before 2nd print of CV4 (15g), so didn't wait until
- * bracket closed to acquire.
- */
-
- }
-
- MPI_Barrier( MPI_COMM_WORLD );
-
- /* Release the read handle and close read context on open CVs */
- fprintf( stderr, "M6.2-r%d: Release read handle on cv 3 (Step 16)\n", my_rank );
- if ( 0 == my_rank ) {
- ret = H5RCrelease( rc_id3, H5_EVENT_STACK_NULL ); ASSERT_RET;
- }
- ret = H5RCclose( rc_id3 ); ASSERT_RET;
-
- fprintf( stderr, "M6.2-r%d: Release read handle on cv 5\n", my_rank );
- if ( 0 == my_rank ) {
- ret = H5RCrelease( rc_id5, H5_EVENT_STACK_NULL ); ASSERT_RET;
- }
- ret = H5RCclose( rc_id5 ); ASSERT_RET;
-
- /* Close the file, then barrier to make sure all have closed it. */
- fprintf( stderr, "M6.2-r%d: close the container (Step 17)\n", my_rank );
- ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- MPI_Barrier( MPI_COMM_WORLD );
-
- /* Now, reopen file and show contents of all available CVs */
- hid_t last_rc_id;
- uint64_t last_version, v;
-
- /* Reopen the file Read/Write */
- fprintf( stderr, "M6.2-r%d: open the container\n", my_rank );
-
- /* Get latest CV on open */
- file_id = H5Fopen_ff( file_name, H5F_ACC_RDWR, fapl_id, &last_rc_id, H5_EVENT_STACK_NULL );
- assert( file_id >= 0 );
-
- H5RCget_version( last_rc_id, &last_version );
- fprintf( stderr, "M6.2-r%d: Latest CV in file is cv %d\n", my_rank, (int)last_version );
-
- if ( last_cv_only ) {
- v = last_version;
- } else {
- v = 1;
- }
- for ( v; v <= last_version; v++ ) {
- version = v;
- if ( 0 == my_rank ) {
- if(v < last_version) {
- fprintf( stderr, "M6.2-r%d: Try to acquire read context for cv %d\n", my_rank, (int)version );
- H5E_BEGIN_TRY
- rc_id = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- H5E_END_TRY
- }
- else
- rc_id = last_rc_id;
- }
-
- MPI_Bcast( &rc_id, 1, MPI_INT64_T, 0, MPI_COMM_WORLD );
-
- if ( rc_id < 0 ) {
- if ( 0 == my_rank ) {
- fprintf( stderr, "M6.2-r%d: Failed to acquire read context for cv %d\n", my_rank, (int)v );
- }
- } else {
- if ( 0 != my_rank ) {
- rc_id = H5RCcreate( file_id, version );
- }
- assert ( version == v );
- print_container_contents( file_id, rc_id, "/", my_rank );
-
- MPI_Barrier( MPI_COMM_WORLD );
- if(v < last_version) {
- if ( 0 == my_rank ) {
- ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- }
- ret = H5RCclose( rc_id ); ASSERT_RET;
- }
- }
- }
-
- /* Release the read handle and close read context on cv obtained from H5Fopen_ff (by all ranks) */
- ret = H5RCrelease( last_rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5RCclose( last_rc_id ); ASSERT_RET;
-
- /* Close 2 H5 Objects that are still open */
- fprintf( stderr, "M6.2-r%d: close all h5 objects that are still open\n", my_rank );
- ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Pclose( fapl_id ); ASSERT_RET;
-
- fprintf( stderr, "M6.2-r%d: Finalize EFF stack\n", my_rank );
-
- /* Perform wrap-up operations */
- MPI_Barrier( MPI_COMM_WORLD );
- EFF_finalize();
- MPI_Finalize();
-
- return 0;
-}
-
-
-/*
- * Helper function used to create string attribute "attr_name"
- * for object identified by "obj_id"
- * in transaction identified by "tr_id".
- * "obj_path" and "my_rank" and "tr_num" are used to create identifiable string value
- */
-void
-create_string_attribute( hid_t obj_id, const char* attr_name, hid_t tr_id, const char* obj_path, int my_rank, uint64_t tr_num ){
- herr_t ret;
- hsize_t dim_cnt;
- hid_t space_id;
- hid_t dtype_id;
- char attr_val[ATTR_STR_LEN];
- hsize_t val_size;
- hid_t attr_id;
-
- dim_cnt = 1;
- space_id = H5Screate_simple( 1, &dim_cnt, NULL ); assert( space_id >= 0 );
- dtype_id = H5Tcopy( H5T_C_S1 ); assert( dtype_id >= 0 );
-
- val_size = sprintf( attr_val, "%s @ %s by rank %d in tr %d", attr_name, obj_path, my_rank, (int)tr_num );
- assert( val_size < ATTR_STR_LEN );
- ret = H5Tset_size( dtype_id, ATTR_STR_LEN ); ASSERT_RET;
-
- attr_id = H5Acreate_ff( obj_id, attr_name, dtype_id, space_id, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- if ( attr_id >= 0 ) {
- fprintf( stderr, "M6.2-r%d: Create %s with value %s - %s\n", my_rank, attr_name, attr_val, "OK" );
- ret = H5Awrite_ff( attr_id, dtype_id, attr_val, tr_id, H5_EVENT_STACK_NULL );
-
- ret = H5Aclose_ff( attr_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- } else {
- fprintf( stderr, "M6.2-r%d: Create %s with value %s - %s\n", my_rank, attr_name, attr_val, "FAILED" );
- }
-
- ret = H5Tclose( dtype_id ); ASSERT_RET;
- ret = H5Sclose( space_id ); ASSERT_RET;
-
- return;
-}
-
-/*
- * Helper function used to create group "group_name"
- * in object identified by "obj_id"
- * in transaction identified by "tr_id".
- * "obj_path" and "my_rank" and "tr_num" are used in the status output.
- */
-void
-create_group( hid_t obj_id, const char* group_name, hid_t tr_id, const char* obj_path, int my_rank, uint64_t tr_num ){
- herr_t ret;
- hid_t group_id;
-
- group_id = H5Gcreate_ff( obj_id, group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- if ( group_id >= 0 ) {
- fprintf( stderr, "M6.2-r%d: Create %s in %s in tr %d - %s\n", my_rank, group_name, obj_path, (int)tr_num, "OK" );
- ret = H5Gclose_ff( group_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- } else {
- fprintf( stderr, "M6.2-r%d: Create %s in %s in tr %d - %s\n", my_rank, group_name, obj_path, (int)tr_num, "FAILED" );
- }
-
- return;
-}
-
-/*
- * Helper function used to create and initialize dataset "dset_name"
- * in object identified by "obj_id"
- * in transaction identified by "tr_id".
- * All datasets are 1D Arrays, start with 4 elements, and have a max of 10 elements.
- * "obj_path" and "my_rank" and "tr_num" and "ordinal" are used in the status output.
- * Cell values are computed using the formula:
- * data[i] = ordinal*1000 + tr_num*100 + my_rank*10 + i;
- * where ordinal can be set by the calling function to denote where the creation of this dataset occurs relative
- * to the creation of other datasets in "obj_id". It is used solely to generate identifiable cell values.
- */
-void
-create_dataset( hid_t obj_id, const char* dset_name, hid_t tr_id, const char* obj_path, int my_rank, uint64_t tr_num, int ordinal ){
- herr_t ret;
- hsize_t current_size = 4;
- hsize_t max_size = 10;
- int data[4]; /* sized to accomodate current_size elements */
- hid_t space_id;
- hid_t dtype_id;
- hid_t dset_id;
- int i;
-
- /* Set cell values */
- for ( i = 0; i < current_size; i++ ) {
- data[i] = ordinal*1000 + tr_num*100 + my_rank*10 + i;
- }
-
- space_id = H5Screate_simple( 1, &current_size, &max_size ); assert( space_id >= 0 );
- dtype_id = H5Tcopy( H5T_NATIVE_INT ); assert( dtype_id >= 0 );
-
- dset_id = H5Dcreate_ff( obj_id, dset_name, dtype_id, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL );
- if ( dset_id >= 0 ) {
- fprintf( stderr, "M6.2-r%d: Create %s in %s in tr %d; ordinal is %d; data values are %d %d %d %d - %s\n",
- my_rank, dset_name, obj_path, (int)tr_num, ordinal, data[0], data[1], data[2], data[3], "OK" );
- ret = H5Dwrite_ff( dset_id, dtype_id, space_id, space_id, H5P_DEFAULT, data, tr_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- } else {
- fprintf( stderr, "M6.2-r%d: Create %s in %s in tr %d; ordinal is %d; data values are %d %d %d %d - %s\n",
- my_rank, dset_name, obj_path, (int)tr_num, ordinal, data[0], data[1], data[2], data[3], "FAILED" );
- }
-
- ret = H5Tclose( dtype_id ); ASSERT_RET;
- ret = H5Sclose( space_id ); ASSERT_RET;
-
- return;
-}
-
-/*
- * Helper function used to create 6x4 2D dataset "dset_name"
- * in object identified by "obj_id"
- * in transaction identified by "tr_id".
- * "obj_path" and "my_rank" and "tr_num" are used in the status output.
- */
-void
-create_dataset2( hid_t obj_id, const char* dset_name, hid_t tr_id, const char* obj_path, int my_rank, uint64_t tr_num ){
- herr_t ret;
- hsize_t current_sizes[2] = {6, 4};
- hsize_t max_sizes[2] = {6, 4};
- hid_t space_id;
- hid_t dset_id;
-
- space_id = H5Screate_simple( 2, current_sizes, max_sizes ); assert( space_id >= 0 );
-
- dset_id = H5Dcreate_ff( obj_id, dset_name, H5T_NATIVE_INT, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL );
- if ( dset_id >= 0 ) {
- fprintf( stderr, "M6.2-r%d: Create %s in %s in tr %d - %s\n",
- my_rank, dset_name, obj_path, (int)tr_num, "OK" );
- ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- } else {
- fprintf( stderr, "M6.2-r%d: Create %s in %s in tr %d - %s\n",
- my_rank, dset_name, obj_path, (int)tr_num, "FAILED" );
- }
-
- ret = H5Sclose( space_id ); ASSERT_RET;
-
- return;
-}
-
-
-/*
- * Helper function used to create committed datatype "type_name"
- * in group identified by "obj_id"
- * in transaction identified by "tr_id".
- * "obj_path" and "my_rank" and "tr_num" are used in the status output.
- */
-void
-create_committed_datatype( hid_t obj_id, const char* type_name, hid_t tr_id, const char* obj_path, int my_rank, uint64_t tr_num ){
- herr_t ret;
- hid_t dtype_id;
-
- dtype_id = H5Tcopy( H5T_NATIVE_INT ); assert( dtype_id >= 0 );
- ret = H5Tcommit_ff( obj_id, type_name, dtype_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- fprintf( stderr, "M6.2-r%d: Create CDT %s in %s tr %d - %s\n", my_rank, type_name, obj_path, (int)tr_num, STATUS );
-
- ret = H5Tclose( dtype_id ); ASSERT_RET;
-
- return;
-}
-
-/*
- * Helper function used to update raw data in dataset "dset_name"
- * in object identified by "obj_id"
- * in transaction identified by "tr_id".
- * "rc_id" for "tr_id" passed in explicitly for now - later add H5 function to extract it from tr_id.
- * cell[my_rank] will be updated, using the formula
- * value = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
- * "obj_path" and "my_rank" and "tr_num" and "ordinal" are used in the status output.
- */
-void
-update_dataset( hid_t obj_id, const char* dset_name, hid_t tr_id, hid_t rc_id, const char* obj_path, int my_rank,
- uint64_t tr_num, int ordinal ){
- herr_t ret;
- hid_t dset_id;
-
- /* Open the dataset, confirm number of dimensions, get current & max size */
- dset_id = H5Dopen_ff( obj_id, dset_name, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
-
- if ( dset_id >= 0 ) {
- hid_t filespace_id;
- int nDims;
- hsize_t current_size, max_size;
-
- filespace_id = H5Dget_space( dset_id ); assert ( filespace_id >= 0 );
- nDims = H5Sget_simple_extent_dims( filespace_id, &current_size, &max_size );
- assert( nDims == 1 );
-
- if ( my_rank < current_size ) {
- int cell_data; /* sized to hold data for a single cell in the dataset*/
- hsize_t mem_size = 1;
- hid_t memspace_id;
-
- /* set data value, create memory data space, then select the element to update */
- cell_data = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
- memspace_id = H5Screate_simple( 1, &mem_size, &mem_size ); assert( memspace_id >= 0 );
-
- hsize_t start, stride, count, block;
- start = (hsize_t)my_rank;
- stride = count = block = 1;
-
- ret = H5Sselect_hyperslab( filespace_id, H5S_SELECT_SET, &start, &stride, &count, &block ); ASSERT_RET;
-
- ret = H5Dwrite_ff( dset_id, H5T_NATIVE_INT, memspace_id, filespace_id, H5P_DEFAULT, &cell_data, tr_id,
- H5_EVENT_STACK_NULL );
- fprintf( stderr, "M6.2-r%d: Update %s[%d] in %s in tr %d to have value %d - %s\n",
- my_rank, dset_name, my_rank, obj_path, (int)tr_num, cell_data, STATUS );
- ret = H5Sclose( memspace_id ); ASSERT_RET;
-
- } else {
- fprintf( stderr, "M6.2-r%d: No update to %s in %s in tr %d for this rank.\n", my_rank, dset_name, obj_path, (int)tr_num );
- }
-
- ret = H5Sclose( filespace_id ); ASSERT_RET;
- ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- } else {
- fprintf( stderr, "M6.2-r%d: Update %s[%d] in %s in tr %d - %s\n",
- my_rank, dset_name, my_rank, obj_path, (int)tr_num, "FAILED" );
- }
-
- return;
-}
-
-/*
- * Helper function used to append data at end of 2D dataset "dset_name"
- * in object identified by "obj_id"
- * in transaction identified by "tr_id".
- * "rc_id" for "tr_id" passed in explicitly for now - later add H5 function to extract it from tr_id.
- * cell[tr_num][my_rank] will be updated, using the formula
- * value = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
- * "obj_path" and "my_rank" and "tr_num" are used in the status output.
- */
-void
-append_dataset2( hid_t obj_id, const char* dset_name, hid_t tr_id, hid_t rc_id, const char* obj_path, int my_rank,
- uint64_t tr_num ){
- herr_t ret;
- hid_t dset_id;
-
- /* Open the dataset, confirm number of dimensions, get current & max size */
- dset_id = H5Dopen_ff( obj_id, dset_name, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
-
- if ( dset_id >= 0 ) {
- hid_t filespace_id;
- int nDims;
- hsize_t current_size[2];
- hsize_t max_size[2];
-
- filespace_id = H5Dget_space( dset_id ); assert ( filespace_id >= 0 );
- nDims = H5Sget_simple_extent_dims( filespace_id, current_size, max_size );
- assert( nDims == 2 );
-
- if ( (int)tr_num < current_size[0] ) {
- hsize_t nCols;
- int* data;
- int i;
- hid_t memspace_id;
-
- nCols = current_size[1];
- data = (int *)calloc( nCols, sizeof(int) ); assert( data != NULL );
-
- /* set data value, create memory data space, then select the element to update */
- for ( i = 0; i < nCols; i++ ) {
- data[i] = 10*tr_num + i;
- }
- memspace_id = H5Screate_simple( 1, &nCols, &nCols ); assert( memspace_id >= 0 );
-
- hsize_t start[2], stride[2], count[2], block[2];
- start[0] = tr_num;
- start[1] = 0;
- stride[0] = 1;
- stride[1] = 1;
- count[0] = 1;
- count[1] = nCols;
- block[0] = 1;
- block[1] = 1;
-
- ret = H5Sselect_hyperslab( filespace_id, H5S_SELECT_SET, start, stride, count, block ); ASSERT_RET;
-
- ret = H5Dwrite_ff( dset_id, H5T_NATIVE_INT, memspace_id, filespace_id, H5P_DEFAULT, data, tr_id, H5_EVENT_STACK_NULL );
- fprintf( stderr, "M6.2-r%d: Update %s[%d][*] in %s in tr %d to have values %d %d %d %d - %s\n",
- my_rank, dset_name, (int)tr_num, obj_path, (int)tr_num, data[0], data[1], data[2], data[3], STATUS );
- ret = H5Sclose( memspace_id ); ASSERT_RET;
-
- free(data);
-
- } else {
- fprintf( stderr, "M6.2-r%d: No update to %s in %s in tr %d for this rank.\n", my_rank, dset_name, obj_path, (int)tr_num );
- }
-
- ret = H5Sclose( filespace_id ); ASSERT_RET;
- ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- } else {
- fprintf( stderr, "M6.2-r%d: Update %s[%d] in %s in tr %d - %s\n",
- my_rank, dset_name, my_rank, obj_path, (int)tr_num, "FAILED" );
- }
-
- return;
-}
-
-/*
- * Helper function used to recursively print container contents
- * for container identified by "file_id"
- * in read context identified by "rc_id"
- * with path to current level in "grp_path"
- * and "my_rank" used to identify the process doing the reading / printing.
- */
-void
-print_container_contents( hid_t file_id, hid_t rc_id, const char* grp_path, int my_rank )
-{
- herr_t ret;
- uint64_t cv;
- htri_t exists;
- char path_to_object[1024];
- hid_t obj_id;
- char name[3];
- int i;
-
- static int lvl = 0; /* level in recursion - used to format printing */
- char preface[128];
- char line[1024];
-
- /* Get the container version for the read context */
- ret = H5RCget_version( rc_id, &cv ); ASSERT_RET;
-
- /* Set up the preface and adding version number */
- sprintf( preface, "M6.2-r%d: cv %d: ", my_rank, (int)cv );
-
- /* Start the printing */
- if ( lvl == 0 ) {
- fprintf( stderr, "%s ----- Container Contents ------------\n", preface );
- }
-
- /* Attributes */
- for ( i = 1; i < 3; i++ ) {
- if ( i == 1 ) {
- strcpy( name, "AA" );
- } else {
- strcpy( name, "AB" );
- }
- ret = H5Aexists_by_name_ff( file_id, grp_path, name, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- if ( exists ) {
- hid_t attr_id, atype_id;
- char attr_val[ATTR_STR_LEN];
-
- attr_id = H5Aopen_by_name_ff( file_id, grp_path, name, H5P_DEFAULT, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( attr_id >= 0 );
-
- atype_id = H5Tcopy( H5T_C_S1 ); assert( atype_id >= 0 );
- ret = H5Tset_size( atype_id, ATTR_STR_LEN ); ASSERT_RET;
-
- ret = H5Aread_ff( attr_id, atype_id, attr_val, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- fprintf( stderr, "%s %s%s value: %s\n", preface, grp_path, name, attr_val );
-
- ret = H5Tclose( atype_id ); ASSERT_RET;
- ret = H5Aclose_ff( attr_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- }
- }
-
- /* Datasets */
- for ( i = 1; i < 4; i++ ) {
- if ( i == 1 ) {
- strcpy( name, "DA" );
- } else if ( i == 2 ){
- strcpy( name, "DB" );
- } else {
- strcpy( name, "DC" );
- }
-
- sprintf( path_to_object, "%s%s", grp_path, name );
- ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
-
- if ( exists ) {
- hid_t dset_id;
- hid_t space_id;
- int nDims;
- hsize_t current_size[2];
- hsize_t max_size[2];
- hsize_t totalSize;
- int *data;
- int i;
-
- dset_id = H5Dopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( dset_id >= 0 );
-
- space_id = H5Dget_space( dset_id ); assert ( space_id >= 0 );
- nDims = H5Sget_simple_extent_dims( space_id, current_size, max_size );
- assert( ( nDims == 1 ) || ( nDims == 2 ) );
-
- if ( nDims == 1 ) {
- totalSize = current_size[0];
- } else {
- totalSize = current_size[0] * current_size[1];
- }
- data = (int *)calloc( totalSize, sizeof(int) ); assert( data != NULL );
-
- ret = H5Dread_ff( dset_id, H5T_NATIVE_INT, space_id, space_id, H5P_DEFAULT, data, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- sprintf( line, "%s %s values: ", preface, path_to_object );
- if ( nDims == 1 ) {
- for ( i = 0; i < totalSize; i++ ) {
- sprintf( line, "%s%d ", line, data[i] );
- }
- } else {
- int r, c;
- i = 0;
- for ( r = 0; r < current_size[0]; r++ ) {
- sprintf( line, "%srow %d: ", line, r );
- for ( c = 0; c < current_size[1]; c++ ) {
- sprintf( line, "%s%d ", line, data[i] );
- i++;
- }
- }
- }
-
- fprintf( stderr, "%s\n", line );
- free( data );
-
- ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Sclose( space_id ); ASSERT_RET;
- }
- }
-
- /* Committed datatypes */
- for ( i = 1; i < 3; i++ ) {
- if ( i == 1 ) {
- strcpy( name, "TA" );
- } else {
- strcpy( name, "TB" );
- }
-
- sprintf( path_to_object, "%s%s", grp_path, name );
- ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
- if ( exists ) {
- fprintf( stderr, "%s %s\n", preface, path_to_object );
- }
- }
-
- /* Groups - if found, descend */
- for ( i = 1; i < 3; i++ ) {
- if ( i == 1 ) {
- strcpy( name, "GA" );
- } else {
- strcpy( name, "GB" );
- }
-
- sprintf( path_to_object, "%s%s/", grp_path, name );
- ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
- if ( exists ) {
- fprintf( stderr, "%s %s\n", preface, path_to_object );
- lvl++;
- print_container_contents( file_id, rc_id, path_to_object, my_rank );
- lvl--;
- }
- }
-
- /* End printing */
- if ( lvl == 0 ) {
- fprintf( stderr, "%s -----------------\n", preface );
- }
-
- return;
-}
-
-/*
- * parse_options - helper function to parse the command line options.
- */
-int
-parse_options( int argc, char** argv, int my_rank ) {
- int i, n;
-
- while ( --argc ) {
- if ( **(++argv) != '-' ) {
- break;
- } else {
- switch( *(*argv+1) ) {
- case 'a':
- abort4 = 1;
- break;
- case 'l':
- last_cv_only = 1;
- break;
- case 'v':
- verbose = 1;
- break;
- default:
- if ( my_rank == 0 ) {
- printf( "Usage: h5ff_client_M6.2_demo [-aiv]\n" );
- printf( "\ta: abort transaction 4\n" );
- printf( "\tl: last container version contents are the only ones printed on reopen\n" );
- printf( "\tv: verbose printing\n" );
- }
- return( 1 );
- }
- }
- }
- return( 0 );
-}
+/*
+ * M6.2-demo.c: This is the demo program for Milestone 6.2 - HDF5 and IO Dispatcher Container Versioning Demonstration.
+ * This program runs on one or more compute nodes and makes calls to the HDF5 API.
+ * The Function Shipper server should be running before this demo program is started.
+ *
+ * March, 2014: Updated to start user-initated transactions at number 1, not 0, to reflect DAOS numbering requirements.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+
+/* define filename for this app, and max size after username prepended */
+#define FILENAME_APP "m6.2-eff_container.h5"
+#define FILENAME_SIZE 256
+
+/* max length of attributes */
+#define ATTR_STR_LEN 128
+
+/* macros related to error reporting */
+#define STATUS (ret >= 0) ? "OK" : "FAILED"
+#define ASSERT_RET assert( ret >= 0 )
+
+/* option flags */
+int verbose = 0; // Verbose (print CV contents between every transaction) defaults to no
+int last_cv_only = 0; // Only print contents of last container version when file re-opened - defaults to no
+int abort4 = 0; // Abort transaction 4 defaults to no
+
+/* prototypes for helper functions */
+void create_string_attribute( hid_t, const char*, hid_t, const char*, int, uint64_t );
+void create_group( hid_t, const char*, hid_t, const char*, int, uint64_t );
+void create_dataset( hid_t, const char*, hid_t, const char*, int, uint64_t, int );
+void create_dataset2( hid_t, const char*, hid_t, const char*, int, uint64_t );
+void create_committed_datatype( hid_t, const char*, hid_t, const char*, int, uint64_t );
+void update_dataset( hid_t, const char*, hid_t, hid_t, const char*, int, uint64_t, int );
+void append_dataset2( hid_t, const char*, hid_t, hid_t, const char*, int, uint64_t );
+void print_container_contents( hid_t, hid_t, const char*, int );
+int parse_options( int, char**, int );
+
+int main( int argc, char **argv ) {
+
+ char *user_name; /* We'll prepend username to make filename unique */
+ char file_name[FILENAME_SIZE]; /* Actual filename to be opened will be user_name + FILENAME_APP */
+
+ int my_rank, comm_size;
+ int provided;
+
+ hid_t fapl_id;
+ hid_t file_id;
+ herr_t ret;
+
+ uint64_t version;
+ hid_t rc_id, rc_id1, rc_id2, rc_id3, rc_id4, rc_id5;
+
+ /* Check for MPI multi-thread support */
+ MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
+ if ( MPI_THREAD_MULTIPLE != provided ) {
+ fprintf( stderr, "M6.2: ERROR: MPI does not have MPI_THREAD_MULTIPLE support\n" );
+ exit( 1 );
+ }
+
+ MPI_Comm_rank( MPI_COMM_WORLD, &my_rank );
+ MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
+
+ if(comm_size < 2) {
+ fprintf(stderr, "Need at least 2 MPI ranks to run this test. Aborting..\n");
+ MPI_Abort(MPI_COMM_WORLD, 1);
+ }
+
+ fprintf( stderr, "M6.2-r%d: Number of MPI processes = %d\n", my_rank, comm_size );
+
+ /* Parse command-line options controlling behavior */
+ if ( parse_options( argc, argv, my_rank ) != 0 ) {
+ exit( 1 );
+ }
+
+ /* Initialize the EFF stack. */
+ fprintf( stderr, "M6.2-r%d: Initialize EFF stack (Step 1)\n", my_rank );
+ EFF_init( MPI_COMM_WORLD, MPI_INFO_NULL );
+
+ /* Specify that the IOD VOL plugin should be used and create H5File (EFF container) */
+ user_name = getenv( "USER" );
+ snprintf( file_name, FILENAME_SIZE, "%s_%s", user_name, FILENAME_APP );
+ fprintf( stderr, "M6.2-r%d: Create the container %s (Step 2)\n", my_rank, file_name );
+ fapl_id = H5Pcreate( H5P_FILE_ACCESS ); assert( fapl_id >= 0 );
+ ret = H5Pset_fapl_iod( fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL ); ASSERT_RET;
+ file_id = H5Fcreate_ff( file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL ); assert( file_id >= 0 );
+
+ /* Acquire a read handle for container version 1 and create a read context. */
+ version = 1;
+ if ( 0 == my_rank ) {
+ fprintf( stderr, "M6.2-r%d: Acquire read context for container version %d (Step 3)\n", my_rank, (int)version );
+ rc_id1 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ } else {
+ rc_id1 = H5RCcreate( file_id, version);
+ }
+ assert( rc_id1 >= 0 ); assert( version == 1 );
+ MPI_Barrier( MPI_COMM_WORLD );
+
+ /* Print container contents at this point */
+ fprintf( stderr, "M6.2-r%d: 1st call to print container contents (Step 4)\n", my_rank );
+ if ( verbose ) print_container_contents( file_id, rc_id1, "/", my_rank );
+
+ /*
+ * Transaction 2 is created, started, updated, and committed by rank 0.
+ * Root group attributes AA and AB are created and populated.
+ * Groups /GA and /GB are created.
+ * Datasets /DA and /DB are created and populated.
+ * Dataset /DC (2-d) and attribute AC@/DC are created & initialized.
+ * Committed datatypes /TA and /TB are created.
+ */
+ if ( my_rank == 0 ) {
+
+ uint64_t tr_num = 2;
+ hid_t tr_id;
+
+ /* Create a local transaction for transaction number. */
+ fprintf( stderr, "M6.2-r%d: Create tr %d (Step 5 - begin) \n", my_rank, (int)tr_num );
+ tr_id = H5TRcreate( file_id, rc_id1, (uint64_t)tr_num ); assert( tr_id >= 0 );
+
+ /* Start transaction 2 with single transaction leader (the default) */
+ fprintf( stderr, "M6.2-r%d: Start tr %d\n", my_rank, (int)tr_num );
+ ret = H5TRstart( tr_id, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Add updates to the transaction */
+
+ /* 2 Attributes */
+ create_string_attribute( file_id, "AA", tr_id, "/", my_rank, tr_num );
+ create_string_attribute( file_id, "AB", tr_id, "/", my_rank, tr_num );
+
+ /* 3 Datasets */
+ create_dataset( file_id, "DA", tr_id, "/", my_rank, tr_num, 1 );
+ create_dataset( file_id, "DB", tr_id, "/", my_rank, tr_num, 2 );
+ create_dataset2( file_id, "DC", tr_id, "/", my_rank, tr_num );
+
+ /* 2 committed datatypes */
+ create_committed_datatype( file_id, "TA", tr_id, "/", my_rank, tr_num );
+ create_committed_datatype( file_id, "TB", tr_id, "/", my_rank, tr_num );
+
+ /* 2 Groups */
+ create_group( file_id, "GA", tr_id, "/", my_rank, tr_num );
+ create_group( file_id, "GB", tr_id, "/", my_rank, tr_num );
+
+ /* Finish and commit transaction 2, causing the updates to appear in container version 2. */
+ fprintf( stderr, "M6.2-r%d: Finish and commit tr %d (Step 5 - end)\n", my_rank, (int)tr_num );
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Close the local transaction. */
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ }
+
+ /* Print container contents at this point */
+ fprintf( stderr, "M6.2-r%d: 2nd call to print container contents (Step 6)\n", my_rank );
+ if (verbose) print_container_contents( file_id, rc_id1, "/", my_rank );
+
+
+ /* Acquire a read handle for container version 2 and create a read context. */
+ version = 2;
+ if ( 0 == my_rank ) {
+ fprintf( stderr, "M6.2-r%d: Try to acquire read context for cv %d (Step 7)\n", my_rank, (int)version );
+ H5E_BEGIN_TRY {
+ rc_id2 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ while ( rc_id2 < 0 ) {
+ fprintf( stderr, "M6.2-r%d: Failed to acquire read context for cv %d - sleep then retry\n", my_rank, (int)version );
+ sleep( 1 );
+ version = 2;
+ rc_id2 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ }
+ } H5E_END_TRY;
+ } else {
+ rc_id2 = H5RCcreate( file_id, version );
+ }
+ assert( rc_id2 >= 0 ); assert ( version == 2 );
+ fprintf( stderr, "M6.2-r%d: Acquired read context for cv 2\n", my_rank );
+ MPI_Barrier( MPI_COMM_WORLD );
+
+ fprintf( stderr, "M6.2-r%d: 3rd call to print container contents (Step 8)\n", my_rank );
+ if (verbose) print_container_contents( file_id, rc_id2, "/", my_rank );
+
+ MPI_Barrier( MPI_COMM_WORLD );
+ /* Release the read handle and close read context on cv 1 */
+ fprintf( stderr, "M6.2-r%d: Release read handle on cv 1 (Step 9)\n", my_rank );
+ if ( 0 == my_rank ) {
+ ret = H5RCrelease( rc_id1, H5_EVENT_STACK_NULL); ASSERT_RET;
+ }
+ ret = H5RCclose( rc_id1 ); ASSERT_RET;
+
+ /*
+ * Transaction 3 is created, started, updated, and committed by ranks 0 & 1, if 2 or more MPI processes.
+ * Group /GA attributes AA and AB are created and populated by rank 0.
+ * Group /GB attributes AA and AB are created and populated by rank 1.
+ * Groups /GA/GA and /GA/GB are created by rank 0.
+ * Groups /GB/GA and /GB/GB are created by rank 1.
+ * Datasets /GA/DA and /GA/DB are created and populated by rank 0.
+ * Datasets /GB/DA and /GB/DB are created and populated by rank 1.
+ * Committed Datatypes /GA/AA and /GA/TB are created by rank 0.
+ * Committed Datatypes /GB/AA and /GB/TB are created by rank 1.
+ * Dataset /DC is appended to by rank 0.
+ */
+ if ( (my_rank == 0) || (my_rank == 1) ) {
+
+ uint64_t tr_num = 3;
+ hid_t tr_id;
+ hid_t trspl_id;
+ hid_t gr_id;
+ char gr_path[128];
+
+ /* Create a local transaction for transaction number 3. */
+ tr_id = H5TRcreate( file_id, rc_id2, tr_num ); assert( tr_id >= 0 );
+
+ /* Start transaction 3 with two transaction leaders (unless there is only 1 MPI process) */
+ fprintf( stderr, "M6.2-r%d: Start tr %d (Step 10 - begin)\n", my_rank, (int)tr_num );
+ trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
+ if ( comm_size >= 2 ) {
+ ret = H5Pset_trspl_num_peers( trspl_id, 2 ); ASSERT_RET;
+ } else {
+ ret = H5Pset_trspl_num_peers( trspl_id, 1 ); ASSERT_RET;
+ }
+ ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Add updates to the transaction */
+ if ( my_rank == 0 ) {
+ gr_id = H5Gopen_ff( file_id, "GA", H5P_DEFAULT, rc_id2, H5_EVENT_STACK_NULL ); assert( gr_id >= 0 );
+ sprintf( gr_path, "/GA/" );
+ } else {
+ gr_id = H5Gopen_ff( file_id, "GB", H5P_DEFAULT, rc_id2, H5_EVENT_STACK_NULL ); assert( gr_id >= 0 );
+ sprintf( gr_path, "/GB/" );
+ }
+
+ /* 2 Attributes in each group */
+ create_string_attribute( gr_id, "AA", tr_id, gr_path, my_rank, tr_num );
+ create_string_attribute( gr_id, "AB", tr_id, gr_path, my_rank, tr_num );
+
+ /* 2 1-D Datasets */
+ create_dataset( gr_id, "DA", tr_id, gr_path, my_rank, tr_num, 1 );
+ create_dataset( gr_id, "DB", tr_id, gr_path, my_rank, tr_num, 2 );
+
+ /* 2 committed datatypes */
+ create_committed_datatype( gr_id, "TA", tr_id, gr_path, my_rank, tr_num );
+ create_committed_datatype( gr_id, "TB", tr_id, gr_path, my_rank, tr_num );
+
+ /* 2 Groups */
+ create_group( gr_id, "GA", tr_id, gr_path, my_rank, tr_num );
+ create_group( gr_id, "GB", tr_id, gr_path, my_rank, tr_num );
+
+ /* Append to dataset /DC */
+ if ( my_rank == 0 ) {
+ append_dataset2( file_id, "/DC", tr_id, rc_id2, "/", my_rank, tr_num );
+ }
+
+ /* Close the group */
+ ret = H5Gclose_ff ( gr_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Finish and commit transaction 3, causing the updates to appear in container version 3. */
+ fprintf( stderr, "M6.2-r%d: Finish and commit tr %d (Step 10 - end)\n", my_rank, (int)tr_num );
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Close the local transaction and transaction start property list. */
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ ret = H5Pclose( trspl_id ); ASSERT_RET;
+ }
+
+ fprintf( stderr, "M6.2-r%d: 4th call to print container contents (Step 11)\n", my_rank );
+ if (verbose) print_container_contents( file_id, rc_id2, "/", my_rank );
+
+ version = 3;
+ if ( 0 == my_rank ) {
+ fprintf( stderr, "M6.2-r%d: Try to acquire read context for cv %d (Step 12)\n", my_rank, (int)version );
+ H5E_BEGIN_TRY {
+ rc_id3 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ while ( rc_id3 < 0 ) {
+ fprintf( stderr, "M6.2-r%d: Failed to acquire read context for cv 3 - sleep then retry\n", my_rank );
+ sleep( 1 );
+ version = 3;
+ rc_id3 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ }
+ } H5E_END_TRY;
+ } else {
+ rc_id3 = H5RCcreate( file_id, version );
+ }
+ assert( rc_id3 >= 0 ); assert( version == 3 );
+ fprintf( stderr, "M6.2-r%d: Acquired read context for cv 3\n", my_rank );
+ MPI_Barrier( MPI_COMM_WORLD );
+
+ fprintf( stderr, "M6.2-r%d: 5th call to print container contents (Step 13)\n", my_rank );
+ if (verbose) print_container_contents( file_id, rc_id3, "/", my_rank );
+
+ MPI_Barrier( MPI_COMM_WORLD );
+ /* Release the read handle and close read context on cv 2 */
+ fprintf( stderr, "M6.2-r%d: Release read handle on cv 2 (Step 14)\n", my_rank );
+ if ( 0 == my_rank) {
+ ret = H5RCrelease( rc_id2, H5_EVENT_STACK_NULL); ASSERT_RET;
+ }
+ ret = H5RCclose( rc_id2 ); ASSERT_RET;
+
+ /*
+ * Transaction 5 is created and started by all MPI processes.
+ * Transaction 4 is created and started by all MPI processes.
+ * Updates are added to the transactions in an interleaved manner,
+ * with the ranks progressing independent of each other, and
+ * additional updates to transaction 5 after transaction 4
+ * is finished and committed.
+ * Transaction 4 may be aborted
+ */
+ { /* Bracket to localize variables, and to later make it easier to restrict ranks participating if desired. */
+ uint64_t tr_num4 = 4;
+ uint64_t tr_num5 = 5;
+ hid_t tr_id4, tr_id5;
+ hid_t trspl_id;
+
+ /* Set up transaction start property list with the number of transaction leaders */
+ trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
+ ret = H5Pset_trspl_num_peers( trspl_id, comm_size ); ASSERT_RET;
+
+ /* Create & start transaction 5, based on cv 3 */
+ tr_id5 = H5TRcreate( file_id, rc_id3, tr_num5 ); assert( tr_id5 >= 0 );
+ ret = H5TRstart( tr_id5, trspl_id, H5_EVENT_STACK_NULL );
+ fprintf( stderr, "M6.2-r%d: Start tr %d (Step 15a - begin) - %s\n", my_rank, (int)tr_num5, STATUS );
+
+ /* Create & start transaction 4, based on cv 3 */
+ tr_id4 = H5TRcreate( file_id, rc_id3, tr_num4 ); assert( tr_id4 >= 0 );
+ ret = H5TRstart( tr_id4, trspl_id, H5_EVENT_STACK_NULL );
+ fprintf( stderr, "M6.2-r%d: Start tr %d (Step 15b - begin) - %s\n", my_rank, (int)tr_num4, STATUS );
+
+ /* Close the transaction start property list. */
+ ret = H5Pclose( trspl_id ); ASSERT_RET;
+
+ /* Add updates to the transactions */
+ /* 1) /GA/GA/GB added to Tr 5 by rank 0 */
+ if ( my_rank == 0 ) {
+ create_group( file_id, "/GA/GA/GB", tr_id5, "/", my_rank, tr_num5 );
+ }
+ /* 2) /GB/GB/GB added to Tr 5 by rank 1 */
+ if ( my_rank == 1 ) {
+ create_group( file_id, "/GB/GB/GB", tr_id5, "/", my_rank, tr_num5 );
+ }
+ /* 3) AA@/ deleted in Tr 4 by rank 0 */
+ if ( my_rank == 0 ) {
+ ret = H5Adelete_by_name_ff( file_id, ".", "AA", H5P_DEFAULT, tr_id4, H5_EVENT_STACK_NULL);
+ fprintf( stderr, "M6.2-r%d: delete AA @ / in tr %d - %s\n", my_rank, (int)tr_num4, STATUS );
+ }
+ /* 4) /GA/DA deleted in Tr 4 by rank 1 */
+ if ( my_rank == 1 ) {
+ ret = H5Ldelete_ff( file_id, "/GA/DA", H5P_DEFAULT, tr_id4, H5_EVENT_STACK_NULL);
+ fprintf( stderr, "M6.2-r%d: delete /GA/DA in tr %d - %s\n", my_rank, (int)tr_num4, STATUS);
+ }
+ /* 5) /GA/TB deleted in Tr 5 by rank 0 */
+ if ( my_rank == 0 ) {
+ ret = H5Ldelete_ff( file_id, "/GA/TB", H5P_DEFAULT, tr_id5, H5_EVENT_STACK_NULL);
+ fprintf( stderr, "M6.2-r%d: delete /GA/TB in tr %d - %s\n", my_rank, (int)tr_num5, STATUS );
+ }
+ /* 6) /DB deleted in Tr 5 by rank 1 */
+ if ( my_rank == 1 ) {
+ ret = H5Ldelete_ff( file_id, "/DB", H5P_DEFAULT, tr_id5, H5_EVENT_STACK_NULL);
+ fprintf( stderr, "M6.2-r%d: delete /DB in tr %d - %s \n", my_rank, (int)tr_num5, STATUS );
+ }
+ /* 7) /GB/GA deleted in Tr 4 by rank 0 */
+ if ( my_rank == 0 ) {
+ ret = H5Ldelete_ff( file_id, "/GB/GA", H5P_DEFAULT, tr_id4, H5_EVENT_STACK_NULL);
+ fprintf( stderr, "M6.2-r%d: delete /GB/GA in tr %d - %s\n", my_rank, (int)tr_num4, STATUS );
+ }
+ /* 8) /GA/TA deleted in Tr 4 by rank 1 */
+ if ( my_rank == 1 ) {
+ ret = H5Ldelete_ff( file_id, "/GA/TA", H5P_DEFAULT, tr_id4, H5_EVENT_STACK_NULL);
+ fprintf( stderr, "M6.2-r%d: delete /GA/TA in tr %d - %s\n", my_rank, (int)tr_num4, STATUS );
+ }
+ /* 9) /GA/GA/GA added in Tr 4 by rank 0 */
+ if ( my_rank == 0 ) {
+ create_group( file_id, "/GA/GA/GA", tr_id4, "/", my_rank, tr_num4 );
+ }
+ /* 10) /GB/TA deleted in Tr 4 by rank 1 */
+ if ( my_rank == 1 ) {
+ ret = H5Ldelete_ff( file_id, "/GB/TA", H5P_DEFAULT, tr_id4, H5_EVENT_STACK_NULL);
+ fprintf( stderr, "M6.2-r%d: delete /GB/TA in tr %d - %s\n", my_rank, (int)tr_num4, STATUS );
+ }
+ /* 11) /GB/GB/DB added in Tr 4 by rank 1 */
+ if ( my_rank == 1 ) {
+ create_dataset( file_id, "/GB/GB/DB", tr_id4, "/", my_rank, tr_num4, 2 );
+ }
+ /* 12) /DA updated in Tr 4 by all ranks */
+ update_dataset( file_id, "/DA", tr_id4, rc_id3, "/", my_rank, tr_num4, 1 );
+
+ /* 13) /GB/DA updated in Tr 4 by rank 0 */
+ if ( my_rank == 0 ) {
+ update_dataset( file_id, "/GB/DA", tr_id4, rc_id3, "/", my_rank, tr_num4, 1 );
+ }
+
+ /* 14) Append to dataset /DC in Tr 4 by rank 0 */
+ if ( my_rank == 0 ) {
+ append_dataset2( file_id, "/DC", tr_id4, rc_id3, "/", my_rank, tr_num4 );
+ }
+
+ /* Abort - one or more processes should be able to abort the Transaction with the same effect. */
+ if ( abort4 ) {
+ /* All ranks call TRabort; the order of actual calls is not determined by the code.
+ * The first abort to reach IOD is the one that aborts the transaction, and abort need not be called by all ranks. */
+ ret = H5TRabort( tr_id4, H5_EVENT_STACK_NULL );
+ fprintf( stderr, "M6.2-r%d: ABORT tr %d (Step 15b - abort) - %s\n", my_rank, (int)tr_num4, STATUS );
+ }
+ else {
+ /* Finish and commit transaction 4, causing the updates to appear in container version 4. */
+ ret = H5TRfinish( tr_id4, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL );
+ fprintf( stderr, "M6.2-r%d: Finish and commit tr %d (Step 15b - end) - %s\n", my_rank, (int)tr_num4, STATUS );
+ }
+
+ ret = H5TRclose( tr_id4 ); ASSERT_RET;
+
+ /* Get read context for CV 4, then print the contents of container */
+ /* Limit the number of tries, because if it was aborted we'll never get it */
+ int max_tries = 4;
+ int current_try = 0;
+ version = 4;
+ if ( 0 == my_rank ) {
+ fprintf( stderr, "M6.2-r%d: Try to acquire read context for cv %d (Step 15c)\n", my_rank, (int)version );
+ H5E_BEGIN_TRY {
+ rc_id4 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ while ( (rc_id4 < 0) && (current_try < max_tries) ) {
+ fprintf( stderr, "M6.2-r%d: Failed to acquire read context for cv 4 - sleep then retry\n", my_rank );
+ sleep( 1 );
+ current_try++;
+ version = 4;
+ rc_id4 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ }
+ } H5E_END_TRY;
+ }
+
+ MPI_Bcast( &rc_id4, 1, MPI_INT64_T, 0, MPI_COMM_WORLD );
+
+ if ( rc_id4 >= 0 ) {
+ if ( 0 != my_rank ) {
+ rc_id4 = H5RCcreate( file_id, version );
+ }
+ assert( version == 4 );
+ fprintf( stderr, "M6.2-r%d: Acquired read context for cv 4\n", my_rank );
+
+ fprintf( stderr, "M6.2-r%d: 6th call to print container contents (Step 15d)\n", my_rank );
+ if (verbose) print_container_contents( file_id, rc_id4, "/", my_rank ); ASSERT_RET;
+
+ /* Release read handle & close read context for CV 4 */
+ fprintf( stderr, "M6.2-r%d: Release read handle on cv 4 (Step 15e)\n", my_rank );
+ MPI_Barrier( MPI_COMM_WORLD );
+ if ( 0 == my_rank ) {
+ ret = H5RCrelease( rc_id4, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ }
+ ret = H5RCclose( rc_id4 ); ASSERT_RET;
+ } else {
+ if ( 0 == my_rank ) {
+ fprintf( stderr, "M6.2-r%d: Failed %d times to aquire read context for cv 4 - continuing\n", my_rank, max_tries );
+ }
+ }
+
+ /* 15) /GB/GB/GA added in Tr 5 by rank 1 */
+ if ( my_rank == 1 ) {
+ create_group( file_id, "/GB/GB/GA", tr_id5, "/", my_rank, tr_num5 );
+ }
+
+ /* 16) /DA updated in Tr 5 by all ranks */
+ update_dataset( file_id, "/DA", tr_id5, rc_id3, "/", my_rank, tr_num5, 1 );
+
+ /* 17) Append to dataset /DC in Tr 5 by rank 0*/
+ if ( my_rank == 0 ) {
+ append_dataset2( file_id, "/DC", tr_id5, rc_id3, "/", my_rank, tr_num5 );
+ }
+
+ /* Finish and commit transaction 5, causing the updates to appear in container version 5. */
+ fprintf( stderr, "M6.2-r%d: Finish and commit tr %d (Step 15a - end)\n", my_rank, (int)tr_num5 );
+ ret = H5TRfinish( tr_id5, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5TRclose( tr_id5 ); ASSERT_RET;
+
+ /* Get read context for CV 5, then print the contents of the container */
+ version = 5;
+ if ( 0 == my_rank) {
+ fprintf( stderr, "M6.2-r%d: Try to acquire read context for cv %d (Step 15f)\n", my_rank, (int)version );
+ H5E_BEGIN_TRY {
+ rc_id5 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ while ( rc_id5 < 0 ) {
+ fprintf( stderr, "M6.2-r%d: Failed to acquire read context for cv 5 - sleep then retry\n", my_rank );
+ sleep( 1 );
+ version = 5;
+ rc_id5 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ }
+ } H5E_END_TRY;
+ } else {
+ rc_id5 = H5RCcreate(file_id, version);
+ }
+ assert( rc_id5 >= 0 ); assert( version == 5 );
+ fprintf( stderr, "M6.2-r%d: Acquired read context for cv 5\n", my_rank );
+ MPI_Barrier( MPI_COMM_WORLD );
+
+ fprintf( stderr, "M6.2-r%d: 7th call to print container contents (Step 15g)\n", my_rank );
+ if (verbose) print_container_contents( file_id, rc_id5, "/", my_rank ); ASSERT_RET;
+
+
+ /*
+ * Get read context for CV 4 from rank 0.
+ * Should be there first time if Transaction 4 was finished & committed (because we know Transaction 5 is done)
+ * If Transaction 4 was aborted, it will never be there.
+ */
+ version = 4;
+ if ( 0 == my_rank) {
+ fprintf( stderr, "M6.2-r%d: Once again, try to acquire read context for cv %d (Step 15h)\n", my_rank, (int)version );
+ rc_id4 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ if ( rc_id4 < 0 ) {
+ fprintf( stderr, "M6.2-r%d: Read context for cv 4 not available.\n", my_rank );
+ } else {
+ assert( rc_id4 >= 0 ); assert( version == 4 );
+ fprintf( stderr, "M6.2-r%d: Acquired read context for cv 4\n", my_rank );
+ fprintf( stderr, "M6.2-r%d: 8th call to print container contents (Step 15i)\n", my_rank );
+ if (verbose) print_container_contents( file_id, rc_id4, "/", my_rank ); ASSERT_RET;
+
+ /* Release read handle & close read context for CV 4 */
+ fprintf( stderr, "M6.2-r%d: Release read handle on cv 4 (Step 15j)\n", my_rank );
+ ret = H5RCrelease( rc_id4, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5RCclose( rc_id4 ); ASSERT_RET;
+ }
+ }
+
+ /*
+ * Note that rc_id5 was obtained in this bracketed code, but we aren't releasing it until later.
+ * Inelegant but I want it again & don't want to have to acquire again.
+ * Wanted to use it in print above (15f) before 2nd print of CV4 (15g), so didn't wait until
+ * bracket closed to acquire.
+ */
+
+ }
+
+ MPI_Barrier( MPI_COMM_WORLD );
+
+ /* Release the read handle and close read context on open CVs */
+ fprintf( stderr, "M6.2-r%d: Release read handle on cv 3 (Step 16)\n", my_rank );
+ if ( 0 == my_rank ) {
+ ret = H5RCrelease( rc_id3, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ }
+ ret = H5RCclose( rc_id3 ); ASSERT_RET;
+
+ fprintf( stderr, "M6.2-r%d: Release read handle on cv 5\n", my_rank );
+ if ( 0 == my_rank ) {
+ ret = H5RCrelease( rc_id5, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ }
+ ret = H5RCclose( rc_id5 ); ASSERT_RET;
+
+ /* Close the file, then barrier to make sure all have closed it. */
+ fprintf( stderr, "M6.2-r%d: close the container (Step 17)\n", my_rank );
+ ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ MPI_Barrier( MPI_COMM_WORLD );
+
+ /* Now, reopen file and show contents of all available CVs */
+ hid_t last_rc_id;
+ uint64_t last_version, v;
+
+ /* Reopen the file Read/Write */
+ fprintf( stderr, "M6.2-r%d: open the container\n", my_rank );
+
+ /* Get latest CV on open */
+ file_id = H5Fopen_ff( file_name, H5F_ACC_RDWR, fapl_id, &last_rc_id, H5_EVENT_STACK_NULL );
+ assert( file_id >= 0 );
+
+ H5RCget_version( last_rc_id, &last_version );
+ fprintf( stderr, "M6.2-r%d: Latest CV in file is cv %d\n", my_rank, (int)last_version );
+
+ if ( last_cv_only ) {
+ v = last_version;
+ } else {
+ v = 1;
+ }
+ for ( v; v <= last_version; v++ ) {
+ version = v;
+ if ( 0 == my_rank ) {
+ if(v < last_version) {
+ fprintf( stderr, "M6.2-r%d: Try to acquire read context for cv %d\n", my_rank, (int)version );
+ H5E_BEGIN_TRY
+ rc_id = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ H5E_END_TRY
+ }
+ else
+ rc_id = last_rc_id;
+ }
+
+ MPI_Bcast( &rc_id, 1, MPI_INT64_T, 0, MPI_COMM_WORLD );
+
+ if ( rc_id < 0 ) {
+ if ( 0 == my_rank ) {
+ fprintf( stderr, "M6.2-r%d: Failed to acquire read context for cv %d\n", my_rank, (int)v );
+ }
+ } else {
+ if ( 0 != my_rank ) {
+ rc_id = H5RCcreate( file_id, version );
+ }
+ assert ( version == v );
+ print_container_contents( file_id, rc_id, "/", my_rank );
+
+ MPI_Barrier( MPI_COMM_WORLD );
+ if(v < last_version) {
+ if ( 0 == my_rank ) {
+ ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ }
+ ret = H5RCclose( rc_id ); ASSERT_RET;
+ }
+ }
+ }
+
+ /* Release the read handle and close read context on cv obtained from H5Fopen_ff (by all ranks) */
+ ret = H5RCrelease( last_rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5RCclose( last_rc_id ); ASSERT_RET;
+
+ /* Close 2 H5 Objects that are still open */
+ fprintf( stderr, "M6.2-r%d: close all h5 objects that are still open\n", my_rank );
+ ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Pclose( fapl_id ); ASSERT_RET;
+
+ fprintf( stderr, "M6.2-r%d: Finalize EFF stack\n", my_rank );
+
+ /* Perform wrap-up operations */
+ MPI_Barrier( MPI_COMM_WORLD );
+ EFF_finalize();
+ MPI_Finalize();
+
+ return 0;
+}
+
+
+/*
+ * Helper function used to create string attribute "attr_name"
+ * for object identified by "obj_id"
+ * in transaction identified by "tr_id".
+ * "obj_path" and "my_rank" and "tr_num" are used to create identifiable string value
+ */
+void
+create_string_attribute( hid_t obj_id, const char* attr_name, hid_t tr_id, const char* obj_path, int my_rank, uint64_t tr_num ){
+ herr_t ret;
+ hsize_t dim_cnt;
+ hid_t space_id;
+ hid_t dtype_id;
+ char attr_val[ATTR_STR_LEN];
+ hsize_t val_size;
+ hid_t attr_id;
+
+ dim_cnt = 1;
+ space_id = H5Screate_simple( 1, &dim_cnt, NULL ); assert( space_id >= 0 );
+ dtype_id = H5Tcopy( H5T_C_S1 ); assert( dtype_id >= 0 );
+
+ val_size = sprintf( attr_val, "%s @ %s by rank %d in tr %d", attr_name, obj_path, my_rank, (int)tr_num );
+ assert( val_size < ATTR_STR_LEN );
+ ret = H5Tset_size( dtype_id, ATTR_STR_LEN ); ASSERT_RET;
+
+ attr_id = H5Acreate_ff( obj_id, attr_name, dtype_id, space_id, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ if ( attr_id >= 0 ) {
+ fprintf( stderr, "M6.2-r%d: Create %s with value %s - %s\n", my_rank, attr_name, attr_val, "OK" );
+ ret = H5Awrite_ff( attr_id, dtype_id, attr_val, tr_id, H5_EVENT_STACK_NULL );
+
+ ret = H5Aclose_ff( attr_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ } else {
+ fprintf( stderr, "M6.2-r%d: Create %s with value %s - %s\n", my_rank, attr_name, attr_val, "FAILED" );
+ }
+
+ ret = H5Tclose( dtype_id ); ASSERT_RET;
+ ret = H5Sclose( space_id ); ASSERT_RET;
+
+ return;
+}
+
+/*
+ * Helper function used to create group "group_name"
+ * in object identified by "obj_id"
+ * in transaction identified by "tr_id".
+ * "obj_path" and "my_rank" and "tr_num" are used in the status output.
+ */
+void
+create_group( hid_t obj_id, const char* group_name, hid_t tr_id, const char* obj_path, int my_rank, uint64_t tr_num ){
+ herr_t ret;
+ hid_t group_id;
+
+ group_id = H5Gcreate_ff( obj_id, group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ if ( group_id >= 0 ) {
+ fprintf( stderr, "M6.2-r%d: Create %s in %s in tr %d - %s\n", my_rank, group_name, obj_path, (int)tr_num, "OK" );
+ ret = H5Gclose_ff( group_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ } else {
+ fprintf( stderr, "M6.2-r%d: Create %s in %s in tr %d - %s\n", my_rank, group_name, obj_path, (int)tr_num, "FAILED" );
+ }
+
+ return;
+}
+
+/*
+ * Helper function used to create and initialize dataset "dset_name"
+ * in object identified by "obj_id"
+ * in transaction identified by "tr_id".
+ * All datasets are 1D Arrays, start with 4 elements, and have a max of 10 elements.
+ * "obj_path" and "my_rank" and "tr_num" and "ordinal" are used in the status output.
+ * Cell values are computed using the formula:
+ * data[i] = ordinal*1000 + tr_num*100 + my_rank*10 + i;
+ * where ordinal can be set by the calling function to denote where the creation of this dataset occurs relative
+ * to the creation of other datasets in "obj_id". It is used solely to generate identifiable cell values.
+ */
+void
+create_dataset( hid_t obj_id, const char* dset_name, hid_t tr_id, const char* obj_path, int my_rank, uint64_t tr_num, int ordinal ){
+ herr_t ret;
+ hsize_t current_size = 4;
+ hsize_t max_size = 10;
+ int data[4]; /* sized to accomodate current_size elements */
+ hid_t space_id;
+ hid_t dtype_id;
+ hid_t dset_id;
+ int i;
+
+ /* Set cell values */
+ for ( i = 0; i < current_size; i++ ) {
+ data[i] = ordinal*1000 + tr_num*100 + my_rank*10 + i;
+ }
+
+ space_id = H5Screate_simple( 1, &current_size, &max_size ); assert( space_id >= 0 );
+ dtype_id = H5Tcopy( H5T_NATIVE_INT ); assert( dtype_id >= 0 );
+
+ dset_id = H5Dcreate_ff( obj_id, dset_name, dtype_id, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL );
+ if ( dset_id >= 0 ) {
+ fprintf( stderr, "M6.2-r%d: Create %s in %s in tr %d; ordinal is %d; data values are %d %d %d %d - %s\n",
+ my_rank, dset_name, obj_path, (int)tr_num, ordinal, data[0], data[1], data[2], data[3], "OK" );
+ ret = H5Dwrite_ff( dset_id, dtype_id, space_id, space_id, H5P_DEFAULT, data, tr_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ } else {
+ fprintf( stderr, "M6.2-r%d: Create %s in %s in tr %d; ordinal is %d; data values are %d %d %d %d - %s\n",
+ my_rank, dset_name, obj_path, (int)tr_num, ordinal, data[0], data[1], data[2], data[3], "FAILED" );
+ }
+
+ ret = H5Tclose( dtype_id ); ASSERT_RET;
+ ret = H5Sclose( space_id ); ASSERT_RET;
+
+ return;
+}
+
+/*
+ * Helper function used to create 6x4 2D dataset "dset_name"
+ * in object identified by "obj_id"
+ * in transaction identified by "tr_id".
+ * "obj_path" and "my_rank" and "tr_num" are used in the status output.
+ */
+void
+create_dataset2( hid_t obj_id, const char* dset_name, hid_t tr_id, const char* obj_path, int my_rank, uint64_t tr_num ){
+ herr_t ret;
+ hsize_t current_sizes[2] = {6, 4};
+ hsize_t max_sizes[2] = {6, 4};
+ hid_t space_id;
+ hid_t dset_id;
+
+ space_id = H5Screate_simple( 2, current_sizes, max_sizes ); assert( space_id >= 0 );
+
+ dset_id = H5Dcreate_ff( obj_id, dset_name, H5T_NATIVE_INT, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL );
+ if ( dset_id >= 0 ) {
+ fprintf( stderr, "M6.2-r%d: Create %s in %s in tr %d - %s\n",
+ my_rank, dset_name, obj_path, (int)tr_num, "OK" );
+ ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ } else {
+ fprintf( stderr, "M6.2-r%d: Create %s in %s in tr %d - %s\n",
+ my_rank, dset_name, obj_path, (int)tr_num, "FAILED" );
+ }
+
+ ret = H5Sclose( space_id ); ASSERT_RET;
+
+ return;
+}
+
+
+/*
+ * Helper function used to create committed datatype "type_name"
+ * in group identified by "obj_id"
+ * in transaction identified by "tr_id".
+ * "obj_path" and "my_rank" and "tr_num" are used in the status output.
+ */
+void
+create_committed_datatype( hid_t obj_id, const char* type_name, hid_t tr_id, const char* obj_path, int my_rank, uint64_t tr_num ){
+ herr_t ret;
+ hid_t dtype_id;
+
+ dtype_id = H5Tcopy( H5T_NATIVE_INT ); assert( dtype_id >= 0 );
+ ret = H5Tcommit_ff( obj_id, type_name, dtype_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ fprintf( stderr, "M6.2-r%d: Create CDT %s in %s tr %d - %s\n", my_rank, type_name, obj_path, (int)tr_num, STATUS );
+
+ ret = H5Tclose( dtype_id ); ASSERT_RET;
+
+ return;
+}
+
+/*
+ * Helper function used to update raw data in dataset "dset_name"
+ * in object identified by "obj_id"
+ * in transaction identified by "tr_id".
+ * "rc_id" for "tr_id" passed in explicitly for now - later add H5 function to extract it from tr_id.
+ * cell[my_rank] will be updated, using the formula
+ * value = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
+ * "obj_path" and "my_rank" and "tr_num" and "ordinal" are used in the status output.
+ */
+void
+update_dataset( hid_t obj_id, const char* dset_name, hid_t tr_id, hid_t rc_id, const char* obj_path, int my_rank,
+ uint64_t tr_num, int ordinal ){
+ herr_t ret;
+ hid_t dset_id;
+
+ /* Open the dataset, confirm number of dimensions, get current & max size */
+ dset_id = H5Dopen_ff( obj_id, dset_name, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+
+ if ( dset_id >= 0 ) {
+ hid_t filespace_id;
+ int nDims;
+ hsize_t current_size, max_size;
+
+ filespace_id = H5Dget_space( dset_id ); assert ( filespace_id >= 0 );
+ nDims = H5Sget_simple_extent_dims( filespace_id, &current_size, &max_size );
+ assert( nDims == 1 );
+
+ if ( my_rank < current_size ) {
+ int cell_data; /* sized to hold data for a single cell in the dataset*/
+ hsize_t mem_size = 1;
+ hid_t memspace_id;
+
+ /* set data value, create memory data space, then select the element to update */
+ cell_data = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
+ memspace_id = H5Screate_simple( 1, &mem_size, &mem_size ); assert( memspace_id >= 0 );
+
+ hsize_t start, stride, count, block;
+ start = (hsize_t)my_rank;
+ stride = count = block = 1;
+
+ ret = H5Sselect_hyperslab( filespace_id, H5S_SELECT_SET, &start, &stride, &count, &block ); ASSERT_RET;
+
+ ret = H5Dwrite_ff( dset_id, H5T_NATIVE_INT, memspace_id, filespace_id, H5P_DEFAULT, &cell_data, tr_id,
+ H5_EVENT_STACK_NULL );
+ fprintf( stderr, "M6.2-r%d: Update %s[%d] in %s in tr %d to have value %d - %s\n",
+ my_rank, dset_name, my_rank, obj_path, (int)tr_num, cell_data, STATUS );
+ ret = H5Sclose( memspace_id ); ASSERT_RET;
+
+ } else {
+ fprintf( stderr, "M6.2-r%d: No update to %s in %s in tr %d for this rank.\n", my_rank, dset_name, obj_path, (int)tr_num );
+ }
+
+ ret = H5Sclose( filespace_id ); ASSERT_RET;
+ ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ } else {
+ fprintf( stderr, "M6.2-r%d: Update %s[%d] in %s in tr %d - %s\n",
+ my_rank, dset_name, my_rank, obj_path, (int)tr_num, "FAILED" );
+ }
+
+ return;
+}
+
+/*
+ * Helper function used to append data at end of 2D dataset "dset_name"
+ * in object identified by "obj_id"
+ * in transaction identified by "tr_id".
+ * "rc_id" for "tr_id" passed in explicitly for now - later add H5 function to extract it from tr_id.
+ * cell[tr_num][my_rank] will be updated, using the formula
+ * value = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
+ * "obj_path" and "my_rank" and "tr_num" are used in the status output.
+ */
+void
+append_dataset2( hid_t obj_id, const char* dset_name, hid_t tr_id, hid_t rc_id, const char* obj_path, int my_rank,
+ uint64_t tr_num ){
+ herr_t ret;
+ hid_t dset_id;
+
+ /* Open the dataset, confirm number of dimensions, get current & max size */
+ dset_id = H5Dopen_ff( obj_id, dset_name, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+
+ if ( dset_id >= 0 ) {
+ hid_t filespace_id;
+ int nDims;
+ hsize_t current_size[2];
+ hsize_t max_size[2];
+
+ filespace_id = H5Dget_space( dset_id ); assert ( filespace_id >= 0 );
+ nDims = H5Sget_simple_extent_dims( filespace_id, current_size, max_size );
+ assert( nDims == 2 );
+
+ if ( (int)tr_num < current_size[0] ) {
+ hsize_t nCols;
+ int* data;
+ int i;
+ hid_t memspace_id;
+
+ nCols = current_size[1];
+ data = (int *)calloc( nCols, sizeof(int) ); assert( data != NULL );
+
+ /* set data value, create memory data space, then select the element to update */
+ for ( i = 0; i < nCols; i++ ) {
+ data[i] = 10*tr_num + i;
+ }
+ memspace_id = H5Screate_simple( 1, &nCols, &nCols ); assert( memspace_id >= 0 );
+
+ hsize_t start[2], stride[2], count[2], block[2];
+ start[0] = tr_num;
+ start[1] = 0;
+ stride[0] = 1;
+ stride[1] = 1;
+ count[0] = 1;
+ count[1] = nCols;
+ block[0] = 1;
+ block[1] = 1;
+
+ ret = H5Sselect_hyperslab( filespace_id, H5S_SELECT_SET, start, stride, count, block ); ASSERT_RET;
+
+ ret = H5Dwrite_ff( dset_id, H5T_NATIVE_INT, memspace_id, filespace_id, H5P_DEFAULT, data, tr_id, H5_EVENT_STACK_NULL );
+ fprintf( stderr, "M6.2-r%d: Update %s[%d][*] in %s in tr %d to have values %d %d %d %d - %s\n",
+ my_rank, dset_name, (int)tr_num, obj_path, (int)tr_num, data[0], data[1], data[2], data[3], STATUS );
+ ret = H5Sclose( memspace_id ); ASSERT_RET;
+
+ free(data);
+
+ } else {
+ fprintf( stderr, "M6.2-r%d: No update to %s in %s in tr %d for this rank.\n", my_rank, dset_name, obj_path, (int)tr_num );
+ }
+
+ ret = H5Sclose( filespace_id ); ASSERT_RET;
+ ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ } else {
+ fprintf( stderr, "M6.2-r%d: Update %s[%d] in %s in tr %d - %s\n",
+ my_rank, dset_name, my_rank, obj_path, (int)tr_num, "FAILED" );
+ }
+
+ return;
+}
+
+/*
+ * Helper function used to recursively print container contents
+ * for container identified by "file_id"
+ * in read context identified by "rc_id"
+ * with path to current level in "grp_path"
+ * and "my_rank" used to identify the process doing the reading / printing.
+ */
+void
+print_container_contents( hid_t file_id, hid_t rc_id, const char* grp_path, int my_rank )
+{
+ herr_t ret;
+ uint64_t cv;
+ htri_t exists;
+ char path_to_object[1024];
+ hid_t obj_id;
+ char name[3];
+ int i;
+
+ static int lvl = 0; /* level in recursion - used to format printing */
+ char preface[128];
+ char line[1024];
+
+ /* Get the container version for the read context */
+ ret = H5RCget_version( rc_id, &cv ); ASSERT_RET;
+
+ /* Set up the preface and adding version number */
+ sprintf( preface, "M6.2-r%d: cv %d: ", my_rank, (int)cv );
+
+ /* Start the printing */
+ if ( lvl == 0 ) {
+ fprintf( stderr, "%s ----- Container Contents ------------\n", preface );
+ }
+
+ /* Attributes */
+ for ( i = 1; i < 3; i++ ) {
+ if ( i == 1 ) {
+ strcpy( name, "AA" );
+ } else {
+ strcpy( name, "AB" );
+ }
+ ret = H5Aexists_by_name_ff( file_id, grp_path, name, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ if ( exists ) {
+ hid_t attr_id, atype_id;
+ char attr_val[ATTR_STR_LEN];
+
+ attr_id = H5Aopen_by_name_ff( file_id, grp_path, name, H5P_DEFAULT, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( attr_id >= 0 );
+
+ atype_id = H5Tcopy( H5T_C_S1 ); assert( atype_id >= 0 );
+ ret = H5Tset_size( atype_id, ATTR_STR_LEN ); ASSERT_RET;
+
+ ret = H5Aread_ff( attr_id, atype_id, attr_val, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ fprintf( stderr, "%s %s%s value: %s\n", preface, grp_path, name, attr_val );
+
+ ret = H5Tclose( atype_id ); ASSERT_RET;
+ ret = H5Aclose_ff( attr_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ }
+ }
+
+ /* Datasets */
+ for ( i = 1; i < 4; i++ ) {
+ if ( i == 1 ) {
+ strcpy( name, "DA" );
+ } else if ( i == 2 ){
+ strcpy( name, "DB" );
+ } else {
+ strcpy( name, "DC" );
+ }
+
+ sprintf( path_to_object, "%s%s", grp_path, name );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+
+ if ( exists ) {
+ hid_t dset_id;
+ hid_t space_id;
+ int nDims;
+ hsize_t current_size[2];
+ hsize_t max_size[2];
+ hsize_t totalSize;
+ int *data;
+ int i;
+
+ dset_id = H5Dopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( dset_id >= 0 );
+
+ space_id = H5Dget_space( dset_id ); assert ( space_id >= 0 );
+ nDims = H5Sget_simple_extent_dims( space_id, current_size, max_size );
+ assert( ( nDims == 1 ) || ( nDims == 2 ) );
+
+ if ( nDims == 1 ) {
+ totalSize = current_size[0];
+ } else {
+ totalSize = current_size[0] * current_size[1];
+ }
+ data = (int *)calloc( totalSize, sizeof(int) ); assert( data != NULL );
+
+ ret = H5Dread_ff( dset_id, H5T_NATIVE_INT, space_id, space_id, H5P_DEFAULT, data, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ sprintf( line, "%s %s values: ", preface, path_to_object );
+ if ( nDims == 1 ) {
+ for ( i = 0; i < totalSize; i++ ) {
+ sprintf( line, "%s%d ", line, data[i] );
+ }
+ } else {
+ int r, c;
+ i = 0;
+ for ( r = 0; r < current_size[0]; r++ ) {
+ sprintf( line, "%srow %d: ", line, r );
+ for ( c = 0; c < current_size[1]; c++ ) {
+ sprintf( line, "%s%d ", line, data[i] );
+ i++;
+ }
+ }
+ }
+
+ fprintf( stderr, "%s\n", line );
+ free( data );
+
+ ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Sclose( space_id ); ASSERT_RET;
+ }
+ }
+
+ /* Committed datatypes */
+ for ( i = 1; i < 3; i++ ) {
+ if ( i == 1 ) {
+ strcpy( name, "TA" );
+ } else {
+ strcpy( name, "TB" );
+ }
+
+ sprintf( path_to_object, "%s%s", grp_path, name );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+ if ( exists ) {
+ fprintf( stderr, "%s %s\n", preface, path_to_object );
+ }
+ }
+
+ /* Groups - if found, descend */
+ for ( i = 1; i < 3; i++ ) {
+ if ( i == 1 ) {
+ strcpy( name, "GA" );
+ } else {
+ strcpy( name, "GB" );
+ }
+
+ sprintf( path_to_object, "%s%s/", grp_path, name );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+ if ( exists ) {
+ fprintf( stderr, "%s %s\n", preface, path_to_object );
+ lvl++;
+ print_container_contents( file_id, rc_id, path_to_object, my_rank );
+ lvl--;
+ }
+ }
+
+ /* End printing */
+ if ( lvl == 0 ) {
+ fprintf( stderr, "%s -----------------\n", preface );
+ }
+
+ return;
+}
+
+/*
+ * parse_options - helper function to parse the command line options.
+ */
+int
+parse_options( int argc, char** argv, int my_rank ) {
+ int i, n;
+
+ while ( --argc ) {
+ if ( **(++argv) != '-' ) {
+ break;
+ } else {
+ switch( *(*argv+1) ) {
+ case 'a':
+ abort4 = 1;
+ break;
+ case 'l':
+ last_cv_only = 1;
+ break;
+ case 'v':
+ verbose = 1;
+ break;
+ default:
+ if ( my_rank == 0 ) {
+ printf( "Usage: h5ff_client_M6.2_demo [-aiv]\n" );
+ printf( "\ta: abort transaction 4\n" );
+ printf( "\tl: last container version contents are the only ones printed on reopen\n" );
+ printf( "\tv: verbose printing\n" );
+ }
+ return( 1 );
+ }
+ }
+ }
+ return( 0 );
+}
diff --git a/examples/h5ff_client_M7.2-pep_demo.c b/examples/h5ff_client_M7.2-pep_demo.c
index a2b9b38..c4f60b5 100644
--- a/examples/h5ff_client_M7.2-pep_demo.c
+++ b/examples/h5ff_client_M7.2-pep_demo.c
@@ -1,1288 +1,1288 @@
-/*
- * h5ff_client_M7.2-pep_demo.c: Demo Persist / Evict / Prefetch
- * Part of Milestone 7.2 - Functionality of Burst Buffer.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include <sys/time.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-/* define filename for this app, and max size after username prepended */
-#define FILENAME_APP "eff_pep.h5"
-#define FILENAME_SIZE 256
-
-/* macros related to error reporting */
-#define STATUS (ret >= 0) ? " " : " - FAILED"
-#define ASSERT_RET assert( ret >= 0 )
-
-/* option flags */
-int use_daos_lustre = 1; // Use DAOS Lustre - defaults to yes
-int verbose = 0; // Verbose output - defaults to no
-int pause = 0; // Pause to allow out-of-band space check - defaults to 0.
-int time_reads = 0; // Time reads after container re-open
-
-/* global variables and macros used to make timing easier */
-struct timeval tv_start;
-struct timeval tv_end;
-#define START_TIME gettimeofday( &tv_start, NULL )
-#define END_TIME gettimeofday( &tv_end, NULL )
-#define ELAPSED_TIME (ulong)( (tv_end.tv_usec + 1000000*tv_end.tv_sec) - (tv_start.tv_usec + 1000000*tv_start.tv_sec) )
-
-/* function prototypes */
-void create_dataset( hid_t, const char*, hid_t, const char*, int, int );
-void create_map( hid_t, const char*, hid_t, const char*, int, int );
-void create_named_datatype( hid_t, const char*, hid_t, const char*, int );
-void update_dataset( hid_t, const char*, hid_t, hid_t, const char*, int, int );
-void update_map( hid_t, const char*, hid_t, hid_t, const char*, int, int );
-void evict_group_members_updates( hid_t, hid_t, const char*, int );
-void print_container_contents( hid_t, hid_t, const char*, int );
-int parse_options( int, char**, int );
-void usage( const char* );
-
-/**************************************************************************************************************/
-int main( int argc, char **argv ) {
-
- /* MPI */
- int my_rank, comm_size, acquire_rank;
- int provided;
-
- /* Container */
- char *user_name; /* We'll prepend username to make filename unique */
- char file_name[FILENAME_SIZE]; /* Actual filename to be opened will be user_name + FILENAME_APP */
- hid_t fapl_id;
- hid_t file_id;
-
- /* Groups */
- hid_t grp_l_id;
- hid_t grp_p_id;
- hid_t grp_s_id;
-
- /* Container Version & Read Contexts */
- uint64_t version;
- hid_t rc_id1, rc_id2, rc_id3, rc_id4, rc_id5, rc_id;
-
- /* Transactions */
- uint64_t tr_num;
- hid_t tr_id;
- hid_t trspl_id;
- int num_tr_leaders;
-
- herr_t ret;
- int i;
-
- /****
- * Initialize and create container
- ****/
-
- /* Confirm needed MPI functionality is available */
- MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
- if ( MPI_THREAD_MULTIPLE != provided ) {
- fprintf( stderr, "APP: ERROR: MPI does not have MPI_THREAD_MULTIPLE support\n" );
- exit( 1 );
- }
-
- /* Find MPI rank & communicator size */
- MPI_Comm_rank( MPI_COMM_WORLD, &my_rank );
- MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
- if ( my_rank == 0 ) {
- fprintf( stderr, "APP-r%d: Number of MPI processes = %d\n", my_rank, comm_size );
- }
-
- if ( comm_size > 3 ) {
- acquire_rank = 3;
- } else {
- acquire_rank = 0;
- }
-
- /* Parse command-line options controlling behavior */
- if ( parse_options( argc, argv, my_rank ) != 0 ) {
- exit( 1 );
- }
-
- /* Initialize the EFF stack, starting FS client, registering HDF5 VOL calls, requesting IOD init */
- fprintf( stderr, "APP-r%d: Initialize EFF stack\n", my_rank );
- EFF_init( MPI_COMM_WORLD, MPI_INFO_NULL );
-
- /* Specify the IOD VOL plugin should be used and create H5File (EFF container) */
- user_name = getenv( "USER" );
- snprintf( file_name, FILENAME_SIZE, "%s_%s", user_name, FILENAME_APP );
- fprintf( stderr, "APP-r%d: Create %s\n", my_rank, file_name );
- fapl_id = H5Pcreate( H5P_FILE_ACCESS ); assert( fapl_id >= 0 );
- ret = H5Pset_fapl_iod( fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL ); ASSERT_RET;
- file_id = H5Fcreate_ff( file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL ); assert( file_id >= 0 );
-
- /****
- * Transaction 2: Rank 0 creates three H5Groups in the H5File
- ****/
-
- if ( my_rank == 0 ) {
- /* Get read context use for Tr 2 */
- version = 1;
- rc_id1 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- assert( rc_id1 >= 0 ); assert( version == 1 );
- fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
-
- /* Create a local transaction for transaction number and start it with a single tr leader (the default). */
- tr_num = 2;
- fprintf( stderr, "APP-r%d: tr %lu - Start\n", my_rank, tr_num );
- tr_id = H5TRcreate( file_id, rc_id1, tr_num ); assert( tr_id >= 0 );
- ret = H5TRstart( tr_id, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Add updates to the transaction for Group creates */
- fprintf( stderr, "APP-r%d: tr %d - Create /G-logged /G-prefetched /G-stored\n", my_rank, tr_num );
- grp_l_id = H5Gcreate_ff( file_id, "G-logged", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- assert( grp_l_id >= 0 );
- grp_p_id = H5Gcreate_ff( file_id, "G-prefetched", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- assert( grp_p_id >= 0 );
- grp_s_id = H5Gcreate_ff( file_id, "G-stored", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- assert( grp_s_id >= 0 );
-
- /* Finish and commit TR and get RC */
- fprintf( stderr, "APP-r%d: tr %lu - Finish\n", my_rank, tr_num );
- ret = H5TRfinish( tr_id, H5P_DEFAULT, &rc_id2, H5_EVENT_STACK_NULL ); ASSERT_RET;
- assert( rc_id2 >= 0 );
- version = 2;
- fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
- ret = H5TRclose( tr_id ); ASSERT_RET;
-
- /* Release the read handle & close read context on previous CV - only held by rank 0 */
- ret = H5RCrelease( rc_id1, H5_EVENT_STACK_NULL); ASSERT_RET;
- ret = H5RCclose( rc_id1 ); ASSERT_RET;
-
- /* If verbose, rank 0 prints the container contents */
- if ( verbose ) print_container_contents( file_id, rc_id2, "/", my_rank );
- }
-
- /* Rank 0 tells others that RC 2 has been acquired; they create local RC object and open the Groups */
- MPI_Bcast( &version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
- if ( my_rank != 0 ) {
- rc_id2 = H5RCcreate( file_id, version );
- assert( rc_id2 >= 0 ); assert ( version == 2 );
- fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
-
- grp_l_id = H5Gopen_ff( file_id, "G-logged", H5P_DEFAULT, rc_id2, H5_EVENT_STACK_NULL ); assert( grp_l_id >= 0 );
- grp_p_id = H5Gopen_ff( file_id, "G-prefetched", H5P_DEFAULT, rc_id2, H5_EVENT_STACK_NULL ); assert( grp_p_id >= 0 );
- grp_s_id = H5Gopen_ff( file_id, "G-stored", H5P_DEFAULT, rc_id2, H5_EVENT_STACK_NULL ); assert( grp_s_id >= 0 );
- }
-
- /****
- * Transaction 3 - In each of the 3 Groups, create "D"(rnk 0), "M"(rnk 1 or 0), "T" (rnk 2 or 1 or 0)
- ****/
-
- if ( my_rank < 3 ) {
- tr_num = 3;
- num_tr_leaders = ( (comm_size < 4) ? comm_size : 3 );
-
- /* Create a local transaction for transaction number and start it. */
- fprintf( stderr, "APP-r%d: tr %lu - Start with %d leaders\n", my_rank, tr_num, num_tr_leaders );
- tr_id = H5TRcreate( file_id, rc_id2, tr_num ); assert( tr_id >= 0 );
- trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
- ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
- ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- char glog_path[128], gpre_path[128], gsto_path[128];
-
- /* Set pathnames to groups used in status messages */
- sprintf( glog_path, "/G-logged/" );
- sprintf( gpre_path, "/G-prefetched/" );
- sprintf( gsto_path, "/G-stored/" );
-
- /* Add updates to the transaction */
- if ( my_rank == 0 ) {
- create_dataset( grp_l_id, "D", tr_id, glog_path, my_rank, 1 );
- create_dataset( grp_p_id, "D", tr_id, gpre_path, my_rank, 2 );
- create_dataset( grp_s_id, "D", tr_id, gsto_path, my_rank, 3 );
- }
- if ( ( my_rank == 1 ) || ( ( my_rank == 0 ) && ( comm_size == 1 ) ) ) {
- create_map( grp_l_id, "M", tr_id, glog_path, my_rank, 1 );
- create_map( grp_p_id, "M", tr_id, gpre_path, my_rank, 2 );
- create_map( grp_s_id, "M", tr_id, gsto_path, my_rank, 3 );
- }
- if ( ( my_rank == 2 ) || ( ( my_rank == 1 ) && ( comm_size == 2 ) ) || ( ( my_rank == 0 ) && ( comm_size == 1 ) ) ) {
- create_named_datatype( grp_l_id, "T", tr_id, glog_path, my_rank );
- create_named_datatype( grp_p_id, "T", tr_id, gpre_path, my_rank );
- create_named_datatype( grp_s_id, "T", tr_id, gsto_path, my_rank );
- }
-
- /* Finish, commit, and close transaction */
- fprintf( stderr, "APP-r%d: tr %lu - Finish\n", my_rank, tr_num );
- ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5TRclose( tr_id ); ASSERT_RET;
- ret = H5Pclose( trspl_id ); ASSERT_RET;
- }
-
- /* Acquire a read handle for container version and create a read context. */
- version = 3;
- if ( verbose ) fprintf( stderr, "APP-r%d: rc %lu - Try to acquire\n", my_rank, version );
-
- if ( acquire_rank == my_rank ) {
- H5E_BEGIN_TRY {
- rc_id3 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- while ( rc_id3 < 0 ) {
- if ( verbose ) fprintf( stderr, "APP-r%d: rc %lu - Failed to acquire; sleep then retry\n", my_rank, version );
- sleep( 1 );
- rc_id3 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- }
- } H5E_END_TRY;
- }
-
- MPI_Bcast( &version, 1, MPI_UINT64_T, acquire_rank, MPI_COMM_WORLD );
- if ( acquire_rank != my_rank ) {
- rc_id3 = H5RCcreate( file_id, version );
- }
- assert( rc_id3 >= 0 ); assert ( version == 3 );
- fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
-
- /* Release the read handle and close read context on previous CV */
- if ( my_rank == 0 ) {
- ret = H5RCrelease( rc_id2, H5_EVENT_STACK_NULL); ASSERT_RET;
- fprintf( stderr, "APP-r%d: rc 2 - Released\n", my_rank );
- }
- ret = H5RCclose( rc_id2 ); ASSERT_RET;
-
- /****
- * Rank 0 persists CV 3
- *
- * Satisfies: An application will create multiple transactions in a container, then persist them
- * from the BB to persistent storage.
- ****/
-
- if ( my_rank == 0 ) {
- fprintf( stderr, "APP-r%d: cv 3 - Persist\n", my_rank );
- ret = H5RCpersist(rc_id3, H5_EVENT_STACK_NULL); ASSERT_RET;
- print_container_contents( file_id, rc_id3, "/", my_rank );
- }
-
- if ( verbose ) print_container_contents( file_id, rc_id3, "/", my_rank );
-
- /****
- * Transaction 4 - All ranks update Dataset and Map objects in the 3 Groups
- ****/
-
- tr_num = 4;
- num_tr_leaders = comm_size;
-
- /* Create a local transaction for transaction number and start it. */
- fprintf( stderr, "APP-r%d: tr %lu - Start with %d leaders\n", my_rank, tr_num, num_tr_leaders );
- tr_id = H5TRcreate( file_id, rc_id3, tr_num ); assert( tr_id >= 0 );
- trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
- ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
- ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Add updates to the transaction */
- update_dataset( file_id, "G-logged/D", tr_id, rc_id3, "/", my_rank, 1 );
- update_dataset( file_id, "G-prefetched/D", tr_id, rc_id3, "/", my_rank, 2 );
- update_dataset( file_id, "G-stored/D", tr_id, rc_id3, "/", my_rank, 3 );
-
- update_map( file_id, "G-logged/M", tr_id, rc_id3, "/", my_rank, 1 );
- update_map( file_id, "G-prefetched/M", tr_id, rc_id3, "/", my_rank, 2 );
- update_map( file_id, "G-stored/M", tr_id, rc_id3, "/", my_rank, 3 );
-
- /* Finish and close transaction */
- fprintf( stderr, "APP-r%d: tr %lu - Finish\n", my_rank, tr_num );
- ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5TRclose( tr_id ); ASSERT_RET;
- ret = H5Pclose( trspl_id ); ASSERT_RET;
-
- /* If verbose mode, get RC 4 and print CV 4 */
- if ( verbose ) {
- version = 4;
- if ( my_rank == 0 ) {
- fprintf( stderr, "APP-r%d: rc %lu - Try to acquire\n", my_rank, version );
- H5E_BEGIN_TRY {
- rc_id4 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- while ( rc_id4 < 0 ) {
- fprintf( stderr, "APP-r%d: rc %lu - Failed to acquire; sleep then retry\n", my_rank, version );
- sleep( 1 );
- rc_id4 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- }
- } H5E_END_TRY;
- }
- MPI_Bcast( &version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
- if ( my_rank != 0 ) {
- rc_id4 = H5RCcreate(file_id, version);
- }
- assert( rc_id4 >= 0 ); assert ( version == 4 );
- fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
- print_container_contents( file_id, rc_id4, "/", my_rank );
-
- /* Release the read handle and close read context on CV 4 */
- MPI_Barrier( MPI_COMM_WORLD );
- if ( my_rank == 0 ) {
- ret = H5RCrelease( rc_id4, H5_EVENT_STACK_NULL); ASSERT_RET;
- }
- ret = H5RCclose( rc_id4 ); ASSERT_RET;
- fprintf( stderr, "APP-r%d: rc 4 - Released\n", my_rank );
- }
-
- /****
- * Transaction 5 (based on CV 3) - All ranks update Dataset and Map objects in the 3 Groups
- ****/
-
- tr_num = 5;
- num_tr_leaders = comm_size;
-
- /* Create a local transaction for transaction number and start it. */
- fprintf( stderr, "APP-r%d: tr %lu - Start with %d leaders\n", my_rank, tr_num, num_tr_leaders );
- tr_id = H5TRcreate( file_id, rc_id3, tr_num ); assert( tr_id >= 0 );
- trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
- ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
- ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Add updates to the transaction */
- update_dataset( file_id, "G-logged/D", tr_id, rc_id3, "/", my_rank, 1 );
- update_dataset( file_id, "G-prefetched/D", tr_id, rc_id3, "/", my_rank, 2 );
- update_dataset( file_id, "G-stored/D", tr_id, rc_id3, "/", my_rank, 3 );
-
- update_map( file_id, "G-logged/M", tr_id, rc_id3, "/", my_rank, 1 );
- update_map( file_id, "G-prefetched/M", tr_id, rc_id3, "/", my_rank, 2 );
- update_map( file_id, "G-stored/M", tr_id, rc_id3, "/", my_rank, 3 );
-
- /* Finish, commit, and close transaction */
- fprintf( stderr, "APP-r%d: tr %lu - Finish\n", my_rank, tr_num );
- ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5TRclose( tr_id ); ASSERT_RET;
- ret = H5Pclose( trspl_id ); ASSERT_RET;
-
- /* Acquire a read handle for container version and create a read context. */
- version = 5;
-
- if ( my_rank == 0 ) {
- if ( verbose ) fprintf( stderr, "APP-r%d: rc %lu - Try to acquire\n", my_rank, version );
- H5E_BEGIN_TRY {
- rc_id5 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- while ( rc_id5 < 0 ) {
- if ( verbose ) fprintf( stderr, "APP-r%d: rc %lu - Failed to acquire; sleep then retry\n", my_rank, version );
- sleep( 1 );
- rc_id5 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- }
- } H5E_END_TRY;
- }
- MPI_Bcast( &version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
- if ( my_rank != 0 ) {
- rc_id5 = H5RCcreate( file_id, version );
- }
- assert( rc_id5 >= 0 ); assert ( version == 5 );
- fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
-
- /* Release the read handle and close read context on CV 3 */
- if ( my_rank == acquire_rank ) {
- ret = H5RCrelease( rc_id3, H5_EVENT_STACK_NULL); ASSERT_RET;
- }
- ret = H5RCclose( rc_id3 ); ASSERT_RET;
- fprintf( stderr, "APP-r%d: rc 3 - Released \n", my_rank );
-
- if ( verbose ) print_container_contents( file_id, rc_id5, "/", my_rank );
-
- /****
- * Highest rank persists CV 5, then evicts objects under G-prefetched and G-stored.
- *
- * Satisfies: An application will create multiple transactions in a container, then persist them
- * from the burst buffer storage to persistent storage, then evict several of the objects in the IOD
- * container, releasing space in the burst buffer.
- ****/
-
- /* Optionally, pause to show change in BB space using out-of-band check. All ranks paused. */
- if ( pause ) {
- MPI_Barrier( MPI_COMM_WORLD );
- if ( my_rank == 0 ) {
- fprintf( stderr, ">>> PAUSED before persist/evict to allow BB space usage to be checked.\n" );
- fprintf( stderr, ">>> Press return to continue..." );
- getchar();
- }
- MPI_Barrier( MPI_COMM_WORLD );
- }
- else
- MPI_Barrier( MPI_COMM_WORLD );
-
- if ( my_rank == comm_size-1 ) {
- fprintf( stderr, "APP-r%d: cv 5 - Persist\n", my_rank );
- ret = H5RCpersist(rc_id5, H5_EVENT_STACK_NULL); ASSERT_RET;
- print_container_contents( file_id, rc_id5, "/", my_rank );
-
- evict_group_members_updates( file_id, rc_id5, "/G-prefetched", my_rank );
- evict_group_members_updates( file_id, rc_id5, "/G-stored", my_rank );
- }
-
- /* Optionally, pause to show change in BB space using out-of-band check. All ranks paused. */
- if ( pause ) {
- MPI_Barrier( MPI_COMM_WORLD );
- if ( my_rank == 0 ) {
- fprintf( stderr, ">>> PAUSED after persist/evict to allow BB space usage to be checked.\n" );
- fprintf( stderr, ">>> Press return to continue..." );
- getchar();
- }
- MPI_Barrier( MPI_COMM_WORLD );
- }
- else
- MPI_Barrier( MPI_COMM_WORLD );
-
- /* All ranks print container here. For rank == comm_size, this will be after objects have been evicted */
- if ( verbose ) print_container_contents( file_id, rc_id5, "/", my_rank );
-
- /* Release the read handle and close read context on CV 5 */
- MPI_Barrier( MPI_COMM_WORLD );
- if ( my_rank == 0 ) {
- ret = H5RCrelease( rc_id5, H5_EVENT_STACK_NULL); ASSERT_RET;
- }
- ret = H5RCclose( rc_id5 ); ASSERT_RET;
- fprintf( stderr, "APP-r%d: rc 5 - Released \n", my_rank );
-
- /****
- * Close the H5File. Reopen, pre-fetch D, M, and T in /G-prefetched.
- *
- * Satisfies: "... then closing the container. The application will then re-open the container and asynchronously
- * pre-fetch multiple objects from persistent storage to burst buffer storage.
- ****/
-
- ret = H5Gclose_ff( grp_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Gclose_ff( grp_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Gclose_ff( grp_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Close the container, then barrier to make sure all ranks have closed it before continuing */
- fprintf( stderr, "APP-r%d: Close %s\n", my_rank, file_name );
- ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Pclose( fapl_id ); ASSERT_RET;
- MPI_Barrier( MPI_COMM_WORLD );
-
- /* Define variables here so easier to move this into separate routine if desired (note: still rely on some defined earlier */
- hid_t dset_l_id, dset_p_id, dset_s_id;
- hid_t space_l_id, space_p_id, space_s_id;
- hrpl_t dset_p_replica;
- hid_t dxpl_p_id;
- int data_l[4], data_p[4], data_s[4];
-
- hid_t map_l_id, map_p_id, map_s_id;
- hrpl_t map_p_replica;
- hid_t mxpl_p_id;
- int map_entries; /* know same number in all maps */
- int *value_l, *value_p, *value_s;
-
- hid_t type_l_id, type_p_id, type_s_id;
- hrpl_t type_p_replica;
- hid_t txpl_p_id;
-
- /* Event stack */
- hid_t d_es_id; /* for prefetch of Dataset */
- hid_t mt_es_id; /* for prefetch of Map and dataType (shared event stack) */
- H5ES_status_t status;
-
- /* Create event stacks for managing asynchronous requests. */
- d_es_id = H5EScreate(); assert( d_es_id >= 0 );
- mt_es_id = H5EScreate(); assert( mt_es_id >= 0 );
-
- /* Reopen the container read-only and get a read context for highest CV */
- fapl_id = H5Pcreate( H5P_FILE_ACCESS ); assert( fapl_id >= 0 );
- ret = H5Pset_fapl_iod( fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL ); ASSERT_RET;
- file_id = H5Fopen_ff( file_name, H5F_ACC_RDONLY, fapl_id, &rc_id, H5_EVENT_STACK_NULL ); assert( file_id >= 0 );
- H5RCget_version( rc_id, &version );
- fprintf( stderr, "APP-r%d: Re-open %s\n", my_rank, file_name );
-
- /* Open Datasets, Maps and DataTypes in all 3 Groups */
- dset_l_id = H5Dopen_ff( file_id, "/G-logged/D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( dset_l_id >= 0 );
- space_l_id = H5Dget_space( dset_l_id ); assert ( space_l_id >= 0 );
-
- dset_p_id = H5Dopen_ff( file_id, "/G-prefetched/D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( dset_p_id >= 0 );
- space_p_id = H5Dget_space( dset_p_id ); assert ( space_p_id >= 0 );
-
- dset_s_id = H5Dopen_ff( file_id, "/G-stored/D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( dset_s_id >= 0 );
- space_s_id = H5Dget_space( dset_s_id ); assert ( space_s_id >= 0 );
-
- map_l_id = H5Mopen_ff( file_id, "/G-logged/M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( map_l_id >= 0 );
-
- map_p_id = H5Mopen_ff( file_id, "/G-prefetched/M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( map_p_id >= 0 );
-
- map_s_id = H5Mopen_ff( file_id, "/G-stored/M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( map_s_id >= 0 );
-
- type_l_id = H5Topen_ff( file_id, "G-logged/T", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( type_l_id >= 0 );
-
- type_p_id = H5Topen_ff( file_id, "G-prefetched/T", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( type_p_id >= 0 );
-
- type_s_id = H5Topen_ff( file_id, "G-stored/T", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( type_s_id >= 0 );
-
-
- /* Optionally, pause to show change in BB space using out-of-band check. All ranks paused. */
- if ( pause ) {
- MPI_Barrier( MPI_COMM_WORLD );
- if ( my_rank == 0 ) {
- fprintf( stderr, ">>> PAUSED before prefetch of replicas allow BB space usage to be checked.\n" );
- fprintf( stderr, ">>> Press return to continue..." );
- getchar();
- }
- MPI_Barrier( MPI_COMM_WORLD );
- }
-
- /* Prefetch objects in the /G-prefetched group. */
- if ( use_daos_lustre ) {
-
- /* Rank 0 prefetches asynchronously, using 2 different stacks */
- if ( my_rank == 0 ) {
- fprintf( stderr, "APP-r%d: Prefetch /G-prefetched/D asynchronously\n", my_rank );
- ret = H5Dprefetch_ff( dset_p_id, rc_id, &dset_p_replica, H5P_DEFAULT, d_es_id ); ASSERT_RET;
-
- fprintf( stderr, "APP-r%d: Prefetch /G-prefetched/M asynchronously\n", my_rank );
- ret = H5Mprefetch_ff( map_p_id, rc_id, &map_p_replica, H5P_DEFAULT, mt_es_id ); ASSERT_RET;
-
- fprintf( stderr, "APP-r%d: Prefetch /G-prefetched/T asynchronously\n", my_rank );
- ret = H5Tprefetch_ff( type_p_id, rc_id, &type_p_replica, H5P_DEFAULT, mt_es_id ); ASSERT_RET;
- }
-
- /* Note: could start some reads of non-prefetched objects here, but since I want to offer pause & read timing, I didn't */
-
- /* After dataset prefetch completes, bcast the replica ID and set up properties */
- if ( my_rank == 0 ) {
- H5ESwait_all( d_es_id, &status );
- fprintf( stderr, "APP-r%d: asynchronous prefetch of D completed with status %d\n", my_rank, status );
- }
- MPI_Bcast( &dset_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
- dxpl_p_id = H5Pcreate( H5P_DATASET_XFER ); /* Used in Read & Evict*/
- ret = H5Pset_dxpl_replica( dxpl_p_id, dset_p_replica ); ASSERT_RET;
-
- /* After both map and datatype prefetches complete, bcast the replica IDs and set up properties */
- if ( my_rank == 0 ) {
- H5ESwait_all( mt_es_id, &status );
- fprintf( stderr, "APP-r%d: asynchronous prefetch of M and T completed with status %d\n", my_rank, status );
- }
- MPI_Bcast( &map_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
- mxpl_p_id = H5Pcreate( H5P_DATASET_XFER ); /* Used in Get & Evict*/
- ret = H5Pset_dxpl_replica( mxpl_p_id, map_p_replica ); ASSERT_RET;
-
- MPI_Bcast( &type_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
- txpl_p_id = H5Pcreate( H5P_DATASET_XFER ); /* Used in Evict */
- ret = H5Pset_dxpl_replica( txpl_p_id, type_p_replica ); ASSERT_RET;
-
- } else {
- /* Without daos-lustre, we don't do prefetch, so default properties */
- dxpl_p_id = H5P_DEFAULT;
- mxpl_p_id = H5P_DEFAULT;
- txpl_p_id = H5P_DEFAULT;
- }
-
- /* Optionally, pause to show change in BB space using out-of-band check. All ranks paused. */
- if ( pause ) {
- MPI_Barrier( MPI_COMM_WORLD );
- if ( my_rank == 0 ) {
- fprintf( stderr, ">>> PAUSED after prefetch of replicas to allow BB space usage to be checked.\n" );
- fprintf( stderr, ">>> Press return to continue..." );
- getchar();
- }
- MPI_Barrier( MPI_COMM_WORLD );
- }
-
- /* Find entries in first Map, then use to create value vectors for all three Map objects - we know they have same size. */
- ret = H5Mget_count_ff( map_l_id, &map_entries, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- value_l = (int *)calloc( map_entries, sizeof(int) ); assert( value_l != NULL );
- value_p = (int *)calloc( map_entries, sizeof(int) ); assert( value_p != NULL );
- value_s = (int *)calloc( map_entries, sizeof(int) ); assert( value_s != NULL );
-
- /* Read Datasets and then Maps in all 3 Groups, measuring time it takes to read data for each*/
-
- if ( time_reads ) {
- ulong usec_dl, usec_dp, usec_ds; // per-read elapsed time for for datasets
- ulong usec_ml, usec_mp, usec_ms; // per-read elapsed time for maps
-
- ulong all_usec_dl, all_usec_dp, all_usec_ds; // total read time for datasets
- ulong all_usec_ml, all_usec_mp, all_usec_ms; // total read time for maps
-
- all_usec_dl = all_usec_dp = all_usec_ds = 0;
- all_usec_ml = all_usec_mp = all_usec_ms = 0;
-
- int t;
-
- for ( t = 0; t < time_reads; t++ ) {
-
- START_TIME;
- ret = H5Dread_ff( dset_l_id, H5T_NATIVE_INT, space_l_id, space_l_id, H5P_DEFAULT, data_l, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- usec_dl = ELAPSED_TIME;
- all_usec_dl += usec_dl;
- fprintf( stderr, "APP-r%d Read Time %d for /G-logged/D: %lu usec\n", my_rank, t, usec_dl );
-
- START_TIME;
- ret = H5Dread_ff( dset_p_id, H5T_NATIVE_INT, space_p_id, space_p_id, dxpl_p_id, data_p, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- usec_dp = ELAPSED_TIME;
- all_usec_dp += usec_dp;
- fprintf( stderr, "APP-r%d Read Time %d for /G-prefetched/D: %lu usec\n", my_rank, t, usec_dp );
-
- START_TIME;
- ret = H5Dread_ff( dset_s_id, H5T_NATIVE_INT, space_s_id, space_s_id, H5P_DEFAULT, data_s, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- usec_ds = ELAPSED_TIME;
- all_usec_ds += usec_ds;
- fprintf( stderr, "APP-r%d Read Time %d for /G-stored/D: %lu usec\n", my_rank, t, usec_ds );
-
- START_TIME;
- for ( i = 0; i < map_entries; i++ ) {
- ret = H5Mget_ff( map_l_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value_l[i], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- }
- END_TIME;
- usec_ml = ELAPSED_TIME;
- all_usec_ml += usec_ml;
- fprintf( stderr, "APP-r%d Read Time %d for /G-logged/M: %lu usec\n", my_rank, t, usec_ml );
-
- START_TIME;
- for ( i = 0; i < map_entries; i++ ) {
- ret = H5Mget_ff( map_p_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value_p[i], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- }
- END_TIME;
- usec_mp = ELAPSED_TIME;
- all_usec_mp += usec_mp;
- fprintf( stderr, "APP-r%d Read Time %d for /G-prefetched/M: %lu usec\n", my_rank, t, usec_mp );
-
- START_TIME;
- for ( i = 0; i < map_entries; i++ ) {
- ret = H5Mget_ff( map_s_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value_s[i], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- }
- END_TIME;
- usec_ms = ELAPSED_TIME;
- all_usec_ms += usec_ms;
- fprintf( stderr, "APP-r%d Read Time %d for /G-stored/M: %lu usec\n", my_rank, t, usec_ms );
-
- }
-
- if ( time_reads > 1 ) {
- fprintf( stderr, "APP-r%d Read Time Avg for /G-logged/D: %lu usec\n", my_rank, all_usec_dl/time_reads );
- fprintf( stderr, "APP-r%d Read Time Avg for /G-prefetched/D: %lu usec\n", my_rank, all_usec_dp/time_reads );
- fprintf( stderr, "APP-r%d Read Time Avg for /G-stored/D: %lu usec\n", my_rank, all_usec_ds/time_reads );
- fprintf( stderr, "APP-r%d Read Time Avg for /G-logged/M: %lu usec\n", my_rank, all_usec_ml/time_reads );
- fprintf( stderr, "APP-r%d Read Time Avg for /G-prefetched/M: %lu usec\n", my_rank, all_usec_mp/time_reads );
- fprintf( stderr, "APP-r%d Read Time Avg for /G-stored/M: %lu usec\n", my_rank, all_usec_ms/time_reads );
- }
-
- } else {
- fprintf( stderr, "APP-r%d Read /G-logged/D\n", my_rank );
- ret = H5Dread_ff( dset_l_id, H5T_NATIVE_INT, space_l_id, space_l_id, H5P_DEFAULT, data_l, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
-
- fprintf( stderr, "APP-r%d Read /G-prefetched/D\n", my_rank );
- ret = H5Dread_ff( dset_p_id, H5T_NATIVE_INT, space_p_id, space_p_id, dxpl_p_id, data_p, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
-
- fprintf( stderr, "APP-r%d Read /G-stored/D\n", my_rank );
- ret = H5Dread_ff( dset_s_id, H5T_NATIVE_INT, space_s_id, space_s_id, H5P_DEFAULT, data_s, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
-
- fprintf( stderr, "APP-r%d Read /G-logged/M\n", my_rank );
- for ( i = 0; i < map_entries; i++ ) {
- ret = H5Mget_ff( map_l_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value_l[i], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- }
-
- fprintf( stderr, "APP-r%d Read /G-prefetched/M\n", my_rank );
- for ( i = 0; i < map_entries; i++ ) {
- ret = H5Mget_ff( map_p_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value_p[i], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- }
-
- fprintf( stderr, "APP-r%d Read /G-stored/M\n", my_rank );
- for ( i = 0; i < map_entries; i++ ) {
- ret = H5Mget_ff( map_s_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value_s[i], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- }
- }
-
- /* Print Dataset and Map values for all 3 groups */
- fprintf( stderr, "APP-r%d: /G-logged/D data: %d %d %d %d\n", my_rank, data_l[0], data_l[1], data_l[2], data_l[3] );
- fprintf( stderr, "APP-r%d: /G-prefetched/D data: %d %d %d %d\n", my_rank, data_p[0], data_p[1], data_p[2], data_p[3]);
- fprintf( stderr, "APP-r%d: /G-stored/D data: %d %d %d %d\n", my_rank, data_s[0], data_s[1], data_s[2], data_s[3] );
-
- if ( map_entries == 4 ) {
- fprintf( stderr, "APP-r%d: /G-logged/M key/values: 0/%d 1/%d 2/%d 3/%d\n", my_rank,
- value_l[0], value_l[1], value_l[2], value_l[3] );
- fprintf( stderr, "APP-r%d: /G-prefetched/M key/values: 0/%d 1/%d 2/%d 3/%d\n", my_rank,
- value_p[0], value_p[1], value_p[2], value_p[3] );
- fprintf( stderr, "APP-r%d: /G-stored/M key/values: 0/%d 1/%d 2/%d 3/%d\n", my_rank,
- value_s[0], value_s[1], value_s[2], value_s[3] );
- } else {
- fprintf( stderr, "APP-r%d: /G-logged/M key/values: 0/%d 1/%d 2/%d 3/%d ... %d/%d\n", my_rank,
- value_l[0], value_l[1], value_l[2], value_l[3], map_entries-1, value_l[map_entries-1] );
- fprintf( stderr, "APP-r%d: /G-prefetched/M key/values: 0/%d 1/%d 2/%d 3/%d ... %d/%d\n", my_rank,
- value_p[0], value_p[1], value_p[2], value_p[3], map_entries-1, value_p[map_entries-1] );
- fprintf( stderr, "APP-r%d: /G-stored/M key/values: 0/%d 1/%d 2/%d 3/%d ... %d/%d\n", my_rank,
- value_s[0], value_s[1], value_s[2], value_s[3], map_entries-1, value_s[map_entries-1] );
- }
- free( value_l );
- free( value_p );
- free( value_s );
-
- /* wait for all ranks to complete read before evicting replicas */
- MPI_Barrier( MPI_COMM_WORLD );
-
- /* Rank 0 evicts replicas */
- if ( my_rank == 0 ) {
- if ( use_daos_lustre ) {
- fprintf( stderr, "APP-r%d: Evict replica of /G-prefetched/D\n", my_rank );
- ret = H5Devict_ff( dset_p_id, version, dxpl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET; /* Note: CV redundant for replica */
-
- fprintf( stderr, "APP-r%d: Evict replica of /G-prefetched/M (Q7 note: currently a no-op in IOD)\n", my_rank );
- ret = H5Mevict_ff( map_p_id, version, mxpl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET; /* Note: CV redundant for replica */
-
- fprintf( stderr, "APP-r%d: Evict replica of /G-prefetched/T\n", my_rank );
- ret = H5Tevict_ff( type_p_id, version, txpl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET; /* Note: CV redundant for replica */
- } else {
- fprintf( stderr, "APP-r%d DAOS-POSIX - No replicas to evict\n", my_rank );
- }
- }
-
- /* Optionally, pause to show change in BB space using out-of-band check. All ranks paused. */
- if ( pause ) {
- MPI_Barrier( MPI_COMM_WORLD );
- if ( my_rank == 0 ) {
- fprintf( stderr, ">>> PAUSED after evict of replicas to allow BB space usage to be checked.\n" );
- fprintf( stderr, ">>> Press return to continue..." );
- getchar();
- }
- MPI_Barrier( MPI_COMM_WORLD );
- }
-
- /* Close Objects*/
-
- if ( use_daos_lustre ) {
- ret = H5Pclose( mxpl_p_id ); ASSERT_RET;
- ret = H5Pclose( dxpl_p_id ); ASSERT_RET;
- ret = H5Pclose( txpl_p_id ); ASSERT_RET;
- }
-
- ret = H5Dclose_ff( dset_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Dclose_ff( dset_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Dclose_ff( dset_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- ret = H5Sclose( space_l_id ); ASSERT_RET;
- ret = H5Sclose( space_p_id ); ASSERT_RET;
- ret = H5Sclose( space_s_id ); ASSERT_RET;
-
- ret = H5Mclose_ff( map_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Mclose_ff( map_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Mclose_ff( map_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- ret = H5Tclose_ff( type_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Tclose_ff( type_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Tclose_ff( type_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Release the read handle and close read context */
- ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
- ret = H5RCclose( rc_id ); ASSERT_RET;
-
- /* Close H5 Objects that are still open */
- fprintf( stderr, "APP-r%d: Close %s\n", my_rank, file_name );
- MPI_Barrier( MPI_COMM_WORLD );
- ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Pclose( fapl_id ); ASSERT_RET;
-
- /* Perform wrap-up operations */
- fprintf( stderr, "APP-r%d: Finalize EFF stack\n", my_rank );
- MPI_Barrier( MPI_COMM_WORLD );
- EFF_finalize();
- MPI_Finalize();
-
- return 0;
-}
-
-
-/*
- * Helper function used to create and initialize dataset "dset_name"
- * in object identified by "obj_id"
- * in transaction identified by "tr_id".
- * "obj_path" and "my_rank" are used in the status output.
- * "ordinal" is used as a multiplier in computing the cell value.
- *
- * All datasets are 1D Arrays with 4 elements.
- * Cell values are computed using the formula:
- * data[i] = ordinal*1000 + tr_num*100 + my_rank*10 + i;
- */
-void
-create_dataset( hid_t obj_id, const char* dset_name, hid_t tr_id, const char* obj_path, int my_rank, int ordinal ) {
-
- herr_t ret;
- uint64_t tr_num;
- int i;
- hsize_t current_size = 4; /* For now, we hard-code to small number */
- hsize_t max_size = 4;
- int data[4];
- hid_t space_id;
- hid_t dtype_id;
- hid_t dset_id;
-
- /* Setup */
- ret = H5TRget_trans_num( tr_id, &tr_num ); ASSERT_RET;
- for ( i = 0; i < current_size; i++ ) {
- data[i] = ordinal*1000 + tr_num*100 + my_rank*10 + i;
- }
- space_id = H5Screate_simple( 1, &current_size, &max_size ); assert( space_id >= 0 );
- dtype_id = H5Tcopy( H5T_NATIVE_INT ); assert( dtype_id >= 0 );
-
- /* Create & Populate */
- dset_id = H5Dcreate_ff( obj_id, dset_name, dtype_id, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL );
- if ( dset_id >= 0 ) {
- fprintf( stderr, "APP-r%d: tr %lu - Create %s%s with data: %d %d %d %d\n",
- my_rank, tr_num, obj_path, dset_name, data[0], data[1], data[2], data[3] );
- ret = H5Dwrite_ff( dset_id, dtype_id, space_id, space_id, H5P_DEFAULT, data, tr_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- } else {
- fprintf( stderr, "APP-r%d: tr %lu - Create %s%s - FAILED\n", my_rank, tr_num, obj_path, dset_name );
- }
- ret = H5Tclose( dtype_id ); ASSERT_RET;
- ret = H5Sclose( space_id ); ASSERT_RET;
- return;
-}
-
-/*
- * Helper function used to create and initialize map "map_name"
- * in object identified by "obj_id"
- * in transaction identified by "tr_id".
- * "obj_path" and "my_rank" are used in the status output.
- * "ordinal" is used as a multiplier in computing the value.
- *
- * All maps are created with 4 initial KV pairs.
- * Keys/values are computed using the formula:
- * key=i / value=(ordinal*1000 + tr_num*100 + my_rank*10 + i)
- */
-void
-create_map( hid_t obj_id, const char* map_name, hid_t tr_id, const char* obj_path, int my_rank, int ordinal ) {
-
- herr_t ret;
- uint64_t tr_num;
- int i;
- int num_kvs = 4; /* Start with 4 KV pairs */
- int value[4];
- hid_t map_id;
-
- /* Setup */
- ret = H5TRget_trans_num( tr_id, &tr_num ); ASSERT_RET;
- for ( i = 0; i < num_kvs; i++ ) {
- value[i] = ordinal*1000 + tr_num*100 + my_rank*10 + i;
- }
-
- /* Create & Populate */
- map_id = H5Mcreate_ff( obj_id, map_name, H5T_STD_I32LE, H5T_STD_I32LE, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL );
- if ( map_id >= 0 ) {
- fprintf( stderr, "APP-r%d: tr %lu - Create %s%s with keys/values: 0/%d 1/%d 2/%d 3/%d\n",
- my_rank, tr_num, obj_path, map_name, value[0], value[1], value[2], value[3] );
- for ( i = 0; i < num_kvs; i++ ) {
- ret = H5Mset_ff( map_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value[i], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- }
- ret = H5Mclose_ff( map_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- } else {
- fprintf( stderr, "APP-r%d: tr %lu - Create %s%s - FAILED\n", my_rank, tr_num, obj_path, map_name );
- }
- return;
-}
-
-/*
- * Helper function used to create named datatype "type_name"
- * in group identified by "obj_id"
- * in transaction identified by "tr_id".
- * "obj_path" and "my_rank" are used in the status output.
- */
-void
-create_named_datatype( hid_t obj_id, const char* type_name, hid_t tr_id, const char* obj_path, int my_rank ) {
-
- herr_t ret;
- uint64_t tr_num;
- hid_t dtype_id;
-
- /* Setup */
- ret = H5TRget_trans_num( tr_id, &tr_num ); ASSERT_RET;
- dtype_id = H5Tcopy( H5T_NATIVE_INT ); assert( dtype_id >= 0 );
-
- /* Create */
- ret = H5Tcommit_ff( obj_id, type_name, dtype_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- fprintf( stderr, "APP-r%d: tr %lu - Create %s%s %s\n", my_rank, tr_num, obj_path, type_name, STATUS );
- ret = H5Tclose( dtype_id ); ASSERT_RET;
- return;
-}
-
-/*
- * Helper function used to update raw data in "dset_name"
- * in object identified by "obj_id"
- * in transaction identified by "tr_id".
- * "rc_id" for "tr_id" passed in explicitly for now - later may be able to get it from info in tr_id (cv & file_id)
- * "obj_path" and "my_rank" are used in the status output.
- * "ordinal" is used as a multiplier in computing the cell value.
- *
- * Cell value to be updated is computed using the formula:
- * data[my_rank] = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
- *
- * If my_rank >= max_size of data, then no update done in call.
- */
-void
-update_dataset( hid_t obj_id, const char* dset_name, hid_t tr_id, hid_t rc_id, const char* obj_path, int my_rank, int ordinal ){
- herr_t ret;
- uint64_t tr_num;
- hid_t dset_id;
-
- /* Setup */
- ret = H5TRget_trans_num( tr_id, &tr_num ); ASSERT_RET;
-
- dset_id = H5Dopen_ff( obj_id, dset_name, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- if ( dset_id >= 0 ) {
-
- /* Dataset opened; confirm as we expect */
- hid_t filespace_id;
- int nDims;
- hsize_t current_size, max_size;
-
- filespace_id = H5Dget_space( dset_id ); assert ( filespace_id >= 0 );
- nDims = H5Sget_simple_extent_dims( filespace_id, &current_size, &max_size );
- assert( nDims == 1 );
- assert( current_size == 4 );
- assert( max_size == 4 );
-
- if ( my_rank < current_size ) {
- /* If rank in bounds of array, do the update */
- int cell_data;
- hsize_t mem_size = 1;
- hid_t memspace_id;
- hsize_t index;
-
- cell_data = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
-
- memspace_id = H5Screate_simple( 1, &mem_size, &mem_size ); assert( memspace_id >= 0 );
- index = (hsize_t)my_rank;
- ret = H5Sselect_elements( filespace_id, H5S_SELECT_SET, 1, &index ); ASSERT_RET;
-
- ret = H5Dwrite_ff( dset_id, H5T_NATIVE_INT, memspace_id, filespace_id, H5P_DEFAULT, &cell_data, tr_id,
- H5_EVENT_STACK_NULL ); ASSERT_RET;
- fprintf( stderr, "APP-r%d: tr %lu - Update %s%s[%d] to %d %s\n",
- my_rank, tr_num, obj_path, dset_name, my_rank, cell_data, STATUS );
- ret = H5Sclose( memspace_id ); ASSERT_RET;
-
- } else {
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: tr %lu - No Update to %s%s for this rank \n", my_rank, tr_num, obj_path, dset_name );
- }
- }
-
- ret = H5Sclose( filespace_id ); ASSERT_RET;
- ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- } else {
- fprintf( stderr, "APP-r%d: tr %lu - Update %s%s - FAILED\n", my_rank, tr_num, obj_path, dset_name );
- }
- return;
-}
-
-/*
- * Helper function used to update (or add) key/value data in "map_name"
- * in object identified by "obj_id"
- * in transaction identified by "tr_id".
- * "rc_id" for "tr_id" passed in explicitly for now - later may be able to get it from info in tr_id (cv & file_id)
- * "obj_path" and "my_rank" are used in the status output.
- * key/value set according to:
- * key = my_rank
- * value = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
- */
-void
-update_map( hid_t obj_id, const char* map_name, hid_t tr_id, hid_t rc_id, const char* obj_path, int my_rank, int ordinal ){
- herr_t ret;
- uint64_t tr_num;
- hid_t map_id;
- int value;
-
- /* Setup */
- ret = H5TRget_trans_num( tr_id, &tr_num ); ASSERT_RET;
-
- /* Open the map */
- map_id = H5Mopen_ff( obj_id, map_name, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
-
- if ( map_id >= 0 ) {
- value = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
- ret = H5Mset_ff( map_id, H5T_STD_I32LE, &my_rank, H5T_STD_I32LE, &value, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- if ( verbose || ( ret < 0 ) ) {
- fprintf( stderr, "APP-r%d: tr %lu - Update %s%s with key/value %d/%d %s\n",
- my_rank, tr_num, obj_path, map_name, my_rank, value, STATUS );
- }
-
- ret = H5Mclose_ff( map_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- } else {
- fprintf( stderr, "APP-r%d: tr %lu - Update %s%s - FAILED\n", my_rank, tr_num, obj_path, map_name );
- }
- return;
-}
-
-/*
- * evict_group_members_updates - evict logged updates for (expected) members of specified group
- * parameters:
- * file_id: identifies container
- * rc_id: identifies read context
- * grp_path: path that specifies group
- * my_rank: identifies rank doing the evict
- */
-void
-evict_group_members_updates( hid_t file_id, hid_t rc_id, const char* grp_path, int my_rank )
-{
- uint64_t cv; char preface[128];
- char path_to_object[1024];
-
- hid_t dset_id, map_id, type_id;
- herr_t ret;
- int i;
-
- /* Get the container version for the read context and set up print preface */
- ret = H5RCget_version( rc_id, &cv ); ASSERT_RET;
- sprintf( preface, "APP-r%d: cv %lu -", my_rank, cv );
-
- /* Dataset */
- sprintf( path_to_object, "%s/%s", grp_path, "D" );
- dset_id = H5Dopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( dset_id >= 0 );
-
- if ( use_daos_lustre ) {
- ret = H5Devict_ff( dset_id, cv, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
- fprintf( stderr, "%s Evicted updates for %s\n", preface, path_to_object );
- } else {
- fprintf( stderr, "%s DAOS-POSIX - Won't evict updates for object %s\n", preface, path_to_object );
- }
- ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Map */
- sprintf( path_to_object, "%s/%s", grp_path, "M" );
- map_id = H5Mopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( map_id >= 0 );
-
- if ( use_daos_lustre ) {
- ret = H5Mevict_ff( map_id, cv, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
- fprintf( stderr, "%s Evicted updates for %s (Q7 note: currently a no-op in IOD)\n", preface, path_to_object );
- } else {
- fprintf( stderr, "%s DAOS-POSIX - Won't evict updates for object %s\n", preface, path_to_object );
- }
- ret = H5Mclose_ff( map_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Named Datatype */
- sprintf( path_to_object, "%s/%s", grp_path, "T" );
- type_id = H5Topen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( type_id >= 0 );
-
- if ( use_daos_lustre ) {
- ret = H5Tevict_ff( type_id, cv, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
- fprintf( stderr, "%s Evicted updates for %s\n", preface, path_to_object );
- } else {
- fprintf( stderr, "%s DAOS-POSIX - Won't evict updates for object %s\n", preface, path_to_object );
- }
- ret = H5Tclose_ff( type_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- return;
-}
-
-
-/*
- * Helper function used to recursively read and print container contents
- * for container identified by "file_id"
- * in read context identified by "rc_id"
- * with path to current level in "grp_path"
- * and "my_rank" used to identify the process doing the reading / printing.
- */
-void
-print_container_contents( hid_t file_id, hid_t rc_id, const char* grp_path, int my_rank )
-{
- herr_t ret;
- uint64_t cv;
- htri_t exists;
- char path_to_object[1024];
- char name[30];
- int i;
-
- static int lvl = 0; /* level in recursion - used to format printing */
- char preface[128];
- char line[1024];
-
- /* Get the container version for the read context */
- ret = H5RCget_version( rc_id, &cv ); ASSERT_RET;
-
- /* Set up the preface and adding version number */
- sprintf( preface, "APP-r%d: cv %d: ", my_rank, (int)cv );
-
- /* Start the printing */
- if ( lvl == 0 ) {
- fprintf( stderr, "%s ----- Container Contents ------------\n", preface );
- }
-
- /* Datasets */
- sprintf( path_to_object, "%s%s", grp_path, "D" );
- ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
-
- if ( exists ) {
- hid_t dset_id;
- hid_t space_id;
- int nDims;
- hsize_t current_size[1];
- hsize_t max_size[1];
- hsize_t totalSize;
- int *data;
- int i;
-
- dset_id = H5Dopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( dset_id >= 0 );
-
- space_id = H5Dget_space( dset_id ); assert ( space_id >= 0 );
- nDims = H5Sget_simple_extent_dims( space_id, current_size, max_size );
- assert( nDims == 1 );
-
- totalSize = current_size[0];
- data = (int *)calloc( totalSize, sizeof(int) ); assert( data != NULL );
-
- ret = H5Dread_ff( dset_id, H5T_NATIVE_INT, space_id, space_id, H5P_DEFAULT, data, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- sprintf( line, "%s %s data: ", preface, path_to_object );
- for ( i = 0; i < totalSize; i++ ) {
- sprintf( line, "%s%d ", line, data[i] );
- }
-
- fprintf( stderr, "%s\n", line );
- free( data );
-
- ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Sclose( space_id ); ASSERT_RET;
- }
-
- /* Maps */
- sprintf( path_to_object, "%s%s", grp_path, "M" );
- ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
-
- if ( exists ) {
- hid_t map_id;
- hsize_t totalCount;
- int *value;
- int i;
-
- map_id = H5Mopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( map_id >= 0 );
-
- ret = H5Mget_count_ff( map_id, &totalCount, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- value = (int *)calloc( totalCount, sizeof(int) ); assert( value != NULL );
-
- for ( i = 0; i < totalCount; i++ ) {
- ret = H5Mget_ff( map_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value[i], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- }
-
- sprintf( line, "%s %s key/values: ", preface, path_to_object );
- for ( i = 0; i < totalCount; i++ ) {
- sprintf( line, "%s %d/%d ", line, i, value[i] );
- }
-
- fprintf( stderr, "%s\n", line );
- free( value );
-
- ret = H5Mclose_ff( map_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- }
-
- /* Named Datatypes */
- sprintf( path_to_object, "%s%s", grp_path, "T" );
- ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
-
- if ( exists ) {
- fprintf( stderr, "%s %s\n", preface, path_to_object );
- }
-
- /* Groups - if found, descend */
- for ( i = 1; i < 4; i++ ) {
- if ( i == 1 ) {
- strcpy( name, "G-logged" );
- } else if ( i == 2 ) {
- strcpy( name, "G-prefetched" );
- } else if ( i == 3 ) {
- strcpy( name, "G-stored" );
- }
-
- sprintf( path_to_object, "%s%s/", grp_path, name );
- ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
- if ( exists ) {
- fprintf( stderr, "%s %s\n", preface, path_to_object );
- lvl++;
- print_container_contents( file_id, rc_id, path_to_object, my_rank );
- lvl--;
- }
- }
-
- /* End printing */
- if ( lvl == 0 ) {
- fprintf( stderr, "%s -----------------\n", preface );
- }
-
- return;
-}
-
-/*
- * parse_options - parse command line options
- */
-int
-parse_options( int argc, char** argv, int my_rank ) {
- int i, n;
- char* app = argv[0];
-
- while ( --argc ) {
- if ( **(++argv) != '-' ) {
- break;
- } else {
- switch( *(*argv+1) ) {
- case 'l':
- use_daos_lustre = 0;
- break;
- case 'p':
- pause = 1;
- break;
- case 't':
- if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
- if ( my_rank == 0 ) {
- printf( "Error: No repeat_cnt specified after -t option.\n" );
- usage( app );
- }
- return( 1 );
- } else {
- ++argv;
- time_reads = atoi( *argv );
- if ( my_rank == 0 ) {
- printf( "Will repeat read sequence %d times to get average elapsed time per read.\n", time_reads );
- }
- }
- break;
- case 'v':
- verbose = 1;
- break;
- default:
- if ( my_rank == 0 ) {
- usage( app );
- }
- return( 1 );
- }
- }
- }
- return( 0 );
-}
-
-/*
- * usage - display usage message
- */
-void
-usage( const char* app ) {
- printf( "Usage: %s [-l] [-p] [-t repeat_cnt] [-v] [-w]\n", app );
- printf( "\tl: don't use DAOS Lustre\n" );
- printf( "\tp: pause to allow out-of-band BB space check\n" );
- printf( "\tt: time reads after container re-open, performing read sequence 'repeat_cnt' times\n" );
- printf( "\tv: verbose output\n" );
-}
+/*
+ * h5ff_client_M7.2-pep_demo.c: Demo Persist / Evict / Prefetch
+ * Part of Milestone 7.2 - Functionality of Burst Buffer.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include <sys/time.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+/* define filename for this app, and max size after username prepended */
+#define FILENAME_APP "eff_pep.h5"
+#define FILENAME_SIZE 256
+
+/* macros related to error reporting */
+#define STATUS (ret >= 0) ? " " : " - FAILED"
+#define ASSERT_RET assert( ret >= 0 )
+
+/* option flags */
+int use_daos_lustre = 1; // Use DAOS Lustre - defaults to yes
+int verbose = 0; // Verbose output - defaults to no
+int pause = 0; // Pause to allow out-of-band space check - defaults to 0.
+int time_reads = 0; // Time reads after container re-open
+
+/* global variables and macros used to make timing easier */
+struct timeval tv_start;
+struct timeval tv_end;
+#define START_TIME gettimeofday( &tv_start, NULL )
+#define END_TIME gettimeofday( &tv_end, NULL )
+#define ELAPSED_TIME (ulong)( (tv_end.tv_usec + 1000000*tv_end.tv_sec) - (tv_start.tv_usec + 1000000*tv_start.tv_sec) )
+
+/* function prototypes */
+void create_dataset( hid_t, const char*, hid_t, const char*, int, int );
+void create_map( hid_t, const char*, hid_t, const char*, int, int );
+void create_named_datatype( hid_t, const char*, hid_t, const char*, int );
+void update_dataset( hid_t, const char*, hid_t, hid_t, const char*, int, int );
+void update_map( hid_t, const char*, hid_t, hid_t, const char*, int, int );
+void evict_group_members_updates( hid_t, hid_t, const char*, int );
+void print_container_contents( hid_t, hid_t, const char*, int );
+int parse_options( int, char**, int );
+void usage( const char* );
+
+/**************************************************************************************************************/
+int main( int argc, char **argv ) {
+
+ /* MPI */
+ int my_rank, comm_size, acquire_rank;
+ int provided;
+
+ /* Container */
+ char *user_name; /* We'll prepend username to make filename unique */
+ char file_name[FILENAME_SIZE]; /* Actual filename to be opened will be user_name + FILENAME_APP */
+ hid_t fapl_id;
+ hid_t file_id;
+
+ /* Groups */
+ hid_t grp_l_id;
+ hid_t grp_p_id;
+ hid_t grp_s_id;
+
+ /* Container Version & Read Contexts */
+ uint64_t version;
+ hid_t rc_id1, rc_id2, rc_id3, rc_id4, rc_id5, rc_id;
+
+ /* Transactions */
+ uint64_t tr_num;
+ hid_t tr_id;
+ hid_t trspl_id;
+ int num_tr_leaders;
+
+ herr_t ret;
+ int i;
+
+ /****
+ * Initialize and create container
+ ****/
+
+ /* Confirm needed MPI functionality is available */
+ MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
+ if ( MPI_THREAD_MULTIPLE != provided ) {
+ fprintf( stderr, "APP: ERROR: MPI does not have MPI_THREAD_MULTIPLE support\n" );
+ exit( 1 );
+ }
+
+ /* Find MPI rank & communicator size */
+ MPI_Comm_rank( MPI_COMM_WORLD, &my_rank );
+ MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
+ if ( my_rank == 0 ) {
+ fprintf( stderr, "APP-r%d: Number of MPI processes = %d\n", my_rank, comm_size );
+ }
+
+ if ( comm_size > 3 ) {
+ acquire_rank = 3;
+ } else {
+ acquire_rank = 0;
+ }
+
+ /* Parse command-line options controlling behavior */
+ if ( parse_options( argc, argv, my_rank ) != 0 ) {
+ exit( 1 );
+ }
+
+ /* Initialize the EFF stack, starting FS client, registering HDF5 VOL calls, requesting IOD init */
+ fprintf( stderr, "APP-r%d: Initialize EFF stack\n", my_rank );
+ EFF_init( MPI_COMM_WORLD, MPI_INFO_NULL );
+
+ /* Specify the IOD VOL plugin should be used and create H5File (EFF container) */
+ user_name = getenv( "USER" );
+ snprintf( file_name, FILENAME_SIZE, "%s_%s", user_name, FILENAME_APP );
+ fprintf( stderr, "APP-r%d: Create %s\n", my_rank, file_name );
+ fapl_id = H5Pcreate( H5P_FILE_ACCESS ); assert( fapl_id >= 0 );
+ ret = H5Pset_fapl_iod( fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL ); ASSERT_RET;
+ file_id = H5Fcreate_ff( file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL ); assert( file_id >= 0 );
+
+ /****
+ * Transaction 2: Rank 0 creates three H5Groups in the H5File
+ ****/
+
+ if ( my_rank == 0 ) {
+ /* Get read context use for Tr 2 */
+ version = 1;
+ rc_id1 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ assert( rc_id1 >= 0 ); assert( version == 1 );
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
+
+ /* Create a local transaction for transaction number and start it with a single tr leader (the default). */
+ tr_num = 2;
+ fprintf( stderr, "APP-r%d: tr %lu - Start\n", my_rank, tr_num );
+ tr_id = H5TRcreate( file_id, rc_id1, tr_num ); assert( tr_id >= 0 );
+ ret = H5TRstart( tr_id, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Add updates to the transaction for Group creates */
+ fprintf( stderr, "APP-r%d: tr %d - Create /G-logged /G-prefetched /G-stored\n", my_rank, tr_num );
+ grp_l_id = H5Gcreate_ff( file_id, "G-logged", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ assert( grp_l_id >= 0 );
+ grp_p_id = H5Gcreate_ff( file_id, "G-prefetched", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ assert( grp_p_id >= 0 );
+ grp_s_id = H5Gcreate_ff( file_id, "G-stored", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ assert( grp_s_id >= 0 );
+
+ /* Finish and commit TR and get RC */
+ fprintf( stderr, "APP-r%d: tr %lu - Finish\n", my_rank, tr_num );
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, &rc_id2, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ assert( rc_id2 >= 0 );
+ version = 2;
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+
+ /* Release the read handle & close read context on previous CV - only held by rank 0 */
+ ret = H5RCrelease( rc_id1, H5_EVENT_STACK_NULL); ASSERT_RET;
+ ret = H5RCclose( rc_id1 ); ASSERT_RET;
+
+ /* If verbose, rank 0 prints the container contents */
+ if ( verbose ) print_container_contents( file_id, rc_id2, "/", my_rank );
+ }
+
+ /* Rank 0 tells others that RC 2 has been acquired; they create local RC object and open the Groups */
+ MPI_Bcast( &version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+ if ( my_rank != 0 ) {
+ rc_id2 = H5RCcreate( file_id, version );
+ assert( rc_id2 >= 0 ); assert ( version == 2 );
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
+
+ grp_l_id = H5Gopen_ff( file_id, "G-logged", H5P_DEFAULT, rc_id2, H5_EVENT_STACK_NULL ); assert( grp_l_id >= 0 );
+ grp_p_id = H5Gopen_ff( file_id, "G-prefetched", H5P_DEFAULT, rc_id2, H5_EVENT_STACK_NULL ); assert( grp_p_id >= 0 );
+ grp_s_id = H5Gopen_ff( file_id, "G-stored", H5P_DEFAULT, rc_id2, H5_EVENT_STACK_NULL ); assert( grp_s_id >= 0 );
+ }
+
+ /****
+ * Transaction 3 - In each of the 3 Groups, create "D"(rnk 0), "M"(rnk 1 or 0), "T" (rnk 2 or 1 or 0)
+ ****/
+
+ if ( my_rank < 3 ) {
+ tr_num = 3;
+ num_tr_leaders = ( (comm_size < 4) ? comm_size : 3 );
+
+ /* Create a local transaction for transaction number and start it. */
+ fprintf( stderr, "APP-r%d: tr %lu - Start with %d leaders\n", my_rank, tr_num, num_tr_leaders );
+ tr_id = H5TRcreate( file_id, rc_id2, tr_num ); assert( tr_id >= 0 );
+ trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
+ ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
+ ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ char glog_path[128], gpre_path[128], gsto_path[128];
+
+ /* Set pathnames to groups used in status messages */
+ sprintf( glog_path, "/G-logged/" );
+ sprintf( gpre_path, "/G-prefetched/" );
+ sprintf( gsto_path, "/G-stored/" );
+
+ /* Add updates to the transaction */
+ if ( my_rank == 0 ) {
+ create_dataset( grp_l_id, "D", tr_id, glog_path, my_rank, 1 );
+ create_dataset( grp_p_id, "D", tr_id, gpre_path, my_rank, 2 );
+ create_dataset( grp_s_id, "D", tr_id, gsto_path, my_rank, 3 );
+ }
+ if ( ( my_rank == 1 ) || ( ( my_rank == 0 ) && ( comm_size == 1 ) ) ) {
+ create_map( grp_l_id, "M", tr_id, glog_path, my_rank, 1 );
+ create_map( grp_p_id, "M", tr_id, gpre_path, my_rank, 2 );
+ create_map( grp_s_id, "M", tr_id, gsto_path, my_rank, 3 );
+ }
+ if ( ( my_rank == 2 ) || ( ( my_rank == 1 ) && ( comm_size == 2 ) ) || ( ( my_rank == 0 ) && ( comm_size == 1 ) ) ) {
+ create_named_datatype( grp_l_id, "T", tr_id, glog_path, my_rank );
+ create_named_datatype( grp_p_id, "T", tr_id, gpre_path, my_rank );
+ create_named_datatype( grp_s_id, "T", tr_id, gsto_path, my_rank );
+ }
+
+ /* Finish, commit, and close transaction */
+ fprintf( stderr, "APP-r%d: tr %lu - Finish\n", my_rank, tr_num );
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ ret = H5Pclose( trspl_id ); ASSERT_RET;
+ }
+
+ /* Acquire a read handle for container version and create a read context. */
+ version = 3;
+ if ( verbose ) fprintf( stderr, "APP-r%d: rc %lu - Try to acquire\n", my_rank, version );
+
+ if ( acquire_rank == my_rank ) {
+ H5E_BEGIN_TRY {
+ rc_id3 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ while ( rc_id3 < 0 ) {
+ if ( verbose ) fprintf( stderr, "APP-r%d: rc %lu - Failed to acquire; sleep then retry\n", my_rank, version );
+ sleep( 1 );
+ rc_id3 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ }
+ } H5E_END_TRY;
+ }
+
+ MPI_Bcast( &version, 1, MPI_UINT64_T, acquire_rank, MPI_COMM_WORLD );
+ if ( acquire_rank != my_rank ) {
+ rc_id3 = H5RCcreate( file_id, version );
+ }
+ assert( rc_id3 >= 0 ); assert ( version == 3 );
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
+
+ /* Release the read handle and close read context on previous CV */
+ if ( my_rank == 0 ) {
+ ret = H5RCrelease( rc_id2, H5_EVENT_STACK_NULL); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc 2 - Released\n", my_rank );
+ }
+ ret = H5RCclose( rc_id2 ); ASSERT_RET;
+
+ /****
+ * Rank 0 persists CV 3
+ *
+ * Satisfies: An application will create multiple transactions in a container, then persist them
+ * from the BB to persistent storage.
+ ****/
+
+ if ( my_rank == 0 ) {
+ fprintf( stderr, "APP-r%d: cv 3 - Persist\n", my_rank );
+ ret = H5RCpersist(rc_id3, H5_EVENT_STACK_NULL); ASSERT_RET;
+ print_container_contents( file_id, rc_id3, "/", my_rank );
+ }
+
+ if ( verbose ) print_container_contents( file_id, rc_id3, "/", my_rank );
+
+ /****
+ * Transaction 4 - All ranks update Dataset and Map objects in the 3 Groups
+ ****/
+
+ tr_num = 4;
+ num_tr_leaders = comm_size;
+
+ /* Create a local transaction for transaction number and start it. */
+ fprintf( stderr, "APP-r%d: tr %lu - Start with %d leaders\n", my_rank, tr_num, num_tr_leaders );
+ tr_id = H5TRcreate( file_id, rc_id3, tr_num ); assert( tr_id >= 0 );
+ trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
+ ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
+ ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Add updates to the transaction */
+ update_dataset( file_id, "G-logged/D", tr_id, rc_id3, "/", my_rank, 1 );
+ update_dataset( file_id, "G-prefetched/D", tr_id, rc_id3, "/", my_rank, 2 );
+ update_dataset( file_id, "G-stored/D", tr_id, rc_id3, "/", my_rank, 3 );
+
+ update_map( file_id, "G-logged/M", tr_id, rc_id3, "/", my_rank, 1 );
+ update_map( file_id, "G-prefetched/M", tr_id, rc_id3, "/", my_rank, 2 );
+ update_map( file_id, "G-stored/M", tr_id, rc_id3, "/", my_rank, 3 );
+
+ /* Finish and close transaction */
+ fprintf( stderr, "APP-r%d: tr %lu - Finish\n", my_rank, tr_num );
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ ret = H5Pclose( trspl_id ); ASSERT_RET;
+
+ /* If verbose mode, get RC 4 and print CV 4 */
+ if ( verbose ) {
+ version = 4;
+ if ( my_rank == 0 ) {
+ fprintf( stderr, "APP-r%d: rc %lu - Try to acquire\n", my_rank, version );
+ H5E_BEGIN_TRY {
+ rc_id4 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ while ( rc_id4 < 0 ) {
+ fprintf( stderr, "APP-r%d: rc %lu - Failed to acquire; sleep then retry\n", my_rank, version );
+ sleep( 1 );
+ rc_id4 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ }
+ } H5E_END_TRY;
+ }
+ MPI_Bcast( &version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+ if ( my_rank != 0 ) {
+ rc_id4 = H5RCcreate(file_id, version);
+ }
+ assert( rc_id4 >= 0 ); assert ( version == 4 );
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
+ print_container_contents( file_id, rc_id4, "/", my_rank );
+
+ /* Release the read handle and close read context on CV 4 */
+ MPI_Barrier( MPI_COMM_WORLD );
+ if ( my_rank == 0 ) {
+ ret = H5RCrelease( rc_id4, H5_EVENT_STACK_NULL); ASSERT_RET;
+ }
+ ret = H5RCclose( rc_id4 ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc 4 - Released\n", my_rank );
+ }
+
+ /****
+ * Transaction 5 (based on CV 3) - All ranks update Dataset and Map objects in the 3 Groups
+ ****/
+
+ tr_num = 5;
+ num_tr_leaders = comm_size;
+
+ /* Create a local transaction for transaction number and start it. */
+ fprintf( stderr, "APP-r%d: tr %lu - Start with %d leaders\n", my_rank, tr_num, num_tr_leaders );
+ tr_id = H5TRcreate( file_id, rc_id3, tr_num ); assert( tr_id >= 0 );
+ trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
+ ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
+ ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Add updates to the transaction */
+ update_dataset( file_id, "G-logged/D", tr_id, rc_id3, "/", my_rank, 1 );
+ update_dataset( file_id, "G-prefetched/D", tr_id, rc_id3, "/", my_rank, 2 );
+ update_dataset( file_id, "G-stored/D", tr_id, rc_id3, "/", my_rank, 3 );
+
+ update_map( file_id, "G-logged/M", tr_id, rc_id3, "/", my_rank, 1 );
+ update_map( file_id, "G-prefetched/M", tr_id, rc_id3, "/", my_rank, 2 );
+ update_map( file_id, "G-stored/M", tr_id, rc_id3, "/", my_rank, 3 );
+
+ /* Finish, commit, and close transaction */
+ fprintf( stderr, "APP-r%d: tr %lu - Finish\n", my_rank, tr_num );
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ ret = H5Pclose( trspl_id ); ASSERT_RET;
+
+ /* Acquire a read handle for container version and create a read context. */
+ version = 5;
+
+ if ( my_rank == 0 ) {
+ if ( verbose ) fprintf( stderr, "APP-r%d: rc %lu - Try to acquire\n", my_rank, version );
+ H5E_BEGIN_TRY {
+ rc_id5 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ while ( rc_id5 < 0 ) {
+ if ( verbose ) fprintf( stderr, "APP-r%d: rc %lu - Failed to acquire; sleep then retry\n", my_rank, version );
+ sleep( 1 );
+ rc_id5 = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ }
+ } H5E_END_TRY;
+ }
+ MPI_Bcast( &version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+ if ( my_rank != 0 ) {
+ rc_id5 = H5RCcreate( file_id, version );
+ }
+ assert( rc_id5 >= 0 ); assert ( version == 5 );
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
+
+ /* Release the read handle and close read context on CV 3 */
+ if ( my_rank == acquire_rank ) {
+ ret = H5RCrelease( rc_id3, H5_EVENT_STACK_NULL); ASSERT_RET;
+ }
+ ret = H5RCclose( rc_id3 ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc 3 - Released \n", my_rank );
+
+ if ( verbose ) print_container_contents( file_id, rc_id5, "/", my_rank );
+
+ /****
+ * Highest rank persists CV 5, then evicts objects under G-prefetched and G-stored.
+ *
+ * Satisfies: An application will create multiple transactions in a container, then persist them
+ * from the burst buffer storage to persistent storage, then evict several of the objects in the IOD
+ * container, releasing space in the burst buffer.
+ ****/
+
+ /* Optionally, pause to show change in BB space using out-of-band check. All ranks paused. */
+ if ( pause ) {
+ MPI_Barrier( MPI_COMM_WORLD );
+ if ( my_rank == 0 ) {
+ fprintf( stderr, ">>> PAUSED before persist/evict to allow BB space usage to be checked.\n" );
+ fprintf( stderr, ">>> Press return to continue..." );
+ getchar();
+ }
+ MPI_Barrier( MPI_COMM_WORLD );
+ }
+ else
+ MPI_Barrier( MPI_COMM_WORLD );
+
+ if ( my_rank == comm_size-1 ) {
+ fprintf( stderr, "APP-r%d: cv 5 - Persist\n", my_rank );
+ ret = H5RCpersist(rc_id5, H5_EVENT_STACK_NULL); ASSERT_RET;
+ print_container_contents( file_id, rc_id5, "/", my_rank );
+
+ evict_group_members_updates( file_id, rc_id5, "/G-prefetched", my_rank );
+ evict_group_members_updates( file_id, rc_id5, "/G-stored", my_rank );
+ }
+
+ /* Optionally, pause to show change in BB space using out-of-band check. All ranks paused. */
+ if ( pause ) {
+ MPI_Barrier( MPI_COMM_WORLD );
+ if ( my_rank == 0 ) {
+ fprintf( stderr, ">>> PAUSED after persist/evict to allow BB space usage to be checked.\n" );
+ fprintf( stderr, ">>> Press return to continue..." );
+ getchar();
+ }
+ MPI_Barrier( MPI_COMM_WORLD );
+ }
+ else
+ MPI_Barrier( MPI_COMM_WORLD );
+
+ /* All ranks print container here. For rank == comm_size, this will be after objects have been evicted */
+ if ( verbose ) print_container_contents( file_id, rc_id5, "/", my_rank );
+
+ /* Release the read handle and close read context on CV 5 */
+ MPI_Barrier( MPI_COMM_WORLD );
+ if ( my_rank == 0 ) {
+ ret = H5RCrelease( rc_id5, H5_EVENT_STACK_NULL); ASSERT_RET;
+ }
+ ret = H5RCclose( rc_id5 ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc 5 - Released \n", my_rank );
+
+ /****
+ * Close the H5File. Reopen, pre-fetch D, M, and T in /G-prefetched.
+ *
+ * Satisfies: "... then closing the container. The application will then re-open the container and asynchronously
+ * pre-fetch multiple objects from persistent storage to burst buffer storage.
+ ****/
+
+ ret = H5Gclose_ff( grp_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Gclose_ff( grp_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Gclose_ff( grp_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Close the container, then barrier to make sure all ranks have closed it before continuing */
+ fprintf( stderr, "APP-r%d: Close %s\n", my_rank, file_name );
+ ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Pclose( fapl_id ); ASSERT_RET;
+ MPI_Barrier( MPI_COMM_WORLD );
+
+ /* Define variables here so easier to move this into separate routine if desired (note: still rely on some defined earlier */
+ hid_t dset_l_id, dset_p_id, dset_s_id;
+ hid_t space_l_id, space_p_id, space_s_id;
+ hrpl_t dset_p_replica;
+ hid_t dxpl_p_id;
+ int data_l[4], data_p[4], data_s[4];
+
+ hid_t map_l_id, map_p_id, map_s_id;
+ hrpl_t map_p_replica;
+ hid_t mxpl_p_id;
+ int map_entries; /* know same number in all maps */
+ int *value_l, *value_p, *value_s;
+
+ hid_t type_l_id, type_p_id, type_s_id;
+ hrpl_t type_p_replica;
+ hid_t txpl_p_id;
+
+ /* Event stack */
+ hid_t d_es_id; /* for prefetch of Dataset */
+ hid_t mt_es_id; /* for prefetch of Map and dataType (shared event stack) */
+ H5ES_status_t status;
+
+ /* Create event stacks for managing asynchronous requests. */
+ d_es_id = H5EScreate(); assert( d_es_id >= 0 );
+ mt_es_id = H5EScreate(); assert( mt_es_id >= 0 );
+
+ /* Reopen the container read-only and get a read context for highest CV */
+ fapl_id = H5Pcreate( H5P_FILE_ACCESS ); assert( fapl_id >= 0 );
+ ret = H5Pset_fapl_iod( fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL ); ASSERT_RET;
+ file_id = H5Fopen_ff( file_name, H5F_ACC_RDONLY, fapl_id, &rc_id, H5_EVENT_STACK_NULL ); assert( file_id >= 0 );
+ H5RCget_version( rc_id, &version );
+ fprintf( stderr, "APP-r%d: Re-open %s\n", my_rank, file_name );
+
+ /* Open Datasets, Maps and DataTypes in all 3 Groups */
+ dset_l_id = H5Dopen_ff( file_id, "/G-logged/D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( dset_l_id >= 0 );
+ space_l_id = H5Dget_space( dset_l_id ); assert ( space_l_id >= 0 );
+
+ dset_p_id = H5Dopen_ff( file_id, "/G-prefetched/D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( dset_p_id >= 0 );
+ space_p_id = H5Dget_space( dset_p_id ); assert ( space_p_id >= 0 );
+
+ dset_s_id = H5Dopen_ff( file_id, "/G-stored/D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( dset_s_id >= 0 );
+ space_s_id = H5Dget_space( dset_s_id ); assert ( space_s_id >= 0 );
+
+ map_l_id = H5Mopen_ff( file_id, "/G-logged/M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( map_l_id >= 0 );
+
+ map_p_id = H5Mopen_ff( file_id, "/G-prefetched/M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( map_p_id >= 0 );
+
+ map_s_id = H5Mopen_ff( file_id, "/G-stored/M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( map_s_id >= 0 );
+
+ type_l_id = H5Topen_ff( file_id, "G-logged/T", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( type_l_id >= 0 );
+
+ type_p_id = H5Topen_ff( file_id, "G-prefetched/T", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( type_p_id >= 0 );
+
+ type_s_id = H5Topen_ff( file_id, "G-stored/T", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( type_s_id >= 0 );
+
+
+ /* Optionally, pause to show change in BB space using out-of-band check. All ranks paused. */
+ if ( pause ) {
+ MPI_Barrier( MPI_COMM_WORLD );
+ if ( my_rank == 0 ) {
+ fprintf( stderr, ">>> PAUSED before prefetch of replicas allow BB space usage to be checked.\n" );
+ fprintf( stderr, ">>> Press return to continue..." );
+ getchar();
+ }
+ MPI_Barrier( MPI_COMM_WORLD );
+ }
+
+ /* Prefetch objects in the /G-prefetched group. */
+ if ( use_daos_lustre ) {
+
+ /* Rank 0 prefetches asynchronously, using 2 different stacks */
+ if ( my_rank == 0 ) {
+ fprintf( stderr, "APP-r%d: Prefetch /G-prefetched/D asynchronously\n", my_rank );
+ ret = H5Dprefetch_ff( dset_p_id, rc_id, &dset_p_replica, H5P_DEFAULT, d_es_id ); ASSERT_RET;
+
+ fprintf( stderr, "APP-r%d: Prefetch /G-prefetched/M asynchronously\n", my_rank );
+ ret = H5Mprefetch_ff( map_p_id, rc_id, &map_p_replica, H5P_DEFAULT, mt_es_id ); ASSERT_RET;
+
+ fprintf( stderr, "APP-r%d: Prefetch /G-prefetched/T asynchronously\n", my_rank );
+ ret = H5Tprefetch_ff( type_p_id, rc_id, &type_p_replica, H5P_DEFAULT, mt_es_id ); ASSERT_RET;
+ }
+
+ /* Note: could start some reads of non-prefetched objects here, but since I want to offer pause & read timing, I didn't */
+
+ /* After dataset prefetch completes, bcast the replica ID and set up properties */
+ if ( my_rank == 0 ) {
+ H5ESwait_all( d_es_id, &status );
+ fprintf( stderr, "APP-r%d: asynchronous prefetch of D completed with status %d\n", my_rank, status );
+ }
+ MPI_Bcast( &dset_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+ dxpl_p_id = H5Pcreate( H5P_DATASET_XFER ); /* Used in Read & Evict*/
+ ret = H5Pset_dxpl_replica( dxpl_p_id, dset_p_replica ); ASSERT_RET;
+
+ /* After both map and datatype prefetches complete, bcast the replica IDs and set up properties */
+ if ( my_rank == 0 ) {
+ H5ESwait_all( mt_es_id, &status );
+ fprintf( stderr, "APP-r%d: asynchronous prefetch of M and T completed with status %d\n", my_rank, status );
+ }
+ MPI_Bcast( &map_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+ mxpl_p_id = H5Pcreate( H5P_DATASET_XFER ); /* Used in Get & Evict*/
+ ret = H5Pset_dxpl_replica( mxpl_p_id, map_p_replica ); ASSERT_RET;
+
+ MPI_Bcast( &type_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+ txpl_p_id = H5Pcreate( H5P_DATASET_XFER ); /* Used in Evict */
+ ret = H5Pset_dxpl_replica( txpl_p_id, type_p_replica ); ASSERT_RET;
+
+ } else {
+ /* Without daos-lustre, we don't do prefetch, so default properties */
+ dxpl_p_id = H5P_DEFAULT;
+ mxpl_p_id = H5P_DEFAULT;
+ txpl_p_id = H5P_DEFAULT;
+ }
+
+ /* Optionally, pause to show change in BB space using out-of-band check. All ranks paused. */
+ if ( pause ) {
+ MPI_Barrier( MPI_COMM_WORLD );
+ if ( my_rank == 0 ) {
+ fprintf( stderr, ">>> PAUSED after prefetch of replicas to allow BB space usage to be checked.\n" );
+ fprintf( stderr, ">>> Press return to continue..." );
+ getchar();
+ }
+ MPI_Barrier( MPI_COMM_WORLD );
+ }
+
+ /* Find entries in first Map, then use to create value vectors for all three Map objects - we know they have same size. */
+ ret = H5Mget_count_ff( map_l_id, &map_entries, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ value_l = (int *)calloc( map_entries, sizeof(int) ); assert( value_l != NULL );
+ value_p = (int *)calloc( map_entries, sizeof(int) ); assert( value_p != NULL );
+ value_s = (int *)calloc( map_entries, sizeof(int) ); assert( value_s != NULL );
+
+ /* Read Datasets and then Maps in all 3 Groups, measuring time it takes to read data for each*/
+
+ if ( time_reads ) {
+ ulong usec_dl, usec_dp, usec_ds; // per-read elapsed time for for datasets
+ ulong usec_ml, usec_mp, usec_ms; // per-read elapsed time for maps
+
+ ulong all_usec_dl, all_usec_dp, all_usec_ds; // total read time for datasets
+ ulong all_usec_ml, all_usec_mp, all_usec_ms; // total read time for maps
+
+ all_usec_dl = all_usec_dp = all_usec_ds = 0;
+ all_usec_ml = all_usec_mp = all_usec_ms = 0;
+
+ int t;
+
+ for ( t = 0; t < time_reads; t++ ) {
+
+ START_TIME;
+ ret = H5Dread_ff( dset_l_id, H5T_NATIVE_INT, space_l_id, space_l_id, H5P_DEFAULT, data_l, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ usec_dl = ELAPSED_TIME;
+ all_usec_dl += usec_dl;
+ fprintf( stderr, "APP-r%d Read Time %d for /G-logged/D: %lu usec\n", my_rank, t, usec_dl );
+
+ START_TIME;
+ ret = H5Dread_ff( dset_p_id, H5T_NATIVE_INT, space_p_id, space_p_id, dxpl_p_id, data_p, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ usec_dp = ELAPSED_TIME;
+ all_usec_dp += usec_dp;
+ fprintf( stderr, "APP-r%d Read Time %d for /G-prefetched/D: %lu usec\n", my_rank, t, usec_dp );
+
+ START_TIME;
+ ret = H5Dread_ff( dset_s_id, H5T_NATIVE_INT, space_s_id, space_s_id, H5P_DEFAULT, data_s, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ usec_ds = ELAPSED_TIME;
+ all_usec_ds += usec_ds;
+ fprintf( stderr, "APP-r%d Read Time %d for /G-stored/D: %lu usec\n", my_rank, t, usec_ds );
+
+ START_TIME;
+ for ( i = 0; i < map_entries; i++ ) {
+ ret = H5Mget_ff( map_l_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value_l[i], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+ END_TIME;
+ usec_ml = ELAPSED_TIME;
+ all_usec_ml += usec_ml;
+ fprintf( stderr, "APP-r%d Read Time %d for /G-logged/M: %lu usec\n", my_rank, t, usec_ml );
+
+ START_TIME;
+ for ( i = 0; i < map_entries; i++ ) {
+ ret = H5Mget_ff( map_p_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value_p[i], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+ END_TIME;
+ usec_mp = ELAPSED_TIME;
+ all_usec_mp += usec_mp;
+ fprintf( stderr, "APP-r%d Read Time %d for /G-prefetched/M: %lu usec\n", my_rank, t, usec_mp );
+
+ START_TIME;
+ for ( i = 0; i < map_entries; i++ ) {
+ ret = H5Mget_ff( map_s_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value_s[i], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+ END_TIME;
+ usec_ms = ELAPSED_TIME;
+ all_usec_ms += usec_ms;
+ fprintf( stderr, "APP-r%d Read Time %d for /G-stored/M: %lu usec\n", my_rank, t, usec_ms );
+
+ }
+
+ if ( time_reads > 1 ) {
+ fprintf( stderr, "APP-r%d Read Time Avg for /G-logged/D: %lu usec\n", my_rank, all_usec_dl/time_reads );
+ fprintf( stderr, "APP-r%d Read Time Avg for /G-prefetched/D: %lu usec\n", my_rank, all_usec_dp/time_reads );
+ fprintf( stderr, "APP-r%d Read Time Avg for /G-stored/D: %lu usec\n", my_rank, all_usec_ds/time_reads );
+ fprintf( stderr, "APP-r%d Read Time Avg for /G-logged/M: %lu usec\n", my_rank, all_usec_ml/time_reads );
+ fprintf( stderr, "APP-r%d Read Time Avg for /G-prefetched/M: %lu usec\n", my_rank, all_usec_mp/time_reads );
+ fprintf( stderr, "APP-r%d Read Time Avg for /G-stored/M: %lu usec\n", my_rank, all_usec_ms/time_reads );
+ }
+
+ } else {
+ fprintf( stderr, "APP-r%d Read /G-logged/D\n", my_rank );
+ ret = H5Dread_ff( dset_l_id, H5T_NATIVE_INT, space_l_id, space_l_id, H5P_DEFAULT, data_l, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+
+ fprintf( stderr, "APP-r%d Read /G-prefetched/D\n", my_rank );
+ ret = H5Dread_ff( dset_p_id, H5T_NATIVE_INT, space_p_id, space_p_id, dxpl_p_id, data_p, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+
+ fprintf( stderr, "APP-r%d Read /G-stored/D\n", my_rank );
+ ret = H5Dread_ff( dset_s_id, H5T_NATIVE_INT, space_s_id, space_s_id, H5P_DEFAULT, data_s, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+
+ fprintf( stderr, "APP-r%d Read /G-logged/M\n", my_rank );
+ for ( i = 0; i < map_entries; i++ ) {
+ ret = H5Mget_ff( map_l_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value_l[i], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+
+ fprintf( stderr, "APP-r%d Read /G-prefetched/M\n", my_rank );
+ for ( i = 0; i < map_entries; i++ ) {
+ ret = H5Mget_ff( map_p_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value_p[i], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+
+ fprintf( stderr, "APP-r%d Read /G-stored/M\n", my_rank );
+ for ( i = 0; i < map_entries; i++ ) {
+ ret = H5Mget_ff( map_s_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value_s[i], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+ }
+
+ /* Print Dataset and Map values for all 3 groups */
+ fprintf( stderr, "APP-r%d: /G-logged/D data: %d %d %d %d\n", my_rank, data_l[0], data_l[1], data_l[2], data_l[3] );
+ fprintf( stderr, "APP-r%d: /G-prefetched/D data: %d %d %d %d\n", my_rank, data_p[0], data_p[1], data_p[2], data_p[3]);
+ fprintf( stderr, "APP-r%d: /G-stored/D data: %d %d %d %d\n", my_rank, data_s[0], data_s[1], data_s[2], data_s[3] );
+
+ if ( map_entries == 4 ) {
+ fprintf( stderr, "APP-r%d: /G-logged/M key/values: 0/%d 1/%d 2/%d 3/%d\n", my_rank,
+ value_l[0], value_l[1], value_l[2], value_l[3] );
+ fprintf( stderr, "APP-r%d: /G-prefetched/M key/values: 0/%d 1/%d 2/%d 3/%d\n", my_rank,
+ value_p[0], value_p[1], value_p[2], value_p[3] );
+ fprintf( stderr, "APP-r%d: /G-stored/M key/values: 0/%d 1/%d 2/%d 3/%d\n", my_rank,
+ value_s[0], value_s[1], value_s[2], value_s[3] );
+ } else {
+ fprintf( stderr, "APP-r%d: /G-logged/M key/values: 0/%d 1/%d 2/%d 3/%d ... %d/%d\n", my_rank,
+ value_l[0], value_l[1], value_l[2], value_l[3], map_entries-1, value_l[map_entries-1] );
+ fprintf( stderr, "APP-r%d: /G-prefetched/M key/values: 0/%d 1/%d 2/%d 3/%d ... %d/%d\n", my_rank,
+ value_p[0], value_p[1], value_p[2], value_p[3], map_entries-1, value_p[map_entries-1] );
+ fprintf( stderr, "APP-r%d: /G-stored/M key/values: 0/%d 1/%d 2/%d 3/%d ... %d/%d\n", my_rank,
+ value_s[0], value_s[1], value_s[2], value_s[3], map_entries-1, value_s[map_entries-1] );
+ }
+ free( value_l );
+ free( value_p );
+ free( value_s );
+
+ /* wait for all ranks to complete read before evicting replicas */
+ MPI_Barrier( MPI_COMM_WORLD );
+
+ /* Rank 0 evicts replicas */
+ if ( my_rank == 0 ) {
+ if ( use_daos_lustre ) {
+ fprintf( stderr, "APP-r%d: Evict replica of /G-prefetched/D\n", my_rank );
+ ret = H5Devict_ff( dset_p_id, version, dxpl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET; /* Note: CV redundant for replica */
+
+ fprintf( stderr, "APP-r%d: Evict replica of /G-prefetched/M (Q7 note: currently a no-op in IOD)\n", my_rank );
+ ret = H5Mevict_ff( map_p_id, version, mxpl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET; /* Note: CV redundant for replica */
+
+ fprintf( stderr, "APP-r%d: Evict replica of /G-prefetched/T\n", my_rank );
+ ret = H5Tevict_ff( type_p_id, version, txpl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET; /* Note: CV redundant for replica */
+ } else {
+ fprintf( stderr, "APP-r%d DAOS-POSIX - No replicas to evict\n", my_rank );
+ }
+ }
+
+ /* Optionally, pause to show change in BB space using out-of-band check. All ranks paused. */
+ if ( pause ) {
+ MPI_Barrier( MPI_COMM_WORLD );
+ if ( my_rank == 0 ) {
+ fprintf( stderr, ">>> PAUSED after evict of replicas to allow BB space usage to be checked.\n" );
+ fprintf( stderr, ">>> Press return to continue..." );
+ getchar();
+ }
+ MPI_Barrier( MPI_COMM_WORLD );
+ }
+
+ /* Close Objects*/
+
+ if ( use_daos_lustre ) {
+ ret = H5Pclose( mxpl_p_id ); ASSERT_RET;
+ ret = H5Pclose( dxpl_p_id ); ASSERT_RET;
+ ret = H5Pclose( txpl_p_id ); ASSERT_RET;
+ }
+
+ ret = H5Dclose_ff( dset_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Dclose_ff( dset_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Dclose_ff( dset_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ ret = H5Sclose( space_l_id ); ASSERT_RET;
+ ret = H5Sclose( space_p_id ); ASSERT_RET;
+ ret = H5Sclose( space_s_id ); ASSERT_RET;
+
+ ret = H5Mclose_ff( map_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Mclose_ff( map_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Mclose_ff( map_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ ret = H5Tclose_ff( type_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Tclose_ff( type_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Tclose_ff( type_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Release the read handle and close read context */
+ ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
+ ret = H5RCclose( rc_id ); ASSERT_RET;
+
+ /* Close H5 Objects that are still open */
+ fprintf( stderr, "APP-r%d: Close %s\n", my_rank, file_name );
+ MPI_Barrier( MPI_COMM_WORLD );
+ ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Pclose( fapl_id ); ASSERT_RET;
+
+ /* Perform wrap-up operations */
+ fprintf( stderr, "APP-r%d: Finalize EFF stack\n", my_rank );
+ MPI_Barrier( MPI_COMM_WORLD );
+ EFF_finalize();
+ MPI_Finalize();
+
+ return 0;
+}
+
+
+/*
+ * Helper function used to create and initialize dataset "dset_name"
+ * in object identified by "obj_id"
+ * in transaction identified by "tr_id".
+ * "obj_path" and "my_rank" are used in the status output.
+ * "ordinal" is used as a multiplier in computing the cell value.
+ *
+ * All datasets are 1D Arrays with 4 elements.
+ * Cell values are computed using the formula:
+ * data[i] = ordinal*1000 + tr_num*100 + my_rank*10 + i;
+ */
+void
+create_dataset( hid_t obj_id, const char* dset_name, hid_t tr_id, const char* obj_path, int my_rank, int ordinal ) {
+
+ herr_t ret;
+ uint64_t tr_num;
+ int i;
+ hsize_t current_size = 4; /* For now, we hard-code to small number */
+ hsize_t max_size = 4;
+ int data[4];
+ hid_t space_id;
+ hid_t dtype_id;
+ hid_t dset_id;
+
+ /* Setup */
+ ret = H5TRget_trans_num( tr_id, &tr_num ); ASSERT_RET;
+ for ( i = 0; i < current_size; i++ ) {
+ data[i] = ordinal*1000 + tr_num*100 + my_rank*10 + i;
+ }
+ space_id = H5Screate_simple( 1, &current_size, &max_size ); assert( space_id >= 0 );
+ dtype_id = H5Tcopy( H5T_NATIVE_INT ); assert( dtype_id >= 0 );
+
+ /* Create & Populate */
+ dset_id = H5Dcreate_ff( obj_id, dset_name, dtype_id, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL );
+ if ( dset_id >= 0 ) {
+ fprintf( stderr, "APP-r%d: tr %lu - Create %s%s with data: %d %d %d %d\n",
+ my_rank, tr_num, obj_path, dset_name, data[0], data[1], data[2], data[3] );
+ ret = H5Dwrite_ff( dset_id, dtype_id, space_id, space_id, H5P_DEFAULT, data, tr_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ } else {
+ fprintf( stderr, "APP-r%d: tr %lu - Create %s%s - FAILED\n", my_rank, tr_num, obj_path, dset_name );
+ }
+ ret = H5Tclose( dtype_id ); ASSERT_RET;
+ ret = H5Sclose( space_id ); ASSERT_RET;
+ return;
+}
+
+/*
+ * Helper function used to create and initialize map "map_name"
+ * in object identified by "obj_id"
+ * in transaction identified by "tr_id".
+ * "obj_path" and "my_rank" are used in the status output.
+ * "ordinal" is used as a multiplier in computing the value.
+ *
+ * All maps are created with 4 initial KV pairs.
+ * Keys/values are computed using the formula:
+ * key=i / value=(ordinal*1000 + tr_num*100 + my_rank*10 + i)
+ */
+void
+create_map( hid_t obj_id, const char* map_name, hid_t tr_id, const char* obj_path, int my_rank, int ordinal ) {
+
+ herr_t ret;
+ uint64_t tr_num;
+ int i;
+ int num_kvs = 4; /* Start with 4 KV pairs */
+ int value[4];
+ hid_t map_id;
+
+ /* Setup */
+ ret = H5TRget_trans_num( tr_id, &tr_num ); ASSERT_RET;
+ for ( i = 0; i < num_kvs; i++ ) {
+ value[i] = ordinal*1000 + tr_num*100 + my_rank*10 + i;
+ }
+
+ /* Create & Populate */
+ map_id = H5Mcreate_ff( obj_id, map_name, H5T_STD_I32LE, H5T_STD_I32LE, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL );
+ if ( map_id >= 0 ) {
+ fprintf( stderr, "APP-r%d: tr %lu - Create %s%s with keys/values: 0/%d 1/%d 2/%d 3/%d\n",
+ my_rank, tr_num, obj_path, map_name, value[0], value[1], value[2], value[3] );
+ for ( i = 0; i < num_kvs; i++ ) {
+ ret = H5Mset_ff( map_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value[i], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+ ret = H5Mclose_ff( map_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ } else {
+ fprintf( stderr, "APP-r%d: tr %lu - Create %s%s - FAILED\n", my_rank, tr_num, obj_path, map_name );
+ }
+ return;
+}
+
+/*
+ * Helper function used to create named datatype "type_name"
+ * in group identified by "obj_id"
+ * in transaction identified by "tr_id".
+ * "obj_path" and "my_rank" are used in the status output.
+ */
+void
+create_named_datatype( hid_t obj_id, const char* type_name, hid_t tr_id, const char* obj_path, int my_rank ) {
+
+ herr_t ret;
+ uint64_t tr_num;
+ hid_t dtype_id;
+
+ /* Setup */
+ ret = H5TRget_trans_num( tr_id, &tr_num ); ASSERT_RET;
+ dtype_id = H5Tcopy( H5T_NATIVE_INT ); assert( dtype_id >= 0 );
+
+ /* Create */
+ ret = H5Tcommit_ff( obj_id, type_name, dtype_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ fprintf( stderr, "APP-r%d: tr %lu - Create %s%s %s\n", my_rank, tr_num, obj_path, type_name, STATUS );
+ ret = H5Tclose( dtype_id ); ASSERT_RET;
+ return;
+}
+
+/*
+ * Helper function used to update raw data in "dset_name"
+ * in object identified by "obj_id"
+ * in transaction identified by "tr_id".
+ * "rc_id" for "tr_id" passed in explicitly for now - later may be able to get it from info in tr_id (cv & file_id)
+ * "obj_path" and "my_rank" are used in the status output.
+ * "ordinal" is used as a multiplier in computing the cell value.
+ *
+ * Cell value to be updated is computed using the formula:
+ * data[my_rank] = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
+ *
+ * If my_rank >= max_size of data, then no update done in call.
+ */
+void
+update_dataset( hid_t obj_id, const char* dset_name, hid_t tr_id, hid_t rc_id, const char* obj_path, int my_rank, int ordinal ){
+ herr_t ret;
+ uint64_t tr_num;
+ hid_t dset_id;
+
+ /* Setup */
+ ret = H5TRget_trans_num( tr_id, &tr_num ); ASSERT_RET;
+
+ dset_id = H5Dopen_ff( obj_id, dset_name, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ if ( dset_id >= 0 ) {
+
+ /* Dataset opened; confirm as we expect */
+ hid_t filespace_id;
+ int nDims;
+ hsize_t current_size, max_size;
+
+ filespace_id = H5Dget_space( dset_id ); assert ( filespace_id >= 0 );
+ nDims = H5Sget_simple_extent_dims( filespace_id, &current_size, &max_size );
+ assert( nDims == 1 );
+ assert( current_size == 4 );
+ assert( max_size == 4 );
+
+ if ( my_rank < current_size ) {
+ /* If rank in bounds of array, do the update */
+ int cell_data;
+ hsize_t mem_size = 1;
+ hid_t memspace_id;
+ hsize_t index;
+
+ cell_data = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
+
+ memspace_id = H5Screate_simple( 1, &mem_size, &mem_size ); assert( memspace_id >= 0 );
+ index = (hsize_t)my_rank;
+ ret = H5Sselect_elements( filespace_id, H5S_SELECT_SET, 1, &index ); ASSERT_RET;
+
+ ret = H5Dwrite_ff( dset_id, H5T_NATIVE_INT, memspace_id, filespace_id, H5P_DEFAULT, &cell_data, tr_id,
+ H5_EVENT_STACK_NULL ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: tr %lu - Update %s%s[%d] to %d %s\n",
+ my_rank, tr_num, obj_path, dset_name, my_rank, cell_data, STATUS );
+ ret = H5Sclose( memspace_id ); ASSERT_RET;
+
+ } else {
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: tr %lu - No Update to %s%s for this rank \n", my_rank, tr_num, obj_path, dset_name );
+ }
+ }
+
+ ret = H5Sclose( filespace_id ); ASSERT_RET;
+ ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ } else {
+ fprintf( stderr, "APP-r%d: tr %lu - Update %s%s - FAILED\n", my_rank, tr_num, obj_path, dset_name );
+ }
+ return;
+}
+
+/*
+ * Helper function used to update (or add) key/value data in "map_name"
+ * in object identified by "obj_id"
+ * in transaction identified by "tr_id".
+ * "rc_id" for "tr_id" passed in explicitly for now - later may be able to get it from info in tr_id (cv & file_id)
+ * "obj_path" and "my_rank" are used in the status output.
+ * key/value set according to:
+ * key = my_rank
+ * value = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
+ */
+void
+update_map( hid_t obj_id, const char* map_name, hid_t tr_id, hid_t rc_id, const char* obj_path, int my_rank, int ordinal ){
+ herr_t ret;
+ uint64_t tr_num;
+ hid_t map_id;
+ int value;
+
+ /* Setup */
+ ret = H5TRget_trans_num( tr_id, &tr_num ); ASSERT_RET;
+
+ /* Open the map */
+ map_id = H5Mopen_ff( obj_id, map_name, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+
+ if ( map_id >= 0 ) {
+ value = ordinal*1000 + tr_num*100 + my_rank*10 + my_rank;
+ ret = H5Mset_ff( map_id, H5T_STD_I32LE, &my_rank, H5T_STD_I32LE, &value, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ if ( verbose || ( ret < 0 ) ) {
+ fprintf( stderr, "APP-r%d: tr %lu - Update %s%s with key/value %d/%d %s\n",
+ my_rank, tr_num, obj_path, map_name, my_rank, value, STATUS );
+ }
+
+ ret = H5Mclose_ff( map_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ } else {
+ fprintf( stderr, "APP-r%d: tr %lu - Update %s%s - FAILED\n", my_rank, tr_num, obj_path, map_name );
+ }
+ return;
+}
+
+/*
+ * evict_group_members_updates - evict logged updates for (expected) members of specified group
+ * parameters:
+ * file_id: identifies container
+ * rc_id: identifies read context
+ * grp_path: path that specifies group
+ * my_rank: identifies rank doing the evict
+ */
+void
+evict_group_members_updates( hid_t file_id, hid_t rc_id, const char* grp_path, int my_rank )
+{
+ uint64_t cv; char preface[128];
+ char path_to_object[1024];
+
+ hid_t dset_id, map_id, type_id;
+ herr_t ret;
+ int i;
+
+ /* Get the container version for the read context and set up print preface */
+ ret = H5RCget_version( rc_id, &cv ); ASSERT_RET;
+ sprintf( preface, "APP-r%d: cv %lu -", my_rank, cv );
+
+ /* Dataset */
+ sprintf( path_to_object, "%s/%s", grp_path, "D" );
+ dset_id = H5Dopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( dset_id >= 0 );
+
+ if ( use_daos_lustre ) {
+ ret = H5Devict_ff( dset_id, cv, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ fprintf( stderr, "%s Evicted updates for %s\n", preface, path_to_object );
+ } else {
+ fprintf( stderr, "%s DAOS-POSIX - Won't evict updates for object %s\n", preface, path_to_object );
+ }
+ ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Map */
+ sprintf( path_to_object, "%s/%s", grp_path, "M" );
+ map_id = H5Mopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( map_id >= 0 );
+
+ if ( use_daos_lustre ) {
+ ret = H5Mevict_ff( map_id, cv, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ fprintf( stderr, "%s Evicted updates for %s (Q7 note: currently a no-op in IOD)\n", preface, path_to_object );
+ } else {
+ fprintf( stderr, "%s DAOS-POSIX - Won't evict updates for object %s\n", preface, path_to_object );
+ }
+ ret = H5Mclose_ff( map_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Named Datatype */
+ sprintf( path_to_object, "%s/%s", grp_path, "T" );
+ type_id = H5Topen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( type_id >= 0 );
+
+ if ( use_daos_lustre ) {
+ ret = H5Tevict_ff( type_id, cv, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ fprintf( stderr, "%s Evicted updates for %s\n", preface, path_to_object );
+ } else {
+ fprintf( stderr, "%s DAOS-POSIX - Won't evict updates for object %s\n", preface, path_to_object );
+ }
+ ret = H5Tclose_ff( type_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ return;
+}
+
+
+/*
+ * Helper function used to recursively read and print container contents
+ * for container identified by "file_id"
+ * in read context identified by "rc_id"
+ * with path to current level in "grp_path"
+ * and "my_rank" used to identify the process doing the reading / printing.
+ */
+void
+print_container_contents( hid_t file_id, hid_t rc_id, const char* grp_path, int my_rank )
+{
+ herr_t ret;
+ uint64_t cv;
+ htri_t exists;
+ char path_to_object[1024];
+ char name[30];
+ int i;
+
+ static int lvl = 0; /* level in recursion - used to format printing */
+ char preface[128];
+ char line[1024];
+
+ /* Get the container version for the read context */
+ ret = H5RCget_version( rc_id, &cv ); ASSERT_RET;
+
+ /* Set up the preface and adding version number */
+ sprintf( preface, "APP-r%d: cv %d: ", my_rank, (int)cv );
+
+ /* Start the printing */
+ if ( lvl == 0 ) {
+ fprintf( stderr, "%s ----- Container Contents ------------\n", preface );
+ }
+
+ /* Datasets */
+ sprintf( path_to_object, "%s%s", grp_path, "D" );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+
+ if ( exists ) {
+ hid_t dset_id;
+ hid_t space_id;
+ int nDims;
+ hsize_t current_size[1];
+ hsize_t max_size[1];
+ hsize_t totalSize;
+ int *data;
+ int i;
+
+ dset_id = H5Dopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( dset_id >= 0 );
+
+ space_id = H5Dget_space( dset_id ); assert ( space_id >= 0 );
+ nDims = H5Sget_simple_extent_dims( space_id, current_size, max_size );
+ assert( nDims == 1 );
+
+ totalSize = current_size[0];
+ data = (int *)calloc( totalSize, sizeof(int) ); assert( data != NULL );
+
+ ret = H5Dread_ff( dset_id, H5T_NATIVE_INT, space_id, space_id, H5P_DEFAULT, data, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ sprintf( line, "%s %s data: ", preface, path_to_object );
+ for ( i = 0; i < totalSize; i++ ) {
+ sprintf( line, "%s%d ", line, data[i] );
+ }
+
+ fprintf( stderr, "%s\n", line );
+ free( data );
+
+ ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Sclose( space_id ); ASSERT_RET;
+ }
+
+ /* Maps */
+ sprintf( path_to_object, "%s%s", grp_path, "M" );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+
+ if ( exists ) {
+ hid_t map_id;
+ hsize_t totalCount;
+ int *value;
+ int i;
+
+ map_id = H5Mopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( map_id >= 0 );
+
+ ret = H5Mget_count_ff( map_id, &totalCount, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ value = (int *)calloc( totalCount, sizeof(int) ); assert( value != NULL );
+
+ for ( i = 0; i < totalCount; i++ ) {
+ ret = H5Mget_ff( map_id, H5T_STD_I32LE, &i, H5T_STD_I32LE, &value[i], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+
+ sprintf( line, "%s %s key/values: ", preface, path_to_object );
+ for ( i = 0; i < totalCount; i++ ) {
+ sprintf( line, "%s %d/%d ", line, i, value[i] );
+ }
+
+ fprintf( stderr, "%s\n", line );
+ free( value );
+
+ ret = H5Mclose_ff( map_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ }
+
+ /* Named Datatypes */
+ sprintf( path_to_object, "%s%s", grp_path, "T" );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+
+ if ( exists ) {
+ fprintf( stderr, "%s %s\n", preface, path_to_object );
+ }
+
+ /* Groups - if found, descend */
+ for ( i = 1; i < 4; i++ ) {
+ if ( i == 1 ) {
+ strcpy( name, "G-logged" );
+ } else if ( i == 2 ) {
+ strcpy( name, "G-prefetched" );
+ } else if ( i == 3 ) {
+ strcpy( name, "G-stored" );
+ }
+
+ sprintf( path_to_object, "%s%s/", grp_path, name );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+ if ( exists ) {
+ fprintf( stderr, "%s %s\n", preface, path_to_object );
+ lvl++;
+ print_container_contents( file_id, rc_id, path_to_object, my_rank );
+ lvl--;
+ }
+ }
+
+ /* End printing */
+ if ( lvl == 0 ) {
+ fprintf( stderr, "%s -----------------\n", preface );
+ }
+
+ return;
+}
+
+/*
+ * parse_options - parse command line options
+ */
+int
+parse_options( int argc, char** argv, int my_rank ) {
+ int i, n;
+ char* app = argv[0];
+
+ while ( --argc ) {
+ if ( **(++argv) != '-' ) {
+ break;
+ } else {
+ switch( *(*argv+1) ) {
+ case 'l':
+ use_daos_lustre = 0;
+ break;
+ case 'p':
+ pause = 1;
+ break;
+ case 't':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "Error: No repeat_cnt specified after -t option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ time_reads = atoi( *argv );
+ if ( my_rank == 0 ) {
+ printf( "Will repeat read sequence %d times to get average elapsed time per read.\n", time_reads );
+ }
+ }
+ break;
+ case 'v':
+ verbose = 1;
+ break;
+ default:
+ if ( my_rank == 0 ) {
+ usage( app );
+ }
+ return( 1 );
+ }
+ }
+ }
+ return( 0 );
+}
+
+/*
+ * usage - display usage message
+ */
+void
+usage( const char* app ) {
+ printf( "Usage: %s [-l] [-p] [-t repeat_cnt] [-v] [-w]\n", app );
+ printf( "\tl: don't use DAOS Lustre\n" );
+ printf( "\tp: pause to allow out-of-band BB space check\n" );
+ printf( "\tt: time reads after container re-open, performing read sequence 'repeat_cnt' times\n" );
+ printf( "\tv: verbose output\n" );
+}
diff --git a/examples/h5ff_client_do.c b/examples/h5ff_client_do.c
index 69bc566..3ccebf4 100644
--- a/examples/h5ff_client_do.c
+++ b/examples/h5ff_client_do.c
@@ -1,289 +1,289 @@
-/*
- * test_client_acg.c: Client side of ACG Dynamic Data Structure Support
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-int main(int argc, char **argv) {
- char file_name[]="acg_file.h5";
- hid_t file_id;
- hid_t dsid = -1; /* Dataset ID */
- hid_t sid = -1; /* Dataspace ID */
- hsize_t dim, max_dim, chunk_dim; /* Dataset and chunk dimensions */
- hsize_t curr_size;
- unsigned u; /* Local index variable */
- unsigned write_elem[60], read_elem[60]; /* Element written/read */
- hid_t fapl_id, dxpl_id;
- int my_rank, my_size;
- int provided;
- hid_t e_stack;
- H5ES_status_t *status = NULL;
- int num_requests = 0, i;
- herr_t ret;
- H5_request_t req1;
- H5ES_status_t status;
-
- MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
- if(MPI_THREAD_MULTIPLE != provided) {
- fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
- exit(1);
- }
-
- /* Call EFF_init to initialize the EFF stack.
- As a result of this call, the Function Shipper client is started,
- and HDF5 VOL calls are registered with the function shipper.
- An "IOD init" call is forwarded from the FS client to the FS server
- which should already be running. */
- EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
-
- MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
- MPI_Comm_size(MPI_COMM_WORLD, &my_size);
- fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
-
- fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
- /* Choose the IOD VOL plugin to use with this file.
- First we create a file access property list. Then we call a new routine to set
- the IOD plugin to use with this fapl */
- fapl_id = H5Pcreate (H5P_FILE_ACCESS);
- H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
-
- /* create an event Queue for managing asynchronous requests.
-
- Event Queues will releive the use from managing and completing
- individual requests for every operation. Instead of passing a
- request for every operation, the event queue is passed and
- internally the HDF5 library creates a request and adds it to
- the event queue.
-
- Multiple Event queue can be created used by the application. */
- e_stack = H5EScreate();
- assert(e_stack);
-
- /* create the file. This is asynchronous, but the file_id can be used. */
- file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, e_stack);
- assert(file_id);
-
- /* Create 1-D dataspace */
- dim = 0;
- max_dim = H5S_UNLIMITED;
- if((sid = H5Screate_simple(1, &dim, &max_dim)) < 0) {
- fprintf(stderr, "Failed\n");
- exit(1);
- }
-
- /* Create 1-D chunked dataset */
- if((dsid = H5Dcreate_ff(file_id, "dset", H5T_NATIVE_UINT, sid,
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0 , e_stack)) < 0) {
- fprintf(stderr, "Failed\n");
- exit(1);
- }
-
- /* Close dataspace */
- H5Sclose(sid);
-
- /* Initialize data elements */
- for(u = 0; u < 60; u++)
- write_elem[u] = u;
-
- printf("App Dataset Id = %d File id = %d\n", dsid, file_id);
- ret = H5DOappend_ff(dsid, H5P_DEFAULT, 1, 10, H5T_NATIVE_UINT,
- write_elem, 0 , e_stack);
- assert(ret<0);
-
- /* Append 60 elements to dataset, along proper axis */
- if(H5DOappend_ff(dsid, H5P_DEFAULT, 0, 60, H5T_NATIVE_UINT,
- write_elem, 0 , e_stack) < 0) {
- fprintf(stderr, "Failed\n");
- exit(1);
- }
-
- /* Get the dataset's dataspace now */
- if((sid = H5Dget_space(dsid)) < 0) {
- fprintf(stderr, "Failed\n");
- exit(1);
- }
-
- if(H5Sget_simple_extent_dims(sid, &curr_size, NULL) < 0) {
- fprintf(stderr, "Failed\n");
- exit(1);
- }
-
- /* Verify dataset is correct size */
- if(curr_size != 60) {
- fprintf(stderr, "Failed\n");
- exit(1);
- }
-
- /* Close dataspace */
- if(H5Sclose(sid) < 0) {
- fprintf(stderr, "Failed\n");
- exit(1);
- }
-
- /* Read elements back, with sequence operation */
- memset(read_elem, 0, sizeof(read_elem));
-
- /* Sequence 10 elements from dataset, along bad axis */
- ret = H5DOsequence_ff(dsid, H5P_DEFAULT, 1, 0, 60, H5T_NATIVE_UINT,
- read_elem, 0 , e_stack);
-
- /* Sequence first 60 elements from dataset, along proper axis */
- ret = H5DOsequence_ff(dsid, H5P_DEFAULT, 0, 0, 60, H5T_NATIVE_UINT,
- read_elem, 0 , e_stack);
- assert(ret == 0);
-
- /* close dataset */
- assert(H5Dclose(dsid) == 0);
-
-
- {
- hvl_t wdata[5]; /* Information to write */
- hvl_t rdata[5]; /* Information to write */
- hid_t tid;
- hsize_t dims[1];
- int increment, j, n;
-
- n = 0;
- increment = 4;
- /* Allocate and initialize VL data to write */
- for(i = 0; i < 5; i++) {
- int temp = i*increment + increment;
-
- wdata[i].p = malloc(temp * sizeof(unsigned int));
- wdata[i].len = temp;
- for(j = 0; j < temp; j++)
- ((unsigned int *)wdata[i].p)[j] = n ++;
- } /* end for */
-
- /* Create a datatype to refer to */
- tid = H5Tvlen_create (H5T_NATIVE_UINT);
-
- /* create a dataspace. This is a local Bookeeping operation that
- does not touch the file */
- dims [0] = 5;
- sid = H5Screate_simple(1, dims, NULL);
-
- /* Create Dataset */
- if((dsid = H5Dcreate_ff(file_id, "dset_vl", tid, sid,
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0 , e_stack)) < 0) {
- fprintf(stderr, "Failed\n");
- exit(1);
- }
-
- ret = H5Dwrite(dsid, tid, sid, sid, H5P_DEFAULT, wdata);
- assert(ret == 0);
-
- ret = H5Dread(dsid, tid, sid, sid, H5P_DEFAULT, rdata);
- assert(ret == 0);
-
- /* Print VL DATA */
- for(i = 0; i < 5; i++) {
- int temp = i*increment + increment;
-
- fprintf(stderr, "Element %d size %zu: ", i, rdata[i].len);
- for(j = 0; j < temp; j++)
- fprintf(stderr, "%d ",((unsigned int *)rdata[i].p)[j]);
- fprintf(stderr, "\n");
- } /* end for */
-
- H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, rdata);
- H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, wdata);
-
- H5Sclose(sid);
- H5Tclose(tid);
-
- /* close dataset */
- assert(H5Dclose_ff(dsid, e_stack) == 0);
-
- }
-
- {
- hid_t tid;
- hsize_t dims[1];
- const char *str_wdata[4]= {
- "Four score and seven years ago our forefathers brought forth on this continent a new nation,",
- "conceived in liberty and dedicated to the proposition that all men are created equal.",
- "Now we are engaged in a great civil war,",
- "testing whether that nation or any nation so conceived and so dedicated can long endure."
- }; /* Information to write */
- char *str_rdata[4]; /* Information read in */
-
- /* create a dataspace. This is a local Bookeeping operation that
- does not touch the file */
- dims [0] = 4;
- sid = H5Screate_simple(1, dims, NULL);
-
- /* Create a datatype to refer to */
- tid = H5Tcopy(H5T_C_S1);
- H5Tset_size(tid,H5T_VARIABLE);
-
- /* Create Dataset */
- if((dsid = H5Dcreate_ff(file_id, "dset_vl_str", tid, sid,
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0 , e_stack)) < 0) {
- fprintf(stderr, "Failed\n");
- exit(1);
- }
-
- ret = H5Dwrite(dsid, tid, sid, sid, H5P_DEFAULT, str_wdata);
- assert(ret == 0);
-
- ret = H5Dread(dsid, tid, sid, sid, H5P_DEFAULT, str_rdata);
- assert(ret == 0);
-
- fprintf(stderr, "Reading VL Strings: \n");
- for(i=0 ; i<4 ; i++) {
- fprintf(stderr, "%s\n", str_rdata[i]);
- }
-
- H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, str_rdata);
-
- H5Sclose(sid);
- H5Tclose(tid);
-
- /* close dataset */
- assert(H5Dclose_ff(dsid, e_stack) == 0);
- }
- /* closing the container also acts as a wait all on all pending requests
- on the container. */
- assert(H5Fclose_ff(file_id, e_stack) == 0);
-
- fprintf(stderr, "\n*****************************************************************************************************************\n");
- fprintf(stderr, "Wait on everything in EQ and check Results of operations in EQ\n");
- fprintf(stderr, "*****************************************************************************************************************\n");
-
- /* wait on all requests and print completion status */
- H5EQwait(e_stack, &num_requests, &status);
- fprintf(stderr, "%d requests in event queue. Completions: ", num_requests);
- for(i=0 ; i<num_requests; i++)
- fprintf(stderr, "%d ",status[i]);
- fprintf(stderr, "\n");
- free(status);
-
- /* Verify data read */
- for(u = 0; u < 60; u++) {
- if(read_elem[u] != write_elem[u]) {
- fprintf(stderr, "Failed\n");
- exit(1);
- }
- }
-
- fprintf(stderr, "\n*****************************************************************************************************************\n");
- fprintf(stderr, "Finalize EFF stack\n");
- fprintf(stderr, "*****************************************************************************************************************\n");
-
- H5ESclose(e_stack);
- H5Pclose(fapl_id);
-
- /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
- and shutsdown the FS server (when all clients send the terminate request)
- and client */
- EFF_finalize();
-
- MPI_Finalize();
- return 0;
-}
+/*
+ * test_client_acg.c: Client side of ACG Dynamic Data Structure Support
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+int main(int argc, char **argv) {
+ char file_name[]="acg_file.h5";
+ hid_t file_id;
+ hid_t dsid = -1; /* Dataset ID */
+ hid_t sid = -1; /* Dataspace ID */
+ hsize_t dim, max_dim, chunk_dim; /* Dataset and chunk dimensions */
+ hsize_t curr_size;
+ unsigned u; /* Local index variable */
+ unsigned write_elem[60], read_elem[60]; /* Element written/read */
+ hid_t fapl_id, dxpl_id;
+ int my_rank, my_size;
+ int provided;
+ hid_t e_stack;
+ H5ES_status_t *status = NULL;
+ int num_requests = 0, i;
+ herr_t ret;
+ H5_request_t req1;
+ H5ES_status_t status;
+
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+ if(MPI_THREAD_MULTIPLE != provided) {
+ fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
+ exit(1);
+ }
+
+ /* Call EFF_init to initialize the EFF stack.
+ As a result of this call, the Function Shipper client is started,
+ and HDF5 VOL calls are registered with the function shipper.
+ An "IOD init" call is forwarded from the FS client to the FS server
+ which should already be running. */
+ EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &my_size);
+ fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
+
+ fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
+ /* Choose the IOD VOL plugin to use with this file.
+ First we create a file access property list. Then we call a new routine to set
+ the IOD plugin to use with this fapl */
+ fapl_id = H5Pcreate (H5P_FILE_ACCESS);
+ H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ /* create an event Queue for managing asynchronous requests.
+
+ Event Queues will releive the use from managing and completing
+ individual requests for every operation. Instead of passing a
+ request for every operation, the event queue is passed and
+ internally the HDF5 library creates a request and adds it to
+ the event queue.
+
+ Multiple Event queue can be created used by the application. */
+ e_stack = H5EScreate();
+ assert(e_stack);
+
+ /* create the file. This is asynchronous, but the file_id can be used. */
+ file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, e_stack);
+ assert(file_id);
+
+ /* Create 1-D dataspace */
+ dim = 0;
+ max_dim = H5S_UNLIMITED;
+ if((sid = H5Screate_simple(1, &dim, &max_dim)) < 0) {
+ fprintf(stderr, "Failed\n");
+ exit(1);
+ }
+
+ /* Create 1-D chunked dataset */
+ if((dsid = H5Dcreate_ff(file_id, "dset", H5T_NATIVE_UINT, sid,
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0 , e_stack)) < 0) {
+ fprintf(stderr, "Failed\n");
+ exit(1);
+ }
+
+ /* Close dataspace */
+ H5Sclose(sid);
+
+ /* Initialize data elements */
+ for(u = 0; u < 60; u++)
+ write_elem[u] = u;
+
+ printf("App Dataset Id = %d File id = %d\n", dsid, file_id);
+ ret = H5DOappend_ff(dsid, H5P_DEFAULT, 1, 10, H5T_NATIVE_UINT,
+ write_elem, 0 , e_stack);
+ assert(ret<0);
+
+ /* Append 60 elements to dataset, along proper axis */
+ if(H5DOappend_ff(dsid, H5P_DEFAULT, 0, 60, H5T_NATIVE_UINT,
+ write_elem, 0 , e_stack) < 0) {
+ fprintf(stderr, "Failed\n");
+ exit(1);
+ }
+
+ /* Get the dataset's dataspace now */
+ if((sid = H5Dget_space(dsid)) < 0) {
+ fprintf(stderr, "Failed\n");
+ exit(1);
+ }
+
+ if(H5Sget_simple_extent_dims(sid, &curr_size, NULL) < 0) {
+ fprintf(stderr, "Failed\n");
+ exit(1);
+ }
+
+ /* Verify dataset is correct size */
+ if(curr_size != 60) {
+ fprintf(stderr, "Failed\n");
+ exit(1);
+ }
+
+ /* Close dataspace */
+ if(H5Sclose(sid) < 0) {
+ fprintf(stderr, "Failed\n");
+ exit(1);
+ }
+
+ /* Read elements back, with sequence operation */
+ memset(read_elem, 0, sizeof(read_elem));
+
+ /* Sequence 10 elements from dataset, along bad axis */
+ ret = H5DOsequence_ff(dsid, H5P_DEFAULT, 1, 0, 60, H5T_NATIVE_UINT,
+ read_elem, 0 , e_stack);
+
+ /* Sequence first 60 elements from dataset, along proper axis */
+ ret = H5DOsequence_ff(dsid, H5P_DEFAULT, 0, 0, 60, H5T_NATIVE_UINT,
+ read_elem, 0 , e_stack);
+ assert(ret == 0);
+
+ /* close dataset */
+ assert(H5Dclose(dsid) == 0);
+
+
+ {
+ hvl_t wdata[5]; /* Information to write */
+ hvl_t rdata[5]; /* Information to write */
+ hid_t tid;
+ hsize_t dims[1];
+ int increment, j, n;
+
+ n = 0;
+ increment = 4;
+ /* Allocate and initialize VL data to write */
+ for(i = 0; i < 5; i++) {
+ int temp = i*increment + increment;
+
+ wdata[i].p = malloc(temp * sizeof(unsigned int));
+ wdata[i].len = temp;
+ for(j = 0; j < temp; j++)
+ ((unsigned int *)wdata[i].p)[j] = n ++;
+ } /* end for */
+
+ /* Create a datatype to refer to */
+ tid = H5Tvlen_create (H5T_NATIVE_UINT);
+
+ /* create a dataspace. This is a local Bookeeping operation that
+ does not touch the file */
+ dims [0] = 5;
+ sid = H5Screate_simple(1, dims, NULL);
+
+ /* Create Dataset */
+ if((dsid = H5Dcreate_ff(file_id, "dset_vl", tid, sid,
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0 , e_stack)) < 0) {
+ fprintf(stderr, "Failed\n");
+ exit(1);
+ }
+
+ ret = H5Dwrite(dsid, tid, sid, sid, H5P_DEFAULT, wdata);
+ assert(ret == 0);
+
+ ret = H5Dread(dsid, tid, sid, sid, H5P_DEFAULT, rdata);
+ assert(ret == 0);
+
+ /* Print VL DATA */
+ for(i = 0; i < 5; i++) {
+ int temp = i*increment + increment;
+
+ fprintf(stderr, "Element %d size %zu: ", i, rdata[i].len);
+ for(j = 0; j < temp; j++)
+ fprintf(stderr, "%d ",((unsigned int *)rdata[i].p)[j]);
+ fprintf(stderr, "\n");
+ } /* end for */
+
+ H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, rdata);
+ H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, wdata);
+
+ H5Sclose(sid);
+ H5Tclose(tid);
+
+ /* close dataset */
+ assert(H5Dclose_ff(dsid, e_stack) == 0);
+
+ }
+
+ {
+ hid_t tid;
+ hsize_t dims[1];
+ const char *str_wdata[4]= {
+ "Four score and seven years ago our forefathers brought forth on this continent a new nation,",
+ "conceived in liberty and dedicated to the proposition that all men are created equal.",
+ "Now we are engaged in a great civil war,",
+ "testing whether that nation or any nation so conceived and so dedicated can long endure."
+ }; /* Information to write */
+ char *str_rdata[4]; /* Information read in */
+
+ /* create a dataspace. This is a local Bookeeping operation that
+ does not touch the file */
+ dims [0] = 4;
+ sid = H5Screate_simple(1, dims, NULL);
+
+ /* Create a datatype to refer to */
+ tid = H5Tcopy(H5T_C_S1);
+ H5Tset_size(tid,H5T_VARIABLE);
+
+ /* Create Dataset */
+ if((dsid = H5Dcreate_ff(file_id, "dset_vl_str", tid, sid,
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, 0 , e_stack)) < 0) {
+ fprintf(stderr, "Failed\n");
+ exit(1);
+ }
+
+ ret = H5Dwrite(dsid, tid, sid, sid, H5P_DEFAULT, str_wdata);
+ assert(ret == 0);
+
+ ret = H5Dread(dsid, tid, sid, sid, H5P_DEFAULT, str_rdata);
+ assert(ret == 0);
+
+ fprintf(stderr, "Reading VL Strings: \n");
+ for(i=0 ; i<4 ; i++) {
+ fprintf(stderr, "%s\n", str_rdata[i]);
+ }
+
+ H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, str_rdata);
+
+ H5Sclose(sid);
+ H5Tclose(tid);
+
+ /* close dataset */
+ assert(H5Dclose_ff(dsid, e_stack) == 0);
+ }
+ /* closing the container also acts as a wait all on all pending requests
+ on the container. */
+ assert(H5Fclose_ff(file_id, e_stack) == 0);
+
+ fprintf(stderr, "\n*****************************************************************************************************************\n");
+ fprintf(stderr, "Wait on everything in EQ and check Results of operations in EQ\n");
+ fprintf(stderr, "*****************************************************************************************************************\n");
+
+ /* wait on all requests and print completion status */
+ H5EQwait(e_stack, &num_requests, &status);
+ fprintf(stderr, "%d requests in event queue. Completions: ", num_requests);
+ for(i=0 ; i<num_requests; i++)
+ fprintf(stderr, "%d ",status[i]);
+ fprintf(stderr, "\n");
+ free(status);
+
+ /* Verify data read */
+ for(u = 0; u < 60; u++) {
+ if(read_elem[u] != write_elem[u]) {
+ fprintf(stderr, "Failed\n");
+ exit(1);
+ }
+ }
+
+ fprintf(stderr, "\n*****************************************************************************************************************\n");
+ fprintf(stderr, "Finalize EFF stack\n");
+ fprintf(stderr, "*****************************************************************************************************************\n");
+
+ H5ESclose(e_stack);
+ H5Pclose(fapl_id);
+
+ /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
+ and shutsdown the FS server (when all clients send the terminate request)
+ and client */
+ EFF_finalize();
+
+ MPI_Finalize();
+ return 0;
+}
diff --git a/examples/h5ff_client_evict_deltas.c b/examples/h5ff_client_evict_deltas.c
index 0bf8ab4..87dafe0 100644
--- a/examples/h5ff_client_evict_deltas.c
+++ b/examples/h5ff_client_evict_deltas.c
@@ -1,332 +1,332 @@
-/*
- * test_client_obj.c: Client side of H5O routines
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-int main(int argc, char **argv) {
- char file_name[50];
- hid_t file_id;
- hid_t gid;
- hid_t did, map;
- hid_t sid, dtid;
- hid_t tid1, tid2, rc_id, rid1, rid2;
- hid_t fapl_id, dxpl_id;
- hid_t e_stack;
- htri_t exists = -1;
-
- const unsigned int nelem=60;
- hsize_t dims[1];
-
- uint64_t version;
- uint64_t trans_num;
-
- int my_rank, my_size;
- int provided;
- MPI_Request mpi_req;
-
- int32_t *wdata1 = NULL, *rdata1 = NULL;
- int key, value, i;
- H5ES_status_t status;
- size_t num_events = 0;
- herr_t ret;
-
- sprintf(file_name, "%s_%s", getenv("USER"), "eff_file_evdt.h5");
-
- MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
- if(MPI_THREAD_MULTIPLE != provided) {
- fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
- exit(1);
- }
-
- /* Call EFF_init to initialize the EFF stack.
- As a result of this call, the Function Shipper client is started,
- and HDF5 VOL calls are registered with the function shipper.
- An "IOD init" call is forwarded from the FS client to the FS server
- which should already be running. */
- EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
-
- MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
- MPI_Comm_size(MPI_COMM_WORLD, &my_size);
- fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
-
- wdata1 = malloc (sizeof(int32_t)*nelem);
- rdata1 = malloc (sizeof(int32_t)*nelem);
- for(i=0;i<nelem;++i) {
- wdata1[i] = i;
- rdata1[i] = 0;
- }
-
- fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
- /* Choose the IOD VOL plugin to use with this file.
- First we create a file access property list. Then we call a new routine to set
- the IOD plugin to use with this fapl */
- fapl_id = H5Pcreate (H5P_FILE_ACCESS);
- H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
-
- /* create an event Queue for managing asynchronous requests.
-
- Event Queues will releive the use from managing and completing
- individual requests for every operation. Instead of passing a
- request for every operation, the event queue is passed and
- internally the HDF5 library creates a request and adds it to
- the event queue.
-
- Multiple Event queue can be created used by the application. */
- e_stack = H5EScreate();
- assert(e_stack);
-
- /* create the file. */
- file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL);
- assert(file_id > 0);
-
- /* create 1-D dataspace with 60 elements */
- dims [0] = nelem;
- sid = H5Screate_simple(1, dims, NULL);
- dtid = H5Tcopy(H5T_STD_I32LE);
-
- /* acquire container version 1 - EXACT.
- This can be asynchronous, but here we need the acquired ID
- right after the call to start the transaction so we make synchronous. */
- if(0 == my_rank) {
- version = 1;
- rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- }
- MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
- assert(1 == version);
- if (my_rank != 0)
- rid1 = H5RCcreate(file_id, version);
-
- /* start transaction 1 with default Leader/Delegate model. Leader
- which is rank 0 here starts the transaction. It can be
- asynchronous, but we make it synchronous here so that the
- Leader can tell its delegates that the transaction is
- started. */
- if(0 == my_rank) {
- hid_t rid_temp;
-
- /* create transaction object */
- tid1 = H5TRcreate(file_id, rid1, (uint64_t)2);
- assert(tid1);
- ret = H5TRstart(tid1, H5P_DEFAULT, e_stack);
- assert(0 == ret);
-
- /* create objects */
- gid = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT,
- H5P_DEFAULT, tid1, e_stack);
- assert(gid >= 0);
-
- did = H5Dcreate_ff(gid, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT,
- H5P_DEFAULT, tid1, e_stack);
- assert(did >= 0);
-
- ret = H5Dwrite_ff(did, dtid, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata1, tid1, e_stack);
- assert(ret == 0);
-
- ret = H5Tcommit_ff(file_id, "DT1", dtid, H5P_DEFAULT, H5P_DEFAULT,
- H5P_DEFAULT, tid1, e_stack);
- assert(ret == 0);
-
- map = H5Mcreate_ff(file_id, "MAP1", H5T_STD_I32LE, H5T_STD_I32LE,
- H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
- assert(map >= 0);
-
- /* write some KV pairs to each map object. */
- {
- key = 1;
- value = 1000;
- ret = H5Mset_ff(map, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,
- H5P_DEFAULT, tid1, e_stack);
- assert(ret == 0);
-
- }
-
- ret = H5TRfinish(tid1, H5P_DEFAULT, &rid_temp, e_stack);
- assert(0 == ret);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- /* Close transaction object. Local op */
- ret = H5TRclose(tid1);
- assert(0 == ret);
-
- /* create transaction object */
- tid2 = H5TRcreate(file_id, rid_temp, (uint64_t)3);
- assert(tid2);
- ret = H5TRstart(tid2, H5P_DEFAULT, e_stack);
- assert(0 == ret);
-
- ret = H5Oset_comment_ff(gid, "Testing Object Comment", tid2, e_stack);
- assert(ret == 0);
-
- ret = H5TRfinish(tid2, H5P_DEFAULT, &rid2, e_stack);
- assert(0 == ret);
-
- assert(H5Gclose_ff(gid, e_stack) == 0);
- assert(H5Mclose_ff(map, e_stack) == 0);
- assert(H5Tclose_ff(dtid, e_stack) == 0);
- assert(H5Dclose_ff(did, e_stack) == 0);
-
- /* release container version 2. */
- ret = H5RCrelease(rid_temp, e_stack);
- assert(0 == ret);
- ret = H5RCclose(rid_temp);
- assert(0 == ret);
-
- version = 3;
- }
-
- /* release container version 1. */
- if(0 == my_rank) {
- ret = H5RCrelease(rid1, e_stack);
- assert(0 == ret);
- }
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- if(0 == my_rank) {
- /* Close transaction object. Local op */
- ret = H5TRclose(tid2);
- assert(0 == ret);
- }
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- /* Process 0 tells other procs that container version 2 is acquired */
- MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
- assert(3 == version);
-
- /* other processes just create a read context object; no need to
- acquire it */
- if(0 != my_rank) {
- rid2 = H5RCcreate(file_id, version);
- assert(rid2 > 0);
- }
-
- /* wait on all requests and print completion status */
- gid = H5Oopen_ff(file_id, "G1", H5P_DEFAULT, rid2);
- assert(gid);
- H5ESget_count(e_stack, &num_events);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- dtid = H5Oopen_ff(file_id, "DT1", H5P_DEFAULT, rid2);
- assert(dtid);
- did = H5Oopen_ff(gid,"D1", H5P_DEFAULT, rid2);
- assert(did);
- map = H5Oopen_ff(file_id,"MAP1", H5P_DEFAULT, rid2);
- assert(map);
-
- if(0 == my_rank) {
- ret = H5RCpersist(rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- }
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- if((my_size > 1 && 1 == my_rank) ||
- (my_size == 1 && 0 == my_rank)) {
- ret = H5Tevict_ff(dtid, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- assert(0 == ret);
-
- ret = H5Devict_ff(did, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- assert(0 == ret);
-
- ret = H5Mevict_ff(map, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- assert(0 == ret);
-
- ret = H5Gevict_ff(gid, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- assert(0 == ret);
-
- /* see if we can read after evicting */
- ret = H5Dread_ff(did, dtid, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata1, rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- printf("Read Data1: ");
- for(i=0;i<nelem;++i)
- printf("%d ",rdata1[i]);
- printf("\n");
-
- key = 1;
- value = -1;
- ret = H5Mget_ff(map, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,
- H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL);
- printf("Value recieved = %d\n", value);
- }
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- if(my_rank == 0) {
- /* release container version 3. */
- ret = H5RCrelease(rid2, e_stack);
- assert(0 == ret);
- }
-
- assert(H5Oclose_ff(gid, e_stack) == 0);
- assert(H5Oclose_ff(did, e_stack) == 0);
- assert(H5Oclose_ff(dtid, e_stack) == 0);
- assert(H5Oclose_ff(map, e_stack) == 0);
-
- ret = H5RCclose(rid1);
- assert(0 == ret);
- ret = H5RCclose(rid2);
- assert(0 == ret);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- if(my_rank == 0) {
- ret = H5TRskip(file_id, 4, 10, e_stack);
- assert(0 == ret);
- }
-
- /* closing the container also acts as a wait all on all pending requests
- on the container. */
- assert(H5Fclose_ff(file_id, 1, H5_EVENT_STACK_NULL) == 0);
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- file_id = H5Fopen_ff(file_name, H5F_ACC_RDONLY, fapl_id, &rc_id, H5_EVENT_STACK_NULL);
- assert(file_id);
-
- H5RCget_version( rc_id, &version );
- fprintf(stderr, "APP-r%d: Re-open %s version %d\n", my_rank, file_name, (int)version);
- assert(14 == version);
-
- /* release container version. */
- ret = H5RCrelease(rc_id, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- ret = H5RCclose(rc_id);
- assert(0 == ret);
-
- MPI_Barrier(MPI_COMM_WORLD);
- assert(H5Fclose_ff(file_id, 0, H5_EVENT_STACK_NULL) == 0);
-
- H5Sclose(sid);
- H5Pclose(fapl_id);
- H5ESclose(e_stack);
-
- /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
- and shutsdown the FS server (when all clients send the terminate request)
- and client */
- MPI_Barrier(MPI_COMM_WORLD);
- ret = EFF_finalize();
- assert(ret >= 0);
-
- MPI_Finalize();
- return 0;
-}
+/*
+ * test_client_obj.c: Client side of H5O routines
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+int main(int argc, char **argv) {
+ char file_name[50];
+ hid_t file_id;
+ hid_t gid;
+ hid_t did, map;
+ hid_t sid, dtid;
+ hid_t tid1, tid2, rc_id, rid1, rid2;
+ hid_t fapl_id, dxpl_id;
+ hid_t e_stack;
+ htri_t exists = -1;
+
+ const unsigned int nelem=60;
+ hsize_t dims[1];
+
+ uint64_t version;
+ uint64_t trans_num;
+
+ int my_rank, my_size;
+ int provided;
+ MPI_Request mpi_req;
+
+ int32_t *wdata1 = NULL, *rdata1 = NULL;
+ int key, value, i;
+ H5ES_status_t status;
+ size_t num_events = 0;
+ herr_t ret;
+
+ sprintf(file_name, "%s_%s", getenv("USER"), "eff_file_evdt.h5");
+
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+ if(MPI_THREAD_MULTIPLE != provided) {
+ fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
+ exit(1);
+ }
+
+ /* Call EFF_init to initialize the EFF stack.
+ As a result of this call, the Function Shipper client is started,
+ and HDF5 VOL calls are registered with the function shipper.
+ An "IOD init" call is forwarded from the FS client to the FS server
+ which should already be running. */
+ EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &my_size);
+ fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
+
+ wdata1 = malloc (sizeof(int32_t)*nelem);
+ rdata1 = malloc (sizeof(int32_t)*nelem);
+ for(i=0;i<nelem;++i) {
+ wdata1[i] = i;
+ rdata1[i] = 0;
+ }
+
+ fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
+ /* Choose the IOD VOL plugin to use with this file.
+ First we create a file access property list. Then we call a new routine to set
+ the IOD plugin to use with this fapl */
+ fapl_id = H5Pcreate (H5P_FILE_ACCESS);
+ H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ /* create an event Queue for managing asynchronous requests.
+
+ Event Queues will releive the use from managing and completing
+ individual requests for every operation. Instead of passing a
+ request for every operation, the event queue is passed and
+ internally the HDF5 library creates a request and adds it to
+ the event queue.
+
+ Multiple Event queue can be created used by the application. */
+ e_stack = H5EScreate();
+ assert(e_stack);
+
+ /* create the file. */
+ file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL);
+ assert(file_id > 0);
+
+ /* create 1-D dataspace with 60 elements */
+ dims [0] = nelem;
+ sid = H5Screate_simple(1, dims, NULL);
+ dtid = H5Tcopy(H5T_STD_I32LE);
+
+ /* acquire container version 1 - EXACT.
+ This can be asynchronous, but here we need the acquired ID
+ right after the call to start the transaction so we make synchronous. */
+ if(0 == my_rank) {
+ version = 1;
+ rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ }
+ MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
+ assert(1 == version);
+ if (my_rank != 0)
+ rid1 = H5RCcreate(file_id, version);
+
+ /* start transaction 1 with default Leader/Delegate model. Leader
+ which is rank 0 here starts the transaction. It can be
+ asynchronous, but we make it synchronous here so that the
+ Leader can tell its delegates that the transaction is
+ started. */
+ if(0 == my_rank) {
+ hid_t rid_temp;
+
+ /* create transaction object */
+ tid1 = H5TRcreate(file_id, rid1, (uint64_t)2);
+ assert(tid1);
+ ret = H5TRstart(tid1, H5P_DEFAULT, e_stack);
+ assert(0 == ret);
+
+ /* create objects */
+ gid = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT,
+ H5P_DEFAULT, tid1, e_stack);
+ assert(gid >= 0);
+
+ did = H5Dcreate_ff(gid, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT,
+ H5P_DEFAULT, tid1, e_stack);
+ assert(did >= 0);
+
+ ret = H5Dwrite_ff(did, dtid, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata1, tid1, e_stack);
+ assert(ret == 0);
+
+ ret = H5Tcommit_ff(file_id, "DT1", dtid, H5P_DEFAULT, H5P_DEFAULT,
+ H5P_DEFAULT, tid1, e_stack);
+ assert(ret == 0);
+
+ map = H5Mcreate_ff(file_id, "MAP1", H5T_STD_I32LE, H5T_STD_I32LE,
+ H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
+ assert(map >= 0);
+
+ /* write some KV pairs to each map object. */
+ {
+ key = 1;
+ value = 1000;
+ ret = H5Mset_ff(map, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,
+ H5P_DEFAULT, tid1, e_stack);
+ assert(ret == 0);
+
+ }
+
+ ret = H5TRfinish(tid1, H5P_DEFAULT, &rid_temp, e_stack);
+ assert(0 == ret);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ /* Close transaction object. Local op */
+ ret = H5TRclose(tid1);
+ assert(0 == ret);
+
+ /* create transaction object */
+ tid2 = H5TRcreate(file_id, rid_temp, (uint64_t)3);
+ assert(tid2);
+ ret = H5TRstart(tid2, H5P_DEFAULT, e_stack);
+ assert(0 == ret);
+
+ ret = H5Oset_comment_ff(gid, "Testing Object Comment", tid2, e_stack);
+ assert(ret == 0);
+
+ ret = H5TRfinish(tid2, H5P_DEFAULT, &rid2, e_stack);
+ assert(0 == ret);
+
+ assert(H5Gclose_ff(gid, e_stack) == 0);
+ assert(H5Mclose_ff(map, e_stack) == 0);
+ assert(H5Tclose_ff(dtid, e_stack) == 0);
+ assert(H5Dclose_ff(did, e_stack) == 0);
+
+ /* release container version 2. */
+ ret = H5RCrelease(rid_temp, e_stack);
+ assert(0 == ret);
+ ret = H5RCclose(rid_temp);
+ assert(0 == ret);
+
+ version = 3;
+ }
+
+ /* release container version 1. */
+ if(0 == my_rank) {
+ ret = H5RCrelease(rid1, e_stack);
+ assert(0 == ret);
+ }
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ if(0 == my_rank) {
+ /* Close transaction object. Local op */
+ ret = H5TRclose(tid2);
+ assert(0 == ret);
+ }
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /* Process 0 tells other procs that container version 2 is acquired */
+ MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
+ assert(3 == version);
+
+ /* other processes just create a read context object; no need to
+ acquire it */
+ if(0 != my_rank) {
+ rid2 = H5RCcreate(file_id, version);
+ assert(rid2 > 0);
+ }
+
+ /* wait on all requests and print completion status */
+ gid = H5Oopen_ff(file_id, "G1", H5P_DEFAULT, rid2);
+ assert(gid);
+ H5ESget_count(e_stack, &num_events);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ dtid = H5Oopen_ff(file_id, "DT1", H5P_DEFAULT, rid2);
+ assert(dtid);
+ did = H5Oopen_ff(gid,"D1", H5P_DEFAULT, rid2);
+ assert(did);
+ map = H5Oopen_ff(file_id,"MAP1", H5P_DEFAULT, rid2);
+ assert(map);
+
+ if(0 == my_rank) {
+ ret = H5RCpersist(rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ }
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if((my_size > 1 && 1 == my_rank) ||
+ (my_size == 1 && 0 == my_rank)) {
+ ret = H5Tevict_ff(dtid, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+
+ ret = H5Devict_ff(did, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+
+ ret = H5Mevict_ff(map, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+
+ ret = H5Gevict_ff(gid, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+
+ /* see if we can read after evicting */
+ ret = H5Dread_ff(did, dtid, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata1, rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ printf("Read Data1: ");
+ for(i=0;i<nelem;++i)
+ printf("%d ",rdata1[i]);
+ printf("\n");
+
+ key = 1;
+ value = -1;
+ ret = H5Mget_ff(map, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,
+ H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL);
+ printf("Value recieved = %d\n", value);
+ }
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if(my_rank == 0) {
+ /* release container version 3. */
+ ret = H5RCrelease(rid2, e_stack);
+ assert(0 == ret);
+ }
+
+ assert(H5Oclose_ff(gid, e_stack) == 0);
+ assert(H5Oclose_ff(did, e_stack) == 0);
+ assert(H5Oclose_ff(dtid, e_stack) == 0);
+ assert(H5Oclose_ff(map, e_stack) == 0);
+
+ ret = H5RCclose(rid1);
+ assert(0 == ret);
+ ret = H5RCclose(rid2);
+ assert(0 == ret);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ if(my_rank == 0) {
+ ret = H5TRskip(file_id, 4, 10, e_stack);
+ assert(0 == ret);
+ }
+
+ /* closing the container also acts as a wait all on all pending requests
+ on the container. */
+ assert(H5Fclose_ff(file_id, 1, H5_EVENT_STACK_NULL) == 0);
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ file_id = H5Fopen_ff(file_name, H5F_ACC_RDONLY, fapl_id, &rc_id, H5_EVENT_STACK_NULL);
+ assert(file_id);
+
+ H5RCget_version( rc_id, &version );
+ fprintf(stderr, "APP-r%d: Re-open %s version %d\n", my_rank, file_name, (int)version);
+ assert(14 == version);
+
+ /* release container version. */
+ ret = H5RCrelease(rc_id, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ ret = H5RCclose(rc_id);
+ assert(0 == ret);
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ assert(H5Fclose_ff(file_id, 0, H5_EVENT_STACK_NULL) == 0);
+
+ H5Sclose(sid);
+ H5Pclose(fapl_id);
+ H5ESclose(e_stack);
+
+ /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
+ and shutsdown the FS server (when all clients send the terminate request)
+ and client */
+ MPI_Barrier(MPI_COMM_WORLD);
+ ret = EFF_finalize();
+ assert(ret >= 0);
+
+ MPI_Finalize();
+ return 0;
+}
diff --git a/examples/h5ff_client_links.c b/examples/h5ff_client_links.c
index 415491a..ebaa517 100644
--- a/examples/h5ff_client_links.c
+++ b/examples/h5ff_client_links.c
@@ -1,314 +1,314 @@
-/*
- * test_client_map.c: Client side of Map Object
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-int main(int argc, char **argv) {
- char file_name[50];
- hid_t file_id;
- hid_t gid1, gid2, gid3, gid4, gid5;
- hid_t did1, did2, did3;
- hid_t sid, dtid;
- hid_t tid1, tid2, rid1, rid2, rid3;
- hid_t fapl_id, dxpl_id, trspl_id;
- hid_t e_stack;
-
- const unsigned int nelem=60;
- hsize_t dims[1];
-
- uint64_t version;
- uint64_t trans_num;
-
- int my_rank, my_size;
- int provided;
- MPI_Request mpi_req;
-
- H5ES_status_t status;
- uint32_t cs_scope = 0;
- size_t num_events = 0;
- herr_t ret;
-
- sprintf(file_name, "%s_%s", getenv("USER"), "eff_file_links.h5");
-
- MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
- if(MPI_THREAD_MULTIPLE != provided) {
- fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
- exit(1);
- }
-
- /* Call EFF_init to initialize the EFF stack.
- As a result of this call, the Function Shipper client is started,
- and HDF5 VOL calls are registered with the function shipper.
- An "IOD init" call is forwarded from the FS client to the FS server
- which should already be running. */
- EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
-
- MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
- MPI_Comm_size(MPI_COMM_WORLD, &my_size);
- fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
-
- fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
- /* Choose the IOD VOL plugin to use with this file.
- First we create a file access property list. Then we call a new routine to set
- the IOD plugin to use with this fapl */
- fapl_id = H5Pcreate (H5P_FILE_ACCESS);
- H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
-
- /* set no metadata integrity */
- cs_scope = 0;
- ret = H5Pset_metadata_integrity_scope(fapl_id, cs_scope);
- assert(ret == 0);
-
- /* create an event Queue for managing asynchronous requests.
-
- Event Queues will releive the use from managing and completing
- individual requests for every operation. Instead of passing a
- request for every operation, the event queue is passed and
- internally the HDF5 library creates a request and adds it to
- the event queue.
-
- Multiple Event queue can be created used by the application. */
- e_stack = H5EScreate();
- assert(e_stack);
-
- /* create the file. This is asynchronous, but the file_id can be used. */
- file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
- assert(file_id);
-
- /* create 1-D dataspace with 60 elements */
- dims [0] = nelem;
- sid = H5Screate_simple(1, dims, NULL);
- dtid = H5Tcopy(H5T_STD_I32LE);
-
- /* start transaction 1 with default Leader/Delegate model. Leader
- which is rank 0 here starts the transaction. It can be
- asynchronous, but we make it synchronous here so that the
- Leader can tell its delegates that the transaction is
- started. */
- if(0 == my_rank) {
-
- /* acquire container version 1 - EXACT.
- This can be asynchronous, but here we need the acquired ID
- right after the call to start the transaction so we make synchronous. */
- version = 1;
- rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- assert(1 == version);
-
- /* create transaction object */
- tid1 = H5TRcreate(file_id, rid1, (uint64_t)2);
- assert(tid1);
- ret = H5TRstart(tid1, H5P_DEFAULT, e_stack);
- assert(0 == ret);
-
- /* create objects */
- gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
- gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
- gid3 = H5Gcreate_ff(gid2, "G3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
-
- did1 = H5Dcreate_ff(gid3, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
- did2 = H5Dcreate_ff(gid3, "D2", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
- did3 = H5Dcreate_ff(gid3, "D3", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
-
- gid4 = H5Gcreate_ff(file_id, "G4", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
- gid5 = H5Gcreate_ff(gid4, "G5", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
-
- ret = H5TRfinish(tid1, H5P_DEFAULT, &rid2, e_stack);
- assert(0 == ret);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
- assert(status == H5ES_STATUS_SUCCEED);
-
- /* Close transaction object. Local op */
- ret = H5TRclose(tid1);
- assert(0 == ret);
-
- /* create transaction object */
- tid2 = H5TRcreate(file_id, rid2, (uint64_t)3);
- assert(tid2);
- ret = H5TRstart(tid2, H5P_DEFAULT, e_stack);
- assert(0 == ret);
-
- /* create a hard link to D1 (created previosuly) so that it can be
- accessed from G5/newD1. */
- ret = H5Lcreate_hard_ff(did1, ".", gid5, "newD1", H5P_DEFAULT,
- H5P_DEFAULT, tid2, e_stack);
- assert(ret == 0);
-
- /* Do more updates on transaction 2 */
-
- ret = H5Lcreate_soft_ff("/G1/G2/G3/D4", gid4, "G5/newD2",
- H5P_DEFAULT, H5P_DEFAULT, tid2, e_stack);
- assert(ret == 0);
-
- ret = H5Lmove_ff(gid3, "D3", file_id, "/G4/G5/D3moved",
- H5P_DEFAULT, H5P_DEFAULT, tid2, e_stack);
- assert(ret == 0);
-
- /* need to close object before removing it */
- assert(H5Dclose_ff(did2, e_stack) == 0);
-
- ret = H5Ldelete_ff(file_id, "/G1/G2/G3/D2", H5P_DEFAULT, tid2, e_stack);
- assert(ret == 0);
-
- /* make this synchronous so we know the container version has been acquired */
- ret = H5TRfinish(tid2, H5P_DEFAULT, &rid3, e_stack);
- assert(0 == ret);
-
- H5ESwait_all(e_stack, &status);
-
- /* should succeed */
- did2 = H5Dopen_ff(file_id,"/G1/G2/G3/D2", H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL);
- assert(did2);
- H5Dclose_ff(did2, H5_EVENT_STACK_NULL);
-
- /* should fail */
- H5E_BEGIN_TRY {
- did2 = H5Dopen_ff(file_id,"/G1/G2/G3/D2", H5P_DEFAULT, rid3, H5_EVENT_STACK_NULL);
- } H5E_END_TRY;
- assert(did2 < 0);
-
- /* release container version 1. This is async. */
- ret = H5RCrelease(rid2, e_stack);
- assert(0 == ret);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
- assert(status == H5ES_STATUS_SUCCEED);
-
- /* Close transaction object. Local op */
- ret = H5TRclose(tid2);
- assert(0 == ret);
-
- assert(H5Gclose_ff(gid1, e_stack) == 0);
- assert(H5Gclose_ff(gid2, e_stack) == 0);
- assert(H5Gclose_ff(gid3, e_stack) == 0);
- assert(H5Gclose_ff(gid4, e_stack) == 0);
- assert(H5Gclose_ff(gid5, e_stack) == 0);
- assert(H5Dclose_ff(did1, e_stack) == 0);
- assert(H5Dclose_ff(did3, e_stack) == 0);
- version = 3;
-
- /* release container version 1. This is async. */
- ret = H5RCrelease(rid1, e_stack);
- assert(0 == ret);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
- assert(status == H5ES_STATUS_SUCCEED);
- }
-
- /* Leader tells other procs that container version 2 is acquired */
- MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
-
- /* other processes just create a read context object; no need to
- acquire it */
- if(0 != my_rank) {
- assert(3 == version);
- rid3 = H5RCcreate(file_id, version);
- assert(rid3 > 0);
- }
-
- /* Try and open the dataset. This is asynchronous. */
- did2 = H5Dopen_ff(file_id,"G4/G5/newD1", H5P_DEFAULT, rid3, e_stack);
- assert(did2);
-
- gid4 = H5Gopen_ff(file_id, "G4", H5P_DEFAULT, rid3, e_stack);
- assert(gid4);
-
- {
- H5L_ff_info_t linfo;
- char *link_buf;
- htri_t exists;
-
- ret = H5Lexists_ff(file_id, "/G1/G2/G3/D2", H5P_DEFAULT, &exists,
- rid3, H5_EVENT_STACK_NULL );
- assert(ret == 0);
- assert(0 == exists);
-
- ret = H5Lget_info_ff(gid4, "G5/newD2", &linfo, H5P_DEFAULT,
- rid3, H5_EVENT_STACK_NULL);
- assert(ret == 0);
-
- switch(linfo.type) {
- case H5L_TYPE_SOFT:
- fprintf(stderr,
- "LINK GET INFO return a SOFT LINK with value size: %zu\n",
- linfo.u.val_size);
- break;
- default:
- fprintf(stderr, "Unexpected result from lget_info\n");
- exit(1);
- }
-
- link_buf = malloc(linfo.u.val_size);
-
- ret = H5Lget_val_ff(gid4, "G5/newD2", link_buf, linfo.u.val_size,
- H5P_DEFAULT, rid3, H5_EVENT_STACK_NULL);
- assert(ret == 0);
-
- fprintf(stderr, "Link value = %s\n", link_buf);
-
- free(link_buf);
- }
-
- if(0 == my_rank) {
- /* release container version 3. This is async. */
- ret = H5RCrelease(rid3, H5_EVENT_STACK_NULL);
- assert(0 == ret);
-
- ret = H5RCclose(rid2);
- assert(0 == ret);
-
- ret = H5RCclose(rid1);
- assert(0 == ret);
- }
-
- ret = H5RCclose(rid3);
- assert(0 == ret);
-
- assert(H5Dclose_ff(did2, e_stack) == 0);
- assert(H5Gclose_ff(gid4, e_stack) == 0);
-
- /* closing the container also acts as a wait all on all pending requests
- on the container. */
- assert(H5Fclose_ff(file_id, 1, H5_EVENT_STACK_NULL) == 0);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
- assert(status == H5ES_STATUS_SUCCEED);
-
- assert(0 == ret);
-
- H5Sclose(sid);
- H5Tclose(dtid);
- H5Pclose(fapl_id);
- H5ESclose(e_stack);
-
- MPI_Barrier(MPI_COMM_WORLD);
- /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
- and shutsdown the FS server (when all clients send the terminate request)
- and client */
- EFF_finalize();
-
- MPI_Finalize();
- return 0;
-}
+/*
+ * test_client_map.c: Client side of Map Object
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+int main(int argc, char **argv) {
+ char file_name[50];
+ hid_t file_id;
+ hid_t gid1, gid2, gid3, gid4, gid5;
+ hid_t did1, did2, did3;
+ hid_t sid, dtid;
+ hid_t tid1, tid2, rid1, rid2, rid3;
+ hid_t fapl_id, dxpl_id, trspl_id;
+ hid_t e_stack;
+
+ const unsigned int nelem=60;
+ hsize_t dims[1];
+
+ uint64_t version;
+ uint64_t trans_num;
+
+ int my_rank, my_size;
+ int provided;
+ MPI_Request mpi_req;
+
+ H5ES_status_t status;
+ uint32_t cs_scope = 0;
+ size_t num_events = 0;
+ herr_t ret;
+
+ sprintf(file_name, "%s_%s", getenv("USER"), "eff_file_links.h5");
+
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+ if(MPI_THREAD_MULTIPLE != provided) {
+ fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
+ exit(1);
+ }
+
+ /* Call EFF_init to initialize the EFF stack.
+ As a result of this call, the Function Shipper client is started,
+ and HDF5 VOL calls are registered with the function shipper.
+ An "IOD init" call is forwarded from the FS client to the FS server
+ which should already be running. */
+ EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &my_size);
+ fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
+
+ fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
+ /* Choose the IOD VOL plugin to use with this file.
+ First we create a file access property list. Then we call a new routine to set
+ the IOD plugin to use with this fapl */
+ fapl_id = H5Pcreate (H5P_FILE_ACCESS);
+ H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ /* set no metadata integrity */
+ cs_scope = 0;
+ ret = H5Pset_metadata_integrity_scope(fapl_id, cs_scope);
+ assert(ret == 0);
+
+ /* create an event Queue for managing asynchronous requests.
+
+ Event Queues will releive the use from managing and completing
+ individual requests for every operation. Instead of passing a
+ request for every operation, the event queue is passed and
+ internally the HDF5 library creates a request and adds it to
+ the event queue.
+
+ Multiple Event queue can be created used by the application. */
+ e_stack = H5EScreate();
+ assert(e_stack);
+
+ /* create the file. This is asynchronous, but the file_id can be used. */
+ file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+ assert(file_id);
+
+ /* create 1-D dataspace with 60 elements */
+ dims [0] = nelem;
+ sid = H5Screate_simple(1, dims, NULL);
+ dtid = H5Tcopy(H5T_STD_I32LE);
+
+ /* start transaction 1 with default Leader/Delegate model. Leader
+ which is rank 0 here starts the transaction. It can be
+ asynchronous, but we make it synchronous here so that the
+ Leader can tell its delegates that the transaction is
+ started. */
+ if(0 == my_rank) {
+
+ /* acquire container version 1 - EXACT.
+ This can be asynchronous, but here we need the acquired ID
+ right after the call to start the transaction so we make synchronous. */
+ version = 1;
+ rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ assert(1 == version);
+
+ /* create transaction object */
+ tid1 = H5TRcreate(file_id, rid1, (uint64_t)2);
+ assert(tid1);
+ ret = H5TRstart(tid1, H5P_DEFAULT, e_stack);
+ assert(0 == ret);
+
+ /* create objects */
+ gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+ gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+ gid3 = H5Gcreate_ff(gid2, "G3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+
+ did1 = H5Dcreate_ff(gid3, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+ did2 = H5Dcreate_ff(gid3, "D2", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+ did3 = H5Dcreate_ff(gid3, "D3", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+
+ gid4 = H5Gcreate_ff(file_id, "G4", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+ gid5 = H5Gcreate_ff(gid4, "G5", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+
+ ret = H5TRfinish(tid1, H5P_DEFAULT, &rid2, e_stack);
+ assert(0 == ret);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+ assert(status == H5ES_STATUS_SUCCEED);
+
+ /* Close transaction object. Local op */
+ ret = H5TRclose(tid1);
+ assert(0 == ret);
+
+ /* create transaction object */
+ tid2 = H5TRcreate(file_id, rid2, (uint64_t)3);
+ assert(tid2);
+ ret = H5TRstart(tid2, H5P_DEFAULT, e_stack);
+ assert(0 == ret);
+
+ /* create a hard link to D1 (created previosuly) so that it can be
+ accessed from G5/newD1. */
+ ret = H5Lcreate_hard_ff(did1, ".", gid5, "newD1", H5P_DEFAULT,
+ H5P_DEFAULT, tid2, e_stack);
+ assert(ret == 0);
+
+ /* Do more updates on transaction 2 */
+
+ ret = H5Lcreate_soft_ff("/G1/G2/G3/D4", gid4, "G5/newD2",
+ H5P_DEFAULT, H5P_DEFAULT, tid2, e_stack);
+ assert(ret == 0);
+
+ ret = H5Lmove_ff(gid3, "D3", file_id, "/G4/G5/D3moved",
+ H5P_DEFAULT, H5P_DEFAULT, tid2, e_stack);
+ assert(ret == 0);
+
+ /* need to close object before removing it */
+ assert(H5Dclose_ff(did2, e_stack) == 0);
+
+ ret = H5Ldelete_ff(file_id, "/G1/G2/G3/D2", H5P_DEFAULT, tid2, e_stack);
+ assert(ret == 0);
+
+ /* make this synchronous so we know the container version has been acquired */
+ ret = H5TRfinish(tid2, H5P_DEFAULT, &rid3, e_stack);
+ assert(0 == ret);
+
+ H5ESwait_all(e_stack, &status);
+
+ /* should succeed */
+ did2 = H5Dopen_ff(file_id,"/G1/G2/G3/D2", H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL);
+ assert(did2);
+ H5Dclose_ff(did2, H5_EVENT_STACK_NULL);
+
+ /* should fail */
+ H5E_BEGIN_TRY {
+ did2 = H5Dopen_ff(file_id,"/G1/G2/G3/D2", H5P_DEFAULT, rid3, H5_EVENT_STACK_NULL);
+ } H5E_END_TRY;
+ assert(did2 < 0);
+
+ /* release container version 1. This is async. */
+ ret = H5RCrelease(rid2, e_stack);
+ assert(0 == ret);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+ assert(status == H5ES_STATUS_SUCCEED);
+
+ /* Close transaction object. Local op */
+ ret = H5TRclose(tid2);
+ assert(0 == ret);
+
+ assert(H5Gclose_ff(gid1, e_stack) == 0);
+ assert(H5Gclose_ff(gid2, e_stack) == 0);
+ assert(H5Gclose_ff(gid3, e_stack) == 0);
+ assert(H5Gclose_ff(gid4, e_stack) == 0);
+ assert(H5Gclose_ff(gid5, e_stack) == 0);
+ assert(H5Dclose_ff(did1, e_stack) == 0);
+ assert(H5Dclose_ff(did3, e_stack) == 0);
+ version = 3;
+
+ /* release container version 1. This is async. */
+ ret = H5RCrelease(rid1, e_stack);
+ assert(0 == ret);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+ assert(status == H5ES_STATUS_SUCCEED);
+ }
+
+ /* Leader tells other procs that container version 2 is acquired */
+ MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
+
+ /* other processes just create a read context object; no need to
+ acquire it */
+ if(0 != my_rank) {
+ assert(3 == version);
+ rid3 = H5RCcreate(file_id, version);
+ assert(rid3 > 0);
+ }
+
+ /* Try and open the dataset. This is asynchronous. */
+ did2 = H5Dopen_ff(file_id,"G4/G5/newD1", H5P_DEFAULT, rid3, e_stack);
+ assert(did2);
+
+ gid4 = H5Gopen_ff(file_id, "G4", H5P_DEFAULT, rid3, e_stack);
+ assert(gid4);
+
+ {
+ H5L_ff_info_t linfo;
+ char *link_buf;
+ htri_t exists;
+
+ ret = H5Lexists_ff(file_id, "/G1/G2/G3/D2", H5P_DEFAULT, &exists,
+ rid3, H5_EVENT_STACK_NULL );
+ assert(ret == 0);
+ assert(0 == exists);
+
+ ret = H5Lget_info_ff(gid4, "G5/newD2", &linfo, H5P_DEFAULT,
+ rid3, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+
+ switch(linfo.type) {
+ case H5L_TYPE_SOFT:
+ fprintf(stderr,
+ "LINK GET INFO return a SOFT LINK with value size: %zu\n",
+ linfo.u.val_size);
+ break;
+ default:
+ fprintf(stderr, "Unexpected result from lget_info\n");
+ exit(1);
+ }
+
+ link_buf = malloc(linfo.u.val_size);
+
+ ret = H5Lget_val_ff(gid4, "G5/newD2", link_buf, linfo.u.val_size,
+ H5P_DEFAULT, rid3, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+
+ fprintf(stderr, "Link value = %s\n", link_buf);
+
+ free(link_buf);
+ }
+
+ if(0 == my_rank) {
+ /* release container version 3. This is async. */
+ ret = H5RCrelease(rid3, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+
+ ret = H5RCclose(rid2);
+ assert(0 == ret);
+
+ ret = H5RCclose(rid1);
+ assert(0 == ret);
+ }
+
+ ret = H5RCclose(rid3);
+ assert(0 == ret);
+
+ assert(H5Dclose_ff(did2, e_stack) == 0);
+ assert(H5Gclose_ff(gid4, e_stack) == 0);
+
+ /* closing the container also acts as a wait all on all pending requests
+ on the container. */
+ assert(H5Fclose_ff(file_id, 1, H5_EVENT_STACK_NULL) == 0);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+ assert(status == H5ES_STATUS_SUCCEED);
+
+ assert(0 == ret);
+
+ H5Sclose(sid);
+ H5Tclose(dtid);
+ H5Pclose(fapl_id);
+ H5ESclose(e_stack);
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
+ and shutsdown the FS server (when all clients send the terminate request)
+ and client */
+ EFF_finalize();
+
+ MPI_Finalize();
+ return 0;
+}
diff --git a/examples/h5ff_client_map.c b/examples/h5ff_client_map.c
index dce84d4..27f3e6a 100644
--- a/examples/h5ff_client_map.c
+++ b/examples/h5ff_client_map.c
@@ -1,568 +1,568 @@
-/*
- * test_client_map.c: Client side of Map Object
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-#define NUM_KEYS 20
-
-int main(int argc, char **argv) {
- char file_name[50];
- hid_t file_id;
- hid_t gid1, gid2, dtid1, dtid2;
- hid_t map1, map2, map3;
- hid_t tid1, tid2, tid3, tid4, rid1, rid2, rid3;
- hid_t fapl_id, dxpl_id, trspl_id;
- hid_t e_stack;
-
- const char *str_wdata[5]= {
- "Four score and seven years ago our forefathers brought forth on this continent a new nation,",
- "conceived in liberty and dedicated to the proposition that all men are created equal.",
- "Now we are engaged in a great civil war,",
- "testing whether that nation or any nation so conceived and so dedicated can long endure.",
- "President Abraham Lincoln"
- }; /* Information to write */
- char *str_rdata[5]; /* Information read in */
- hvl_t wdata[5]; /* Information to write */
- hvl_t rdata[5]; /* Information to write */
- int i;
-
- void *map_token1, *map_token2, *map_token3;
- size_t token_size1, token_size2, token_size3;
- uint64_t version;
- uint64_t trans_num;
-
- int my_rank, my_size;
- int provided;
- MPI_Request mpi_req, mpi_reqs[6];
-
- H5ES_status_t status;
- size_t num_events = 0;
- herr_t ret;
-
- hsize_t count = -1;
- int key, value;
- htri_t exists;
-
- sprintf(file_name, "%s_%s", getenv("USER"), "eff_file_map.h5");
-
- MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
- if(MPI_THREAD_MULTIPLE != provided) {
- fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
- exit(1);
- }
-
- /* Call EFF_init to initialize the EFF stack.
- As a result of this call, the Function Shipper client is started,
- and HDF5 VOL calls are registered with the function shipper.
- An "IOD init" call is forwarded from the FS client to the FS server
- which should already be running. */
- EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
-
- MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
- MPI_Comm_size(MPI_COMM_WORLD, &my_size);
- fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
-
- fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
- /* Choose the IOD VOL plugin to use with this file.
- First we create a file access property list. Then we call a new routine to set
- the IOD plugin to use with this fapl */
- fapl_id = H5Pcreate (H5P_FILE_ACCESS);
- H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
-
- /* create an event Queue for managing asynchronous requests.
-
- Event Queues will releive the use from managing and completing
- individual requests for every operation. Instead of passing a
- request for every operation, the event queue is passed and
- internally the HDF5 library creates a request and adds it to
- the event queue.
-
- Multiple Event queue can be created used by the application. */
- e_stack = H5EScreate();
- assert(e_stack);
-
-
- /* set write buffer for VL data*/
- {
- int increment, j, n;
-
- n = 0;
- increment = 4;
- /* Allocate and initialize VL data to write */
- for(i = 0; i < 5; i++) {
- int temp = i*increment + increment;
-
- wdata[i].p = malloc(temp * sizeof(unsigned int));
- wdata[i].len = temp;
- for(j = 0; j < temp; j++)
- ((unsigned int *)wdata[i].p)[j] = n ++;
- } /* end for */
- }
-
- /* Create an HDF5 VL datatype */
- dtid1 = H5Tvlen_create (H5T_NATIVE_UINT);
- /* Create an HDF5 VL string datatype */
- dtid2 = H5Tcopy(H5T_C_S1);
- H5Tset_size(dtid2,H5T_VARIABLE);
-
- /* create the file. */
- file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL);
- assert(file_id > 0);
-
- /* acquire container version 1 - EXACT.
- This can be asynchronous, but here we need the acquired ID
- right after the call to start the transaction so we make synchronous. */
- if(0 == my_rank) {
- version = 1;
- rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- }
- MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
- assert(1 == version);
- if (my_rank != 0)
- rid1 = H5RCcreate(file_id, version);
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- /* create transaction object */
- tid1 = H5TRcreate(file_id, rid1, (uint64_t)2);
- assert(tid1);
-
- /* start transaction 2 with default Leader/Delegate model. Leader
- which is rank 0 here starts the transaction. It can be
- asynchronous, but we make it synchronous here so that the
- Leader can tell its delegates that the transaction is
- started. */
- if(0 == my_rank) {
- trans_num = 2;
- ret = H5TRstart(tid1, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- assert(0 == ret);
-
- /* Leader also create some objects in transaction 1 */
-
- /* create two groups */
- gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
- assert(gid1);
- gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
- assert(gid2);
-
- /* Create 3 Map objects with the key type being 32 bit LE integer */
-
- /* First Map object with a Value type a 32 bit LE integer */
- map1 = H5Mcreate_ff(file_id, "MAP_1", H5T_STD_I32LE, H5T_STD_I32LE,
- H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
- assert(map1);
- /* Second Map object with a Value type being an HDF5 VL datatype */
- map2 = H5Mcreate_ff(gid1, "MAP_2", H5T_STD_I32LE, dtid1,
- H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
- assert(map2);
- /* Third Map object with a Value type being an HDF5 VL string */
- map3 = H5Mcreate_ff(gid2, "MAP_3", H5T_STD_I32LE, dtid2,
- H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
- assert(map3);
-
- /* write some KV pairs to each map object. */
- {
- for(i=0 ; i<NUM_KEYS ; i++) {
- key = i;
- value = 1000 + i;
- ret = H5Mset_ff(map1, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,
- H5P_DEFAULT, tid1, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- }
-
- for(i=0 ; i<5 ; i++) {
- key = i;
- ret = H5Mset_ff(map2, H5T_STD_I32LE, &key, dtid1, &wdata[i],
- H5P_DEFAULT, tid1, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- }
-
- for(i=0 ; i<5 ; i++) {
- key = i;
- ret = H5Mset_ff(map3, H5T_STD_I32LE, &key, dtid2, str_wdata[i],
- H5P_DEFAULT, tid1, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- }
- }
- }
-
- /* Tell Delegates that transaction 1 is started */
- MPI_Ibcast(&trans_num, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD, &mpi_req);
-
- /* Do the local-to-global, global-to-local, so all delegates can
- write the maps created in transaction 1 */
-
- if(0 == my_rank) {
- /* get the token size of each map */
- ret = H5Oget_token(map1, NULL, &token_size1);
- assert(0 == ret);
- ret = H5Oget_token(map2, NULL, &token_size2);
- assert(0 == ret);
- ret = H5Oget_token(map3, NULL, &token_size3);
- assert(0 == ret);
-
- /* allocate buffers for each token */
- map_token1 = malloc(token_size1);
- map_token2 = malloc(token_size2);
- map_token3 = malloc(token_size3);
-
- /* get the token buffer */
- ret = H5Oget_token(map1, map_token1, &token_size1);
- assert(0 == ret);
- ret = H5Oget_token(map2, map_token2, &token_size2);
- assert(0 == ret);
- ret = H5Oget_token(map3, map_token3, &token_size3);
- assert(0 == ret);
-
- /* make sure the create operations have completed before
- telling the delegates to open them */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- /* bcast the token sizes and the tokens */
- MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]);
- MPI_Ibcast(&token_size2, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]);
- MPI_Ibcast(&token_size3, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]);
- MPI_Ibcast(map_token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[3]);
- MPI_Ibcast(map_token2, token_size2, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[4]);
- MPI_Ibcast(map_token3, token_size3, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[5]);
- }
-
- /* Leader can continue writing to transaction 1,
- while others wait for the ibcast to complete */
- if(0 != my_rank) {
-
- /* this wait if for the transaction start */
- MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
- assert(2 == trans_num);
-
- /* recieve the token sizes */
- MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]);
- MPI_Ibcast(&token_size2, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]);
- MPI_Ibcast(&token_size3, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]);
- MPI_Waitall(3, mpi_reqs, MPI_STATUS_IGNORE);
-
- /* allocate buffers for each token */
- map_token1 = malloc(token_size1);
- map_token2 = malloc(token_size2);
- map_token3 = malloc(token_size3);
-
- /* recieve the tokens */
- MPI_Ibcast(map_token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]);
- MPI_Ibcast(map_token2, token_size2, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]);
- MPI_Ibcast(map_token3, token_size3, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]);
- MPI_Waitall(3, mpi_reqs, MPI_STATUS_IGNORE);
-
- map1 = H5Oopen_by_token(map_token1, tid1, e_stack);
- map2 = H5Oopen_by_token(map_token2, tid1, e_stack);
- map3 = H5Oopen_by_token(map_token3, tid1, e_stack);
- }
-
-
- /* none leader procs have to complete operations before notifying the leader */
- if(0 != my_rank) {
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
- }
-
- /* Barrier to make sure all processes are done writing so Process
- 0 can finish transaction 1 and acquire a read context on it. */
- MPI_Barrier(MPI_COMM_WORLD);
-
- /* Leader process finishes/commits the transaction and acquires a
- read context on it */
- if(0 == my_rank) {
- MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
- MPI_Waitall(6, mpi_reqs, MPI_STATUS_IGNORE);
-
- ret = H5TRfinish(tid1, H5P_DEFAULT, &rid2, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- }
-
- /* another barrier so other processes know that container version is acquried */
- MPI_Barrier(MPI_COMM_WORLD);
-
- /* Close the first transaction object. Local op */
- ret = H5TRclose(tid1);
- assert(0 == ret);
-
- /* release container version 1. This is async. */
- if(0 == my_rank) {
- ret = H5RCrelease(rid1, e_stack);
- assert(0 == ret);
- }
-
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- /* Leader process tells delegates that container version 2 is acquired */
- version = 2;
- MPI_Bcast(&version, 2, MPI_UINT64_T, 0, MPI_COMM_WORLD);
-
- /* delegates just create a read context object; no need to acquire
- it, since it has been acquired by the leader. */
- if(0 != my_rank) {
- rid2 = H5RCcreate(file_id, version);
- assert(rid2 > 0);
- }
- if(0 == my_rank) {
- assert(H5Gclose_ff(gid1, e_stack) == 0);
- assert(H5Gclose_ff(gid2, e_stack) == 0);
- }
-
- /* issue some read operations using the read context acquired */
-
- ret = H5Mget_count_ff(map2, &count, rid2, e_stack);
- assert(ret == 0);
-
- key = 1;
- ret = H5Mexists_ff(map1, H5T_STD_I32LE, &key, &exists, rid2, e_stack);
- assert(ret == 0);
-
- /* create & start transaction 3, 4, and 5 with Multiple Leader - No Delegate Model. */
- tid2 = H5TRcreate(file_id, rid2, (uint64_t)3);
- assert(tid2);
- tid3 = H5TRcreate(file_id, rid2, (uint64_t)4);
- assert(tid3);
- tid4 = H5TRcreate(file_id, rid2, (uint64_t)5);
- assert(tid4);
- trspl_id = H5Pcreate (H5P_TR_START);
- ret = H5Pset_trspl_num_peers(trspl_id, my_size);
- assert(0 == ret);
- ret = H5TRstart(tid2, trspl_id, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- ret = H5TRstart(tid3, trspl_id, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- ret = H5TRstart(tid4, trspl_id, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- ret = H5Pclose(trspl_id);
- assert(0 == ret);
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- if((my_size > 1 && 1 == my_rank) ||
- (my_size == 1 && 0 == my_rank)) {
- hid_t temp_id;
-
- /* modify container contents using transaction started. */
- key = 1;
- ret = H5Mdelete_ff(map3, H5T_STD_I32LE, &key, tid2, H5_EVENT_STACK_NULL);
-
- H5E_BEGIN_TRY {
- temp_id = H5Gcreate_ff(file_id, "temp_group", H5P_DEFAULT, H5P_DEFAULT,
- H5P_DEFAULT, tid2, H5_EVENT_STACK_NULL);
- } H5E_END_TRY;
- if(temp_id > 0)
- assert(H5Gclose_ff(temp_id, H5_EVENT_STACK_NULL) ==0);
- }
-
- /* finish transaction 3, 4, 5 */
- if(my_rank == 0) {
- /* set dependencies from 4 and 5 on 3 */
- ret = H5TRset_dependency(tid3, (uint64_t)3, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- ret = H5TRset_dependency(tid4, (uint64_t)3, H5_EVENT_STACK_NULL);
- assert(0 == ret);
-
- ret = H5TRabort(tid2, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- }
- else {
- H5E_BEGIN_TRY {
- ret = H5TRfinish(tid2, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL);
- } H5E_END_TRY;
- if(ret < 0)
- fprintf(stderr, "Transaction finish failed as expected (aborted)\n");
- }
-
- H5E_BEGIN_TRY {
- ret = H5TRfinish(tid3, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL);
- } H5E_END_TRY;
- if(ret < 0)
- fprintf(stderr, "Transaction finish failed (aborted)\n");
-
- H5E_BEGIN_TRY {
- ret = H5TRfinish(tid4, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL);
- } H5E_END_TRY;
- if(ret < 0)
- fprintf(stderr, "Transaction finish failed (aborted)\n");
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- assert(H5Mclose_ff(map1, e_stack) == 0);
- assert(H5Mclose_ff(map2, e_stack) == 0);
- assert(H5Mclose_ff(map3, e_stack) == 0);
-
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- ret = H5RCclose(rid1);
- assert(0 == ret);
- ret = H5TRclose(tid2);
- assert(0 == ret);
- ret = H5TRclose(tid3);
- assert(0 == ret);
- ret = H5TRclose(tid4);
- assert(0 == ret);
-
- /* Barrier so all processes are guranteed to have finished/aborted transaction 2 */
- MPI_Barrier(MPI_COMM_WORLD);
-
- /* acquire container version 3, 4, 5 - EXACT (Should Fail since 2 is aborted) */
- version = 3;
- H5E_BEGIN_TRY {
- rid3 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- } H5E_END_TRY;
- assert(rid3 < 0);
- version = 4;
- H5E_BEGIN_TRY {
- rid3 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- } H5E_END_TRY;
- assert(rid3 < 0);
- version = 5;
- H5E_BEGIN_TRY {
- rid3 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- } H5E_END_TRY;
- assert(rid3 < 0);
-
- /* Now read some data from the container at version 1*/
-
- map1 = H5Mopen_ff(file_id, "MAP_1", H5P_DEFAULT, rid2, e_stack);
- map2 = H5Mopen_ff(file_id, "G1/MAP_2", H5P_DEFAULT, rid2, e_stack);
- map3 = H5Mopen_ff(file_id, "G1/G2/MAP_3", H5P_DEFAULT, rid2, e_stack);
-
- {
- int key, value;
- int increment=4, j=0;
-
- for(i=0 ; i<NUM_KEYS ; i++) {
- key = i;
- value = -1;
- ret = H5Mget_ff(map1, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,
- H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL);
-
- if(value != 1000+i) {
- printf("Key %d: Value recieved = %d\n", key, value);
- assert(0);
- }
- }
-
- for(i=0 ; i<5 ; i++) {
- key = i;
- ret = H5Mget_ff(map2, H5T_STD_I32LE, &key, dtid1, &rdata[i],
- H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- }
-
- /* Print VL DATA */
- fprintf(stderr, "Reading VL Data: \n");
- for(i = 0; i < 5; i++) {
- int temp = i*increment + increment;
-
- fprintf(stderr, "Key %d size %zu: ", i, rdata[i].len);
- for(j = 0; j < temp; j++)
- fprintf(stderr, "%d ",((unsigned int *)rdata[i].p)[j]);
- fprintf(stderr, "\n");
- } /* end for */
-
- for(i=0 ; i<5 ; i++) {
- key = i;
- str_rdata[i] = NULL;
- ret = H5Mget_ff(map3, H5T_STD_I32LE, &key, dtid2, &str_rdata[i],
- H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- }
-
- fprintf(stderr, "Reading VL Strings: \n");
-
- for(i=0 ; i<5 ; i++) {
- fprintf(stderr, "Key %d: %s\n", i, str_rdata[i]);
- free(str_rdata[i]);
- }
- }
-
- ret = H5Tclose(dtid1);
- assert(ret == 0);
- ret = H5Tclose(dtid2);
- assert(ret == 0);
-
- assert(H5Mclose_ff(map1, e_stack) == 0);
- assert(H5Mclose_ff(map2, e_stack) == 0);
- assert(H5Mclose_ff(map3, e_stack) == 0);
-
- if(my_rank == 0) {
- /* release container version 2. This is async. */
- ret = H5RCrelease(rid2, e_stack);
- assert(0 == ret);
- }
- ret = H5RCclose(rid2);
- assert(0 == ret);
-
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- /* closing the container also acts as a wait all on all pending requests
- on the container. */
- assert(H5Fclose_ff(file_id, 1, H5_EVENT_STACK_NULL) == 0);
-
- fprintf(stderr, "\n*******************************************************************\n");
- fprintf(stderr, "Wait on everything in ES and check Results of operations in ES\n");
- fprintf(stderr, "\n*******************************************************************\n");
-
- H5ESget_count(e_stack, &num_events);
-
- H5EStest_all(e_stack, &status);
- printf("%d events in event stack. H5EStest_all Completion status = %d\n", num_events, status);
-
- H5ESwait_all(e_stack, &status);
- printf("%d events in event stack. H5ESwait_all Completion status = %d\n", num_events, status);
-
- H5EStest_all(e_stack, &status);
- printf("%d events in event stack. H5EStest_all Completion status = %d\n", num_events, status);
-
- H5ESclear(e_stack);
-
- assert (count == 5);
- assert (exists > 0);
-
- fprintf(stderr, "\n*****************************************************************************************************************\n");
- fprintf(stderr, "Finalize EFF stack\n");
- fprintf(stderr, "*****************************************************************************************************************\n");
-
- H5ESclose(e_stack);
- H5Pclose(fapl_id);
-
- for(i=0 ; i<5 ; i++) {
- free(wdata[i].p);
- free(rdata[i].p);
- }
-
- free(map_token1);
- free(map_token2);
- free(map_token3);
-
- MPI_Barrier(MPI_COMM_WORLD);
- /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
- and shutsdown the FS server (when all clients send the terminate request)
- and client */
- EFF_finalize();
-
- MPI_Finalize();
- return 0;
-}
+/*
+ * test_client_map.c: Client side of Map Object
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+#define NUM_KEYS 20
+
+int main(int argc, char **argv) {
+ char file_name[50];
+ hid_t file_id;
+ hid_t gid1, gid2, dtid1, dtid2;
+ hid_t map1, map2, map3;
+ hid_t tid1, tid2, tid3, tid4, rid1, rid2, rid3;
+ hid_t fapl_id, dxpl_id, trspl_id;
+ hid_t e_stack;
+
+ const char *str_wdata[5]= {
+ "Four score and seven years ago our forefathers brought forth on this continent a new nation,",
+ "conceived in liberty and dedicated to the proposition that all men are created equal.",
+ "Now we are engaged in a great civil war,",
+ "testing whether that nation or any nation so conceived and so dedicated can long endure.",
+ "President Abraham Lincoln"
+ }; /* Information to write */
+ char *str_rdata[5]; /* Information read in */
+ hvl_t wdata[5]; /* Information to write */
+ hvl_t rdata[5]; /* Information to write */
+ int i;
+
+ void *map_token1, *map_token2, *map_token3;
+ size_t token_size1, token_size2, token_size3;
+ uint64_t version;
+ uint64_t trans_num;
+
+ int my_rank, my_size;
+ int provided;
+ MPI_Request mpi_req, mpi_reqs[6];
+
+ H5ES_status_t status;
+ size_t num_events = 0;
+ herr_t ret;
+
+ hsize_t count = -1;
+ int key, value;
+ htri_t exists;
+
+ sprintf(file_name, "%s_%s", getenv("USER"), "eff_file_map.h5");
+
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+ if(MPI_THREAD_MULTIPLE != provided) {
+ fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
+ exit(1);
+ }
+
+ /* Call EFF_init to initialize the EFF stack.
+ As a result of this call, the Function Shipper client is started,
+ and HDF5 VOL calls are registered with the function shipper.
+ An "IOD init" call is forwarded from the FS client to the FS server
+ which should already be running. */
+ EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &my_size);
+ fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
+
+ fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
+ /* Choose the IOD VOL plugin to use with this file.
+ First we create a file access property list. Then we call a new routine to set
+ the IOD plugin to use with this fapl */
+ fapl_id = H5Pcreate (H5P_FILE_ACCESS);
+ H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ /* create an event Queue for managing asynchronous requests.
+
+ Event Queues will releive the use from managing and completing
+ individual requests for every operation. Instead of passing a
+ request for every operation, the event queue is passed and
+ internally the HDF5 library creates a request and adds it to
+ the event queue.
+
+ Multiple Event queue can be created used by the application. */
+ e_stack = H5EScreate();
+ assert(e_stack);
+
+
+ /* set write buffer for VL data*/
+ {
+ int increment, j, n;
+
+ n = 0;
+ increment = 4;
+ /* Allocate and initialize VL data to write */
+ for(i = 0; i < 5; i++) {
+ int temp = i*increment + increment;
+
+ wdata[i].p = malloc(temp * sizeof(unsigned int));
+ wdata[i].len = temp;
+ for(j = 0; j < temp; j++)
+ ((unsigned int *)wdata[i].p)[j] = n ++;
+ } /* end for */
+ }
+
+ /* Create an HDF5 VL datatype */
+ dtid1 = H5Tvlen_create (H5T_NATIVE_UINT);
+ /* Create an HDF5 VL string datatype */
+ dtid2 = H5Tcopy(H5T_C_S1);
+ H5Tset_size(dtid2,H5T_VARIABLE);
+
+ /* create the file. */
+ file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL);
+ assert(file_id > 0);
+
+ /* acquire container version 1 - EXACT.
+ This can be asynchronous, but here we need the acquired ID
+ right after the call to start the transaction so we make synchronous. */
+ if(0 == my_rank) {
+ version = 1;
+ rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ }
+ MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
+ assert(1 == version);
+ if (my_rank != 0)
+ rid1 = H5RCcreate(file_id, version);
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /* create transaction object */
+ tid1 = H5TRcreate(file_id, rid1, (uint64_t)2);
+ assert(tid1);
+
+ /* start transaction 2 with default Leader/Delegate model. Leader
+ which is rank 0 here starts the transaction. It can be
+ asynchronous, but we make it synchronous here so that the
+ Leader can tell its delegates that the transaction is
+ started. */
+ if(0 == my_rank) {
+ trans_num = 2;
+ ret = H5TRstart(tid1, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+
+ /* Leader also create some objects in transaction 1 */
+
+ /* create two groups */
+ gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+ assert(gid1);
+ gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+ assert(gid2);
+
+ /* Create 3 Map objects with the key type being 32 bit LE integer */
+
+ /* First Map object with a Value type a 32 bit LE integer */
+ map1 = H5Mcreate_ff(file_id, "MAP_1", H5T_STD_I32LE, H5T_STD_I32LE,
+ H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
+ assert(map1);
+ /* Second Map object with a Value type being an HDF5 VL datatype */
+ map2 = H5Mcreate_ff(gid1, "MAP_2", H5T_STD_I32LE, dtid1,
+ H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
+ assert(map2);
+ /* Third Map object with a Value type being an HDF5 VL string */
+ map3 = H5Mcreate_ff(gid2, "MAP_3", H5T_STD_I32LE, dtid2,
+ H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
+ assert(map3);
+
+ /* write some KV pairs to each map object. */
+ {
+ for(i=0 ; i<NUM_KEYS ; i++) {
+ key = i;
+ value = 1000 + i;
+ ret = H5Mset_ff(map1, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,
+ H5P_DEFAULT, tid1, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ }
+
+ for(i=0 ; i<5 ; i++) {
+ key = i;
+ ret = H5Mset_ff(map2, H5T_STD_I32LE, &key, dtid1, &wdata[i],
+ H5P_DEFAULT, tid1, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ }
+
+ for(i=0 ; i<5 ; i++) {
+ key = i;
+ ret = H5Mset_ff(map3, H5T_STD_I32LE, &key, dtid2, str_wdata[i],
+ H5P_DEFAULT, tid1, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ }
+ }
+ }
+
+ /* Tell Delegates that transaction 1 is started */
+ MPI_Ibcast(&trans_num, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD, &mpi_req);
+
+ /* Do the local-to-global, global-to-local, so all delegates can
+ write the maps created in transaction 1 */
+
+ if(0 == my_rank) {
+ /* get the token size of each map */
+ ret = H5Oget_token(map1, NULL, &token_size1);
+ assert(0 == ret);
+ ret = H5Oget_token(map2, NULL, &token_size2);
+ assert(0 == ret);
+ ret = H5Oget_token(map3, NULL, &token_size3);
+ assert(0 == ret);
+
+ /* allocate buffers for each token */
+ map_token1 = malloc(token_size1);
+ map_token2 = malloc(token_size2);
+ map_token3 = malloc(token_size3);
+
+ /* get the token buffer */
+ ret = H5Oget_token(map1, map_token1, &token_size1);
+ assert(0 == ret);
+ ret = H5Oget_token(map2, map_token2, &token_size2);
+ assert(0 == ret);
+ ret = H5Oget_token(map3, map_token3, &token_size3);
+ assert(0 == ret);
+
+ /* make sure the create operations have completed before
+ telling the delegates to open them */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ /* bcast the token sizes and the tokens */
+ MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]);
+ MPI_Ibcast(&token_size2, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]);
+ MPI_Ibcast(&token_size3, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]);
+ MPI_Ibcast(map_token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[3]);
+ MPI_Ibcast(map_token2, token_size2, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[4]);
+ MPI_Ibcast(map_token3, token_size3, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[5]);
+ }
+
+ /* Leader can continue writing to transaction 1,
+ while others wait for the ibcast to complete */
+ if(0 != my_rank) {
+
+ /* this wait if for the transaction start */
+ MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
+ assert(2 == trans_num);
+
+ /* recieve the token sizes */
+ MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]);
+ MPI_Ibcast(&token_size2, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]);
+ MPI_Ibcast(&token_size3, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]);
+ MPI_Waitall(3, mpi_reqs, MPI_STATUS_IGNORE);
+
+ /* allocate buffers for each token */
+ map_token1 = malloc(token_size1);
+ map_token2 = malloc(token_size2);
+ map_token3 = malloc(token_size3);
+
+ /* recieve the tokens */
+ MPI_Ibcast(map_token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]);
+ MPI_Ibcast(map_token2, token_size2, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]);
+ MPI_Ibcast(map_token3, token_size3, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]);
+ MPI_Waitall(3, mpi_reqs, MPI_STATUS_IGNORE);
+
+ map1 = H5Oopen_by_token(map_token1, tid1, e_stack);
+ map2 = H5Oopen_by_token(map_token2, tid1, e_stack);
+ map3 = H5Oopen_by_token(map_token3, tid1, e_stack);
+ }
+
+
+ /* none leader procs have to complete operations before notifying the leader */
+ if(0 != my_rank) {
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+ }
+
+ /* Barrier to make sure all processes are done writing so Process
+ 0 can finish transaction 1 and acquire a read context on it. */
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /* Leader process finishes/commits the transaction and acquires a
+ read context on it */
+ if(0 == my_rank) {
+ MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
+ MPI_Waitall(6, mpi_reqs, MPI_STATUS_IGNORE);
+
+ ret = H5TRfinish(tid1, H5P_DEFAULT, &rid2, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ }
+
+ /* another barrier so other processes know that container version is acquried */
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /* Close the first transaction object. Local op */
+ ret = H5TRclose(tid1);
+ assert(0 == ret);
+
+ /* release container version 1. This is async. */
+ if(0 == my_rank) {
+ ret = H5RCrelease(rid1, e_stack);
+ assert(0 == ret);
+ }
+
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ /* Leader process tells delegates that container version 2 is acquired */
+ version = 2;
+ MPI_Bcast(&version, 2, MPI_UINT64_T, 0, MPI_COMM_WORLD);
+
+ /* delegates just create a read context object; no need to acquire
+ it, since it has been acquired by the leader. */
+ if(0 != my_rank) {
+ rid2 = H5RCcreate(file_id, version);
+ assert(rid2 > 0);
+ }
+ if(0 == my_rank) {
+ assert(H5Gclose_ff(gid1, e_stack) == 0);
+ assert(H5Gclose_ff(gid2, e_stack) == 0);
+ }
+
+ /* issue some read operations using the read context acquired */
+
+ ret = H5Mget_count_ff(map2, &count, rid2, e_stack);
+ assert(ret == 0);
+
+ key = 1;
+ ret = H5Mexists_ff(map1, H5T_STD_I32LE, &key, &exists, rid2, e_stack);
+ assert(ret == 0);
+
+ /* create & start transaction 3, 4, and 5 with Multiple Leader - No Delegate Model. */
+ tid2 = H5TRcreate(file_id, rid2, (uint64_t)3);
+ assert(tid2);
+ tid3 = H5TRcreate(file_id, rid2, (uint64_t)4);
+ assert(tid3);
+ tid4 = H5TRcreate(file_id, rid2, (uint64_t)5);
+ assert(tid4);
+ trspl_id = H5Pcreate (H5P_TR_START);
+ ret = H5Pset_trspl_num_peers(trspl_id, my_size);
+ assert(0 == ret);
+ ret = H5TRstart(tid2, trspl_id, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ ret = H5TRstart(tid3, trspl_id, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ ret = H5TRstart(tid4, trspl_id, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ ret = H5Pclose(trspl_id);
+ assert(0 == ret);
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if((my_size > 1 && 1 == my_rank) ||
+ (my_size == 1 && 0 == my_rank)) {
+ hid_t temp_id;
+
+ /* modify container contents using transaction started. */
+ key = 1;
+ ret = H5Mdelete_ff(map3, H5T_STD_I32LE, &key, tid2, H5_EVENT_STACK_NULL);
+
+ H5E_BEGIN_TRY {
+ temp_id = H5Gcreate_ff(file_id, "temp_group", H5P_DEFAULT, H5P_DEFAULT,
+ H5P_DEFAULT, tid2, H5_EVENT_STACK_NULL);
+ } H5E_END_TRY;
+ if(temp_id > 0)
+ assert(H5Gclose_ff(temp_id, H5_EVENT_STACK_NULL) ==0);
+ }
+
+ /* finish transaction 3, 4, 5 */
+ if(my_rank == 0) {
+ /* set dependencies from 4 and 5 on 3 */
+ ret = H5TRset_dependency(tid3, (uint64_t)3, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ ret = H5TRset_dependency(tid4, (uint64_t)3, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+
+ ret = H5TRabort(tid2, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ }
+ else {
+ H5E_BEGIN_TRY {
+ ret = H5TRfinish(tid2, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL);
+ } H5E_END_TRY;
+ if(ret < 0)
+ fprintf(stderr, "Transaction finish failed as expected (aborted)\n");
+ }
+
+ H5E_BEGIN_TRY {
+ ret = H5TRfinish(tid3, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL);
+ } H5E_END_TRY;
+ if(ret < 0)
+ fprintf(stderr, "Transaction finish failed (aborted)\n");
+
+ H5E_BEGIN_TRY {
+ ret = H5TRfinish(tid4, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL);
+ } H5E_END_TRY;
+ if(ret < 0)
+ fprintf(stderr, "Transaction finish failed (aborted)\n");
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ assert(H5Mclose_ff(map1, e_stack) == 0);
+ assert(H5Mclose_ff(map2, e_stack) == 0);
+ assert(H5Mclose_ff(map3, e_stack) == 0);
+
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ ret = H5RCclose(rid1);
+ assert(0 == ret);
+ ret = H5TRclose(tid2);
+ assert(0 == ret);
+ ret = H5TRclose(tid3);
+ assert(0 == ret);
+ ret = H5TRclose(tid4);
+ assert(0 == ret);
+
+ /* Barrier so all processes are guranteed to have finished/aborted transaction 2 */
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /* acquire container version 3, 4, 5 - EXACT (Should Fail since 2 is aborted) */
+ version = 3;
+ H5E_BEGIN_TRY {
+ rid3 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ } H5E_END_TRY;
+ assert(rid3 < 0);
+ version = 4;
+ H5E_BEGIN_TRY {
+ rid3 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ } H5E_END_TRY;
+ assert(rid3 < 0);
+ version = 5;
+ H5E_BEGIN_TRY {
+ rid3 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ } H5E_END_TRY;
+ assert(rid3 < 0);
+
+ /* Now read some data from the container at version 1*/
+
+ map1 = H5Mopen_ff(file_id, "MAP_1", H5P_DEFAULT, rid2, e_stack);
+ map2 = H5Mopen_ff(file_id, "G1/MAP_2", H5P_DEFAULT, rid2, e_stack);
+ map3 = H5Mopen_ff(file_id, "G1/G2/MAP_3", H5P_DEFAULT, rid2, e_stack);
+
+ {
+ int key, value;
+ int increment=4, j=0;
+
+ for(i=0 ; i<NUM_KEYS ; i++) {
+ key = i;
+ value = -1;
+ ret = H5Mget_ff(map1, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,
+ H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL);
+
+ if(value != 1000+i) {
+ printf("Key %d: Value recieved = %d\n", key, value);
+ assert(0);
+ }
+ }
+
+ for(i=0 ; i<5 ; i++) {
+ key = i;
+ ret = H5Mget_ff(map2, H5T_STD_I32LE, &key, dtid1, &rdata[i],
+ H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ }
+
+ /* Print VL DATA */
+ fprintf(stderr, "Reading VL Data: \n");
+ for(i = 0; i < 5; i++) {
+ int temp = i*increment + increment;
+
+ fprintf(stderr, "Key %d size %zu: ", i, rdata[i].len);
+ for(j = 0; j < temp; j++)
+ fprintf(stderr, "%d ",((unsigned int *)rdata[i].p)[j]);
+ fprintf(stderr, "\n");
+ } /* end for */
+
+ for(i=0 ; i<5 ; i++) {
+ key = i;
+ str_rdata[i] = NULL;
+ ret = H5Mget_ff(map3, H5T_STD_I32LE, &key, dtid2, &str_rdata[i],
+ H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ }
+
+ fprintf(stderr, "Reading VL Strings: \n");
+
+ for(i=0 ; i<5 ; i++) {
+ fprintf(stderr, "Key %d: %s\n", i, str_rdata[i]);
+ free(str_rdata[i]);
+ }
+ }
+
+ ret = H5Tclose(dtid1);
+ assert(ret == 0);
+ ret = H5Tclose(dtid2);
+ assert(ret == 0);
+
+ assert(H5Mclose_ff(map1, e_stack) == 0);
+ assert(H5Mclose_ff(map2, e_stack) == 0);
+ assert(H5Mclose_ff(map3, e_stack) == 0);
+
+ if(my_rank == 0) {
+ /* release container version 2. This is async. */
+ ret = H5RCrelease(rid2, e_stack);
+ assert(0 == ret);
+ }
+ ret = H5RCclose(rid2);
+ assert(0 == ret);
+
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ /* closing the container also acts as a wait all on all pending requests
+ on the container. */
+ assert(H5Fclose_ff(file_id, 1, H5_EVENT_STACK_NULL) == 0);
+
+ fprintf(stderr, "\n*******************************************************************\n");
+ fprintf(stderr, "Wait on everything in ES and check Results of operations in ES\n");
+ fprintf(stderr, "\n*******************************************************************\n");
+
+ H5ESget_count(e_stack, &num_events);
+
+ H5EStest_all(e_stack, &status);
+ printf("%d events in event stack. H5EStest_all Completion status = %d\n", num_events, status);
+
+ H5ESwait_all(e_stack, &status);
+ printf("%d events in event stack. H5ESwait_all Completion status = %d\n", num_events, status);
+
+ H5EStest_all(e_stack, &status);
+ printf("%d events in event stack. H5EStest_all Completion status = %d\n", num_events, status);
+
+ H5ESclear(e_stack);
+
+ assert (count == 5);
+ assert (exists > 0);
+
+ fprintf(stderr, "\n*****************************************************************************************************************\n");
+ fprintf(stderr, "Finalize EFF stack\n");
+ fprintf(stderr, "*****************************************************************************************************************\n");
+
+ H5ESclose(e_stack);
+ H5Pclose(fapl_id);
+
+ for(i=0 ; i<5 ; i++) {
+ free(wdata[i].p);
+ free(rdata[i].p);
+ }
+
+ free(map_token1);
+ free(map_token2);
+ free(map_token3);
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
+ and shutsdown the FS server (when all clients send the terminate request)
+ and client */
+ EFF_finalize();
+
+ MPI_Finalize();
+ return 0;
+}
diff --git a/examples/h5ff_client_multiple_cont.c b/examples/h5ff_client_multiple_cont.c
index b984100..9dda5ca 100644
--- a/examples/h5ff_client_multiple_cont.c
+++ b/examples/h5ff_client_multiple_cont.c
@@ -1,438 +1,438 @@
-/*
- * test_client.c: Client side of Milestone 4.2 Asynchronous I/O and initial
- * IOD VOL plugin demonstration. This is, in effect, the application program that
- * would run on one or more compute nodes and make calls to the HDF5 API.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-int main(int argc, char **argv) {
- char file_name1[50];
- char file_name2[50];
- hid_t fid1, fid2;
- hid_t gid1, gid2;
- hid_t did1, did2;
- hid_t dtid1, dtid2;
- hid_t sid;
- hid_t tid1, tid2, rid1, rid2, rid3, rid4;
- hid_t fapl_id, trspl_id;
- hid_t e_stack;
-
- void *token1, *token2, *token3, *token4, *token5, *token6;
- size_t token_size1, token_size2, token_size3, token_size4, token_size5, token_size6;
- uint64_t version;
- uint64_t trans_num;
-
- const unsigned int nelem=60;
- int *wdata1 = NULL, *wdata2 = NULL;
- int *rdata1 = NULL, *rdata2 = NULL;
- unsigned int i = 0;
- hsize_t dims[1];
- int my_rank, my_size;
- int provided;
- MPI_Request mpi_req, mpi_reqs[12];
-
- H5ES_status_t status;
- size_t num_events = 0;
- herr_t ret;
-
- sprintf(file_name1, "%s_%s", getenv("USER"), "eff_file_2.h5");
- sprintf(file_name2, "%s_%s", getenv("USER"), "eff_file_1.h5");
-
- MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
- if(MPI_THREAD_MULTIPLE != provided) {
- fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
- exit(1);
- }
-
- /* Call EFF_init to initialize the EFF stack.
- As a result of this call, the Function Shipper client is started,
- and HDF5 VOL calls are registered with the function shipper.
- An "IOD init" call is forwarded from the FS client to the FS server
- which should already be running. */
- EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
-
- MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
- MPI_Comm_size(MPI_COMM_WORLD, &my_size);
- fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
-
- fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
- /* Choose the IOD VOL plugin to use with this file.
- First we create a file access property list. Then we call a new routine to set
- the IOD plugin to use with this fapl */
- fapl_id = H5Pcreate (H5P_FILE_ACCESS);
- H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
-
- /* allocate and initialize arrays for dataset & attribute writes and reads.
- The write arrays are intialized to contain 60 integers (0-59).
- The read arrays are intialized to contain 60 integers all 0s. */
- wdata1 = malloc (sizeof(int)*nelem);
- wdata2 = malloc (sizeof(int)*nelem);
- rdata1 = malloc (sizeof(int)*nelem);
- rdata2 = malloc (sizeof(int)*nelem);
- for(i=0;i<nelem;++i) {
- rdata1[i] = 0;
- wdata1[i]=my_rank+1;
- rdata2[i] = 0;
- wdata2[i]=my_rank+1;
- }
-
- /* create a 32 bit integer LE datatype. This is a local operation
- that does not touch the file */
- dtid1 = H5Tcopy(H5T_NATIVE_INT);
- dtid2 = H5Tcopy(H5T_NATIVE_INT);
-
- /* create a dataspace. This is a local Bookeeping operation that
- does not touch the file */
- dims [0] = 60;
- sid = H5Screate_simple(1, dims, NULL);
-
- /* create an event Queue for managing asynchronous requests.
-
- Event Queues will releive the use from managing and completing
- individual requests for every operation. Instead of passing a
- request for every operation, the event queue is passed and
- internally the HDF5 library creates a request and adds it to
- the event queue.
-
- Multiple Event queue can be created used by the application. */
- e_stack = H5EScreate();
- assert(e_stack);
-
- /* create the file. */
- fid1 = H5Fcreate(file_name1, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
- assert(fid1);
- /* create second file */
- fid2 = H5Fcreate(file_name2, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
- assert(fid2);
-
- /* acquire container version 1 on both containers - EXACT
- This can be asynchronous, but here we need the acquired ID
- right after the call to start the transaction so we make synchronous. */
- if(0 == my_rank) {
- version = 1;
- rid1 = H5RCacquire(fid1, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- rid2 = H5RCacquire(fid2, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- }
- MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
- assert(1 == version);
- if (my_rank != 0) {
- rid1 = H5RCcreate(fid1, version);
- rid2 = H5RCcreate(fid2, version);
- }
-
- /* create transaction objects */
- tid1 = H5TRcreate(fid1, rid1, (uint64_t)2);
- assert(tid1);
- tid2 = H5TRcreate(fid2, rid2, (uint64_t)2);
- assert(tid2);
-
- /* start transaction 1 with default Leader/Delegate model. Leader
- which is rank 0 here starts the transaction. It can be
- asynchronous, but we make it synchronous here so that the
- Leader can tell its delegates that the transaction is
- started. */
- if(0 == my_rank) {
- ret = H5TRstart(tid1, H5P_DEFAULT, e_stack);
- assert(0 == ret);
- ret = H5TRstart(tid2, H5P_DEFAULT, e_stack);
- assert(0 == ret);
-
- /* create groups */
- gid1 = H5Gcreate_ff(fid1, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
- gid2 = H5Gcreate_ff(fid2, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid2, e_stack);
-
- /* Commit datatypes to the files */
- H5Tcommit_ff(fid1, "int", dtid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT,
- tid1, e_stack);
- H5Tcommit_ff(fid2, "int", dtid2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT,
- tid2, e_stack);
-
- /* create Datasets */
- did1 = H5Dcreate_ff(gid1, "D1", dtid1, sid,
- H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
- did2 = H5Dcreate_ff(gid2, "D1", dtid2, sid,
- H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid2, e_stack);
-
- /* Raw data write on D1. */
- H5Dwrite_ff(did1, dtid1, sid, sid, H5P_DEFAULT, wdata1, tid1, e_stack);
- H5Dwrite_ff(did2, dtid2, sid, sid, H5P_DEFAULT, wdata2, tid2, e_stack);
-
- trans_num = 2;
- }
-
- /* Tell other procs that transaction 1 is started */
- MPI_Ibcast(&trans_num, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD, &mpi_req);
-
- /* Do the local-to-global, global-to-local, so all delegates can
- write the gids created in transaction 1 */
-
- if(0 == my_rank) {
- /* get the token size of each map */
- ret = H5Oget_token(gid1, NULL, &token_size1);
- assert(0 == ret);
- ret = H5Oget_token(gid2, NULL, &token_size2);
- assert(0 == ret);
- ret = H5Oget_token(dtid1, NULL, &token_size3);
- assert(0 == ret);
- ret = H5Oget_token(dtid2, NULL, &token_size4);
- assert(0 == ret);
- ret = H5Oget_token(did1, NULL, &token_size5);
- assert(0 == ret);
- ret = H5Oget_token(did2, NULL, &token_size6);
- assert(0 == ret);
-
- /* allocate buffers for each token */
- token1 = malloc(token_size1);
- token2 = malloc(token_size2);
- token3 = malloc(token_size3);
- token4 = malloc(token_size4);
- token5 = malloc(token_size5);
- token6 = malloc(token_size6);
-
- /* get the token buffer */
- ret = H5Oget_token(gid1, token1, &token_size1);
- assert(0 == ret);
- ret = H5Oget_token(gid2, token2, &token_size2);
- assert(0 == ret);
- ret = H5Oget_token(dtid1, token3, &token_size3);
- assert(0 == ret);
- ret = H5Oget_token(dtid2, token4, &token_size4);
- assert(0 == ret);
- ret = H5Oget_token(did1, token5, &token_size5);
- assert(0 == ret);
- ret = H5Oget_token(did2, token6, &token_size6);
- assert(0 == ret);
-
- /* make sure the create operations have completed before
- telling the delegates to open them */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- /* bcast the token sizes and the tokens */
- MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]);
- MPI_Ibcast(&token_size2, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]);
- MPI_Ibcast(&token_size3, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]);
- MPI_Ibcast(&token_size4, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[3]);
- MPI_Ibcast(&token_size5, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[4]);
- MPI_Ibcast(&token_size6, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[5]);
- MPI_Ibcast(token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[6]);
- MPI_Ibcast(token2, token_size2, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[7]);
- MPI_Ibcast(token3, token_size3, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[8]);
- MPI_Ibcast(token4, token_size4, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[9]);
- MPI_Ibcast(token5, token_size5, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[10]);
- MPI_Ibcast(token6, token_size6, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[11]);
- }
-
- /* Leader can continue writing to transaction 1,
- while others wait for the ibcast to complete */
- if(0 != my_rank) {
-
- /* this wait if for the transaction start */
- MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
- assert(2 == trans_num);
-
- /* recieve the token sizes */
- MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]);
- MPI_Ibcast(&token_size2, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]);
- MPI_Ibcast(&token_size3, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]);
- MPI_Ibcast(&token_size4, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[3]);
- MPI_Ibcast(&token_size5, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[4]);
- MPI_Ibcast(&token_size6, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[5]);
- MPI_Waitall(6, mpi_reqs, MPI_STATUS_IGNORE);
-
- /* allocate buffers for each token */
- token1 = malloc(token_size1);
- token2 = malloc(token_size2);
- token3 = malloc(token_size3);
- token4 = malloc(token_size4);
- token5 = malloc(token_size5);
- token6 = malloc(token_size6);
-
- /* recieve the tokens */
- MPI_Ibcast(token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]);
- MPI_Ibcast(token2, token_size2, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]);
- MPI_Ibcast(token3, token_size3, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]);
- MPI_Ibcast(token4, token_size4, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[3]);
- MPI_Ibcast(token5, token_size5, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[4]);
- MPI_Ibcast(token6, token_size6, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[5]);
- MPI_Waitall(6, mpi_reqs, MPI_STATUS_IGNORE);
-
- gid1 = H5Oopen_by_token(token1, tid1, e_stack);
- gid2 = H5Oopen_by_token(token2, tid2, e_stack);
- dtid1 = H5Oopen_by_token(token3, tid1, e_stack);
- dtid2 = H5Oopen_by_token(token4, tid2, e_stack);
- did1 = H5Oopen_by_token(token5, tid1, e_stack);
- did2 = H5Oopen_by_token(token6, tid2, e_stack);
-
- /* Raw data write on D1. */
- H5Dwrite_ff(did1, dtid1, sid, sid, H5P_DEFAULT, wdata1, tid1, e_stack);
- H5Dwrite_ff(did2, dtid2, sid, sid, H5P_DEFAULT, wdata2, tid2, e_stack);
-
- /* none leader procs have to complete operations before notifying the leader */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
- }
-
- /* Barrier to make sure all processes are done writing so Process
- 0 can finish transaction 1 and acquire a read context on it. */
- MPI_Barrier(MPI_COMM_WORLD);
-
- /* Leader process finishes/commits the transaction */
- if(0 == my_rank) {
- MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
- MPI_Waitall(12, mpi_reqs, MPI_STATUS_IGNORE);
-
- ret = H5TRfinish(tid1, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- ret = H5TRfinish(tid2, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- }
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- /* Close the first transaction object. Local op */
- ret = H5TRclose(tid1);
- assert(0 == ret);
- ret = H5TRclose(tid2);
- assert(0 == ret);
-
- /* release container version 1. */
- if(0 == my_rank) {
- ret = H5RCrelease(rid1, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- ret = H5RCrelease(rid2, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- }
-
- ret = H5RCclose(rid1);
- assert(0 == ret);
- ret = H5RCclose(rid2);
- assert(0 == ret);
-
-
- if(0 == my_rank) {
- version = 2;
- rid1 = H5RCacquire(fid1, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- rid2 = H5RCacquire(fid2, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- }
- MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
- assert(2 == version);
- if (my_rank != 0) {
- rid1 = H5RCcreate(fid1, version);
- rid2 = H5RCcreate(fid2, version);
- }
-
- {
- hid_t dxpl_id;
- uint64_t cs_scope;
-
- dxpl_id = H5Pcreate (H5P_DATASET_XFER);
- /* tell HDF5 to disable all data integrity checks for this write */
- cs_scope = 0;
- ret = H5Pset_rawdata_integrity_scope(dxpl_id, cs_scope);
- assert(ret == 0);
-
- ret = H5Dread_ff(did1, dtid1, sid, sid, dxpl_id, rdata1, rid1, e_stack);
- assert(ret == 0);
- ret = H5Dread_ff(did2, dtid2, sid, sid, dxpl_id, rdata2, rid2, e_stack);
- assert(ret == 0);
-
- H5Pclose(dxpl_id);
-
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- fprintf(stderr, "Read Data from container 1: ");
- for(i=0;i<nelem;++i)
- fprintf(stderr, "%d ",rdata1[i]);
- fprintf(stderr, "\n");
-
- fprintf(stderr, "Read Data from container 2: ");
- for(i=0;i<nelem;++i)
- fprintf(stderr, "%d ",rdata2[i]);
- fprintf(stderr, "\n");
- }
-
- MPI_Barrier(MPI_COMM_WORLD);
- /* release container version 2. This is async. */
- if(0 == my_rank) {
- ret = H5RCrelease(rid1, e_stack);
- assert(0 == ret);
- ret = H5RCrelease(rid2, e_stack);
- assert(0 == ret);
- }
-
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- fprintf(stderr, "\n*****************************************************************************************************************\n");
- fprintf(stderr, "Close Objects\n");
- fprintf(stderr, "*****************************************************************************************************************\n");
-
- assert(H5Dclose_ff(did1, e_stack) == 0);
- assert(H5Dclose_ff(did2, e_stack) == 0);
-
- assert(H5Tclose_ff(dtid1, e_stack) == 0);
- assert(H5Tclose_ff(dtid2, e_stack) == 0);
-
- assert(H5Gclose_ff(gid1, e_stack) == 0);
- assert(H5Gclose_ff(gid2, e_stack) == 0);
-
- ret = H5RCclose(rid1);
- assert(0 == ret);
- ret = H5RCclose(rid2);
- assert(0 == ret);
-
- assert(H5Fclose_ff(fid1, 1, H5_EVENT_STACK_NULL) == 0);
- assert(H5Fclose_ff(fid2, 1, H5_EVENT_STACK_NULL) == 0);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- H5ESclose(e_stack);
- H5Sclose(sid);
- H5Pclose(fapl_id);
-
- free(token1);
- free(token2);
- free(token3);
- free(token4);
- free(token5);
- free(token6);
-
- free(wdata1);
- free(rdata1);
- free(wdata2);
- free(rdata2);
-
- fprintf(stderr, "\n*****************************************************************************************************************\n");
- fprintf(stderr, "Finalize EFF stack\n");
- fprintf(stderr, "*****************************************************************************************************************\n");
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
- and shutsdown the FS server (when all clients send the terminate request)
- and client */
- EFF_finalize();
- MPI_Finalize();
-
- return 0;
-}
+/*
+ * test_client.c: Client side of Milestone 4.2 Asynchronous I/O and initial
+ * IOD VOL plugin demonstration. This is, in effect, the application program that
+ * would run on one or more compute nodes and make calls to the HDF5 API.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+int main(int argc, char **argv) {
+ char file_name1[50];
+ char file_name2[50];
+ hid_t fid1, fid2;
+ hid_t gid1, gid2;
+ hid_t did1, did2;
+ hid_t dtid1, dtid2;
+ hid_t sid;
+ hid_t tid1, tid2, rid1, rid2, rid3, rid4;
+ hid_t fapl_id, trspl_id;
+ hid_t e_stack;
+
+ void *token1, *token2, *token3, *token4, *token5, *token6;
+ size_t token_size1, token_size2, token_size3, token_size4, token_size5, token_size6;
+ uint64_t version;
+ uint64_t trans_num;
+
+ const unsigned int nelem=60;
+ int *wdata1 = NULL, *wdata2 = NULL;
+ int *rdata1 = NULL, *rdata2 = NULL;
+ unsigned int i = 0;
+ hsize_t dims[1];
+ int my_rank, my_size;
+ int provided;
+ MPI_Request mpi_req, mpi_reqs[12];
+
+ H5ES_status_t status;
+ size_t num_events = 0;
+ herr_t ret;
+
+ sprintf(file_name1, "%s_%s", getenv("USER"), "eff_file_2.h5");
+ sprintf(file_name2, "%s_%s", getenv("USER"), "eff_file_1.h5");
+
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+ if(MPI_THREAD_MULTIPLE != provided) {
+ fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
+ exit(1);
+ }
+
+ /* Call EFF_init to initialize the EFF stack.
+ As a result of this call, the Function Shipper client is started,
+ and HDF5 VOL calls are registered with the function shipper.
+ An "IOD init" call is forwarded from the FS client to the FS server
+ which should already be running. */
+ EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &my_size);
+ fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
+
+ fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
+ /* Choose the IOD VOL plugin to use with this file.
+ First we create a file access property list. Then we call a new routine to set
+ the IOD plugin to use with this fapl */
+ fapl_id = H5Pcreate (H5P_FILE_ACCESS);
+ H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ /* allocate and initialize arrays for dataset & attribute writes and reads.
+ The write arrays are intialized to contain 60 integers (0-59).
+ The read arrays are intialized to contain 60 integers all 0s. */
+ wdata1 = malloc (sizeof(int)*nelem);
+ wdata2 = malloc (sizeof(int)*nelem);
+ rdata1 = malloc (sizeof(int)*nelem);
+ rdata2 = malloc (sizeof(int)*nelem);
+ for(i=0;i<nelem;++i) {
+ rdata1[i] = 0;
+ wdata1[i]=my_rank+1;
+ rdata2[i] = 0;
+ wdata2[i]=my_rank+1;
+ }
+
+ /* create a 32 bit integer LE datatype. This is a local operation
+ that does not touch the file */
+ dtid1 = H5Tcopy(H5T_NATIVE_INT);
+ dtid2 = H5Tcopy(H5T_NATIVE_INT);
+
+ /* create a dataspace. This is a local Bookeeping operation that
+ does not touch the file */
+ dims [0] = 60;
+ sid = H5Screate_simple(1, dims, NULL);
+
+ /* create an event Queue for managing asynchronous requests.
+
+ Event Queues will releive the use from managing and completing
+ individual requests for every operation. Instead of passing a
+ request for every operation, the event queue is passed and
+ internally the HDF5 library creates a request and adds it to
+ the event queue.
+
+ Multiple Event queue can be created used by the application. */
+ e_stack = H5EScreate();
+ assert(e_stack);
+
+ /* create the file. */
+ fid1 = H5Fcreate(file_name1, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+ assert(fid1);
+ /* create second file */
+ fid2 = H5Fcreate(file_name2, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+ assert(fid2);
+
+ /* acquire container version 1 on both containers - EXACT
+ This can be asynchronous, but here we need the acquired ID
+ right after the call to start the transaction so we make synchronous. */
+ if(0 == my_rank) {
+ version = 1;
+ rid1 = H5RCacquire(fid1, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ rid2 = H5RCacquire(fid2, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ }
+ MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
+ assert(1 == version);
+ if (my_rank != 0) {
+ rid1 = H5RCcreate(fid1, version);
+ rid2 = H5RCcreate(fid2, version);
+ }
+
+ /* create transaction objects */
+ tid1 = H5TRcreate(fid1, rid1, (uint64_t)2);
+ assert(tid1);
+ tid2 = H5TRcreate(fid2, rid2, (uint64_t)2);
+ assert(tid2);
+
+ /* start transaction 1 with default Leader/Delegate model. Leader
+ which is rank 0 here starts the transaction. It can be
+ asynchronous, but we make it synchronous here so that the
+ Leader can tell its delegates that the transaction is
+ started. */
+ if(0 == my_rank) {
+ ret = H5TRstart(tid1, H5P_DEFAULT, e_stack);
+ assert(0 == ret);
+ ret = H5TRstart(tid2, H5P_DEFAULT, e_stack);
+ assert(0 == ret);
+
+ /* create groups */
+ gid1 = H5Gcreate_ff(fid1, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+ gid2 = H5Gcreate_ff(fid2, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid2, e_stack);
+
+ /* Commit datatypes to the files */
+ H5Tcommit_ff(fid1, "int", dtid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT,
+ tid1, e_stack);
+ H5Tcommit_ff(fid2, "int", dtid2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT,
+ tid2, e_stack);
+
+ /* create Datasets */
+ did1 = H5Dcreate_ff(gid1, "D1", dtid1, sid,
+ H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
+ did2 = H5Dcreate_ff(gid2, "D1", dtid2, sid,
+ H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid2, e_stack);
+
+ /* Raw data write on D1. */
+ H5Dwrite_ff(did1, dtid1, sid, sid, H5P_DEFAULT, wdata1, tid1, e_stack);
+ H5Dwrite_ff(did2, dtid2, sid, sid, H5P_DEFAULT, wdata2, tid2, e_stack);
+
+ trans_num = 2;
+ }
+
+ /* Tell other procs that transaction 1 is started */
+ MPI_Ibcast(&trans_num, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD, &mpi_req);
+
+ /* Do the local-to-global, global-to-local, so all delegates can
+ write the gids created in transaction 1 */
+
+ if(0 == my_rank) {
+ /* get the token size of each map */
+ ret = H5Oget_token(gid1, NULL, &token_size1);
+ assert(0 == ret);
+ ret = H5Oget_token(gid2, NULL, &token_size2);
+ assert(0 == ret);
+ ret = H5Oget_token(dtid1, NULL, &token_size3);
+ assert(0 == ret);
+ ret = H5Oget_token(dtid2, NULL, &token_size4);
+ assert(0 == ret);
+ ret = H5Oget_token(did1, NULL, &token_size5);
+ assert(0 == ret);
+ ret = H5Oget_token(did2, NULL, &token_size6);
+ assert(0 == ret);
+
+ /* allocate buffers for each token */
+ token1 = malloc(token_size1);
+ token2 = malloc(token_size2);
+ token3 = malloc(token_size3);
+ token4 = malloc(token_size4);
+ token5 = malloc(token_size5);
+ token6 = malloc(token_size6);
+
+ /* get the token buffer */
+ ret = H5Oget_token(gid1, token1, &token_size1);
+ assert(0 == ret);
+ ret = H5Oget_token(gid2, token2, &token_size2);
+ assert(0 == ret);
+ ret = H5Oget_token(dtid1, token3, &token_size3);
+ assert(0 == ret);
+ ret = H5Oget_token(dtid2, token4, &token_size4);
+ assert(0 == ret);
+ ret = H5Oget_token(did1, token5, &token_size5);
+ assert(0 == ret);
+ ret = H5Oget_token(did2, token6, &token_size6);
+ assert(0 == ret);
+
+ /* make sure the create operations have completed before
+ telling the delegates to open them */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ /* bcast the token sizes and the tokens */
+ MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]);
+ MPI_Ibcast(&token_size2, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]);
+ MPI_Ibcast(&token_size3, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]);
+ MPI_Ibcast(&token_size4, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[3]);
+ MPI_Ibcast(&token_size5, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[4]);
+ MPI_Ibcast(&token_size6, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[5]);
+ MPI_Ibcast(token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[6]);
+ MPI_Ibcast(token2, token_size2, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[7]);
+ MPI_Ibcast(token3, token_size3, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[8]);
+ MPI_Ibcast(token4, token_size4, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[9]);
+ MPI_Ibcast(token5, token_size5, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[10]);
+ MPI_Ibcast(token6, token_size6, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[11]);
+ }
+
+ /* Leader can continue writing to transaction 1,
+ while others wait for the ibcast to complete */
+ if(0 != my_rank) {
+
+ /* this wait if for the transaction start */
+ MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
+ assert(2 == trans_num);
+
+ /* recieve the token sizes */
+ MPI_Ibcast(&token_size1, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]);
+ MPI_Ibcast(&token_size2, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]);
+ MPI_Ibcast(&token_size3, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]);
+ MPI_Ibcast(&token_size4, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[3]);
+ MPI_Ibcast(&token_size5, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[4]);
+ MPI_Ibcast(&token_size6, sizeof(size_t), MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[5]);
+ MPI_Waitall(6, mpi_reqs, MPI_STATUS_IGNORE);
+
+ /* allocate buffers for each token */
+ token1 = malloc(token_size1);
+ token2 = malloc(token_size2);
+ token3 = malloc(token_size3);
+ token4 = malloc(token_size4);
+ token5 = malloc(token_size5);
+ token6 = malloc(token_size6);
+
+ /* recieve the tokens */
+ MPI_Ibcast(token1, token_size1, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[0]);
+ MPI_Ibcast(token2, token_size2, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[1]);
+ MPI_Ibcast(token3, token_size3, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[2]);
+ MPI_Ibcast(token4, token_size4, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[3]);
+ MPI_Ibcast(token5, token_size5, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[4]);
+ MPI_Ibcast(token6, token_size6, MPI_BYTE, 0, MPI_COMM_WORLD, &mpi_reqs[5]);
+ MPI_Waitall(6, mpi_reqs, MPI_STATUS_IGNORE);
+
+ gid1 = H5Oopen_by_token(token1, tid1, e_stack);
+ gid2 = H5Oopen_by_token(token2, tid2, e_stack);
+ dtid1 = H5Oopen_by_token(token3, tid1, e_stack);
+ dtid2 = H5Oopen_by_token(token4, tid2, e_stack);
+ did1 = H5Oopen_by_token(token5, tid1, e_stack);
+ did2 = H5Oopen_by_token(token6, tid2, e_stack);
+
+ /* Raw data write on D1. */
+ H5Dwrite_ff(did1, dtid1, sid, sid, H5P_DEFAULT, wdata1, tid1, e_stack);
+ H5Dwrite_ff(did2, dtid2, sid, sid, H5P_DEFAULT, wdata2, tid2, e_stack);
+
+ /* none leader procs have to complete operations before notifying the leader */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+ }
+
+ /* Barrier to make sure all processes are done writing so Process
+ 0 can finish transaction 1 and acquire a read context on it. */
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /* Leader process finishes/commits the transaction */
+ if(0 == my_rank) {
+ MPI_Wait(&mpi_req, MPI_STATUS_IGNORE);
+ MPI_Waitall(12, mpi_reqs, MPI_STATUS_IGNORE);
+
+ ret = H5TRfinish(tid1, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ ret = H5TRfinish(tid2, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ }
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /* Close the first transaction object. Local op */
+ ret = H5TRclose(tid1);
+ assert(0 == ret);
+ ret = H5TRclose(tid2);
+ assert(0 == ret);
+
+ /* release container version 1. */
+ if(0 == my_rank) {
+ ret = H5RCrelease(rid1, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ ret = H5RCrelease(rid2, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ }
+
+ ret = H5RCclose(rid1);
+ assert(0 == ret);
+ ret = H5RCclose(rid2);
+ assert(0 == ret);
+
+
+ if(0 == my_rank) {
+ version = 2;
+ rid1 = H5RCacquire(fid1, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ rid2 = H5RCacquire(fid2, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ }
+ MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
+ assert(2 == version);
+ if (my_rank != 0) {
+ rid1 = H5RCcreate(fid1, version);
+ rid2 = H5RCcreate(fid2, version);
+ }
+
+ {
+ hid_t dxpl_id;
+ uint64_t cs_scope;
+
+ dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+ /* tell HDF5 to disable all data integrity checks for this write */
+ cs_scope = 0;
+ ret = H5Pset_rawdata_integrity_scope(dxpl_id, cs_scope);
+ assert(ret == 0);
+
+ ret = H5Dread_ff(did1, dtid1, sid, sid, dxpl_id, rdata1, rid1, e_stack);
+ assert(ret == 0);
+ ret = H5Dread_ff(did2, dtid2, sid, sid, dxpl_id, rdata2, rid2, e_stack);
+ assert(ret == 0);
+
+ H5Pclose(dxpl_id);
+
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ fprintf(stderr, "Read Data from container 1: ");
+ for(i=0;i<nelem;++i)
+ fprintf(stderr, "%d ",rdata1[i]);
+ fprintf(stderr, "\n");
+
+ fprintf(stderr, "Read Data from container 2: ");
+ for(i=0;i<nelem;++i)
+ fprintf(stderr, "%d ",rdata2[i]);
+ fprintf(stderr, "\n");
+ }
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ /* release container version 2. This is async. */
+ if(0 == my_rank) {
+ ret = H5RCrelease(rid1, e_stack);
+ assert(0 == ret);
+ ret = H5RCrelease(rid2, e_stack);
+ assert(0 == ret);
+ }
+
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ fprintf(stderr, "\n*****************************************************************************************************************\n");
+ fprintf(stderr, "Close Objects\n");
+ fprintf(stderr, "*****************************************************************************************************************\n");
+
+ assert(H5Dclose_ff(did1, e_stack) == 0);
+ assert(H5Dclose_ff(did2, e_stack) == 0);
+
+ assert(H5Tclose_ff(dtid1, e_stack) == 0);
+ assert(H5Tclose_ff(dtid2, e_stack) == 0);
+
+ assert(H5Gclose_ff(gid1, e_stack) == 0);
+ assert(H5Gclose_ff(gid2, e_stack) == 0);
+
+ ret = H5RCclose(rid1);
+ assert(0 == ret);
+ ret = H5RCclose(rid2);
+ assert(0 == ret);
+
+ assert(H5Fclose_ff(fid1, 1, H5_EVENT_STACK_NULL) == 0);
+ assert(H5Fclose_ff(fid2, 1, H5_EVENT_STACK_NULL) == 0);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ H5ESclose(e_stack);
+ H5Sclose(sid);
+ H5Pclose(fapl_id);
+
+ free(token1);
+ free(token2);
+ free(token3);
+ free(token4);
+ free(token5);
+ free(token6);
+
+ free(wdata1);
+ free(rdata1);
+ free(wdata2);
+ free(rdata2);
+
+ fprintf(stderr, "\n*****************************************************************************************************************\n");
+ fprintf(stderr, "Finalize EFF stack\n");
+ fprintf(stderr, "*****************************************************************************************************************\n");
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
+ and shutsdown the FS server (when all clients send the terminate request)
+ and client */
+ EFF_finalize();
+ MPI_Finalize();
+
+ return 0;
+}
diff --git a/examples/h5ff_client_obj.c b/examples/h5ff_client_obj.c
index 187dc2d..6daf617 100644
--- a/examples/h5ff_client_obj.c
+++ b/examples/h5ff_client_obj.c
@@ -1,313 +1,313 @@
-/*
- * test_client_obj.c: Client side of H5O routines
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-int main(int argc, char **argv) {
- char file_name[50];
- hid_t file_id;
- hid_t gid;
- hid_t did, map;
- hid_t sid, dtid;
- hid_t tid1, tid2, rid1, rid2;
- hid_t fapl_id, dxpl_id;
- hid_t e_stack;
- htri_t exists = -1;
-
- const unsigned int nelem=60;
- hsize_t dims[1];
-
- uint64_t version;
- uint64_t trans_num;
-
- int my_rank, my_size;
- int provided;
- MPI_Request mpi_req;
-
- H5ES_status_t status;
- size_t num_events = 0;
- herr_t ret;
-
- sprintf(file_name, "%s_%s", getenv("USER"), "eff_file_obj.h5");
-
- MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
- if(MPI_THREAD_MULTIPLE != provided) {
- fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
- exit(1);
- }
-
- /* Call EFF_init to initialize the EFF stack.
- As a result of this call, the Function Shipper client is started,
- and HDF5 VOL calls are registered with the function shipper.
- An "IOD init" call is forwarded from the FS client to the FS server
- which should already be running. */
- EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
-
- MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
- MPI_Comm_size(MPI_COMM_WORLD, &my_size);
- fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
-
- fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
- /* Choose the IOD VOL plugin to use with this file.
- First we create a file access property list. Then we call a new routine to set
- the IOD plugin to use with this fapl */
- fapl_id = H5Pcreate (H5P_FILE_ACCESS);
- H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
-
- /* create an event Queue for managing asynchronous requests.
-
- Event Queues will releive the use from managing and completing
- individual requests for every operation. Instead of passing a
- request for every operation, the event queue is passed and
- internally the HDF5 library creates a request and adds it to
- the event queue.
-
- Multiple Event queue can be created used by the application. */
- e_stack = H5EScreate();
- assert(e_stack);
-
- /* create the file. */
- file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL);
- assert(file_id > 0);
-
- /* create 1-D dataspace with 60 elements */
- dims [0] = nelem;
- sid = H5Screate_simple(1, dims, NULL);
- dtid = H5Tcopy(H5T_STD_I32LE);
-
- /* start transaction 2 with default Leader/Delegate model. Leader
- which is rank 0 here starts the transaction. It can be
- asynchronous, but we make it synchronous here so that the
- Leader can tell its delegates that the transaction is
- started. */
- if(0 == my_rank) {
- hid_t rid_temp;
- hid_t anon_did;
-
- version = 1;
- rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- assert(1 == version);
-
- /* create transaction object */
- tid1 = H5TRcreate(file_id, rid1, (uint64_t)2);
- assert(tid1);
- ret = H5TRstart(tid1, H5P_DEFAULT, e_stack);
- assert(0 == ret);
-
- /* create objects */
- gid = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT,
- H5P_DEFAULT, tid1, e_stack);
- assert(gid >= 0);
-
- did = H5Dcreate_ff(gid, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT,
- H5P_DEFAULT, tid1, e_stack);
- assert(did >= 0);
-
- ret = H5Tcommit_ff(file_id, "DT1", dtid, H5P_DEFAULT, H5P_DEFAULT,
- H5P_DEFAULT, tid1, e_stack);
- assert(ret == 0);
-
- map = H5Mcreate_ff(file_id, "MAP1", H5T_STD_I32LE, H5T_STD_I32LE,
- H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
- assert(map >= 0);
-
- anon_did = H5Dcreate_anon_ff(file_id, dtid, sid, H5P_DEFAULT, H5P_DEFAULT,
- tid1, e_stack);
- assert(anon_did > 0);
-
- ret = H5Dclose_ff(anon_did, e_stack);
- assert(0 == ret);
-
- ret = H5TRfinish(tid1, H5P_DEFAULT, &rid_temp, e_stack);
- assert(0 == ret);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
- assert(status == H5ES_STATUS_SUCCEED);
-
- /* Close transaction object. Local op */
- ret = H5TRclose(tid1);
- assert(0 == ret);
-
- /* create transaction object */
- tid2 = H5TRcreate(file_id, rid_temp, (uint64_t)3);
- assert(tid2);
- ret = H5TRstart(tid2, H5P_DEFAULT, e_stack);
- assert(0 == ret);
-
- ret = H5Oset_comment_ff(gid, "Testing Object Comment", tid2, e_stack);
- assert(ret == 0);
-
- ret = H5TRfinish(tid2, H5P_DEFAULT, &rid2, e_stack);
- assert(0 == ret);
-
- assert(H5Gclose_ff(gid, e_stack) == 0);
- assert(H5Mclose_ff(map, e_stack) == 0);
- assert(H5Tclose_ff(dtid, e_stack) == 0);
- assert(H5Dclose_ff(did, e_stack) == 0);
-
- /* release container version 2. This is async. */
- ret = H5RCrelease(rid_temp, e_stack);
- assert(0 == ret);
- ret = H5RCclose(rid_temp);
- assert(0 == ret);
-
- ret = H5RCrelease(rid1, e_stack);
- assert(0 == ret);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
- assert(status == H5ES_STATUS_SUCCEED);
-
- /* Close transaction object. Local op */
- ret = H5TRclose(tid2);
- assert(0 == ret);
-
- version = 3;
- }
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- /* Process 0 tells other procs that container version 3 is acquired */
- MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
- assert(3 == version);
-
- /* other processes just create a read context object; no need to
- acquire it */
- if(0 != my_rank) {
- rid2 = H5RCcreate(file_id, version);
- assert(rid2 > 0);
- }
-
- gid = H5Oopen_ff(file_id, "G1", H5P_DEFAULT, rid2);
- assert(gid);
- dtid = H5Oopen_ff(file_id, "DT1", H5P_DEFAULT, rid2);
- assert(dtid);
- did = H5Oopen_ff(gid,"D1", H5P_DEFAULT, rid2);
- assert(did);
- map = H5Oopen_ff(file_id,"MAP1", H5P_DEFAULT, rid2);
- assert(did);
-
- {
- ssize_t ret_size = 0;
- char *comment = NULL;
- H5O_ff_info_t oinfo;
-
- ret = H5Oget_comment_ff(gid, NULL, 0, &ret_size, rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
-
- fprintf(stderr, "size of comment is %d\n", ret_size);
-
- comment = malloc((size_t)ret_size + 1);
-
- ret = H5Oget_comment_ff(gid, comment, (size_t)ret_size + 1,
- &ret_size, rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
-
- fprintf(stderr, "size of comment is %d Comment is %s\n", ret_size, comment);
- free(comment);
-
- ret = H5Oget_info_ff(file_id, &oinfo, rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- assert(H5O_TYPE_GROUP == oinfo.type);
- fprintf(stderr,
- "get_info: Group with OID: %"PRIx64", num attrs = %zu, ref count = %u\n",
- oinfo.addr, oinfo.num_attrs, oinfo.rc);
-
- ret = H5Oget_info_by_name_ff(file_id, ".", &oinfo, H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- assert(H5O_TYPE_GROUP == oinfo.type);
- fprintf(stderr,
- "get_info: Group with OID: %"PRIx64", num attrs = %zu, ref count = %u\n",
- oinfo.addr, oinfo.num_attrs, oinfo.rc);
-
- ret = H5Oget_info_ff(gid, &oinfo, rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- assert(H5O_TYPE_GROUP == oinfo.type);
- fprintf(stderr,
- "get_info: Group with OID: %"PRIx64", num attrs = %zu, ref count = %u\n",
- oinfo.addr, oinfo.num_attrs, oinfo.rc);
-
- ret = H5Oget_info_ff(did, &oinfo, rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- assert(H5O_TYPE_DATASET == oinfo.type);
- fprintf(stderr,
- "get_info: Dataset with OID: %"PRIx64", num attrs = %zu, ref count = %u\n",
- oinfo.addr, oinfo.num_attrs, oinfo.rc);
-
- ret = H5Oget_info_ff(dtid, &oinfo, rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- assert(H5O_TYPE_NAMED_DATATYPE == oinfo.type);
- fprintf(stderr,
- "get_info: Named Datatype with OID: %"PRIx64", num attrs = %zu, ref count = %u\n",
- oinfo.addr, oinfo.num_attrs, oinfo.rc);
-
- ret = H5Oget_info_ff(map, &oinfo, rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- assert(H5O_TYPE_MAP == oinfo.type);
- fprintf(stderr,
- "get_info: MAP with OID: %"PRIx64", num attrs = %zu, ref count = %u\n",
- oinfo.addr, oinfo.num_attrs, oinfo.rc);
- }
-
- /* check if an object exists. This is asynchronous, so checking
- the value should be done after the wait */
- ret = H5Oexists_by_name_ff(file_id, "G1", &exists, H5P_DEFAULT, rid2, e_stack);
- assert(ret == 0);
-
- if(my_rank == 0) {
- /* release container version 3. This is async. */
- ret = H5RCrelease(rid2, e_stack);
- assert(0 == ret);
-
- ret = H5RCclose(rid1);
- assert(0 == ret);
- }
-
- assert(H5Oclose_ff(gid, e_stack) == 0);
- assert(H5Oclose_ff(did, e_stack) == 0);
- assert(H5Oclose_ff(dtid, e_stack) == 0);
- assert(H5Oclose_ff(map, e_stack) == 0);
-
- ret = H5RCclose(rid2);
- assert(0 == ret);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
- assert(status == H5ES_STATUS_SUCCEED);
-
- /* closing the container also acts as a wait all on all pending requests
- on the container. */
- assert(H5Fclose_ff(file_id, 1, H5_EVENT_STACK_NULL) == 0);
-
- assert(exists);
-
- H5Sclose(sid);
- H5Pclose(fapl_id);
- H5ESclose(e_stack);
-
- /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
- and shutsdown the FS server (when all clients send the terminate request)
- and client */
- MPI_Barrier(MPI_COMM_WORLD);
- ret = EFF_finalize();
- assert(ret >= 0);
-
- MPI_Finalize();
- return 0;
-}
+/*
+ * test_client_obj.c: Client side of H5O routines
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+int main(int argc, char **argv) {
+ char file_name[50];
+ hid_t file_id;
+ hid_t gid;
+ hid_t did, map;
+ hid_t sid, dtid;
+ hid_t tid1, tid2, rid1, rid2;
+ hid_t fapl_id, dxpl_id;
+ hid_t e_stack;
+ htri_t exists = -1;
+
+ const unsigned int nelem=60;
+ hsize_t dims[1];
+
+ uint64_t version;
+ uint64_t trans_num;
+
+ int my_rank, my_size;
+ int provided;
+ MPI_Request mpi_req;
+
+ H5ES_status_t status;
+ size_t num_events = 0;
+ herr_t ret;
+
+ sprintf(file_name, "%s_%s", getenv("USER"), "eff_file_obj.h5");
+
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+ if(MPI_THREAD_MULTIPLE != provided) {
+ fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
+ exit(1);
+ }
+
+ /* Call EFF_init to initialize the EFF stack.
+ As a result of this call, the Function Shipper client is started,
+ and HDF5 VOL calls are registered with the function shipper.
+ An "IOD init" call is forwarded from the FS client to the FS server
+ which should already be running. */
+ EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &my_size);
+ fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
+
+ fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
+ /* Choose the IOD VOL plugin to use with this file.
+ First we create a file access property list. Then we call a new routine to set
+ the IOD plugin to use with this fapl */
+ fapl_id = H5Pcreate (H5P_FILE_ACCESS);
+ H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ /* create an event Queue for managing asynchronous requests.
+
+ Event Queues will releive the use from managing and completing
+ individual requests for every operation. Instead of passing a
+ request for every operation, the event queue is passed and
+ internally the HDF5 library creates a request and adds it to
+ the event queue.
+
+ Multiple Event queue can be created used by the application. */
+ e_stack = H5EScreate();
+ assert(e_stack);
+
+ /* create the file. */
+ file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL);
+ assert(file_id > 0);
+
+ /* create 1-D dataspace with 60 elements */
+ dims [0] = nelem;
+ sid = H5Screate_simple(1, dims, NULL);
+ dtid = H5Tcopy(H5T_STD_I32LE);
+
+ /* start transaction 2 with default Leader/Delegate model. Leader
+ which is rank 0 here starts the transaction. It can be
+ asynchronous, but we make it synchronous here so that the
+ Leader can tell its delegates that the transaction is
+ started. */
+ if(0 == my_rank) {
+ hid_t rid_temp;
+ hid_t anon_did;
+
+ version = 1;
+ rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ assert(1 == version);
+
+ /* create transaction object */
+ tid1 = H5TRcreate(file_id, rid1, (uint64_t)2);
+ assert(tid1);
+ ret = H5TRstart(tid1, H5P_DEFAULT, e_stack);
+ assert(0 == ret);
+
+ /* create objects */
+ gid = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT,
+ H5P_DEFAULT, tid1, e_stack);
+ assert(gid >= 0);
+
+ did = H5Dcreate_ff(gid, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT,
+ H5P_DEFAULT, tid1, e_stack);
+ assert(did >= 0);
+
+ ret = H5Tcommit_ff(file_id, "DT1", dtid, H5P_DEFAULT, H5P_DEFAULT,
+ H5P_DEFAULT, tid1, e_stack);
+ assert(ret == 0);
+
+ map = H5Mcreate_ff(file_id, "MAP1", H5T_STD_I32LE, H5T_STD_I32LE,
+ H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
+ assert(map >= 0);
+
+ anon_did = H5Dcreate_anon_ff(file_id, dtid, sid, H5P_DEFAULT, H5P_DEFAULT,
+ tid1, e_stack);
+ assert(anon_did > 0);
+
+ ret = H5Dclose_ff(anon_did, e_stack);
+ assert(0 == ret);
+
+ ret = H5TRfinish(tid1, H5P_DEFAULT, &rid_temp, e_stack);
+ assert(0 == ret);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+ assert(status == H5ES_STATUS_SUCCEED);
+
+ /* Close transaction object. Local op */
+ ret = H5TRclose(tid1);
+ assert(0 == ret);
+
+ /* create transaction object */
+ tid2 = H5TRcreate(file_id, rid_temp, (uint64_t)3);
+ assert(tid2);
+ ret = H5TRstart(tid2, H5P_DEFAULT, e_stack);
+ assert(0 == ret);
+
+ ret = H5Oset_comment_ff(gid, "Testing Object Comment", tid2, e_stack);
+ assert(ret == 0);
+
+ ret = H5TRfinish(tid2, H5P_DEFAULT, &rid2, e_stack);
+ assert(0 == ret);
+
+ assert(H5Gclose_ff(gid, e_stack) == 0);
+ assert(H5Mclose_ff(map, e_stack) == 0);
+ assert(H5Tclose_ff(dtid, e_stack) == 0);
+ assert(H5Dclose_ff(did, e_stack) == 0);
+
+ /* release container version 2. This is async. */
+ ret = H5RCrelease(rid_temp, e_stack);
+ assert(0 == ret);
+ ret = H5RCclose(rid_temp);
+ assert(0 == ret);
+
+ ret = H5RCrelease(rid1, e_stack);
+ assert(0 == ret);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+ assert(status == H5ES_STATUS_SUCCEED);
+
+ /* Close transaction object. Local op */
+ ret = H5TRclose(tid2);
+ assert(0 == ret);
+
+ version = 3;
+ }
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /* Process 0 tells other procs that container version 3 is acquired */
+ MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
+ assert(3 == version);
+
+ /* other processes just create a read context object; no need to
+ acquire it */
+ if(0 != my_rank) {
+ rid2 = H5RCcreate(file_id, version);
+ assert(rid2 > 0);
+ }
+
+ gid = H5Oopen_ff(file_id, "G1", H5P_DEFAULT, rid2);
+ assert(gid);
+ dtid = H5Oopen_ff(file_id, "DT1", H5P_DEFAULT, rid2);
+ assert(dtid);
+ did = H5Oopen_ff(gid,"D1", H5P_DEFAULT, rid2);
+ assert(did);
+ map = H5Oopen_ff(file_id,"MAP1", H5P_DEFAULT, rid2);
+ assert(did);
+
+ {
+ ssize_t ret_size = 0;
+ char *comment = NULL;
+ H5O_ff_info_t oinfo;
+
+ ret = H5Oget_comment_ff(gid, NULL, 0, &ret_size, rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+
+ fprintf(stderr, "size of comment is %d\n", ret_size);
+
+ comment = malloc((size_t)ret_size + 1);
+
+ ret = H5Oget_comment_ff(gid, comment, (size_t)ret_size + 1,
+ &ret_size, rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+
+ fprintf(stderr, "size of comment is %d Comment is %s\n", ret_size, comment);
+ free(comment);
+
+ ret = H5Oget_info_ff(file_id, &oinfo, rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ assert(H5O_TYPE_GROUP == oinfo.type);
+ fprintf(stderr,
+ "get_info: Group with OID: %"PRIx64", num attrs = %zu, ref count = %u\n",
+ oinfo.addr, oinfo.num_attrs, oinfo.rc);
+
+ ret = H5Oget_info_by_name_ff(file_id, ".", &oinfo, H5P_DEFAULT, rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ assert(H5O_TYPE_GROUP == oinfo.type);
+ fprintf(stderr,
+ "get_info: Group with OID: %"PRIx64", num attrs = %zu, ref count = %u\n",
+ oinfo.addr, oinfo.num_attrs, oinfo.rc);
+
+ ret = H5Oget_info_ff(gid, &oinfo, rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ assert(H5O_TYPE_GROUP == oinfo.type);
+ fprintf(stderr,
+ "get_info: Group with OID: %"PRIx64", num attrs = %zu, ref count = %u\n",
+ oinfo.addr, oinfo.num_attrs, oinfo.rc);
+
+ ret = H5Oget_info_ff(did, &oinfo, rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ assert(H5O_TYPE_DATASET == oinfo.type);
+ fprintf(stderr,
+ "get_info: Dataset with OID: %"PRIx64", num attrs = %zu, ref count = %u\n",
+ oinfo.addr, oinfo.num_attrs, oinfo.rc);
+
+ ret = H5Oget_info_ff(dtid, &oinfo, rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ assert(H5O_TYPE_NAMED_DATATYPE == oinfo.type);
+ fprintf(stderr,
+ "get_info: Named Datatype with OID: %"PRIx64", num attrs = %zu, ref count = %u\n",
+ oinfo.addr, oinfo.num_attrs, oinfo.rc);
+
+ ret = H5Oget_info_ff(map, &oinfo, rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ assert(H5O_TYPE_MAP == oinfo.type);
+ fprintf(stderr,
+ "get_info: MAP with OID: %"PRIx64", num attrs = %zu, ref count = %u\n",
+ oinfo.addr, oinfo.num_attrs, oinfo.rc);
+ }
+
+ /* check if an object exists. This is asynchronous, so checking
+ the value should be done after the wait */
+ ret = H5Oexists_by_name_ff(file_id, "G1", &exists, H5P_DEFAULT, rid2, e_stack);
+ assert(ret == 0);
+
+ if(my_rank == 0) {
+ /* release container version 3. This is async. */
+ ret = H5RCrelease(rid2, e_stack);
+ assert(0 == ret);
+
+ ret = H5RCclose(rid1);
+ assert(0 == ret);
+ }
+
+ assert(H5Oclose_ff(gid, e_stack) == 0);
+ assert(H5Oclose_ff(did, e_stack) == 0);
+ assert(H5Oclose_ff(dtid, e_stack) == 0);
+ assert(H5Oclose_ff(map, e_stack) == 0);
+
+ ret = H5RCclose(rid2);
+ assert(0 == ret);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+ assert(status == H5ES_STATUS_SUCCEED);
+
+ /* closing the container also acts as a wait all on all pending requests
+ on the container. */
+ assert(H5Fclose_ff(file_id, 1, H5_EVENT_STACK_NULL) == 0);
+
+ assert(exists);
+
+ H5Sclose(sid);
+ H5Pclose(fapl_id);
+ H5ESclose(e_stack);
+
+ /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
+ and shutsdown the FS server (when all clients send the terminate request)
+ and client */
+ MPI_Barrier(MPI_COMM_WORLD);
+ ret = EFF_finalize();
+ assert(ret >= 0);
+
+ MPI_Finalize();
+ return 0;
+}
diff --git a/examples/h5ff_client_old_api.c b/examples/h5ff_client_old_api.c
index 27e28ba..2659bf8 100644
--- a/examples/h5ff_client_old_api.c
+++ b/examples/h5ff_client_old_api.c
@@ -1,280 +1,280 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-int main(int argc, char **argv) {
- char file_name[]="eff_file.h5";
- hid_t file_id;
- hid_t gid1, gid2, gid3;
- hid_t dataspaceId;
- hid_t did1, did2, did3;
- hid_t aid1, aid2, aid3;
- hid_t fapl_id, dxpl_id;
- const unsigned int nelem=60;
- int *data = NULL, *r_data = NULL, *r2_data = NULL, *data2 = NULL, *data3 = NULL, *a_data = NULL;
- unsigned int i = 0;
- hsize_t dims[1];
- int my_rank, my_size;
- int provided;
- hid_t int_id;
- H5ES_status_t *status = NULL;
- size_t num_events = 0;
- hsize_t extent = 20;
- hbool_t exists;
- H5_request_t req1;
- H5ES_status_t status;
-
- MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
- if(MPI_THREAD_MULTIPLE != provided) {
- fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
- exit(1);
- }
-
- /* Call EFF_init to initialize the EFF stack.
- As a result of this call, the Function Shipper client is started,
- and HDF5 VOL calls are registered with the function shipper.
- An "IOD init" call is forwarded from the FS client to the FS server
- which should already be running. */
- EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
-
- MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
- MPI_Comm_size(MPI_COMM_WORLD, &my_size);
- fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
-
- /* Choose the IOD VOL plugin to use with this file.
- First we create a file access property list. Then we call a new routine to set
- the IOD plugin to use with this fapl */
- fapl_id = H5Pcreate (H5P_FILE_ACCESS);
- H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
-
- /* allocate and initialize 3 arrays for dataset writes, and 2 arrays
- for dataset reads.
- The write arrays are intialized to contain 60 integers (0-59).
- The read arrays are intialized to contain 60 integers all 0s. */
- data = malloc (sizeof(int)*nelem);
- data2 = malloc (sizeof(int)*nelem);
- data3 = malloc (sizeof(int)*nelem);
- r_data = malloc (sizeof(int)*nelem);
- a_data = malloc (sizeof(int)*nelem);
- r2_data = malloc (sizeof(int)*nelem);
- for(i=0;i<nelem;++i) {
- r_data[i] = 0;
- a_data[i] = 0;
- r2_data[i] = 0;
- data[i]=i;
- data2[i]=i;
- data3[i]=i;
- }
-
- /* create the file. */
- file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
- assert(file_id);
-
- gid1 = H5Gcreate2(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- H5Oset_comment(gid1, "Testing Object Comment");
- gid2 = H5Gcreate2(file_id, "G1/G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- gid3 = H5Gcreate2(file_id, "G1/G2/G3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- assert(gid1);
- assert(gid2);
- assert(gid3);
-
- int_id = H5Tcopy(H5T_NATIVE_INT);
- H5Tcommit(file_id, "int", int_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
-
- dims [0] = 60;
- dataspaceId = H5Screate_simple(1, dims, NULL);
-
- aid1 = H5Acreate2(gid1, "ATTR1", H5T_NATIVE_INT, dataspaceId, H5P_DEFAULT, H5P_DEFAULT);
- assert(aid1);
- H5Aclose(aid1);
-
- exists = H5Aexists_by_name(file_id,"G1","ATTR1", H5P_DEFAULT);
- if(exists)
- printf("Attribute ATTR1 exists!\n");
- else
- printf("Attribute ATTR1 does NOT exist. This must be the test without a native backend\n");
-
- assert(H5Adelete_by_name(file_id, "G1", "ATTR1", H5P_DEFAULT) == 0);
- assert(!H5Aexists(gid1, "ATTR1"));
-
- did1 = H5Dcreate(file_id,"G1/G2/G3/D1",int_id,dataspaceId,H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT);
- assert(did1);
-
- /* create an attribute on dataset D1 */
- aid2 = H5Acreate_by_name(file_id, "G1/G2/G3/D1", "ATTR2_tmp", H5T_NATIVE_INT,
- dataspaceId, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- assert(aid2);
- H5Arename(did1, "ATTR2_tmp", "ATTR2");
- H5Awrite(aid2, int_id, data);
-
- did2 = H5Dcreate(file_id,"G1/G2/G3/D2",int_id,dataspaceId,H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT);
- assert(did2);
-
- did3 = H5Dcreate(file_id,"G1/G2/G3/D3",int_id,dataspaceId,H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT);
- assert(did3);
-
- H5Dwrite(did1, int_id, dataspaceId, dataspaceId, H5P_DEFAULT, data);
- H5Dwrite(did2, int_id, dataspaceId, dataspaceId, H5P_DEFAULT, data2);
- H5Dwrite(did3, int_id, dataspaceId, dataspaceId, H5P_DEFAULT, data3);
- H5Dread(did1, int_id, dataspaceId, dataspaceId, H5P_DEFAULT, r_data);
-
- dxpl_id = H5Pcreate (H5P_DATASET_XFER);
- H5Pset_dxpl_inject_corruption(dxpl_id, 1);
- H5Dread(did1, int_id, dataspaceId, dataspaceId, dxpl_id, r2_data);
- H5Pclose(dxpl_id);
-
- /* Issue other operations to query certain metadata values or
- update previously created objects */
- {
- unsigned intent;
- char temp_name[50];
- hid_t plist_id;
-
- H5Fget_intent(file_id, &intent);
- fprintf(stderr, "Intent %d %d\n", intent, H5F_ACC_RDWR);
- H5Fget_name(gid1, temp_name, 50);
- fprintf(stderr, "File name %s %s\n", temp_name, file_name);
-
- plist_id = H5Fget_access_plist(file_id);
- assert(plist_id);
- H5Pclose(plist_id);
-
- plist_id = H5Fget_create_plist(file_id);
- assert(plist_id);
- H5Pclose(plist_id);
-
- /* change the dataset dimensions for Dataset D1. */
- //assert(H5Dset_extent(did1, &extent) == 0);
- }
-
- assert(H5Dclose(did1) == 0);
- assert(H5Dclose(did2) == 0);
- assert(H5Dclose(did3) == 0);
- H5Sclose(dataspaceId);
- assert(H5Aclose(aid2) == 0);
- assert(H5Tclose(int_id) == 0);
- assert(H5Gclose(gid1) == 0);
- assert(H5Gclose(gid2) == 0);
- assert(H5Gclose(gid3) == 0);
-
- /* Test Links */
- gid1 = H5Gcreate2(file_id, "G4", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- gid2 = H5Gcreate2(file_id, "G4/G5", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
-
- H5Lcreate_hard(file_id, "G1/G2/G3/D1", gid1, "G5/newD1", H5P_DEFAULT, H5P_DEFAULT);
- did1 = H5Dopen2(file_id,"G4/G5/newD1", H5P_DEFAULT);
-
- H5Lcreate_soft("/G1/G2/G3/D4", gid1, "G5/newD2", H5P_DEFAULT, H5P_DEFAULT);
-
- H5Lmove(file_id, "/G1/G2/G3/D3", file_id, "/G4/G5/D3moved", H5P_DEFAULT, H5P_DEFAULT);
-
- H5Ldelete(file_id, "/G1/G2/G3/D2", H5P_DEFAULT);
-
- assert(H5Dclose(did1) == 0);
- assert(H5Gclose(gid1) == 0);
- assert(H5Gclose(gid2) == 0);
-
- /* flush all the contents of file to disk. This is asynchronous. */
- assert(H5Fflush(file_id, H5F_SCOPE_GLOBAL) == 0);
- assert(H5Fclose(file_id) == 0);
-
- /* Print the data that has been read, after we have issued a wait
- (in the H5Dclose).
- This should printf the correct array (0-59) */
- fprintf(stderr, "Printing After Waiting ");
- for(i=0;i<nelem;++i)
- fprintf(stderr, "%d ",r_data[i]);
- fprintf(stderr, "\n");
-
- /* Print the data that has been read with an injected fault,
- This should print the array similar to the previous one, but with the
- first value modified to be 10 (the injected error) */
- fprintf(stderr, "Printing Corrupted Data ");
- for(i=0;i<nelem;++i)
- fprintf(stderr, "%d ",r2_data[i]);
- fprintf(stderr, "\n");
-
- /* Now we test the Open routines. Since there is no underneath
- container, the underlying VOL server is just going to "fake"
- open calls and assume they exist. However there is no metadata
- information returned since nothing is stored on disk for
- now. */
-
- file_id = H5Fopen(file_name, H5F_ACC_RDONLY, fapl_id);
- assert(file_id);
-
- assert(H5Ocopy(file_id, "/G1/G2/G3/D1", file_id, "D1_copy", H5P_DEFAULT, H5P_DEFAULT) == 0);
-
- exists = H5Oexists_by_name(file_id, "G1/G2/G3", H5P_DEFAULT);
- if(exists)
- printf("Group G3 exists!\n");
- else
- printf("Group G3 does NOT exist. This must be the test without a native backend\n");
-
- gid1 = H5Oopen(file_id, "G1", H5P_DEFAULT);
- assert(gid1);
- int_id = H5Oopen(file_id, "int", H5P_DEFAULT);
- assert(int_id);
- did1 = H5Oopen(file_id,"G1/G2/G3/D1", H5P_DEFAULT);
- assert(did1);
- aid2 = H5Aopen_by_name(file_id, "G1/G2/G3/D1", "ATTR2", H5P_DEFAULT, H5P_DEFAULT);
- assert(aid2);
-
- assert(H5Aclose(aid2) == 0);
- assert(H5Oclose(did1) == 0);
- assert(H5Oclose(int_id) == 0);
- assert(H5Oclose(gid1) == 0);
-
- gid1 = H5Gopen2(file_id, "G1", H5P_DEFAULT);
- assert(gid1);
- {
- ssize_t ret_size = 0;
- char *comment = NULL;
-
- ret_size = H5Oget_comment(gid1, NULL, 0);
- fprintf(stderr, "size of comment is %zd\n", ret_size);
-
- comment = (char *)malloc(sizeof(char *) * (ret_size + 1));
-
- ret_size = H5Oget_comment(gid1, comment, (size_t)ret_size + 1);
- fprintf(stderr, "size of comment is %zd Comment is %s\n", ret_size, comment);
- free(comment);
- }
- int_id = H5Topen2(file_id, "int", H5P_DEFAULT);
- assert(int_id);
- did1 = H5Dopen2(file_id,"G1/G2/G3/D1", H5P_DEFAULT);
- assert(did1);
-
- aid2 = H5Aopen(did1, "ATTR2", H5P_DEFAULT);
- H5Aread(aid2, int_id, a_data);
- fprintf(stderr, "Printing Attribute data: ");
- for(i=0;i<nelem;++i)
- fprintf(stderr, "%d ",a_data[i]);
- fprintf(stderr, "\n");
-
- assert(H5Aclose(aid2) == 0);
- assert(H5Dclose(did1) == 0);
- assert(H5Tclose(int_id) == 0);
- assert(H5Gclose(gid1) == 0);
-
- assert(H5Fclose(file_id) == 0);
- H5Pclose(fapl_id);
-
- free(data);
- free(r_data);
- free(a_data);
- free(data2);
- free(r2_data);
- free(data3);
-
- /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
- and shutsdown the FS server (when all clients send the terminate request)
- and client */
- EFF_finalize();
-
- MPI_Finalize();
- return 0;
-}
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+int main(int argc, char **argv) {
+ char file_name[]="eff_file.h5";
+ hid_t file_id;
+ hid_t gid1, gid2, gid3;
+ hid_t dataspaceId;
+ hid_t did1, did2, did3;
+ hid_t aid1, aid2, aid3;
+ hid_t fapl_id, dxpl_id;
+ const unsigned int nelem=60;
+ int *data = NULL, *r_data = NULL, *r2_data = NULL, *data2 = NULL, *data3 = NULL, *a_data = NULL;
+ unsigned int i = 0;
+ hsize_t dims[1];
+ int my_rank, my_size;
+ int provided;
+ hid_t int_id;
+ H5ES_status_t *status = NULL;
+ size_t num_events = 0;
+ hsize_t extent = 20;
+ hbool_t exists;
+ H5_request_t req1;
+ H5ES_status_t status;
+
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+ if(MPI_THREAD_MULTIPLE != provided) {
+ fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
+ exit(1);
+ }
+
+ /* Call EFF_init to initialize the EFF stack.
+ As a result of this call, the Function Shipper client is started,
+ and HDF5 VOL calls are registered with the function shipper.
+ An "IOD init" call is forwarded from the FS client to the FS server
+ which should already be running. */
+ EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &my_size);
+ fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
+
+ /* Choose the IOD VOL plugin to use with this file.
+ First we create a file access property list. Then we call a new routine to set
+ the IOD plugin to use with this fapl */
+ fapl_id = H5Pcreate (H5P_FILE_ACCESS);
+ H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ /* allocate and initialize 3 arrays for dataset writes, and 2 arrays
+ for dataset reads.
+ The write arrays are intialized to contain 60 integers (0-59).
+ The read arrays are intialized to contain 60 integers all 0s. */
+ data = malloc (sizeof(int)*nelem);
+ data2 = malloc (sizeof(int)*nelem);
+ data3 = malloc (sizeof(int)*nelem);
+ r_data = malloc (sizeof(int)*nelem);
+ a_data = malloc (sizeof(int)*nelem);
+ r2_data = malloc (sizeof(int)*nelem);
+ for(i=0;i<nelem;++i) {
+ r_data[i] = 0;
+ a_data[i] = 0;
+ r2_data[i] = 0;
+ data[i]=i;
+ data2[i]=i;
+ data3[i]=i;
+ }
+
+ /* create the file. */
+ file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+ assert(file_id);
+
+ gid1 = H5Gcreate2(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Oset_comment(gid1, "Testing Object Comment");
+ gid2 = H5Gcreate2(file_id, "G1/G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ gid3 = H5Gcreate2(file_id, "G1/G2/G3", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ assert(gid1);
+ assert(gid2);
+ assert(gid3);
+
+ int_id = H5Tcopy(H5T_NATIVE_INT);
+ H5Tcommit(file_id, "int", int_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+
+ dims [0] = 60;
+ dataspaceId = H5Screate_simple(1, dims, NULL);
+
+ aid1 = H5Acreate2(gid1, "ATTR1", H5T_NATIVE_INT, dataspaceId, H5P_DEFAULT, H5P_DEFAULT);
+ assert(aid1);
+ H5Aclose(aid1);
+
+ exists = H5Aexists_by_name(file_id,"G1","ATTR1", H5P_DEFAULT);
+ if(exists)
+ printf("Attribute ATTR1 exists!\n");
+ else
+ printf("Attribute ATTR1 does NOT exist. This must be the test without a native backend\n");
+
+ assert(H5Adelete_by_name(file_id, "G1", "ATTR1", H5P_DEFAULT) == 0);
+ assert(!H5Aexists(gid1, "ATTR1"));
+
+ did1 = H5Dcreate(file_id,"G1/G2/G3/D1",int_id,dataspaceId,H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT);
+ assert(did1);
+
+ /* create an attribute on dataset D1 */
+ aid2 = H5Acreate_by_name(file_id, "G1/G2/G3/D1", "ATTR2_tmp", H5T_NATIVE_INT,
+ dataspaceId, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ assert(aid2);
+ H5Arename(did1, "ATTR2_tmp", "ATTR2");
+ H5Awrite(aid2, int_id, data);
+
+ did2 = H5Dcreate(file_id,"G1/G2/G3/D2",int_id,dataspaceId,H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT);
+ assert(did2);
+
+ did3 = H5Dcreate(file_id,"G1/G2/G3/D3",int_id,dataspaceId,H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT);
+ assert(did3);
+
+ H5Dwrite(did1, int_id, dataspaceId, dataspaceId, H5P_DEFAULT, data);
+ H5Dwrite(did2, int_id, dataspaceId, dataspaceId, H5P_DEFAULT, data2);
+ H5Dwrite(did3, int_id, dataspaceId, dataspaceId, H5P_DEFAULT, data3);
+ H5Dread(did1, int_id, dataspaceId, dataspaceId, H5P_DEFAULT, r_data);
+
+ dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+ H5Pset_dxpl_inject_corruption(dxpl_id, 1);
+ H5Dread(did1, int_id, dataspaceId, dataspaceId, dxpl_id, r2_data);
+ H5Pclose(dxpl_id);
+
+ /* Issue other operations to query certain metadata values or
+ update previously created objects */
+ {
+ unsigned intent;
+ char temp_name[50];
+ hid_t plist_id;
+
+ H5Fget_intent(file_id, &intent);
+ fprintf(stderr, "Intent %d %d\n", intent, H5F_ACC_RDWR);
+ H5Fget_name(gid1, temp_name, 50);
+ fprintf(stderr, "File name %s %s\n", temp_name, file_name);
+
+ plist_id = H5Fget_access_plist(file_id);
+ assert(plist_id);
+ H5Pclose(plist_id);
+
+ plist_id = H5Fget_create_plist(file_id);
+ assert(plist_id);
+ H5Pclose(plist_id);
+
+ /* change the dataset dimensions for Dataset D1. */
+ //assert(H5Dset_extent(did1, &extent) == 0);
+ }
+
+ assert(H5Dclose(did1) == 0);
+ assert(H5Dclose(did2) == 0);
+ assert(H5Dclose(did3) == 0);
+ H5Sclose(dataspaceId);
+ assert(H5Aclose(aid2) == 0);
+ assert(H5Tclose(int_id) == 0);
+ assert(H5Gclose(gid1) == 0);
+ assert(H5Gclose(gid2) == 0);
+ assert(H5Gclose(gid3) == 0);
+
+ /* Test Links */
+ gid1 = H5Gcreate2(file_id, "G4", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ gid2 = H5Gcreate2(file_id, "G4/G5", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+
+ H5Lcreate_hard(file_id, "G1/G2/G3/D1", gid1, "G5/newD1", H5P_DEFAULT, H5P_DEFAULT);
+ did1 = H5Dopen2(file_id,"G4/G5/newD1", H5P_DEFAULT);
+
+ H5Lcreate_soft("/G1/G2/G3/D4", gid1, "G5/newD2", H5P_DEFAULT, H5P_DEFAULT);
+
+ H5Lmove(file_id, "/G1/G2/G3/D3", file_id, "/G4/G5/D3moved", H5P_DEFAULT, H5P_DEFAULT);
+
+ H5Ldelete(file_id, "/G1/G2/G3/D2", H5P_DEFAULT);
+
+ assert(H5Dclose(did1) == 0);
+ assert(H5Gclose(gid1) == 0);
+ assert(H5Gclose(gid2) == 0);
+
+ /* flush all the contents of file to disk. This is asynchronous. */
+ assert(H5Fflush(file_id, H5F_SCOPE_GLOBAL) == 0);
+ assert(H5Fclose(file_id) == 0);
+
+ /* Print the data that has been read, after we have issued a wait
+ (in the H5Dclose).
+ This should printf the correct array (0-59) */
+ fprintf(stderr, "Printing After Waiting ");
+ for(i=0;i<nelem;++i)
+ fprintf(stderr, "%d ",r_data[i]);
+ fprintf(stderr, "\n");
+
+ /* Print the data that has been read with an injected fault,
+ This should print the array similar to the previous one, but with the
+ first value modified to be 10 (the injected error) */
+ fprintf(stderr, "Printing Corrupted Data ");
+ for(i=0;i<nelem;++i)
+ fprintf(stderr, "%d ",r2_data[i]);
+ fprintf(stderr, "\n");
+
+ /* Now we test the Open routines. Since there is no underneath
+ container, the underlying VOL server is just going to "fake"
+ open calls and assume they exist. However there is no metadata
+ information returned since nothing is stored on disk for
+ now. */
+
+ file_id = H5Fopen(file_name, H5F_ACC_RDONLY, fapl_id);
+ assert(file_id);
+
+ assert(H5Ocopy(file_id, "/G1/G2/G3/D1", file_id, "D1_copy", H5P_DEFAULT, H5P_DEFAULT) == 0);
+
+ exists = H5Oexists_by_name(file_id, "G1/G2/G3", H5P_DEFAULT);
+ if(exists)
+ printf("Group G3 exists!\n");
+ else
+ printf("Group G3 does NOT exist. This must be the test without a native backend\n");
+
+ gid1 = H5Oopen(file_id, "G1", H5P_DEFAULT);
+ assert(gid1);
+ int_id = H5Oopen(file_id, "int", H5P_DEFAULT);
+ assert(int_id);
+ did1 = H5Oopen(file_id,"G1/G2/G3/D1", H5P_DEFAULT);
+ assert(did1);
+ aid2 = H5Aopen_by_name(file_id, "G1/G2/G3/D1", "ATTR2", H5P_DEFAULT, H5P_DEFAULT);
+ assert(aid2);
+
+ assert(H5Aclose(aid2) == 0);
+ assert(H5Oclose(did1) == 0);
+ assert(H5Oclose(int_id) == 0);
+ assert(H5Oclose(gid1) == 0);
+
+ gid1 = H5Gopen2(file_id, "G1", H5P_DEFAULT);
+ assert(gid1);
+ {
+ ssize_t ret_size = 0;
+ char *comment = NULL;
+
+ ret_size = H5Oget_comment(gid1, NULL, 0);
+ fprintf(stderr, "size of comment is %zd\n", ret_size);
+
+ comment = (char *)malloc(sizeof(char *) * (ret_size + 1));
+
+ ret_size = H5Oget_comment(gid1, comment, (size_t)ret_size + 1);
+ fprintf(stderr, "size of comment is %zd Comment is %s\n", ret_size, comment);
+ free(comment);
+ }
+ int_id = H5Topen2(file_id, "int", H5P_DEFAULT);
+ assert(int_id);
+ did1 = H5Dopen2(file_id,"G1/G2/G3/D1", H5P_DEFAULT);
+ assert(did1);
+
+ aid2 = H5Aopen(did1, "ATTR2", H5P_DEFAULT);
+ H5Aread(aid2, int_id, a_data);
+ fprintf(stderr, "Printing Attribute data: ");
+ for(i=0;i<nelem;++i)
+ fprintf(stderr, "%d ",a_data[i]);
+ fprintf(stderr, "\n");
+
+ assert(H5Aclose(aid2) == 0);
+ assert(H5Dclose(did1) == 0);
+ assert(H5Tclose(int_id) == 0);
+ assert(H5Gclose(gid1) == 0);
+
+ assert(H5Fclose(file_id) == 0);
+ H5Pclose(fapl_id);
+
+ free(data);
+ free(r_data);
+ free(a_data);
+ free(data2);
+ free(r2_data);
+ free(data3);
+
+ /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
+ and shutsdown the FS server (when all clients send the terminate request)
+ and client */
+ EFF_finalize();
+
+ MPI_Finalize();
+ return 0;
+}
diff --git a/examples/h5ff_client_prefetch.c b/examples/h5ff_client_prefetch.c
index 5772f21..9f50b4c 100644
--- a/examples/h5ff_client_prefetch.c
+++ b/examples/h5ff_client_prefetch.c
@@ -1,439 +1,439 @@
-/*
- * test_client_obj.c: Client side of H5O routines
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-#define NELEM 60
-#define NKVP 10
-
-static hid_t gid=-1, did=-1, map=-1, dtid=-1, sid=-1;
-int my_rank, my_size;
-
-static void prefetch (int local_ion, int sub_obj, hid_t rcxt)
-{
- hrpl_t map_replica, dt_replica, dset_replica, grp_replica;
- hid_t dxpl_id;
- int key, value, i;
- int32_t rdata1[NELEM];
- herr_t ret;
-
- if((my_size > 1 && 1 == my_rank) ||
- (my_size == 1 && 0 == my_rank)) {
- /* prefetch objects */
- dxpl_id = H5Pcreate (H5P_DATASET_XFER);
- if(sub_obj) {
- hsize_t start=0, count=NELEM/2;
- H5Sselect_hyperslab(sid, H5S_SELECT_SET, &start, NULL, &count, NULL);
- H5Pset_prefetch_selection(dxpl_id, sid);
- }
- if(local_ion)
- H5Pset_prefetch_layout(dxpl_id, H5_LOCAL_NODE);
-
- ret = H5Dprefetch_ff(did, rcxt, &dset_replica, dxpl_id, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- printf("prefetched dataset with replica id %"PRIx64"\n", dset_replica);
- H5Pclose(dxpl_id);
-
- /* prefetch objects */
- dxpl_id = H5Pcreate (H5P_DATASET_XFER);
- if(sub_obj) {
- int low_key = 3;
- int high_key = 8;
- H5Pset_prefetch_range(dxpl_id, H5T_STD_I32LE, &low_key, &high_key);
- }
- if(local_ion)
- H5Pset_prefetch_layout(dxpl_id, H5_LOCAL_NODE);
- ret = H5Mprefetch_ff(map, rcxt, &map_replica, dxpl_id, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- printf("prefetched map with replica id %"PRIx64"\n", map_replica);
- H5Pclose(dxpl_id);
-
- dxpl_id = H5Pcreate (H5P_DATASET_XFER);
- if(local_ion)
- H5Pset_prefetch_layout(dxpl_id, H5_LOCAL_NODE);
- ret = H5Tprefetch_ff(dtid, rcxt, &dt_replica, dxpl_id, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- printf("prefetched datatype with replica id %"PRIx64"\n", dt_replica);
- ret = H5Gprefetch_ff(gid, rcxt, &grp_replica, dxpl_id, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- printf("prefetched group with replica id %"PRIx64"\n", grp_replica);
- H5Pclose(dxpl_id);
- }
-
- if(my_size > 1) {
- MPI_Bcast(&map_replica, 1, MPI_UINT64_T, 1, MPI_COMM_WORLD);
- MPI_Bcast(&dt_replica, 1, MPI_UINT64_T, 1, MPI_COMM_WORLD);
- MPI_Bcast(&dset_replica, 1, MPI_UINT64_T, 1, MPI_COMM_WORLD);
- MPI_Bcast(&grp_replica, 1, MPI_UINT64_T, 1, MPI_COMM_WORLD);
- }
-
- /* Read from prefetched Replicas */
- /* set dxpl to read from replica in BB */
- dxpl_id = H5Pcreate (H5P_DATASET_XFER);
- ret = H5Pset_dxpl_replica(dxpl_id, dset_replica);
- assert(0 == ret);
- ret = H5Dread_ff(did, dtid, H5S_ALL, H5S_ALL, dxpl_id, rdata1, rcxt, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- printf("Read Data1: ");
- for(i=0;i<NELEM;++i)
- printf("%d ",rdata1[i]);
- printf("\n");
- H5Pclose(dxpl_id);
-
- dxpl_id = H5Pcreate (H5P_DATASET_XFER);
- ret = H5Pset_dxpl_replica(dxpl_id, map_replica);
- assert(0 == ret);
- for(i=0; i<NKVP; i++){
- key = i;
- value = -i;
- ret = H5Mget_ff(map, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,
- dxpl_id, rcxt, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- printf("Value recieved = %d\n", value);
- assert(1000*i == value);
- }
- H5Pclose(dxpl_id);
-
- /* wait for all reads to complete before evicting */
- MPI_Barrier(MPI_COMM_WORLD);
-
- if(0 == my_rank) {
- /* evict Replicas */
- dxpl_id = H5Pcreate (H5P_DATASET_XFER);
- ret = H5Pset_dxpl_replica(dxpl_id, dset_replica);
- assert(0 == ret);
- ret = H5Tevict_ff(dtid, 3, dxpl_id, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- H5Pclose(dxpl_id);
-
- dxpl_id = H5Pcreate (H5P_DATASET_XFER);
- ret = H5Pset_dxpl_replica(dxpl_id, dset_replica);
- assert(0 == ret);
- ret = H5Devict_ff(did, 3, dxpl_id, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- H5Pclose(dxpl_id);
-
- dxpl_id = H5Pcreate (H5P_DATASET_XFER);
- ret = H5Pset_dxpl_replica(dxpl_id, map_replica);
- assert(0 == ret);
- ret = H5Mevict_ff(map, 3, dxpl_id, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- H5Pclose(dxpl_id);
-
- dxpl_id = H5Pcreate (H5P_DATASET_XFER);
- ret = H5Pset_dxpl_replica(dxpl_id, grp_replica);
- assert(0 == ret);
- ret = H5Gevict_ff(gid, 3, dxpl_id, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- H5Pclose(dxpl_id);
- }
-}
-
-int main(int argc, char **argv) {
- char file_name[50];
- hid_t file_id;
- hid_t tid1, tid2, rid1, rid2;
- hid_t fapl_id, dxpl_id;
- hid_t e_stack;
- htri_t exists = -1;
-
- hsize_t dims[1];
-
- uint64_t version;
- uint64_t trans_num;
-
- int provided;
- MPI_Request mpi_req;
-
- int32_t wdata1[NELEM],rdata1[NELEM];
- int key, value, i;
- H5ES_status_t status;
- size_t num_events = 0;
- herr_t ret;
-
- sprintf(file_name, "%s_%s", getenv("USER"), "eff_file_pref.h5");
-
- MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
- if(MPI_THREAD_MULTIPLE != provided) {
- fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
- exit(1);
- }
-
- /* Call EFF_init to initialize the EFF stack.
- As a result of this call, the Function Shipper client is started,
- and HDF5 VOL calls are registered with the function shipper.
- An "IOD init" call is forwarded from the FS client to the FS server
- which should already be running. */
- EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
-
- MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
- MPI_Comm_size(MPI_COMM_WORLD, &my_size);
- fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
-
- for(i=0;i<NELEM;++i) {
- wdata1[i]=i;
- rdata1[i] = 0;
- }
-
- fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
- /* Choose the IOD VOL plugin to use with this file.
- First we create a file access property list. Then we call a new routine to set
- the IOD plugin to use with this fapl */
- fapl_id = H5Pcreate (H5P_FILE_ACCESS);
- H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
-
- /* create an event Queue for managing asynchronous requests.
-
- Event Queues will releive the use from managing and completing
- individual requests for every operation. Instead of passing a
- request for every operation, the event queue is passed and
- internally the HDF5 library creates a request and adds it to
- the event queue.
-
- Multiple Event queue can be created used by the application. */
- e_stack = H5EScreate();
- assert(e_stack);
-
- /* create the file. */
- file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL);
- assert(file_id > 0);
-
- /* create 1-D dataspace with 60 elements */
- dims [0] = NELEM;
- sid = H5Screate_simple(1, dims, NULL);
- dtid = H5Tcopy(H5T_STD_I32LE);
-
- /* acquire container version 1 - EXACT.
- This can be asynchronous, but here we need the acquired ID
- right after the call to start the transaction so we make synchronous. */
- if(0 == my_rank) {
- version = 1;
- rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- }
- MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
- assert(1 == version);
- if (my_rank != 0)
- rid1 = H5RCcreate(file_id, version);
-
- /* start transaction 2 with default Leader/Delegate model. Leader
- which is rank 0 here starts the transaction. It can be
- asynchronous, but we make it synchronous here so that the
- Leader can tell its delegates that the transaction is
- started. */
- if(0 == my_rank) {
- hid_t rid_temp;
-
- /* create transaction object */
- tid1 = H5TRcreate(file_id, rid1, (uint64_t)2);
- assert(tid1);
- ret = H5TRstart(tid1, H5P_DEFAULT, e_stack);
- assert(0 == ret);
-
- /* create objects */
- gid = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT,
- H5P_DEFAULT, tid1, e_stack);
- assert(gid >= 0);
-
- did = H5Dcreate_ff(gid, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT,
- H5P_DEFAULT, tid1, e_stack);
- assert(did >= 0);
-
- ret = H5Dwrite_ff(did, dtid, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata1, tid1, e_stack);
- assert(ret == 0);
-
- ret = H5Tcommit_ff(file_id, "DT1", dtid, H5P_DEFAULT, H5P_DEFAULT,
- H5P_DEFAULT, tid1, e_stack);
- assert(ret == 0);
-
- map = H5Mcreate_ff(file_id, "MAP1", H5T_STD_I32LE, H5T_STD_I32LE,
- H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
- assert(map >= 0);
-
- /* write some KV pairs to each map object. */
- for(i=0; i<NKVP; i++){
- key = i;
- value = 1000*i;
- ret = H5Mset_ff(map, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,
- H5P_DEFAULT, tid1, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- }
-
- ret = H5TRfinish(tid1, H5P_DEFAULT, &rid_temp, e_stack);
- assert(0 == ret);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- /* Close transaction object. Local op */
- ret = H5TRclose(tid1);
- assert(0 == ret);
-
- /* create transaction object */
- tid2 = H5TRcreate(file_id, rid_temp, (uint64_t)3);
- assert(tid2);
- ret = H5TRstart(tid2, H5P_DEFAULT, e_stack);
- assert(0 == ret);
-
- ret = H5Oset_comment_ff(gid, "Testing Object Comment", tid2, e_stack);
- assert(ret == 0);
-
- ret = H5TRfinish(tid2, H5P_DEFAULT, &rid2, e_stack);
- assert(0 == ret);
-
- assert(H5Gclose_ff(gid, e_stack) == 0);
- assert(H5Mclose_ff(map, e_stack) == 0);
- assert(H5Tclose_ff(dtid, e_stack) == 0);
- assert(H5Dclose_ff(did, e_stack) == 0);
-
- /* release container version 2. This is async. */
- ret = H5RCrelease(rid_temp, e_stack);
- assert(0 == ret);
- ret = H5RCclose(rid_temp);
- assert(0 == ret);
-
- version = 3;
- }
-
- if(0 == my_rank) {
- /* release container version 1. This is async. */
- ret = H5RCrelease(rid1, e_stack);
- assert(0 == ret);
- }
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- /* Process 0 tells other procs that container version 3 is acquired */
- MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
- assert(3 == version);
-
- /* other processes just create a read context object; no need to
- acquire it */
- if(0 != my_rank) {
- rid2 = H5RCcreate(file_id, version);
- assert(rid2 > 0);
- }
-
- if(0 == my_rank) {
- /* Close transaction object. Local op */
- ret = H5TRclose(tid2);
- assert(0 == ret);
-
- ret = H5RCpersist(rid2, H5_EVENT_STACK_NULL);
- assert(ret == 0);
- }
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- gid = H5Oopen_ff(file_id, "G1", H5P_DEFAULT, rid2);
- assert(gid);
- dtid = H5Oopen_ff(file_id, "DT1", H5P_DEFAULT, rid2);
- assert(dtid);
- did = H5Oopen_ff(gid,"D1", H5P_DEFAULT, rid2);
- assert(did);
- map = H5Oopen_ff(file_id,"MAP1", H5P_DEFAULT, rid2);
- assert(map);
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- if((my_size > 1 && 1 == my_rank) ||
- (my_size == 1 && 0 == my_rank)) {
- ret = H5Tevict_ff(dtid, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- assert(0 == ret);
-
- ret = H5Devict_ff(did, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- assert(0 == ret);
-
- ret = H5Mevict_ff(map, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- assert(0 == ret);
-
- ret = H5Gevict_ff(gid, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- }
-
- MPI_Barrier(MPI_COMM_WORLD);
-
- if(0 == my_rank) {
- printf("***************************************************************\n");
- printf("TESTING Prefetch - Default layout, Default partition\n");
- }
- MPI_Barrier(MPI_COMM_WORLD);
- prefetch(0, 0, rid2);
- MPI_Barrier(MPI_COMM_WORLD);
-
- if(0 == my_rank) {
- printf("***************************************************************\n");
- printf("TESTING Prefetch - NON-Default layout, Default partition\n");
- }
- MPI_Barrier(MPI_COMM_WORLD);
- prefetch(1, 0, rid2);
- MPI_Barrier(MPI_COMM_WORLD);
-
- if(0 == my_rank) {
- printf("***************************************************************\n");
- printf("TESTING Prefetch - Default layout, NON-Default partition\n");
- }
- MPI_Barrier(MPI_COMM_WORLD);
- prefetch(0, 1, rid2);
- MPI_Barrier(MPI_COMM_WORLD);
-
- if(0 == my_rank) {
- printf("***************************************************************\n");
- printf("TESTING Prefetch - NON-Default layout, NON-Default partition\n");
- }
- MPI_Barrier(MPI_COMM_WORLD);
- prefetch(1, 1, rid2);
- MPI_Barrier(MPI_COMM_WORLD);
-
- if(my_rank == 0) {
- /* release container version 3. */
- ret = H5RCrelease(rid2, H5_EVENT_STACK_NULL);
- assert(0 == ret);
- }
-
- assert(H5Oclose_ff(gid, e_stack) == 0);
- assert(H5Oclose_ff(did, e_stack) == 0);
- assert(H5Oclose_ff(dtid, e_stack) == 0);
- assert(H5Oclose_ff(map, e_stack) == 0);
-
- ret = H5RCclose(rid1);
- assert(0 == ret);
- ret = H5RCclose(rid2);
- assert(0 == ret);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- /* closing the container also acts as a wait all on all pending requests
- on the container. */
- assert(H5Fclose_ff(file_id, 1, H5_EVENT_STACK_NULL) == 0);
-
- H5Sclose(sid);
- H5Pclose(fapl_id);
- H5ESclose(e_stack);
-
- /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
- and shutsdown the FS server (when all clients send the terminate request)
- and client */
- MPI_Barrier(MPI_COMM_WORLD);
- ret = EFF_finalize();
- assert(ret >= 0);
-
- MPI_Finalize();
- return 0;
-}
+/*
+ * test_client_obj.c: Client side of H5O routines
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+#define NELEM 60
+#define NKVP 10
+
+static hid_t gid=-1, did=-1, map=-1, dtid=-1, sid=-1;
+int my_rank, my_size;
+
+static void prefetch (int local_ion, int sub_obj, hid_t rcxt)
+{
+ hrpl_t map_replica, dt_replica, dset_replica, grp_replica;
+ hid_t dxpl_id;
+ int key, value, i;
+ int32_t rdata1[NELEM];
+ herr_t ret;
+
+ if((my_size > 1 && 1 == my_rank) ||
+ (my_size == 1 && 0 == my_rank)) {
+ /* prefetch objects */
+ dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+ if(sub_obj) {
+ hsize_t start=0, count=NELEM/2;
+ H5Sselect_hyperslab(sid, H5S_SELECT_SET, &start, NULL, &count, NULL);
+ H5Pset_prefetch_selection(dxpl_id, sid);
+ }
+ if(local_ion)
+ H5Pset_prefetch_layout(dxpl_id, H5_LOCAL_NODE);
+
+ ret = H5Dprefetch_ff(did, rcxt, &dset_replica, dxpl_id, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ printf("prefetched dataset with replica id %"PRIx64"\n", dset_replica);
+ H5Pclose(dxpl_id);
+
+ /* prefetch objects */
+ dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+ if(sub_obj) {
+ int low_key = 3;
+ int high_key = 8;
+ H5Pset_prefetch_range(dxpl_id, H5T_STD_I32LE, &low_key, &high_key);
+ }
+ if(local_ion)
+ H5Pset_prefetch_layout(dxpl_id, H5_LOCAL_NODE);
+ ret = H5Mprefetch_ff(map, rcxt, &map_replica, dxpl_id, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ printf("prefetched map with replica id %"PRIx64"\n", map_replica);
+ H5Pclose(dxpl_id);
+
+ dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+ if(local_ion)
+ H5Pset_prefetch_layout(dxpl_id, H5_LOCAL_NODE);
+ ret = H5Tprefetch_ff(dtid, rcxt, &dt_replica, dxpl_id, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ printf("prefetched datatype with replica id %"PRIx64"\n", dt_replica);
+ ret = H5Gprefetch_ff(gid, rcxt, &grp_replica, dxpl_id, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ printf("prefetched group with replica id %"PRIx64"\n", grp_replica);
+ H5Pclose(dxpl_id);
+ }
+
+ if(my_size > 1) {
+ MPI_Bcast(&map_replica, 1, MPI_UINT64_T, 1, MPI_COMM_WORLD);
+ MPI_Bcast(&dt_replica, 1, MPI_UINT64_T, 1, MPI_COMM_WORLD);
+ MPI_Bcast(&dset_replica, 1, MPI_UINT64_T, 1, MPI_COMM_WORLD);
+ MPI_Bcast(&grp_replica, 1, MPI_UINT64_T, 1, MPI_COMM_WORLD);
+ }
+
+ /* Read from prefetched Replicas */
+ /* set dxpl to read from replica in BB */
+ dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+ ret = H5Pset_dxpl_replica(dxpl_id, dset_replica);
+ assert(0 == ret);
+ ret = H5Dread_ff(did, dtid, H5S_ALL, H5S_ALL, dxpl_id, rdata1, rcxt, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ printf("Read Data1: ");
+ for(i=0;i<NELEM;++i)
+ printf("%d ",rdata1[i]);
+ printf("\n");
+ H5Pclose(dxpl_id);
+
+ dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+ ret = H5Pset_dxpl_replica(dxpl_id, map_replica);
+ assert(0 == ret);
+ for(i=0; i<NKVP; i++){
+ key = i;
+ value = -i;
+ ret = H5Mget_ff(map, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,
+ dxpl_id, rcxt, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ printf("Value recieved = %d\n", value);
+ assert(1000*i == value);
+ }
+ H5Pclose(dxpl_id);
+
+ /* wait for all reads to complete before evicting */
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if(0 == my_rank) {
+ /* evict Replicas */
+ dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+ ret = H5Pset_dxpl_replica(dxpl_id, dset_replica);
+ assert(0 == ret);
+ ret = H5Tevict_ff(dtid, 3, dxpl_id, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ H5Pclose(dxpl_id);
+
+ dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+ ret = H5Pset_dxpl_replica(dxpl_id, dset_replica);
+ assert(0 == ret);
+ ret = H5Devict_ff(did, 3, dxpl_id, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ H5Pclose(dxpl_id);
+
+ dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+ ret = H5Pset_dxpl_replica(dxpl_id, map_replica);
+ assert(0 == ret);
+ ret = H5Mevict_ff(map, 3, dxpl_id, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ H5Pclose(dxpl_id);
+
+ dxpl_id = H5Pcreate (H5P_DATASET_XFER);
+ ret = H5Pset_dxpl_replica(dxpl_id, grp_replica);
+ assert(0 == ret);
+ ret = H5Gevict_ff(gid, 3, dxpl_id, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ H5Pclose(dxpl_id);
+ }
+}
+
+int main(int argc, char **argv) {
+ char file_name[50];
+ hid_t file_id;
+ hid_t tid1, tid2, rid1, rid2;
+ hid_t fapl_id, dxpl_id;
+ hid_t e_stack;
+ htri_t exists = -1;
+
+ hsize_t dims[1];
+
+ uint64_t version;
+ uint64_t trans_num;
+
+ int provided;
+ MPI_Request mpi_req;
+
+ int32_t wdata1[NELEM],rdata1[NELEM];
+ int key, value, i;
+ H5ES_status_t status;
+ size_t num_events = 0;
+ herr_t ret;
+
+ sprintf(file_name, "%s_%s", getenv("USER"), "eff_file_pref.h5");
+
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+ if(MPI_THREAD_MULTIPLE != provided) {
+ fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
+ exit(1);
+ }
+
+ /* Call EFF_init to initialize the EFF stack.
+ As a result of this call, the Function Shipper client is started,
+ and HDF5 VOL calls are registered with the function shipper.
+ An "IOD init" call is forwarded from the FS client to the FS server
+ which should already be running. */
+ EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &my_size);
+ fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
+
+ for(i=0;i<NELEM;++i) {
+ wdata1[i]=i;
+ rdata1[i] = 0;
+ }
+
+ fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
+ /* Choose the IOD VOL plugin to use with this file.
+ First we create a file access property list. Then we call a new routine to set
+ the IOD plugin to use with this fapl */
+ fapl_id = H5Pcreate (H5P_FILE_ACCESS);
+ H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ /* create an event Queue for managing asynchronous requests.
+
+ Event Queues will releive the use from managing and completing
+ individual requests for every operation. Instead of passing a
+ request for every operation, the event queue is passed and
+ internally the HDF5 library creates a request and adds it to
+ the event queue.
+
+ Multiple Event queue can be created used by the application. */
+ e_stack = H5EScreate();
+ assert(e_stack);
+
+ /* create the file. */
+ file_id = H5Fcreate_ff(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL);
+ assert(file_id > 0);
+
+ /* create 1-D dataspace with 60 elements */
+ dims [0] = NELEM;
+ sid = H5Screate_simple(1, dims, NULL);
+ dtid = H5Tcopy(H5T_STD_I32LE);
+
+ /* acquire container version 1 - EXACT.
+ This can be asynchronous, but here we need the acquired ID
+ right after the call to start the transaction so we make synchronous. */
+ if(0 == my_rank) {
+ version = 1;
+ rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ }
+ MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
+ assert(1 == version);
+ if (my_rank != 0)
+ rid1 = H5RCcreate(file_id, version);
+
+ /* start transaction 2 with default Leader/Delegate model. Leader
+ which is rank 0 here starts the transaction. It can be
+ asynchronous, but we make it synchronous here so that the
+ Leader can tell its delegates that the transaction is
+ started. */
+ if(0 == my_rank) {
+ hid_t rid_temp;
+
+ /* create transaction object */
+ tid1 = H5TRcreate(file_id, rid1, (uint64_t)2);
+ assert(tid1);
+ ret = H5TRstart(tid1, H5P_DEFAULT, e_stack);
+ assert(0 == ret);
+
+ /* create objects */
+ gid = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT,
+ H5P_DEFAULT, tid1, e_stack);
+ assert(gid >= 0);
+
+ did = H5Dcreate_ff(gid, "D1", dtid, sid, H5P_DEFAULT, H5P_DEFAULT,
+ H5P_DEFAULT, tid1, e_stack);
+ assert(did >= 0);
+
+ ret = H5Dwrite_ff(did, dtid, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata1, tid1, e_stack);
+ assert(ret == 0);
+
+ ret = H5Tcommit_ff(file_id, "DT1", dtid, H5P_DEFAULT, H5P_DEFAULT,
+ H5P_DEFAULT, tid1, e_stack);
+ assert(ret == 0);
+
+ map = H5Mcreate_ff(file_id, "MAP1", H5T_STD_I32LE, H5T_STD_I32LE,
+ H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT, tid1, e_stack);
+ assert(map >= 0);
+
+ /* write some KV pairs to each map object. */
+ for(i=0; i<NKVP; i++){
+ key = i;
+ value = 1000*i;
+ ret = H5Mset_ff(map, H5T_STD_I32LE, &key, H5T_STD_I32LE, &value,
+ H5P_DEFAULT, tid1, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ }
+
+ ret = H5TRfinish(tid1, H5P_DEFAULT, &rid_temp, e_stack);
+ assert(0 == ret);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ /* Close transaction object. Local op */
+ ret = H5TRclose(tid1);
+ assert(0 == ret);
+
+ /* create transaction object */
+ tid2 = H5TRcreate(file_id, rid_temp, (uint64_t)3);
+ assert(tid2);
+ ret = H5TRstart(tid2, H5P_DEFAULT, e_stack);
+ assert(0 == ret);
+
+ ret = H5Oset_comment_ff(gid, "Testing Object Comment", tid2, e_stack);
+ assert(ret == 0);
+
+ ret = H5TRfinish(tid2, H5P_DEFAULT, &rid2, e_stack);
+ assert(0 == ret);
+
+ assert(H5Gclose_ff(gid, e_stack) == 0);
+ assert(H5Mclose_ff(map, e_stack) == 0);
+ assert(H5Tclose_ff(dtid, e_stack) == 0);
+ assert(H5Dclose_ff(did, e_stack) == 0);
+
+ /* release container version 2. This is async. */
+ ret = H5RCrelease(rid_temp, e_stack);
+ assert(0 == ret);
+ ret = H5RCclose(rid_temp);
+ assert(0 == ret);
+
+ version = 3;
+ }
+
+ if(0 == my_rank) {
+ /* release container version 1. This is async. */
+ ret = H5RCrelease(rid1, e_stack);
+ assert(0 == ret);
+ }
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ /* Process 0 tells other procs that container version 3 is acquired */
+ MPI_Bcast(&version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD);
+ assert(3 == version);
+
+ /* other processes just create a read context object; no need to
+ acquire it */
+ if(0 != my_rank) {
+ rid2 = H5RCcreate(file_id, version);
+ assert(rid2 > 0);
+ }
+
+ if(0 == my_rank) {
+ /* Close transaction object. Local op */
+ ret = H5TRclose(tid2);
+ assert(0 == ret);
+
+ ret = H5RCpersist(rid2, H5_EVENT_STACK_NULL);
+ assert(ret == 0);
+ }
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ gid = H5Oopen_ff(file_id, "G1", H5P_DEFAULT, rid2);
+ assert(gid);
+ dtid = H5Oopen_ff(file_id, "DT1", H5P_DEFAULT, rid2);
+ assert(dtid);
+ did = H5Oopen_ff(gid,"D1", H5P_DEFAULT, rid2);
+ assert(did);
+ map = H5Oopen_ff(file_id,"MAP1", H5P_DEFAULT, rid2);
+ assert(map);
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if((my_size > 1 && 1 == my_rank) ||
+ (my_size == 1 && 0 == my_rank)) {
+ ret = H5Tevict_ff(dtid, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+
+ ret = H5Devict_ff(did, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+
+ ret = H5Mevict_ff(map, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+
+ ret = H5Gevict_ff(gid, 3, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ }
+
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if(0 == my_rank) {
+ printf("***************************************************************\n");
+ printf("TESTING Prefetch - Default layout, Default partition\n");
+ }
+ MPI_Barrier(MPI_COMM_WORLD);
+ prefetch(0, 0, rid2);
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if(0 == my_rank) {
+ printf("***************************************************************\n");
+ printf("TESTING Prefetch - NON-Default layout, Default partition\n");
+ }
+ MPI_Barrier(MPI_COMM_WORLD);
+ prefetch(1, 0, rid2);
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if(0 == my_rank) {
+ printf("***************************************************************\n");
+ printf("TESTING Prefetch - Default layout, NON-Default partition\n");
+ }
+ MPI_Barrier(MPI_COMM_WORLD);
+ prefetch(0, 1, rid2);
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if(0 == my_rank) {
+ printf("***************************************************************\n");
+ printf("TESTING Prefetch - NON-Default layout, NON-Default partition\n");
+ }
+ MPI_Barrier(MPI_COMM_WORLD);
+ prefetch(1, 1, rid2);
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ if(my_rank == 0) {
+ /* release container version 3. */
+ ret = H5RCrelease(rid2, H5_EVENT_STACK_NULL);
+ assert(0 == ret);
+ }
+
+ assert(H5Oclose_ff(gid, e_stack) == 0);
+ assert(H5Oclose_ff(did, e_stack) == 0);
+ assert(H5Oclose_ff(dtid, e_stack) == 0);
+ assert(H5Oclose_ff(map, e_stack) == 0);
+
+ ret = H5RCclose(rid1);
+ assert(0 == ret);
+ ret = H5RCclose(rid2);
+ assert(0 == ret);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ /* closing the container also acts as a wait all on all pending requests
+ on the container. */
+ assert(H5Fclose_ff(file_id, 1, H5_EVENT_STACK_NULL) == 0);
+
+ H5Sclose(sid);
+ H5Pclose(fapl_id);
+ H5ESclose(e_stack);
+
+ /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
+ and shutsdown the FS server (when all clients send the terminate request)
+ and client */
+ MPI_Barrier(MPI_COMM_WORLD);
+ ret = EFF_finalize();
+ assert(ret >= 0);
+
+ MPI_Finalize();
+ return 0;
+}
diff --git a/examples/h5ff_client_time_datasets.c b/examples/h5ff_client_time_datasets.c
index 5d1954d..7eb1b01 100644
--- a/examples/h5ff_client_time_datasets.c
+++ b/examples/h5ff_client_time_datasets.c
@@ -1,1355 +1,1355 @@
-/*
- * h5ff_time_datasets2.c: Do timings of various write / persist / prefetch / read operations for datasets
- *
- * /DL is dataset that is read back from transaction logs in BB
- * /DP is dataset that is read back from prefetched replica in BB
- * /DS is dataset that is read back from stored data (DAOS)
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include <sys/time.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-/* define filename for this app, and max size after username prepended */
-#define FILENAME_APP "eff_time_datasets2.h5"
-#define FILENAME_SIZE 256
-
-/* define default sizing parameters - can be changed via command line arguments */
-//#define DEFAULT_COLS_PER_ROW 131072 /* = 1 MiB/row => ( 1024*1024 /8 bytes per cell) */
-//#define DEFAULT_ROWS_PER_WRITE 2 /* = 2 MiB writes with 1MiB/row */
-//#define DEFAULT_ROWS_PER_READ 2 /* = 2 MiB writes with 1MiB/row */
-//#define DEFAULT_ROWS_PER_RANK 5120 /* = 5 GiB/rank with 1MiB/row */
-/* Sized for testing - with verbose */
-#define DEFAULT_COLS_PER_ROW 6
-#define DEFAULT_ROWS_PER_WRITE 2
-#define DEFAULT_ROWS_PER_READ 1
-#define DEFAULT_ROWS_PER_RANK 4
-
-/* macros related to error reporting */
-#define STATUS (ret >= 0) ? " " : " - FAILED"
-#define ASSERT_RET assert( ret >= 0 )
-
-/* option flags */
-int cols_per_row = DEFAULT_COLS_PER_ROW;
-int rows_per_write = DEFAULT_ROWS_PER_WRITE;
-int rows_per_read = DEFAULT_ROWS_PER_READ;
-int rows_per_rank = DEFAULT_ROWS_PER_RANK;
-
-int logged_dset = 0; /* Create DL - defaults to no (unless neither DP nor DS are created ) */
-int prefetched_dset = 0; /* Create DP - defaults to no */
-int stored_dset = 0; /* Create DS - defaults to no */
-int num_iterations = 1; /* Number of times to perform the commit/persist/evict/prefetch/read/evict cycle */
-int persist_rate = 1; /* Rate at which transactions are persisted */
-int enable_checksums = 0; /* Enable checksums on raw data - defaults to no */
-int detailed_timing = 0; /* Print detailed timing information for each write & read - defaults to no */
-int verbose = 0; /* Verbose output - defaults to no */
-
-/* global variables and macros used to make timing easier */
-struct timeval tv_start;
-struct timeval tv_end;
-#define START_TIME gettimeofday( &tv_start, NULL )
-#define END_TIME gettimeofday( &tv_end, NULL )
-#define ELAPSED_TIME (ulong)( (tv_end.tv_usec + 1000000*tv_end.tv_sec) - (tv_start.tv_usec + 1000000*tv_start.tv_sec) )
-
-/* function prototypes */
-void fill_stats( int*, ulong*, ulong*, ulong*, ulong );
-void print_container_contents( hid_t, hid_t, int );
-int parse_options( int, char**, int, int );
-void usage( const char* );
-
-/**************************************************************************************************************/
-int main( int argc, char **argv ) {
-
- /* MPI */
- int my_rank, comm_size;
- int provided;
-
- /* Container */
- char *user_name; /* We'll prepend username to make filename unique */
- char file_name[FILENAME_SIZE]; /* Actual filename to be opened will be user_name + FILENAME_APP */
- hid_t fapl_id;
- hid_t file_id;
-
- /* Dataspaces and hyperslab selection parameters */
- hid_t space_l_id;
- hid_t space_p_id;
- hid_t space_s_id;
- hsize_t rank_offset[2], rank_count[2];
-
- /* Datasets */
- hid_t dset_l_id;
- hid_t dset_p_id;
- hid_t dset_s_id;
- hsize_t dset_size[2];
- hid_t space_id;
-
- /* Data creation and transfer properties */
- hid_t dcpl_id;
- hid_t dxpl_l_id;
- hid_t dxpl_p_id;
- hid_t dxpl_s_id;
-
- /* Replicas */
- hrpl_t dset_p_replica;
-
- /* Container Version & Read Contexts */
- uint64_t version, versionH;
- hid_t rc_id, rc_idH;
-
- /* Transactions */
- uint64_t tr_num;
- hid_t tr_id;
- hid_t trspl_id;
- int num_tr_leaders;
-
- /* Memory buffer to hold data for 1 rank and memory hyperslab selection parameters */
- uint64_t *mbuf;
- uint64_t mbuf_offset;
- hsize_t mbuf_size[1];
- hid_t mem_space_id;
- hsize_t mem_offset[1], mem_count[1];
-
- /* Variables used for reading data written by a neighbor rank */
- int neighbor_rank;
- hid_t neighbor_space_l_id;
- hid_t neighbor_space_p_id;
- hid_t neighbor_space_s_id;
- hsize_t neighbor_offset[2], neighbor_count[2];
-
- /* Misc */
- herr_t ret;
- uint64_t u, r, c;
- uint64_t bytesPerCell, bytesPerWrite, bytesPerRead, bytesPerRank, bytesPerDataset, bytesPerContainer;
- double megabytesPerWrite, megabytesPerRead, megabytesPerRank, megabytesPerDataset, megabytesPerContainer;
- int rows_transferred;
- ulong elapsed_time;
- double rate;
- ulong e_min, e_max, e_sum;
- int cnt;
-
- int i;
- int iteration;
- int iter_since_last_persist;
-
- /****
- * Initialize and create container
- ****/
-
- /* Confirm needed MPI functionality is available */
- MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
- if ( MPI_THREAD_MULTIPLE != provided ) {
- fprintf( stderr, "APP: ERROR: MPI does not have MPI_THREAD_MULTIPLE support\n" );
- exit( 1 );
- }
-
- /* Find MPI rank & communicator size */
- MPI_Comm_rank( MPI_COMM_WORLD, &my_rank );
- MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
-
- /* Parse command-line options controlling behavior */
- if ( parse_options( argc, argv, my_rank, comm_size ) != 0 ) {
- exit( 1 );
- }
- if ( my_rank == 0 ) {
- fprintf( stderr, "APP-r%d: Number of MPI processes = %d\n", my_rank, comm_size );
- fprintf( stderr, "APP-r%d: Datasets will have %d columns.\n", my_rank, cols_per_row );
- fprintf( stderr, "APP-r%d: Datasets will be written %d rows at a time.\n", my_rank, rows_per_write );
- fprintf( stderr, "APP-r%d: Datasets will be read %d rows at a time.\n", my_rank, rows_per_read );
- fprintf( stderr, "APP-r%d: Datasets will have %d rows (%d per rank).\n", my_rank, (rows_per_rank*comm_size), rows_per_rank );
- fprintf( stderr, "APP-r%d: There will be %d iterations.\n", my_rank, num_iterations );
- fprintf( stderr, "APP-r%d: Persists will occur every %d iterations.\n", my_rank, persist_rate );
- if ( ( prefetched_dset==0 ) && ( stored_dset==0 ) ) { /* Make sure something is created! */
- logged_dset = 1;
- }
- if ( logged_dset ) {
- fprintf( stderr, "APP-r%d: /DL will be created\n", my_rank );
- }
- if ( prefetched_dset ) {
- fprintf( stderr, "APP-r%d: /DP will be created\n", my_rank );
- }
- if ( stored_dset ) {
- fprintf( stderr, "APP-r%d: /DS will be created\n", my_rank );
- }
- }
-
- /* Initialize the EFF stack, starting FS client, registering HDF5 VOL calls, requesting IOD init */
- fprintf( stderr, "APP-r%d: Initialize EFF stack\n", my_rank );
- EFF_init( MPI_COMM_WORLD, MPI_INFO_NULL );
-
- /* Specify the IOD VOL plugin should be used and create H5File (EFF container), with user name prepended */
- user_name = getenv( "USER" );
- snprintf( file_name, FILENAME_SIZE, "%s_%s", user_name, FILENAME_APP );
- fprintf( stderr, "APP-r%d: Create %s\n", my_rank, file_name );
- fapl_id = H5Pcreate( H5P_FILE_ACCESS ); assert( fapl_id >= 0 );
- ret = H5Pset_fapl_iod( fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL ); ASSERT_RET;
- file_id = H5Fcreate_ff( file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL ); assert( file_id >= 0 );
-
- /* Create the memory buffer that will be used in many places - both for writing and reading */
- bytesPerCell = sizeof( uint64_t );
- bytesPerWrite = bytesPerCell * rows_per_write * cols_per_row; /* Bytes written in a single operation*/
- bytesPerRead = bytesPerCell * rows_per_read * cols_per_row; /* Bytes read in a single operations */
- bytesPerRank = bytesPerCell * rows_per_rank * cols_per_row; /* Bytes per rank to each dset */
- bytesPerDataset = bytesPerRank * comm_size; /* Bytes written by all ranks to each dset */
- bytesPerContainer = bytesPerDataset; /* Bytes written by all ranks to all dsets */
- if ( prefetched_dset ) {
- bytesPerContainer += bytesPerDataset;
- }
- if ( stored_dset ) {
- bytesPerContainer += bytesPerDataset;
- }
-
- megabytesPerWrite = (double)bytesPerWrite/(1024*1024);
- megabytesPerRead = (double)bytesPerRead/(1024*1024);
- megabytesPerRank = (double)bytesPerRank/(1024*1024);
- megabytesPerDataset = (double)bytesPerDataset/(1024*1024);
- megabytesPerContainer = (double)bytesPerContainer/(1024*1024);
-
- if ( my_rank == 0 ) {
- fprintf( stderr, "APP-r%d: Bytes per Cell = %lu\n", my_rank, bytesPerCell );
- fprintf( stderr, "APP-r%d: Bytes per Write = %lu\n", my_rank, bytesPerWrite );
- fprintf( stderr, "APP-r%d: Bytes per Read = %lu\n", my_rank, bytesPerRead );
- fprintf( stderr, "APP-r%d: Bytes per Rank = %lu\n", my_rank, bytesPerRank );
- fprintf( stderr, "APP-r%d: Bytes per Dataset = %lu\n", my_rank, bytesPerDataset );
- fprintf( stderr, "APP-r%d: Bytes per Container = %lu\n", my_rank, bytesPerContainer );
- fprintf( stderr, "APP-r%d: MiB per Write = %f\n", my_rank, megabytesPerWrite );
- fprintf( stderr, "APP-r%d: MiB per Read = %f\n", my_rank, megabytesPerRead );
- fprintf( stderr, "APP-r%d: MiB per Rank = %f\n", my_rank, megabytesPerRank );
- fprintf( stderr, "APP-r%d: MiB per Dataset = %f\n", my_rank, megabytesPerDataset );
- fprintf( stderr, "APP-r%d: MiB per Container = %f\n", my_rank, megabytesPerContainer );
- }
-
- mbuf = (uint64_t *) calloc( bytesPerRank, 1 );
- if ( mbuf == NULL ) {
- fprintf( stderr, "APP-r%d: calloc failed when trying to allocate %lu bytes\n", my_rank, bytesPerRank );
- exit( -1 );
- }
-
- /****
- * Transaction 2: Rank 0 creates H5Objects in the container
- ****/
-
- tr_num = 2; /* Set tr_num here so all ranks can later access the value */
- if ( my_rank == 0 ) {
-
- /* Acquire read context for CV 1 */
- version = 1;
- rc_id = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); assert( rc_id >= 0); assert( version == 1 );
- fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
-
- /* Start a transaction with a single leader (the default) */
- fprintf( stderr, "APP-r%d: tr %lu - Start\n", my_rank, tr_num );
- tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
- ret = H5TRstart( tr_id, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Set Dataset Creation Property List to control use of checksums at IOD level */
- dcpl_id = H5Pcreate( H5P_DATASET_CREATE );
- if ( enable_checksums ) {
- /* enabled by default */
- } else {
- H5Pset_ocpl_enable_checksum( dcpl_id, 0 );
- }
-
- /* Create Dataspace for Datasets */
- dset_size[0] = rows_per_rank * comm_size;
- dset_size[1] = cols_per_row;
- space_id = H5Screate_simple( 2, dset_size, dset_size ); assert( space_id >= 0 );
-
- /* Add updates to the transaction for Dataset creates */
- fprintf( stderr, "APP-r%d: tr %d - Create Dataset(s)\n", my_rank, tr_num );
-
- if ( logged_dset ) {
- dset_l_id = H5Dcreate_ff( file_id, "DL", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL ); assert( dset_l_id >= 0 );
- }
-
- if ( prefetched_dset ) {
- dset_p_id = H5Dcreate_ff( file_id, "DP", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL ); assert( dset_p_id >= 0 );
- }
-
- if ( stored_dset ) {
- dset_s_id = H5Dcreate_ff( file_id, "DS", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL ); assert( dset_s_id >= 0 );
- }
-
- ret = H5Sclose( space_id ); ASSERT_RET;
-
- /* Finish and commit TR and get RC for it */
- ret = H5TRfinish( tr_id, H5P_DEFAULT, &rc_idH, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5TRclose( tr_id ); ASSERT_RET;
- fprintf( stderr, "APP-r%d: tr %lu - Finished\n", my_rank, tr_num );
-
- assert( rc_idH >= 0 );
- versionH = tr_num;
- fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, versionH );
-
- /* Release the read handle and close read context on earlier CV */
- ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
- ret = H5RCclose( rc_id ); ASSERT_RET;
- fprintf( stderr, "APP-r%d: rc %lu - Released and Closed\n", my_rank, version );
-
- /* The just-acquired RC is now the primary rc_id we'll work with (for awhile) */
- rc_id = rc_idH;
- version = versionH;
- }
-
- /* Rank 0 notifies other ranks that transaction 2 committed and RC acquired */
- MPI_Bcast( &version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
-
- /****
- * Read Context 2: Other ranks create RC and open HObjects created by rank 0.
- ****/
- if ( my_rank != 0 ) {
- rc_id = H5RCcreate( file_id, version );
- assert( rc_id >= 0 ); assert ( version == 2 );
- fprintf( stderr, "APP-r%d: rc %lu - Created for rank\n", my_rank, version );
-
- if ( logged_dset ) {
- dset_l_id = H5Dopen_ff( file_id, "DL", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_l_id >= 0 );
- }
- if ( prefetched_dset ) {
- dset_p_id = H5Dopen_ff( file_id, "DP", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_p_id >= 0 );
- }
- if ( stored_dset ) {
- dset_s_id = H5Dopen_ff( file_id, "DS", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_s_id >= 0 );
- }
-
- } else {
- if ( verbose ) print_container_contents( file_id, rc_id, my_rank );
- }
-
- /* Keep track of my_rank's neighbor */
- neighbor_rank = my_rank + 1;
- if ( neighbor_rank == comm_size ) {
- neighbor_rank = 0;
- }
-
- /* Get the dataspaces for the three datasets and make a copy of each for neighbor's */
- if ( logged_dset ) {
- space_l_id = H5Dget_space( dset_l_id ); assert ( space_l_id >= 0 );
- neighbor_space_l_id = H5Scopy( space_l_id );
- }
- if ( prefetched_dset ) {
- space_p_id = H5Dget_space( dset_p_id ); assert ( space_p_id >= 0 );
- neighbor_space_p_id = H5Scopy( space_p_id );
- }
- if ( stored_dset ) {
- space_s_id = H5Dget_space( dset_s_id ); assert ( space_s_id >= 0 );
- neighbor_space_s_id = H5Scopy( space_s_id );
- }
-
-
- /* Create the memory dataspace for one rank worth of data */
- mbuf_size[0] = rows_per_rank * cols_per_row;
- mem_space_id = H5Screate_simple( 1, mbuf_size, mbuf_size ); assert( mem_space_id >= 0 );
-
- /* Create property lists that will be used */
- if ( logged_dset ) {
- dxpl_l_id = H5Pcreate( H5P_DATASET_XFER );
- if ( enable_checksums ) {
- ret = H5Pset_rawdata_integrity_scope( dxpl_l_id, H5_CHECKSUM_ALL ); ASSERT_RET;
- } else {
- ret = H5Pset_rawdata_integrity_scope( dxpl_l_id, H5_CHECKSUM_NONE ); ASSERT_RET;
- }
- }
-
- if ( prefetched_dset ) {
- dxpl_p_id = H5Pcreate( H5P_DATASET_XFER );
- if ( enable_checksums ) {
- ret = H5Pset_rawdata_integrity_scope( dxpl_p_id, H5_CHECKSUM_ALL ); ASSERT_RET;
- } else {
- ret = H5Pset_rawdata_integrity_scope( dxpl_p_id, H5_CHECKSUM_NONE ); ASSERT_RET;
- }
- }
-
- if ( stored_dset ) {
- dxpl_s_id = H5Pcreate( H5P_DATASET_XFER );
- if ( enable_checksums ) {
- ret = H5Pset_rawdata_integrity_scope( dxpl_s_id, H5_CHECKSUM_ALL ); ASSERT_RET;
- } else {
- ret = H5Pset_rawdata_integrity_scope( dxpl_s_id, H5_CHECKSUM_NONE ); ASSERT_RET;
- }
- }
-
- iter_since_last_persist = 0;
-
- for ( iteration = 1; iteration <= num_iterations; iteration++ ) {
-
- /****
- * Cycle of:
- * Tranasction N Start / Update / Commit: All ranks update Dataset(s) then commit
- * CV N Persist: Rank 0
- * Prefetch: Rank 0
- * Read rank data: All Ranks
- * Read neighbor rank data: All Ranks
- * Read all data: All Ranks
- ****/
- tr_num = tr_num + 1;
- num_tr_leaders = comm_size;
-
- /* Create a transaction and start it. */
- fprintf( stderr, "APP-r%d: iter %05d tr %lu - Transaction Start with %d leaders\n",
- my_rank, iteration, tr_num, num_tr_leaders );
- tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
- trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
- ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
- ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Set cell values (in memory) for this rank based on Row, Rank, Column, Transaction */
- u = 0;
- for ( r = 0; r < rows_per_rank; r++ ) {
- for ( c = 0; c < cols_per_row; c++ ) {
- mbuf[u] = ( my_rank*10000 + r*1000 + c*100 + tr_num ) ;
- u++;
- }
- }
-
- fprintf( stderr, "APP-r%d: iter %05d tr %lu - Add updates to transaction.\n", my_rank, iteration, tr_num );
-
- /* Add dataset updates to transaction */
- if ( logged_dset ) {
- cnt = e_min = e_max = e_sum = 0;
- /* Set parameters that will be used to select the initial hyperslabs for this rank */
- mem_offset[0] = 0;
- mem_count[0] = rows_per_write * cols_per_row;
- mbuf_offset = 0;
- rank_offset[0] = my_rank * rows_per_rank;
- rank_offset[1] = 0;
- rank_count[0] = rows_per_write;
- rank_count[1] = cols_per_row;
- rows_transferred = 0;
-
- while ( (rows_transferred+rows_per_write) <= rows_per_rank ) { /* final rows left unfilled if not exact multiple */
- /* Select the hyperslabs for this write */
- ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( space_l_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
-
- START_TIME;
- ret = H5Dwrite_ff( dset_l_id, H5T_NATIVE_UINT64, mem_space_id, space_l_id, dxpl_l_id, &mbuf[mbuf_offset],
- tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
- if ( detailed_timing ) {
- rate = megabytesPerWrite/((double)elapsed_time/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 01: Time to add Write updates for /DL (%lu bytes): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerWrite, elapsed_time, rate );
- }
-
- /* Advance in preparation for next write */
- rows_transferred += rows_per_write;
- rank_offset[0] += rows_per_write;
- mem_offset[0] += ( rows_per_write * cols_per_row );
- mbuf_offset += ( rows_per_write * cols_per_row );
- }
- if ( cnt > 0 ) {
- rate = megabytesPerWrite/((double)(e_sum/cnt)/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 01: Average Time to add Write updates for /DL (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
- my_rank, iteration, bytesPerWrite, e_min, e_max, e_sum/cnt, rate );
- }
- }
-
- if ( prefetched_dset ) {
- cnt = e_min = e_max = e_sum = 0;
- /* Set parameters that will be used to select the initial hyperslabs for this rank */
- mem_offset[0] = 0;
- mem_count[0] = rows_per_write * cols_per_row;
- mbuf_offset = 0;
- rank_offset[0] = my_rank * rows_per_rank;
- rank_offset[1] = 0;
- rank_count[0] = rows_per_write;
- rank_count[1] = cols_per_row;
- rows_transferred = 0;
-
- while ( (rows_transferred+rows_per_write) <= rows_per_rank ) { /* final rows left unfilled if not exact multiple */
- /* Select the hyperslabs for this write */
- ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( space_p_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
-
- START_TIME;
- ret = H5Dwrite_ff( dset_p_id, H5T_NATIVE_UINT64, mem_space_id, space_p_id, dxpl_p_id, &mbuf[mbuf_offset], tr_id,
- H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
- if ( detailed_timing ) {
- rate = megabytesPerWrite/((double)elapsed_time/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 02: Time to add Write updates for /DP (%lu bytes): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerWrite, elapsed_time, rate );
- }
-
- /* Advance in preparation for next write */
- rows_transferred += rows_per_write;
- rank_offset[0] += rows_per_write;
- mem_offset[0] += ( rows_per_write * cols_per_row );
- mbuf_offset += ( rows_per_write * cols_per_row );
- }
- if ( cnt > 0 ) {
- rate = megabytesPerWrite/((double)(e_sum/cnt)/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 02: Average Time to add Write updates for /DP (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
- my_rank, iteration, bytesPerWrite, e_min, e_max, e_sum/cnt, rate );
- }
- }
-
- if ( stored_dset ) {
- cnt = e_min = e_max = e_sum = 0;
- /* Set parameters that will be used to select the initial hyperslabs for this rank */
- mem_offset[0] = 0;
- mem_count[0] = rows_per_write * cols_per_row;
- mbuf_offset = 0;
- rank_offset[0] = my_rank * rows_per_rank;
- rank_offset[1] = 0;
- rank_count[0] = rows_per_write;
- rank_count[1] = cols_per_row;
- rows_transferred = 0;
-
- while ( (rows_transferred+rows_per_write) <= rows_per_rank ) { /* final rows left unfilled if not exact multiple */
- /* Select the hyperslabs for this write */
- ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( space_s_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
-
- START_TIME;
- ret = H5Dwrite_ff( dset_s_id, H5T_NATIVE_UINT64, mem_space_id, space_s_id, dxpl_s_id, &mbuf[mbuf_offset], tr_id,
- H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
- if ( detailed_timing ) {
- rate = megabytesPerWrite/((double)elapsed_time/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 03: Time to add Write updates for /DS (%lu bytes): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerWrite, elapsed_time, rate );
- }
-
- /* Advance in preparation for next write */
- rows_transferred += rows_per_write;
- rank_offset[0] += rows_per_write;
- mem_offset[0] += ( rows_per_write * cols_per_row );
- mbuf_offset += ( rows_per_write * cols_per_row );
- }
- if ( cnt > 0 ) {
- rate = megabytesPerWrite/((double)(e_sum/cnt)/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 03: Average Time to add Write updates for /DS (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
- my_rank, iteration, bytesPerWrite, e_min, e_max, e_sum/cnt, rate );
- }
- }
-
- /* Finish, (commit), and close transaction */
- fprintf( stderr, "APP-r%d: iter %05d tr %lu - Finish and Commit\n", my_rank, iteration, tr_num );
- ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5TRclose( tr_id ); ASSERT_RET;
- ret = H5Pclose( trspl_id ); ASSERT_RET;
-
- /* After commit completes on all ranks, acquire RC on rank 0 and create local handle for RC on other ranks */
- MPI_Barrier( MPI_COMM_WORLD );
- versionH = tr_num;
- if ( my_rank == 0 ) {
- rc_idH = H5RCacquire( file_id, &versionH, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- assert( rc_idH >= 0 ); assert ( versionH == tr_num );
- fprintf( stderr, "APP-r%d: iter %05d rc %lu - Acquired\n", my_rank, iteration, versionH );
- } else {
- rc_idH = H5RCcreate( file_id, versionH );
- assert( rc_idH >= 0 ); assert ( versionH == tr_num );
- fprintf( stderr, "APP-r%d: iter %05d rc %lu - Created\n", my_rank, iteration, versionH );
- }
- MPI_Barrier( MPI_COMM_WORLD ); /* Wait to make sure rank 0 has acquired from IOD before proceeding */
-
- /* Release previous Read Context */
- if ( my_rank == 0 ) {
- ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
- fprintf( stderr, "APP-r%d: iter %05d rc %lu - Released\n", my_rank, iteration, version );
- }
- ret = H5RCclose( rc_id ); ASSERT_RET;
- fprintf( stderr, "APP-r%d: iter %05d rc %lu - Closed\n", my_rank, iteration, version );
-
- /* The just-acquired RC is now the primary rc_id we'll work with (for awhile) */
- rc_id = rc_idH;
- version = versionH;
-
- iter_since_last_persist++; /* We've done the commit of a TR, so advance iter ctr */
-
- /* Rank 0:
- if this is a persist iteration:
- - persists the container
- - evicts DP & DS
- - prefetches DP
- */
- if ( my_rank == 0 ) {
- if ( verbose ) print_container_contents( file_id, rc_id, my_rank );
-
- if ( iter_since_last_persist == persist_rate ) {
- START_TIME;
- ret = H5RCpersist( rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- rate = megabytesPerContainer/((double)elapsed_time/1000000);
- fprintf( stderr, "APP-r%d: iter %05d step 04: Time to Persist container (%lu bytes + KVs): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerContainer, elapsed_time, rate );
-
- if ( prefetched_dset ) {
- START_TIME;
- ret = H5Devict_ff( dset_p_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- rate = megabytesPerDataset/((double)elapsed_time/1000000);
- fprintf( stderr, "APP-r%d: iter %05d step 05: Time to Evict /DP (%lu bytes): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerDataset, elapsed_time, rate );
- }
-
- if ( stored_dset ) {
- START_TIME;
- ret = H5Devict_ff( dset_s_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- rate = megabytesPerDataset/((double)elapsed_time/1000000);
- fprintf( stderr, "APP-r%d: iter %05d step 06: Time to Evict /DS (%lu bytes): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerDataset, elapsed_time, rate );
- }
-
- if ( prefetched_dset ) {
- START_TIME;
- ret = H5Dprefetch_ff( dset_p_id, rc_id, &dset_p_replica, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- rate = megabytesPerDataset/((double)elapsed_time/1000000);
- fprintf( stderr, "APP-r%d: iter %05d step 07: Time to Prefetch (%lu bytes) /DP: %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerDataset, elapsed_time, rate );
- }
- }
- }
-
- /* We only prefetch if a persist occurred */
- if ( ( prefetched_dset ) && ( iter_since_last_persist == persist_rate ) ) {
- MPI_Bcast( &dset_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
- ret = H5Pset_dxpl_replica( dxpl_p_id, dset_p_replica ); ASSERT_RET;
- }
-
- MPI_Barrier( MPI_COMM_WORLD ); /* Make sure all are here before continuing, (esp in case of no prefetch Bcast) */
-
- /*
- * All ranks read 'their' part of the written dataset (s)
- * For /DL, read every iteration.
- * For /DP and DS, only read on persisted iterations.
- */
-
- if ( logged_dset ) {
- /* Zero the memory buf so we can be sure we are reading */
- if ( verbose ) {
- bzero( (void*) mbuf, bytesPerRank );
- }
-
- cnt = e_min = e_max = e_sum = 0;
- /* Set parameters that will be used to select the initial hyperslabs for this rank */
- mem_offset[0] = 0;
- mem_count[0] = rows_per_read * cols_per_row;
- mbuf_offset = 0;
- rank_offset[0] = my_rank * rows_per_rank;
- rank_offset[1] = 0;
- rank_count[0] = rows_per_read;
- rank_count[1] = cols_per_row;
- rows_transferred = 0;
-
- while ( (rows_transferred+rows_per_read) <= rows_per_rank ) { /* final rows left unread if not exact multiple */
- /* Select the hyperslabs for this read */
- ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( space_l_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
-
- START_TIME;
- ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, mem_space_id, space_l_id, dxpl_l_id, &mbuf[mbuf_offset], rc_id,
- H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
- if ( detailed_timing ) {
- rate = megabytesPerRead/((double)elapsed_time/1000000);
- fprintf( stderr, "APP-r%d: iter %05d step 08: Time to Read my entries for /DL (%lu bytes): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerRead, elapsed_time, rate );
- }
-
- fprintf( stderr, "APP-r%d: first value read in /DL: %09lu\n", my_rank, mbuf[mbuf_offset] );
- if ( ( verbose ) && ( my_rank == 0 ) ) { /* print all mbuf on rank 0 to confirm correctness while developing */
- u = 0;
- for ( r = 0; r < rows_per_rank; r++ ) {
- for ( c = 0; c < cols_per_row; c++ ) {
- fprintf( stderr, "%lu ", mbuf[u] );
- u++;
- }
- fprintf( stderr, "\n" );
- }
- }
-
- /* Advance in preparation for next read */
- rows_transferred += rows_per_read;
- rank_offset[0] += rows_per_read;
- mem_offset[0] += ( rows_per_read * cols_per_row );
- mbuf_offset += ( rows_per_read * cols_per_row );
- }
-
- if ( cnt > 0 ) {
- rate = megabytesPerRead/((double)(e_sum/cnt)/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 08: Average Time to Read my entries for /DL (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
- my_rank, iteration, bytesPerRead, e_min, e_max, e_sum/cnt, rate );
- }
- }
-
- if ( ( prefetched_dset ) && ( iter_since_last_persist == persist_rate ) ) {
- /* Zero the memory buf so we can be sure we are reading */
- if ( verbose ) {
- bzero( (void*) mbuf, bytesPerRank );
- }
-
- cnt = e_min = e_max = e_sum = 0;
- /* Set parameters that will be used to select the initial hyperslabs for this rank */
- mem_offset[0] = 0;
- mem_count[0] = rows_per_read * cols_per_row;
- mbuf_offset = 0;
- rank_offset[0] = my_rank * rows_per_rank;
- rank_offset[1] = 0;
- rank_count[0] = rows_per_read;
- rank_count[1] = cols_per_row;
- rows_transferred = 0;
-
- while ( (rows_transferred+rows_per_read) <= rows_per_rank ) { /* final rows left unread if not exact multiple */
- /* Select the hyperslabs for this read */
- ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( space_p_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
-
- START_TIME;
- ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, mem_space_id, space_p_id, dxpl_p_id, &mbuf[mbuf_offset], rc_id,
- H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
- if ( detailed_timing ) {
- rate = megabytesPerRead/((double)elapsed_time/1000000);
- fprintf( stderr, "APP-r%d: iter %05d step 09: Time to Read my entries for /DP (%lu bytes): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerRead, elapsed_time, rate );
- }
-
- fprintf( stderr, "APP-r%d: first value read in /DP: %09lu\n", my_rank, mbuf[mbuf_offset] );
- if ( ( verbose ) && ( my_rank == 0 ) ) { /* print all mbuf on rank 0 to confirm correctness while developing */
- u = 0;
- for ( r = 0; r < rows_per_rank; r++ ) {
- for ( c = 0; c < cols_per_row; c++ ) {
- fprintf( stderr, "%lu ", mbuf[u] );
- u++;
- }
- fprintf( stderr, "\n" );
- }
- }
-
- /* Advance in preparation for next read */
- rows_transferred += rows_per_read;
- rank_offset[0] += rows_per_read;
- mem_offset[0] += ( rows_per_read * cols_per_row );
- mbuf_offset += ( rows_per_read * cols_per_row );
-
- }
- if ( cnt > 0 ) {
- rate = megabytesPerRead/((double)(e_sum/cnt)/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 09: Average Time to Read my entries for /DP (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
- my_rank, iteration, bytesPerRead, e_min, e_max, e_sum/cnt, rate );
- }
- }
-
- if ( ( stored_dset ) && ( iter_since_last_persist == persist_rate ) ) {
- /* Zero the memory buf so we can be sure we are reading */
- if ( verbose ) {
- bzero( (void*) mbuf, bytesPerRank );
- }
-
- cnt = e_min = e_max = e_sum = 0;
- /* Set parameters that will be used to select the initial hyperslabs for this rank */
- mem_offset[0] = 0;
- mem_count[0] = rows_per_read * cols_per_row;
- mbuf_offset = 0;
- rank_offset[0] = my_rank * rows_per_rank;
- rank_offset[1] = 0;
- rank_count[0] = rows_per_read;
- rank_count[1] = cols_per_row;
- rows_transferred = 0;
-
- while ( (rows_transferred+rows_per_read) <= rows_per_rank ) { /* final rows left unread if not exact multiple */
- /* Select the hyperslabs for this read */
- ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( space_s_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
-
- START_TIME;
- ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, mem_space_id, space_s_id, dxpl_s_id, &mbuf[mbuf_offset], rc_id,
- H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
- if ( detailed_timing ) {
- rate = megabytesPerRead/((double)elapsed_time/1000000);
- fprintf( stderr, "APP-r%d: iter %05d step 10: Time to Read my entries for /DS (%lu bytes): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerRead, elapsed_time, rate );
- }
-
- fprintf( stderr, "APP-r%d: first value read in /DS: %09lu\n", my_rank, mbuf[mbuf_offset] );
- if ( ( verbose ) && ( my_rank == 0 ) ) { /* print all mbuf on rank 0 to confirm correctness while developing */
- u = 0;
- for ( r = 0; r < rows_per_rank; r++ ) {
- for ( c = 0; c < cols_per_row; c++ ) {
- fprintf( stderr, "%lu ", mbuf[u] );
- u++;
- }
- fprintf( stderr, "\n" );
- }
- }
-
- /* Advance in preparation for next read */
- rows_transferred += rows_per_read;
- rank_offset[0] += rows_per_read;
- mem_offset[0] += ( rows_per_read * cols_per_row );
- mbuf_offset += ( rows_per_read * cols_per_row );
- }
-
- if ( cnt > 0 ) {
- rate = megabytesPerRead/((double)(e_sum/cnt)/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 10: Average Time to Read my entries for /DS (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
- my_rank, iteration, bytesPerRead, e_min, e_max, e_sum/cnt, rate );
- }
- }
-
- /*
- * All ranks read the values "next rank" neighbor updated in last transaction
- * For /DL, read every iteration.
- * For /DP and DS, only read on persisted iterations.
- */
-
- if ( logged_dset && ( my_rank != neighbor_rank ) ) {
- /* Zero the memory buf so we can be sure we are reading */
- if ( verbose ) {
- bzero( (void*) mbuf, bytesPerRank );
- }
-
- cnt = e_min = e_max = e_sum = 0;
- /* Set parameters that will be used to select the initial hyperslabs for this rank */
- mem_offset[0] = 0;
- mem_count[0] = rows_per_read * cols_per_row;
- mbuf_offset = 0;
- neighbor_offset[0] = neighbor_rank * rows_per_rank;
- neighbor_offset[1] = 0;
- neighbor_count[0] = rows_per_read;
- neighbor_count[1] = cols_per_row;
- rows_transferred = 0;
-
- while ( (rows_transferred+rows_per_read) <= rows_per_rank ) { /* final rows left unread if not exact multiple */
- /* Select the hyperslabs for this read */
- ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( neighbor_space_l_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL );
- ASSERT_RET;
-
- START_TIME;
- ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, mem_space_id, neighbor_space_l_id, dxpl_l_id, &mbuf[mbuf_offset], rc_id,
- H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
- if ( detailed_timing ) {
- rate = megabytesPerRead/((double)elapsed_time/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 11: Time to Read neighbor's entries for /DL (%lu bytes): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerRead, elapsed_time, rate );
- }
-
- fprintf( stderr, "APP-r%d: neighbor's first value read in /DL: %09lu\n", my_rank, mbuf[mbuf_offset] );
- if ( ( verbose ) && ( my_rank == 0 ) ) { /* print all mbuf on rank 0 to confirm correctness while developing */
- u = 0;
- for ( r = 0; r < rows_per_rank; r++ ) {
- for ( c = 0; c < cols_per_row; c++ ) {
- fprintf( stderr, "%lu ", mbuf[u] );
- u++;
- }
- fprintf( stderr, "\n" );
- }
- }
-
- /* Advance in preparation for next read */
- rows_transferred += rows_per_read;
- neighbor_offset[0] += rows_per_read;
- mem_offset[0] += ( rows_per_read * cols_per_row );
- mbuf_offset += ( rows_per_read * cols_per_row );
- }
-
- if ( cnt > 0 ) {
- rate = megabytesPerRead/((double)(e_sum/cnt)/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 11: Average Time to Read neighbor's entries for /DL (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
- my_rank, iteration, bytesPerRead, e_min, e_max, e_sum/cnt, rate );
- }
- }
-
- if ( ( prefetched_dset ) && ( iter_since_last_persist == persist_rate ) && ( my_rank != neighbor_rank ) ) {
- /* Zero the memory buf so we can be sure we are reading */
- if ( verbose ) {
- bzero( (void*) mbuf, bytesPerRank );
- }
-
- cnt = e_min = e_max = e_sum = 0;
- /* Set parameters that will be used to select the initial hyperslabs for this rank */
- mem_offset[0] = 0;
- mem_count[0] = rows_per_read * cols_per_row;
- mbuf_offset = 0;
- neighbor_offset[0] = neighbor_rank * rows_per_rank;
- neighbor_offset[1] = 0;
- neighbor_count[0] = rows_per_read;
- neighbor_count[1] = cols_per_row;
- rows_transferred = 0;
-
- while ( (rows_transferred+rows_per_read) <= rows_per_rank ) { /* final rows left unread if not exact multiple */
- /* Select the hyperslabs for this read */
- ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( neighbor_space_p_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL );
- ASSERT_RET;
-
- START_TIME;
- ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, mem_space_id, neighbor_space_p_id, dxpl_p_id, &mbuf[mbuf_offset], rc_id,
- H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
- if ( detailed_timing ) {
- rate = megabytesPerRead/((double)elapsed_time/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 12: Time to Read neighbor's entries for /DP (%lu bytes): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerRead, elapsed_time, rate );
- }
-
- fprintf( stderr, "APP-r%d: neighbor's first value read in /DP: %09lu\n", my_rank, mbuf[mbuf_offset] );
- if ( ( verbose ) && ( my_rank == 0 ) ) { /* print all mbuf on rank 0 to confirm correctness while developing */
- u = 0;
- for ( r = 0; r < rows_per_rank; r++ ) {
- for ( c = 0; c < cols_per_row; c++ ) {
- fprintf( stderr, "%lu ", mbuf[u] );
- u++;
- }
- fprintf( stderr, "\n" );
- }
- }
-
- /* Advance in preparation for next read */
- rows_transferred += rows_per_read;
- neighbor_offset[0] += rows_per_read;
- mem_offset[0] += ( rows_per_read * cols_per_row );
- mbuf_offset += ( rows_per_read * cols_per_row );
- }
-
- if ( cnt > 0 ) {
- rate = megabytesPerRead/((double)(e_sum/cnt)/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 12: Average Time to Read neighbor's entries for /DP (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
- my_rank, iteration, bytesPerRead, e_min, e_max, e_sum/cnt, rate );
- }
- }
-
- if ( ( stored_dset ) && ( iter_since_last_persist == persist_rate ) && ( my_rank != neighbor_rank ) ) {
- /* Zero the memory buf so we can be sure we are reading */
- if ( verbose ) {
- bzero( (void*) mbuf, bytesPerRank );
- }
-
- cnt = e_min = e_max = e_sum = 0;
- /* Set parameters that will be used to select the initial hyperslabs for this rank */
- mem_offset[0] = 0;
- mem_count[0] = rows_per_read * cols_per_row;
- mbuf_offset = 0;
- neighbor_offset[0] = neighbor_rank * rows_per_rank;
- neighbor_offset[1] = 0;
- neighbor_count[0] = rows_per_read;
- neighbor_count[1] = cols_per_row;
- rows_transferred = 0;
-
- while ( (rows_transferred+rows_per_read) <= rows_per_rank ) { /* final rows left unread if not exact multiple */
- /* Select the hyperslabs for this read */
- ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( neighbor_space_s_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL );
- ASSERT_RET;
-
- START_TIME;
- ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, mem_space_id, neighbor_space_s_id, dxpl_s_id, &mbuf[mbuf_offset], rc_id,
- H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
- if ( detailed_timing ) {
- rate = megabytesPerRead/((double)elapsed_time/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 13: Time to Read neighbor's entries for /DS (%lu bytes): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerRead, elapsed_time, rate );
- }
-
- fprintf( stderr, "APP-r%d: neighbor's first value read in /DS: %09lu\n", my_rank, mbuf[mbuf_offset] );
- if ( ( verbose ) && ( my_rank == 0 ) ) { /* print all mbuf on rank 0 to confirm correctness while developing */
- u = 0;
- for ( r = 0; r < rows_per_rank; r++ ) {
- for ( c = 0; c < cols_per_row; c++ ) {
- fprintf( stderr, "%lu ", mbuf[u] );
- u++;
- }
- fprintf( stderr, "\n" );
- }
- }
-
- /* Advance in preparation for next read */
- rows_transferred += rows_per_read;
- neighbor_offset[0] += rows_per_read;
- mem_offset[0] += ( rows_per_read * cols_per_row );
- mbuf_offset += ( rows_per_read * cols_per_row );
- }
-
- if ( cnt > 0 ) {
- rate = megabytesPerRead/((double)(e_sum/cnt)/1000000);
- fprintf( stderr,
- "APP-r%d: iter %05d step 13: Average Time to Read neighbor's entries for /DS (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
- my_rank, iteration, bytesPerRead, e_min, e_max, e_sum/cnt, rate );
- }
-
- }
-
- /* For persisted iterations, reset the counter & if we prefetched a replica, evict it */
- if ( iter_since_last_persist == persist_rate ) {
- iter_since_last_persist = 0;
-
- MPI_Barrier( MPI_COMM_WORLD ); /* Make sure all ranks done reading the replica before evicting */
-
- /* Rank 0 evicts the replica */
- if ( ( my_rank == 0 ) && ( prefetched_dset ) ) {
- START_TIME;
- ret = H5Devict_ff( dset_p_id, version, dxpl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- elapsed_time = ELAPSED_TIME;
- rate = megabytesPerDataset/((double)elapsed_time/1000000);
- fprintf( stderr, "APP-r%d: iter %05d step 17: Time to Evict Replica of /DP (%lu bytes): %lu usec - %f MiB/sec\n",
- my_rank, iteration, bytesPerDataset, elapsed_time, rate );
- }
- }
- }
-
- /*
- * Wrap-up
- */
-
- if ( my_rank == 0 ) {
- if ( verbose ) print_container_contents( file_id, rc_id, my_rank );
- }
-
- fprintf( stderr, "APP-r%d: Closing H5Objects\n", my_rank );
-
- ret = H5Sclose( mem_space_id ); ASSERT_RET;
-
- if ( logged_dset ) {
- ret = H5Dclose_ff( dset_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Sclose( space_l_id ); ASSERT_RET;
- ret = H5Sclose( neighbor_space_l_id ); ASSERT_RET;
- ret = H5Pclose( dxpl_l_id ); ASSERT_RET;
- }
- if ( prefetched_dset ) {
- ret = H5Dclose_ff( dset_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Sclose( space_p_id ); ASSERT_RET;
- ret = H5Sclose( neighbor_space_p_id ); ASSERT_RET;
- ret = H5Pclose( dxpl_p_id ); ASSERT_RET;
- }
- if ( stored_dset ) {
- ret = H5Dclose_ff( dset_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Sclose( space_s_id ); ASSERT_RET;
- ret = H5Sclose( neighbor_space_s_id ); ASSERT_RET;
- ret = H5Pclose( dxpl_s_id ); ASSERT_RET;
- }
-
-
- /* Release the read handle and close read context */
- MPI_Barrier( MPI_COMM_WORLD ); /* Make sure all ranks done with RC */
- if ( my_rank == 0 ) {
- ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
- fprintf( stderr, "APP-r%d: rc %lu - Released\n", my_rank, version );
- }
- ret = H5RCclose( rc_id ); ASSERT_RET;
- fprintf( stderr, "APP-r%d: rc %lu - Closed\n", my_rank, version );
-
- /* Close H5 Objects that are still open */
- fprintf( stderr, "APP-r%d: Closing %s\n", my_rank, file_name );
- MPI_Barrier( MPI_COMM_WORLD );
- ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Pclose( fapl_id ); ASSERT_RET;
-
- /* Perform wrap-up operations */
- fprintf( stderr, "APP-r%d: Finalize EFF stack\n", my_rank );
- MPI_Barrier( MPI_COMM_WORLD );
- EFF_finalize();
- MPI_Finalize();
-
- return 0;
-}
-
-/*
- * Helper function to keep track of timing info
- */
-void
-fill_stats( int* cnt, ulong* min, ulong* max, ulong* sum, ulong elapsed )
-{
- if ( *min == 0 ) {
- *min = elapsed;
- *max = elapsed;
- } else {
- if ( elapsed < *min ) {
- *min = elapsed;
- }
- if ( elapsed > *max ) {
- *max = elapsed;
- }
- }
- *sum += elapsed;
- *cnt += 1;
-}
-
-/*
- * Helper function used to recursively read and print container contents
- * for container identified by "file_id"
- * in read context identified by "rc_id"
- * "my_rank" used to identify the process doing the reading / printing.
- */
-void
-print_container_contents( hid_t file_id, hid_t rc_id, int my_rank )
-{
- herr_t ret;
- uint64_t cv;
- htri_t exists;
- char path_to_object[1024];
- char name[30];
- int i, d;
-
- char preface[128];
-
- /* Get the container version for the read context */
- ret = H5RCget_version( rc_id, &cv ); ASSERT_RET;
-
- /* Set up the preface and adding version number */
- sprintf( preface, "APP-r%d: cv %d: ", my_rank, (int)cv );
-
- /* Start the printing */
- fprintf( stderr, "%s ----- Container Contents ------------\n", preface );
-
- for ( d = 0; d < 3; d++ ) {
- if ( d == 0 ) {
- sprintf( path_to_object, "%s", "/DL" );
- } else if ( d == 1 ) {
- sprintf( path_to_object, "%s", "/DP" );
- } else if ( d == 2 ) {
- sprintf( path_to_object, "%s", "/DS" );
- }
-
- ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
-
- if ( exists ) {
- hid_t dset_id;
- hid_t space_id;
- int nDims;
- hsize_t current_size[2];
- hsize_t max_size[2];
- hsize_t totalSize;
- hid_t dxpl_id;
- uint64_t *data;
- uint64_t u, e;
-
- dset_id = H5Dopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( dset_id >= 0 );
-
- space_id = H5Dget_space( dset_id ); assert ( space_id >= 0 );
- nDims = H5Sget_simple_extent_dims( space_id, current_size, max_size );
- assert( nDims == 2 );
-
- totalSize = current_size[0] * current_size[1];
- data = (uint64_t *)calloc( totalSize, sizeof(uint64_t) ); assert( data != NULL );
-
- dxpl_id = H5Pcreate( H5P_DATASET_XFER );
- if ( enable_checksums ) {
- ret = H5Pset_rawdata_integrity_scope( dxpl_id, H5_CHECKSUM_ALL ); ASSERT_RET;
- } else {
- ret = H5Pset_rawdata_integrity_scope( dxpl_id, H5_CHECKSUM_NONE ); ASSERT_RET;
- }
-
- ret = H5Dread_ff( dset_id, H5T_NATIVE_UINT64, space_id, space_id, dxpl_id, data, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
-
- /* note that by printing piece-by-piece, other output may be interspersed. trade-off since won't know line size */
- fprintf( stderr, "%s %s data:\n", preface, path_to_object );
- e = 0;
- for ( i = 0; i < totalSize; i++ ) {
- if ( e == 0 ) fprintf( stderr, "%s\t", preface );
- fprintf( stderr, "%09lu ", data[i] );
- if ( ++e == cols_per_row ) {
- fprintf( stderr, "\n" );
- e = 0;
- }
- }
-
- free( data );
- sleep( 1 ); /* give output a chance to be printed before diag messages from lower layers on close */
- ret = H5Pclose( dxpl_id ); ASSERT_RET;
- ret = H5Sclose( space_id ); ASSERT_RET;
- ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- }
- }
-
- /* End printing */
- fprintf( stderr, "%s -----------------\n", preface );
-
- return;
-}
-
-/*
- * parse_options - parse command line options
- */
-int
-parse_options( int argc, char** argv, int my_rank, int comm_size ) {
- int i, n;
- char* app = argv[0];
-
- while ( --argc ) {
- if ( **(++argv) != '-' ) {
- break;
- } else {
- switch( *(*argv+1) ) {
- case 'c':
- if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
- if ( my_rank == 0 ) {
- printf( "Error: No number specified after -c option.\n" );
- usage( app );
- }
- return( 1 );
- } else {
- ++argv;
- cols_per_row = atoi( *argv );
- }
- break;
- case 'w':
- if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
- if ( my_rank == 0 ) {
- printf( "Error: No number specified after -w option.\n" );
- usage( app );
- }
- return( 1 );
- } else {
- ++argv;
- rows_per_write = atoi( *argv );
- }
- break;
- case 'r':
- if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
- if ( my_rank == 0 ) {
- printf( "Error: No number specified after -r option.\n" );
- usage( app );
- }
- return( 1 );
- } else {
- ++argv;
- rows_per_read = atoi( *argv );
- }
- break;
- case 'b':
- if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
- if ( my_rank == 0 ) {
- printf( "Error: No number specified after -b option.\n" );
- usage( app );
- }
- return( 1 );
- } else {
- ++argv;
- rows_per_rank = atoi( *argv );
- }
- break;
- case 'i':
- if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
- if ( my_rank == 0 ) {
- printf( "APP-r0: Error: No number specified after -i option.\n" );
- usage( app );
- }
- return( 1 );
- } else {
- ++argv;
- num_iterations = atoi( *argv );
- }
- break;
- case 'p':
- if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
- if ( my_rank == 0 ) {
- printf( "APP-r0: Error: No number specified after -p option.\n" );
- usage( app );
- }
- return( 1 );
- } else {
- ++argv;
- persist_rate = atoi( *argv );
- }
- break;
- case 'e':
- enable_checksums = 1;
- break;
- case 'L':
- logged_dset = 1;
- break;
- case 'P':
- prefetched_dset = 1;
- break;
- case 'S':
- stored_dset = 1;
- break;
- case 'd':
- detailed_timing = 1;
- break;
- case 'v':
- verbose = 1;
- break;
- default:
- if ( my_rank == 0 ) {
- usage( app );
- }
- return( 1 );
- }
- }
- }
- return( 0 );
-}
-
-/*
- * usage - display usage message
- */
-void
-usage( const char* app ) {
- printf( "Usage: %s [-c cols_per_row] [-w rows_per_write] [-r rows_per_read] [-b rows_per_rank] [-i num_iterations] [-p persist_rate] [-e] [-L] [-P] [-S] [-d] [-v] \n", app );
- printf( "\tc: number of columns per row (each cell is 8 bytes)\n" );
- printf( "\tw: number of rows per write per rank\n" );
- printf( "\tr: number of rows per read per rank\n" );
- printf( "\tb: number of rows in each rank's block (if not multiple of w or r, may not be fully written or read)\n" );
- printf( "\ti: number of iterations to do writes/commit/[persist/evict/prefetch/reads/evict]\n" );
- printf( "\tp: persist rate; for /DP and /DS, also impacts evict/prefetch/read/evict\n" );
- printf( "\te: enable checksums on raw data in H5Datasets\n" );
- printf( "\tL: create /DL dataset - data will not be evicted (created by default if no /DP nor /DS)\n" );
- printf( "\tP: create /DP dataset - data will be evicted, replica prefetched, reads from replica\n" );
- printf( "\tS: create /DS dataset - data will be evicted, reads from storage (DAOS)\n" );
- printf( "\td: print detailed timing information for each read and write operation\n" );
- printf( "\tv: verbose output of data values - intended for use only with small arrays and few iterations\n" );
-}
+/*
+ * h5ff_time_datasets2.c: Do timings of various write / persist / prefetch / read operations for datasets
+ *
+ * /DL is dataset that is read back from transaction logs in BB
+ * /DP is dataset that is read back from prefetched replica in BB
+ * /DS is dataset that is read back from stored data (DAOS)
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include <sys/time.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+/* define filename for this app, and max size after username prepended */
+#define FILENAME_APP "eff_time_datasets2.h5"
+#define FILENAME_SIZE 256
+
+/* define default sizing parameters - can be changed via command line arguments */
+//#define DEFAULT_COLS_PER_ROW 131072 /* = 1 MiB/row => ( 1024*1024 /8 bytes per cell) */
+//#define DEFAULT_ROWS_PER_WRITE 2 /* = 2 MiB writes with 1MiB/row */
+//#define DEFAULT_ROWS_PER_READ 2 /* = 2 MiB writes with 1MiB/row */
+//#define DEFAULT_ROWS_PER_RANK 5120 /* = 5 GiB/rank with 1MiB/row */
+/* Sized for testing - with verbose */
+#define DEFAULT_COLS_PER_ROW 6
+#define DEFAULT_ROWS_PER_WRITE 2
+#define DEFAULT_ROWS_PER_READ 1
+#define DEFAULT_ROWS_PER_RANK 4
+
+/* macros related to error reporting */
+#define STATUS (ret >= 0) ? " " : " - FAILED"
+#define ASSERT_RET assert( ret >= 0 )
+
+/* option flags */
+int cols_per_row = DEFAULT_COLS_PER_ROW;
+int rows_per_write = DEFAULT_ROWS_PER_WRITE;
+int rows_per_read = DEFAULT_ROWS_PER_READ;
+int rows_per_rank = DEFAULT_ROWS_PER_RANK;
+
+int logged_dset = 0; /* Create DL - defaults to no (unless neither DP nor DS are created ) */
+int prefetched_dset = 0; /* Create DP - defaults to no */
+int stored_dset = 0; /* Create DS - defaults to no */
+int num_iterations = 1; /* Number of times to perform the commit/persist/evict/prefetch/read/evict cycle */
+int persist_rate = 1; /* Rate at which transactions are persisted */
+int enable_checksums = 0; /* Enable checksums on raw data - defaults to no */
+int detailed_timing = 0; /* Print detailed timing information for each write & read - defaults to no */
+int verbose = 0; /* Verbose output - defaults to no */
+
+/* global variables and macros used to make timing easier */
+struct timeval tv_start;
+struct timeval tv_end;
+#define START_TIME gettimeofday( &tv_start, NULL )
+#define END_TIME gettimeofday( &tv_end, NULL )
+#define ELAPSED_TIME (ulong)( (tv_end.tv_usec + 1000000*tv_end.tv_sec) - (tv_start.tv_usec + 1000000*tv_start.tv_sec) )
+
+/* function prototypes */
+void fill_stats( int*, ulong*, ulong*, ulong*, ulong );
+void print_container_contents( hid_t, hid_t, int );
+int parse_options( int, char**, int, int );
+void usage( const char* );
+
+/**************************************************************************************************************/
+int main( int argc, char **argv ) {
+
+ /* MPI */
+ int my_rank, comm_size;
+ int provided;
+
+ /* Container */
+ char *user_name; /* We'll prepend username to make filename unique */
+ char file_name[FILENAME_SIZE]; /* Actual filename to be opened will be user_name + FILENAME_APP */
+ hid_t fapl_id;
+ hid_t file_id;
+
+ /* Dataspaces and hyperslab selection parameters */
+ hid_t space_l_id;
+ hid_t space_p_id;
+ hid_t space_s_id;
+ hsize_t rank_offset[2], rank_count[2];
+
+ /* Datasets */
+ hid_t dset_l_id;
+ hid_t dset_p_id;
+ hid_t dset_s_id;
+ hsize_t dset_size[2];
+ hid_t space_id;
+
+ /* Data creation and transfer properties */
+ hid_t dcpl_id;
+ hid_t dxpl_l_id;
+ hid_t dxpl_p_id;
+ hid_t dxpl_s_id;
+
+ /* Replicas */
+ hrpl_t dset_p_replica;
+
+ /* Container Version & Read Contexts */
+ uint64_t version, versionH;
+ hid_t rc_id, rc_idH;
+
+ /* Transactions */
+ uint64_t tr_num;
+ hid_t tr_id;
+ hid_t trspl_id;
+ int num_tr_leaders;
+
+ /* Memory buffer to hold data for 1 rank and memory hyperslab selection parameters */
+ uint64_t *mbuf;
+ uint64_t mbuf_offset;
+ hsize_t mbuf_size[1];
+ hid_t mem_space_id;
+ hsize_t mem_offset[1], mem_count[1];
+
+ /* Variables used for reading data written by a neighbor rank */
+ int neighbor_rank;
+ hid_t neighbor_space_l_id;
+ hid_t neighbor_space_p_id;
+ hid_t neighbor_space_s_id;
+ hsize_t neighbor_offset[2], neighbor_count[2];
+
+ /* Misc */
+ herr_t ret;
+ uint64_t u, r, c;
+ uint64_t bytesPerCell, bytesPerWrite, bytesPerRead, bytesPerRank, bytesPerDataset, bytesPerContainer;
+ double megabytesPerWrite, megabytesPerRead, megabytesPerRank, megabytesPerDataset, megabytesPerContainer;
+ int rows_transferred;
+ ulong elapsed_time;
+ double rate;
+ ulong e_min, e_max, e_sum;
+ int cnt;
+
+ int i;
+ int iteration;
+ int iter_since_last_persist;
+
+ /****
+ * Initialize and create container
+ ****/
+
+ /* Confirm needed MPI functionality is available */
+ MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
+ if ( MPI_THREAD_MULTIPLE != provided ) {
+ fprintf( stderr, "APP: ERROR: MPI does not have MPI_THREAD_MULTIPLE support\n" );
+ exit( 1 );
+ }
+
+ /* Find MPI rank & communicator size */
+ MPI_Comm_rank( MPI_COMM_WORLD, &my_rank );
+ MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
+
+ /* Parse command-line options controlling behavior */
+ if ( parse_options( argc, argv, my_rank, comm_size ) != 0 ) {
+ exit( 1 );
+ }
+ if ( my_rank == 0 ) {
+ fprintf( stderr, "APP-r%d: Number of MPI processes = %d\n", my_rank, comm_size );
+ fprintf( stderr, "APP-r%d: Datasets will have %d columns.\n", my_rank, cols_per_row );
+ fprintf( stderr, "APP-r%d: Datasets will be written %d rows at a time.\n", my_rank, rows_per_write );
+ fprintf( stderr, "APP-r%d: Datasets will be read %d rows at a time.\n", my_rank, rows_per_read );
+ fprintf( stderr, "APP-r%d: Datasets will have %d rows (%d per rank).\n", my_rank, (rows_per_rank*comm_size), rows_per_rank );
+ fprintf( stderr, "APP-r%d: There will be %d iterations.\n", my_rank, num_iterations );
+ fprintf( stderr, "APP-r%d: Persists will occur every %d iterations.\n", my_rank, persist_rate );
+ if ( ( prefetched_dset==0 ) && ( stored_dset==0 ) ) { /* Make sure something is created! */
+ logged_dset = 1;
+ }
+ if ( logged_dset ) {
+ fprintf( stderr, "APP-r%d: /DL will be created\n", my_rank );
+ }
+ if ( prefetched_dset ) {
+ fprintf( stderr, "APP-r%d: /DP will be created\n", my_rank );
+ }
+ if ( stored_dset ) {
+ fprintf( stderr, "APP-r%d: /DS will be created\n", my_rank );
+ }
+ }
+
+ /* Initialize the EFF stack, starting FS client, registering HDF5 VOL calls, requesting IOD init */
+ fprintf( stderr, "APP-r%d: Initialize EFF stack\n", my_rank );
+ EFF_init( MPI_COMM_WORLD, MPI_INFO_NULL );
+
+ /* Specify the IOD VOL plugin should be used and create H5File (EFF container), with user name prepended */
+ user_name = getenv( "USER" );
+ snprintf( file_name, FILENAME_SIZE, "%s_%s", user_name, FILENAME_APP );
+ fprintf( stderr, "APP-r%d: Create %s\n", my_rank, file_name );
+ fapl_id = H5Pcreate( H5P_FILE_ACCESS ); assert( fapl_id >= 0 );
+ ret = H5Pset_fapl_iod( fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL ); ASSERT_RET;
+ file_id = H5Fcreate_ff( file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL ); assert( file_id >= 0 );
+
+ /* Create the memory buffer that will be used in many places - both for writing and reading */
+ bytesPerCell = sizeof( uint64_t );
+ bytesPerWrite = bytesPerCell * rows_per_write * cols_per_row; /* Bytes written in a single operation*/
+ bytesPerRead = bytesPerCell * rows_per_read * cols_per_row; /* Bytes read in a single operations */
+ bytesPerRank = bytesPerCell * rows_per_rank * cols_per_row; /* Bytes per rank to each dset */
+ bytesPerDataset = bytesPerRank * comm_size; /* Bytes written by all ranks to each dset */
+ bytesPerContainer = bytesPerDataset; /* Bytes written by all ranks to all dsets */
+ if ( prefetched_dset ) {
+ bytesPerContainer += bytesPerDataset;
+ }
+ if ( stored_dset ) {
+ bytesPerContainer += bytesPerDataset;
+ }
+
+ megabytesPerWrite = (double)bytesPerWrite/(1024*1024);
+ megabytesPerRead = (double)bytesPerRead/(1024*1024);
+ megabytesPerRank = (double)bytesPerRank/(1024*1024);
+ megabytesPerDataset = (double)bytesPerDataset/(1024*1024);
+ megabytesPerContainer = (double)bytesPerContainer/(1024*1024);
+
+ if ( my_rank == 0 ) {
+ fprintf( stderr, "APP-r%d: Bytes per Cell = %lu\n", my_rank, bytesPerCell );
+ fprintf( stderr, "APP-r%d: Bytes per Write = %lu\n", my_rank, bytesPerWrite );
+ fprintf( stderr, "APP-r%d: Bytes per Read = %lu\n", my_rank, bytesPerRead );
+ fprintf( stderr, "APP-r%d: Bytes per Rank = %lu\n", my_rank, bytesPerRank );
+ fprintf( stderr, "APP-r%d: Bytes per Dataset = %lu\n", my_rank, bytesPerDataset );
+ fprintf( stderr, "APP-r%d: Bytes per Container = %lu\n", my_rank, bytesPerContainer );
+ fprintf( stderr, "APP-r%d: MiB per Write = %f\n", my_rank, megabytesPerWrite );
+ fprintf( stderr, "APP-r%d: MiB per Read = %f\n", my_rank, megabytesPerRead );
+ fprintf( stderr, "APP-r%d: MiB per Rank = %f\n", my_rank, megabytesPerRank );
+ fprintf( stderr, "APP-r%d: MiB per Dataset = %f\n", my_rank, megabytesPerDataset );
+ fprintf( stderr, "APP-r%d: MiB per Container = %f\n", my_rank, megabytesPerContainer );
+ }
+
+ mbuf = (uint64_t *) calloc( bytesPerRank, 1 );
+ if ( mbuf == NULL ) {
+ fprintf( stderr, "APP-r%d: calloc failed when trying to allocate %lu bytes\n", my_rank, bytesPerRank );
+ exit( -1 );
+ }
+
+ /****
+ * Transaction 2: Rank 0 creates H5Objects in the container
+ ****/
+
+ tr_num = 2; /* Set tr_num here so all ranks can later access the value */
+ if ( my_rank == 0 ) {
+
+ /* Acquire read context for CV 1 */
+ version = 1;
+ rc_id = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); assert( rc_id >= 0); assert( version == 1 );
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
+
+ /* Start a transaction with a single leader (the default) */
+ fprintf( stderr, "APP-r%d: tr %lu - Start\n", my_rank, tr_num );
+ tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
+ ret = H5TRstart( tr_id, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Set Dataset Creation Property List to control use of checksums at IOD level */
+ dcpl_id = H5Pcreate( H5P_DATASET_CREATE );
+ if ( enable_checksums ) {
+ /* enabled by default */
+ } else {
+ H5Pset_ocpl_enable_checksum( dcpl_id, 0 );
+ }
+
+ /* Create Dataspace for Datasets */
+ dset_size[0] = rows_per_rank * comm_size;
+ dset_size[1] = cols_per_row;
+ space_id = H5Screate_simple( 2, dset_size, dset_size ); assert( space_id >= 0 );
+
+ /* Add updates to the transaction for Dataset creates */
+ fprintf( stderr, "APP-r%d: tr %d - Create Dataset(s)\n", my_rank, tr_num );
+
+ if ( logged_dset ) {
+ dset_l_id = H5Dcreate_ff( file_id, "DL", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( dset_l_id >= 0 );
+ }
+
+ if ( prefetched_dset ) {
+ dset_p_id = H5Dcreate_ff( file_id, "DP", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( dset_p_id >= 0 );
+ }
+
+ if ( stored_dset ) {
+ dset_s_id = H5Dcreate_ff( file_id, "DS", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( dset_s_id >= 0 );
+ }
+
+ ret = H5Sclose( space_id ); ASSERT_RET;
+
+ /* Finish and commit TR and get RC for it */
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, &rc_idH, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: tr %lu - Finished\n", my_rank, tr_num );
+
+ assert( rc_idH >= 0 );
+ versionH = tr_num;
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, versionH );
+
+ /* Release the read handle and close read context on earlier CV */
+ ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
+ ret = H5RCclose( rc_id ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc %lu - Released and Closed\n", my_rank, version );
+
+ /* The just-acquired RC is now the primary rc_id we'll work with (for awhile) */
+ rc_id = rc_idH;
+ version = versionH;
+ }
+
+ /* Rank 0 notifies other ranks that transaction 2 committed and RC acquired */
+ MPI_Bcast( &version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+
+ /****
+ * Read Context 2: Other ranks create RC and open HObjects created by rank 0.
+ ****/
+ if ( my_rank != 0 ) {
+ rc_id = H5RCcreate( file_id, version );
+ assert( rc_id >= 0 ); assert ( version == 2 );
+ fprintf( stderr, "APP-r%d: rc %lu - Created for rank\n", my_rank, version );
+
+ if ( logged_dset ) {
+ dset_l_id = H5Dopen_ff( file_id, "DL", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_l_id >= 0 );
+ }
+ if ( prefetched_dset ) {
+ dset_p_id = H5Dopen_ff( file_id, "DP", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_p_id >= 0 );
+ }
+ if ( stored_dset ) {
+ dset_s_id = H5Dopen_ff( file_id, "DS", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_s_id >= 0 );
+ }
+
+ } else {
+ if ( verbose ) print_container_contents( file_id, rc_id, my_rank );
+ }
+
+ /* Keep track of my_rank's neighbor */
+ neighbor_rank = my_rank + 1;
+ if ( neighbor_rank == comm_size ) {
+ neighbor_rank = 0;
+ }
+
+ /* Get the dataspaces for the three datasets and make a copy of each for neighbor's */
+ if ( logged_dset ) {
+ space_l_id = H5Dget_space( dset_l_id ); assert ( space_l_id >= 0 );
+ neighbor_space_l_id = H5Scopy( space_l_id );
+ }
+ if ( prefetched_dset ) {
+ space_p_id = H5Dget_space( dset_p_id ); assert ( space_p_id >= 0 );
+ neighbor_space_p_id = H5Scopy( space_p_id );
+ }
+ if ( stored_dset ) {
+ space_s_id = H5Dget_space( dset_s_id ); assert ( space_s_id >= 0 );
+ neighbor_space_s_id = H5Scopy( space_s_id );
+ }
+
+
+ /* Create the memory dataspace for one rank worth of data */
+ mbuf_size[0] = rows_per_rank * cols_per_row;
+ mem_space_id = H5Screate_simple( 1, mbuf_size, mbuf_size ); assert( mem_space_id >= 0 );
+
+ /* Create property lists that will be used */
+ if ( logged_dset ) {
+ dxpl_l_id = H5Pcreate( H5P_DATASET_XFER );
+ if ( enable_checksums ) {
+ ret = H5Pset_rawdata_integrity_scope( dxpl_l_id, H5_CHECKSUM_ALL ); ASSERT_RET;
+ } else {
+ ret = H5Pset_rawdata_integrity_scope( dxpl_l_id, H5_CHECKSUM_NONE ); ASSERT_RET;
+ }
+ }
+
+ if ( prefetched_dset ) {
+ dxpl_p_id = H5Pcreate( H5P_DATASET_XFER );
+ if ( enable_checksums ) {
+ ret = H5Pset_rawdata_integrity_scope( dxpl_p_id, H5_CHECKSUM_ALL ); ASSERT_RET;
+ } else {
+ ret = H5Pset_rawdata_integrity_scope( dxpl_p_id, H5_CHECKSUM_NONE ); ASSERT_RET;
+ }
+ }
+
+ if ( stored_dset ) {
+ dxpl_s_id = H5Pcreate( H5P_DATASET_XFER );
+ if ( enable_checksums ) {
+ ret = H5Pset_rawdata_integrity_scope( dxpl_s_id, H5_CHECKSUM_ALL ); ASSERT_RET;
+ } else {
+ ret = H5Pset_rawdata_integrity_scope( dxpl_s_id, H5_CHECKSUM_NONE ); ASSERT_RET;
+ }
+ }
+
+ iter_since_last_persist = 0;
+
+ for ( iteration = 1; iteration <= num_iterations; iteration++ ) {
+
+ /****
+ * Cycle of:
+ * Tranasction N Start / Update / Commit: All ranks update Dataset(s) then commit
+ * CV N Persist: Rank 0
+ * Prefetch: Rank 0
+ * Read rank data: All Ranks
+ * Read neighbor rank data: All Ranks
+ * Read all data: All Ranks
+ ****/
+ tr_num = tr_num + 1;
+ num_tr_leaders = comm_size;
+
+ /* Create a transaction and start it. */
+ fprintf( stderr, "APP-r%d: iter %05d tr %lu - Transaction Start with %d leaders\n",
+ my_rank, iteration, tr_num, num_tr_leaders );
+ tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
+ trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
+ ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
+ ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Set cell values (in memory) for this rank based on Row, Rank, Column, Transaction */
+ u = 0;
+ for ( r = 0; r < rows_per_rank; r++ ) {
+ for ( c = 0; c < cols_per_row; c++ ) {
+ mbuf[u] = ( my_rank*10000 + r*1000 + c*100 + tr_num ) ;
+ u++;
+ }
+ }
+
+ fprintf( stderr, "APP-r%d: iter %05d tr %lu - Add updates to transaction.\n", my_rank, iteration, tr_num );
+
+ /* Add dataset updates to transaction */
+ if ( logged_dset ) {
+ cnt = e_min = e_max = e_sum = 0;
+ /* Set parameters that will be used to select the initial hyperslabs for this rank */
+ mem_offset[0] = 0;
+ mem_count[0] = rows_per_write * cols_per_row;
+ mbuf_offset = 0;
+ rank_offset[0] = my_rank * rows_per_rank;
+ rank_offset[1] = 0;
+ rank_count[0] = rows_per_write;
+ rank_count[1] = cols_per_row;
+ rows_transferred = 0;
+
+ while ( (rows_transferred+rows_per_write) <= rows_per_rank ) { /* final rows left unfilled if not exact multiple */
+ /* Select the hyperslabs for this write */
+ ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_l_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
+
+ START_TIME;
+ ret = H5Dwrite_ff( dset_l_id, H5T_NATIVE_UINT64, mem_space_id, space_l_id, dxpl_l_id, &mbuf[mbuf_offset],
+ tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
+ if ( detailed_timing ) {
+ rate = megabytesPerWrite/((double)elapsed_time/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 01: Time to add Write updates for /DL (%lu bytes): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerWrite, elapsed_time, rate );
+ }
+
+ /* Advance in preparation for next write */
+ rows_transferred += rows_per_write;
+ rank_offset[0] += rows_per_write;
+ mem_offset[0] += ( rows_per_write * cols_per_row );
+ mbuf_offset += ( rows_per_write * cols_per_row );
+ }
+ if ( cnt > 0 ) {
+ rate = megabytesPerWrite/((double)(e_sum/cnt)/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 01: Average Time to add Write updates for /DL (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
+ my_rank, iteration, bytesPerWrite, e_min, e_max, e_sum/cnt, rate );
+ }
+ }
+
+ if ( prefetched_dset ) {
+ cnt = e_min = e_max = e_sum = 0;
+ /* Set parameters that will be used to select the initial hyperslabs for this rank */
+ mem_offset[0] = 0;
+ mem_count[0] = rows_per_write * cols_per_row;
+ mbuf_offset = 0;
+ rank_offset[0] = my_rank * rows_per_rank;
+ rank_offset[1] = 0;
+ rank_count[0] = rows_per_write;
+ rank_count[1] = cols_per_row;
+ rows_transferred = 0;
+
+ while ( (rows_transferred+rows_per_write) <= rows_per_rank ) { /* final rows left unfilled if not exact multiple */
+ /* Select the hyperslabs for this write */
+ ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_p_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
+
+ START_TIME;
+ ret = H5Dwrite_ff( dset_p_id, H5T_NATIVE_UINT64, mem_space_id, space_p_id, dxpl_p_id, &mbuf[mbuf_offset], tr_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
+ if ( detailed_timing ) {
+ rate = megabytesPerWrite/((double)elapsed_time/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 02: Time to add Write updates for /DP (%lu bytes): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerWrite, elapsed_time, rate );
+ }
+
+ /* Advance in preparation for next write */
+ rows_transferred += rows_per_write;
+ rank_offset[0] += rows_per_write;
+ mem_offset[0] += ( rows_per_write * cols_per_row );
+ mbuf_offset += ( rows_per_write * cols_per_row );
+ }
+ if ( cnt > 0 ) {
+ rate = megabytesPerWrite/((double)(e_sum/cnt)/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 02: Average Time to add Write updates for /DP (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
+ my_rank, iteration, bytesPerWrite, e_min, e_max, e_sum/cnt, rate );
+ }
+ }
+
+ if ( stored_dset ) {
+ cnt = e_min = e_max = e_sum = 0;
+ /* Set parameters that will be used to select the initial hyperslabs for this rank */
+ mem_offset[0] = 0;
+ mem_count[0] = rows_per_write * cols_per_row;
+ mbuf_offset = 0;
+ rank_offset[0] = my_rank * rows_per_rank;
+ rank_offset[1] = 0;
+ rank_count[0] = rows_per_write;
+ rank_count[1] = cols_per_row;
+ rows_transferred = 0;
+
+ while ( (rows_transferred+rows_per_write) <= rows_per_rank ) { /* final rows left unfilled if not exact multiple */
+ /* Select the hyperslabs for this write */
+ ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_s_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
+
+ START_TIME;
+ ret = H5Dwrite_ff( dset_s_id, H5T_NATIVE_UINT64, mem_space_id, space_s_id, dxpl_s_id, &mbuf[mbuf_offset], tr_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
+ if ( detailed_timing ) {
+ rate = megabytesPerWrite/((double)elapsed_time/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 03: Time to add Write updates for /DS (%lu bytes): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerWrite, elapsed_time, rate );
+ }
+
+ /* Advance in preparation for next write */
+ rows_transferred += rows_per_write;
+ rank_offset[0] += rows_per_write;
+ mem_offset[0] += ( rows_per_write * cols_per_row );
+ mbuf_offset += ( rows_per_write * cols_per_row );
+ }
+ if ( cnt > 0 ) {
+ rate = megabytesPerWrite/((double)(e_sum/cnt)/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 03: Average Time to add Write updates for /DS (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
+ my_rank, iteration, bytesPerWrite, e_min, e_max, e_sum/cnt, rate );
+ }
+ }
+
+ /* Finish, (commit), and close transaction */
+ fprintf( stderr, "APP-r%d: iter %05d tr %lu - Finish and Commit\n", my_rank, iteration, tr_num );
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ ret = H5Pclose( trspl_id ); ASSERT_RET;
+
+ /* After commit completes on all ranks, acquire RC on rank 0 and create local handle for RC on other ranks */
+ MPI_Barrier( MPI_COMM_WORLD );
+ versionH = tr_num;
+ if ( my_rank == 0 ) {
+ rc_idH = H5RCacquire( file_id, &versionH, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ assert( rc_idH >= 0 ); assert ( versionH == tr_num );
+ fprintf( stderr, "APP-r%d: iter %05d rc %lu - Acquired\n", my_rank, iteration, versionH );
+ } else {
+ rc_idH = H5RCcreate( file_id, versionH );
+ assert( rc_idH >= 0 ); assert ( versionH == tr_num );
+ fprintf( stderr, "APP-r%d: iter %05d rc %lu - Created\n", my_rank, iteration, versionH );
+ }
+ MPI_Barrier( MPI_COMM_WORLD ); /* Wait to make sure rank 0 has acquired from IOD before proceeding */
+
+ /* Release previous Read Context */
+ if ( my_rank == 0 ) {
+ ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: iter %05d rc %lu - Released\n", my_rank, iteration, version );
+ }
+ ret = H5RCclose( rc_id ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: iter %05d rc %lu - Closed\n", my_rank, iteration, version );
+
+ /* The just-acquired RC is now the primary rc_id we'll work with (for awhile) */
+ rc_id = rc_idH;
+ version = versionH;
+
+ iter_since_last_persist++; /* We've done the commit of a TR, so advance iter ctr */
+
+ /* Rank 0:
+ if this is a persist iteration:
+ - persists the container
+ - evicts DP & DS
+ - prefetches DP
+ */
+ if ( my_rank == 0 ) {
+ if ( verbose ) print_container_contents( file_id, rc_id, my_rank );
+
+ if ( iter_since_last_persist == persist_rate ) {
+ START_TIME;
+ ret = H5RCpersist( rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ rate = megabytesPerContainer/((double)elapsed_time/1000000);
+ fprintf( stderr, "APP-r%d: iter %05d step 04: Time to Persist container (%lu bytes + KVs): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerContainer, elapsed_time, rate );
+
+ if ( prefetched_dset ) {
+ START_TIME;
+ ret = H5Devict_ff( dset_p_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ rate = megabytesPerDataset/((double)elapsed_time/1000000);
+ fprintf( stderr, "APP-r%d: iter %05d step 05: Time to Evict /DP (%lu bytes): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerDataset, elapsed_time, rate );
+ }
+
+ if ( stored_dset ) {
+ START_TIME;
+ ret = H5Devict_ff( dset_s_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ rate = megabytesPerDataset/((double)elapsed_time/1000000);
+ fprintf( stderr, "APP-r%d: iter %05d step 06: Time to Evict /DS (%lu bytes): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerDataset, elapsed_time, rate );
+ }
+
+ if ( prefetched_dset ) {
+ START_TIME;
+ ret = H5Dprefetch_ff( dset_p_id, rc_id, &dset_p_replica, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ rate = megabytesPerDataset/((double)elapsed_time/1000000);
+ fprintf( stderr, "APP-r%d: iter %05d step 07: Time to Prefetch (%lu bytes) /DP: %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerDataset, elapsed_time, rate );
+ }
+ }
+ }
+
+ /* We only prefetch if a persist occurred */
+ if ( ( prefetched_dset ) && ( iter_since_last_persist == persist_rate ) ) {
+ MPI_Bcast( &dset_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+ ret = H5Pset_dxpl_replica( dxpl_p_id, dset_p_replica ); ASSERT_RET;
+ }
+
+ MPI_Barrier( MPI_COMM_WORLD ); /* Make sure all are here before continuing, (esp in case of no prefetch Bcast) */
+
+ /*
+ * All ranks read 'their' part of the written dataset (s)
+ * For /DL, read every iteration.
+ * For /DP and DS, only read on persisted iterations.
+ */
+
+ if ( logged_dset ) {
+ /* Zero the memory buf so we can be sure we are reading */
+ if ( verbose ) {
+ bzero( (void*) mbuf, bytesPerRank );
+ }
+
+ cnt = e_min = e_max = e_sum = 0;
+ /* Set parameters that will be used to select the initial hyperslabs for this rank */
+ mem_offset[0] = 0;
+ mem_count[0] = rows_per_read * cols_per_row;
+ mbuf_offset = 0;
+ rank_offset[0] = my_rank * rows_per_rank;
+ rank_offset[1] = 0;
+ rank_count[0] = rows_per_read;
+ rank_count[1] = cols_per_row;
+ rows_transferred = 0;
+
+ while ( (rows_transferred+rows_per_read) <= rows_per_rank ) { /* final rows left unread if not exact multiple */
+ /* Select the hyperslabs for this read */
+ ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_l_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
+
+ START_TIME;
+ ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, mem_space_id, space_l_id, dxpl_l_id, &mbuf[mbuf_offset], rc_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
+ if ( detailed_timing ) {
+ rate = megabytesPerRead/((double)elapsed_time/1000000);
+ fprintf( stderr, "APP-r%d: iter %05d step 08: Time to Read my entries for /DL (%lu bytes): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerRead, elapsed_time, rate );
+ }
+
+ fprintf( stderr, "APP-r%d: first value read in /DL: %09lu\n", my_rank, mbuf[mbuf_offset] );
+ if ( ( verbose ) && ( my_rank == 0 ) ) { /* print all mbuf on rank 0 to confirm correctness while developing */
+ u = 0;
+ for ( r = 0; r < rows_per_rank; r++ ) {
+ for ( c = 0; c < cols_per_row; c++ ) {
+ fprintf( stderr, "%lu ", mbuf[u] );
+ u++;
+ }
+ fprintf( stderr, "\n" );
+ }
+ }
+
+ /* Advance in preparation for next read */
+ rows_transferred += rows_per_read;
+ rank_offset[0] += rows_per_read;
+ mem_offset[0] += ( rows_per_read * cols_per_row );
+ mbuf_offset += ( rows_per_read * cols_per_row );
+ }
+
+ if ( cnt > 0 ) {
+ rate = megabytesPerRead/((double)(e_sum/cnt)/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 08: Average Time to Read my entries for /DL (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
+ my_rank, iteration, bytesPerRead, e_min, e_max, e_sum/cnt, rate );
+ }
+ }
+
+ if ( ( prefetched_dset ) && ( iter_since_last_persist == persist_rate ) ) {
+ /* Zero the memory buf so we can be sure we are reading */
+ if ( verbose ) {
+ bzero( (void*) mbuf, bytesPerRank );
+ }
+
+ cnt = e_min = e_max = e_sum = 0;
+ /* Set parameters that will be used to select the initial hyperslabs for this rank */
+ mem_offset[0] = 0;
+ mem_count[0] = rows_per_read * cols_per_row;
+ mbuf_offset = 0;
+ rank_offset[0] = my_rank * rows_per_rank;
+ rank_offset[1] = 0;
+ rank_count[0] = rows_per_read;
+ rank_count[1] = cols_per_row;
+ rows_transferred = 0;
+
+ while ( (rows_transferred+rows_per_read) <= rows_per_rank ) { /* final rows left unread if not exact multiple */
+ /* Select the hyperslabs for this read */
+ ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_p_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
+
+ START_TIME;
+ ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, mem_space_id, space_p_id, dxpl_p_id, &mbuf[mbuf_offset], rc_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
+ if ( detailed_timing ) {
+ rate = megabytesPerRead/((double)elapsed_time/1000000);
+ fprintf( stderr, "APP-r%d: iter %05d step 09: Time to Read my entries for /DP (%lu bytes): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerRead, elapsed_time, rate );
+ }
+
+ fprintf( stderr, "APP-r%d: first value read in /DP: %09lu\n", my_rank, mbuf[mbuf_offset] );
+ if ( ( verbose ) && ( my_rank == 0 ) ) { /* print all mbuf on rank 0 to confirm correctness while developing */
+ u = 0;
+ for ( r = 0; r < rows_per_rank; r++ ) {
+ for ( c = 0; c < cols_per_row; c++ ) {
+ fprintf( stderr, "%lu ", mbuf[u] );
+ u++;
+ }
+ fprintf( stderr, "\n" );
+ }
+ }
+
+ /* Advance in preparation for next read */
+ rows_transferred += rows_per_read;
+ rank_offset[0] += rows_per_read;
+ mem_offset[0] += ( rows_per_read * cols_per_row );
+ mbuf_offset += ( rows_per_read * cols_per_row );
+
+ }
+ if ( cnt > 0 ) {
+ rate = megabytesPerRead/((double)(e_sum/cnt)/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 09: Average Time to Read my entries for /DP (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
+ my_rank, iteration, bytesPerRead, e_min, e_max, e_sum/cnt, rate );
+ }
+ }
+
+ if ( ( stored_dset ) && ( iter_since_last_persist == persist_rate ) ) {
+ /* Zero the memory buf so we can be sure we are reading */
+ if ( verbose ) {
+ bzero( (void*) mbuf, bytesPerRank );
+ }
+
+ cnt = e_min = e_max = e_sum = 0;
+ /* Set parameters that will be used to select the initial hyperslabs for this rank */
+ mem_offset[0] = 0;
+ mem_count[0] = rows_per_read * cols_per_row;
+ mbuf_offset = 0;
+ rank_offset[0] = my_rank * rows_per_rank;
+ rank_offset[1] = 0;
+ rank_count[0] = rows_per_read;
+ rank_count[1] = cols_per_row;
+ rows_transferred = 0;
+
+ while ( (rows_transferred+rows_per_read) <= rows_per_rank ) { /* final rows left unread if not exact multiple */
+ /* Select the hyperslabs for this read */
+ ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_s_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
+
+ START_TIME;
+ ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, mem_space_id, space_s_id, dxpl_s_id, &mbuf[mbuf_offset], rc_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
+ if ( detailed_timing ) {
+ rate = megabytesPerRead/((double)elapsed_time/1000000);
+ fprintf( stderr, "APP-r%d: iter %05d step 10: Time to Read my entries for /DS (%lu bytes): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerRead, elapsed_time, rate );
+ }
+
+ fprintf( stderr, "APP-r%d: first value read in /DS: %09lu\n", my_rank, mbuf[mbuf_offset] );
+ if ( ( verbose ) && ( my_rank == 0 ) ) { /* print all mbuf on rank 0 to confirm correctness while developing */
+ u = 0;
+ for ( r = 0; r < rows_per_rank; r++ ) {
+ for ( c = 0; c < cols_per_row; c++ ) {
+ fprintf( stderr, "%lu ", mbuf[u] );
+ u++;
+ }
+ fprintf( stderr, "\n" );
+ }
+ }
+
+ /* Advance in preparation for next read */
+ rows_transferred += rows_per_read;
+ rank_offset[0] += rows_per_read;
+ mem_offset[0] += ( rows_per_read * cols_per_row );
+ mbuf_offset += ( rows_per_read * cols_per_row );
+ }
+
+ if ( cnt > 0 ) {
+ rate = megabytesPerRead/((double)(e_sum/cnt)/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 10: Average Time to Read my entries for /DS (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
+ my_rank, iteration, bytesPerRead, e_min, e_max, e_sum/cnt, rate );
+ }
+ }
+
+ /*
+ * All ranks read the values "next rank" neighbor updated in last transaction
+ * For /DL, read every iteration.
+ * For /DP and DS, only read on persisted iterations.
+ */
+
+ if ( logged_dset && ( my_rank != neighbor_rank ) ) {
+ /* Zero the memory buf so we can be sure we are reading */
+ if ( verbose ) {
+ bzero( (void*) mbuf, bytesPerRank );
+ }
+
+ cnt = e_min = e_max = e_sum = 0;
+ /* Set parameters that will be used to select the initial hyperslabs for this rank */
+ mem_offset[0] = 0;
+ mem_count[0] = rows_per_read * cols_per_row;
+ mbuf_offset = 0;
+ neighbor_offset[0] = neighbor_rank * rows_per_rank;
+ neighbor_offset[1] = 0;
+ neighbor_count[0] = rows_per_read;
+ neighbor_count[1] = cols_per_row;
+ rows_transferred = 0;
+
+ while ( (rows_transferred+rows_per_read) <= rows_per_rank ) { /* final rows left unread if not exact multiple */
+ /* Select the hyperslabs for this read */
+ ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( neighbor_space_l_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL );
+ ASSERT_RET;
+
+ START_TIME;
+ ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, mem_space_id, neighbor_space_l_id, dxpl_l_id, &mbuf[mbuf_offset], rc_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
+ if ( detailed_timing ) {
+ rate = megabytesPerRead/((double)elapsed_time/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 11: Time to Read neighbor's entries for /DL (%lu bytes): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerRead, elapsed_time, rate );
+ }
+
+ fprintf( stderr, "APP-r%d: neighbor's first value read in /DL: %09lu\n", my_rank, mbuf[mbuf_offset] );
+ if ( ( verbose ) && ( my_rank == 0 ) ) { /* print all mbuf on rank 0 to confirm correctness while developing */
+ u = 0;
+ for ( r = 0; r < rows_per_rank; r++ ) {
+ for ( c = 0; c < cols_per_row; c++ ) {
+ fprintf( stderr, "%lu ", mbuf[u] );
+ u++;
+ }
+ fprintf( stderr, "\n" );
+ }
+ }
+
+ /* Advance in preparation for next read */
+ rows_transferred += rows_per_read;
+ neighbor_offset[0] += rows_per_read;
+ mem_offset[0] += ( rows_per_read * cols_per_row );
+ mbuf_offset += ( rows_per_read * cols_per_row );
+ }
+
+ if ( cnt > 0 ) {
+ rate = megabytesPerRead/((double)(e_sum/cnt)/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 11: Average Time to Read neighbor's entries for /DL (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
+ my_rank, iteration, bytesPerRead, e_min, e_max, e_sum/cnt, rate );
+ }
+ }
+
+ if ( ( prefetched_dset ) && ( iter_since_last_persist == persist_rate ) && ( my_rank != neighbor_rank ) ) {
+ /* Zero the memory buf so we can be sure we are reading */
+ if ( verbose ) {
+ bzero( (void*) mbuf, bytesPerRank );
+ }
+
+ cnt = e_min = e_max = e_sum = 0;
+ /* Set parameters that will be used to select the initial hyperslabs for this rank */
+ mem_offset[0] = 0;
+ mem_count[0] = rows_per_read * cols_per_row;
+ mbuf_offset = 0;
+ neighbor_offset[0] = neighbor_rank * rows_per_rank;
+ neighbor_offset[1] = 0;
+ neighbor_count[0] = rows_per_read;
+ neighbor_count[1] = cols_per_row;
+ rows_transferred = 0;
+
+ while ( (rows_transferred+rows_per_read) <= rows_per_rank ) { /* final rows left unread if not exact multiple */
+ /* Select the hyperslabs for this read */
+ ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( neighbor_space_p_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL );
+ ASSERT_RET;
+
+ START_TIME;
+ ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, mem_space_id, neighbor_space_p_id, dxpl_p_id, &mbuf[mbuf_offset], rc_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
+ if ( detailed_timing ) {
+ rate = megabytesPerRead/((double)elapsed_time/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 12: Time to Read neighbor's entries for /DP (%lu bytes): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerRead, elapsed_time, rate );
+ }
+
+ fprintf( stderr, "APP-r%d: neighbor's first value read in /DP: %09lu\n", my_rank, mbuf[mbuf_offset] );
+ if ( ( verbose ) && ( my_rank == 0 ) ) { /* print all mbuf on rank 0 to confirm correctness while developing */
+ u = 0;
+ for ( r = 0; r < rows_per_rank; r++ ) {
+ for ( c = 0; c < cols_per_row; c++ ) {
+ fprintf( stderr, "%lu ", mbuf[u] );
+ u++;
+ }
+ fprintf( stderr, "\n" );
+ }
+ }
+
+ /* Advance in preparation for next read */
+ rows_transferred += rows_per_read;
+ neighbor_offset[0] += rows_per_read;
+ mem_offset[0] += ( rows_per_read * cols_per_row );
+ mbuf_offset += ( rows_per_read * cols_per_row );
+ }
+
+ if ( cnt > 0 ) {
+ rate = megabytesPerRead/((double)(e_sum/cnt)/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 12: Average Time to Read neighbor's entries for /DP (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
+ my_rank, iteration, bytesPerRead, e_min, e_max, e_sum/cnt, rate );
+ }
+ }
+
+ if ( ( stored_dset ) && ( iter_since_last_persist == persist_rate ) && ( my_rank != neighbor_rank ) ) {
+ /* Zero the memory buf so we can be sure we are reading */
+ if ( verbose ) {
+ bzero( (void*) mbuf, bytesPerRank );
+ }
+
+ cnt = e_min = e_max = e_sum = 0;
+ /* Set parameters that will be used to select the initial hyperslabs for this rank */
+ mem_offset[0] = 0;
+ mem_count[0] = rows_per_read * cols_per_row;
+ mbuf_offset = 0;
+ neighbor_offset[0] = neighbor_rank * rows_per_rank;
+ neighbor_offset[1] = 0;
+ neighbor_count[0] = rows_per_read;
+ neighbor_count[1] = cols_per_row;
+ rows_transferred = 0;
+
+ while ( (rows_transferred+rows_per_read) <= rows_per_rank ) { /* final rows left unread if not exact multiple */
+ /* Select the hyperslabs for this read */
+ ret = H5Sselect_hyperslab( mem_space_id, H5S_SELECT_SET, mem_offset, NULL, mem_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( neighbor_space_s_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL );
+ ASSERT_RET;
+
+ START_TIME;
+ ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, mem_space_id, neighbor_space_s_id, dxpl_s_id, &mbuf[mbuf_offset], rc_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ fill_stats( &cnt, &e_min, &e_max, &e_sum, elapsed_time );
+ if ( detailed_timing ) {
+ rate = megabytesPerRead/((double)elapsed_time/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 13: Time to Read neighbor's entries for /DS (%lu bytes): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerRead, elapsed_time, rate );
+ }
+
+ fprintf( stderr, "APP-r%d: neighbor's first value read in /DS: %09lu\n", my_rank, mbuf[mbuf_offset] );
+ if ( ( verbose ) && ( my_rank == 0 ) ) { /* print all mbuf on rank 0 to confirm correctness while developing */
+ u = 0;
+ for ( r = 0; r < rows_per_rank; r++ ) {
+ for ( c = 0; c < cols_per_row; c++ ) {
+ fprintf( stderr, "%lu ", mbuf[u] );
+ u++;
+ }
+ fprintf( stderr, "\n" );
+ }
+ }
+
+ /* Advance in preparation for next read */
+ rows_transferred += rows_per_read;
+ neighbor_offset[0] += rows_per_read;
+ mem_offset[0] += ( rows_per_read * cols_per_row );
+ mbuf_offset += ( rows_per_read * cols_per_row );
+ }
+
+ if ( cnt > 0 ) {
+ rate = megabytesPerRead/((double)(e_sum/cnt)/1000000);
+ fprintf( stderr,
+ "APP-r%d: iter %05d step 13: Average Time to Read neighbor's entries for /DS (%lu bytes): Min: %lu usec; Max: %lu usec; Avg: %lu usec - Avg %f MiB/sec\n",
+ my_rank, iteration, bytesPerRead, e_min, e_max, e_sum/cnt, rate );
+ }
+
+ }
+
+ /* For persisted iterations, reset the counter & if we prefetched a replica, evict it */
+ if ( iter_since_last_persist == persist_rate ) {
+ iter_since_last_persist = 0;
+
+ MPI_Barrier( MPI_COMM_WORLD ); /* Make sure all ranks done reading the replica before evicting */
+
+ /* Rank 0 evicts the replica */
+ if ( ( my_rank == 0 ) && ( prefetched_dset ) ) {
+ START_TIME;
+ ret = H5Devict_ff( dset_p_id, version, dxpl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ elapsed_time = ELAPSED_TIME;
+ rate = megabytesPerDataset/((double)elapsed_time/1000000);
+ fprintf( stderr, "APP-r%d: iter %05d step 17: Time to Evict Replica of /DP (%lu bytes): %lu usec - %f MiB/sec\n",
+ my_rank, iteration, bytesPerDataset, elapsed_time, rate );
+ }
+ }
+ }
+
+ /*
+ * Wrap-up
+ */
+
+ if ( my_rank == 0 ) {
+ if ( verbose ) print_container_contents( file_id, rc_id, my_rank );
+ }
+
+ fprintf( stderr, "APP-r%d: Closing H5Objects\n", my_rank );
+
+ ret = H5Sclose( mem_space_id ); ASSERT_RET;
+
+ if ( logged_dset ) {
+ ret = H5Dclose_ff( dset_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Sclose( space_l_id ); ASSERT_RET;
+ ret = H5Sclose( neighbor_space_l_id ); ASSERT_RET;
+ ret = H5Pclose( dxpl_l_id ); ASSERT_RET;
+ }
+ if ( prefetched_dset ) {
+ ret = H5Dclose_ff( dset_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Sclose( space_p_id ); ASSERT_RET;
+ ret = H5Sclose( neighbor_space_p_id ); ASSERT_RET;
+ ret = H5Pclose( dxpl_p_id ); ASSERT_RET;
+ }
+ if ( stored_dset ) {
+ ret = H5Dclose_ff( dset_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Sclose( space_s_id ); ASSERT_RET;
+ ret = H5Sclose( neighbor_space_s_id ); ASSERT_RET;
+ ret = H5Pclose( dxpl_s_id ); ASSERT_RET;
+ }
+
+
+ /* Release the read handle and close read context */
+ MPI_Barrier( MPI_COMM_WORLD ); /* Make sure all ranks done with RC */
+ if ( my_rank == 0 ) {
+ ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc %lu - Released\n", my_rank, version );
+ }
+ ret = H5RCclose( rc_id ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc %lu - Closed\n", my_rank, version );
+
+ /* Close H5 Objects that are still open */
+ fprintf( stderr, "APP-r%d: Closing %s\n", my_rank, file_name );
+ MPI_Barrier( MPI_COMM_WORLD );
+ ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Pclose( fapl_id ); ASSERT_RET;
+
+ /* Perform wrap-up operations */
+ fprintf( stderr, "APP-r%d: Finalize EFF stack\n", my_rank );
+ MPI_Barrier( MPI_COMM_WORLD );
+ EFF_finalize();
+ MPI_Finalize();
+
+ return 0;
+}
+
+/*
+ * Helper function to keep track of timing info
+ */
+void
+fill_stats( int* cnt, ulong* min, ulong* max, ulong* sum, ulong elapsed )
+{
+ if ( *min == 0 ) {
+ *min = elapsed;
+ *max = elapsed;
+ } else {
+ if ( elapsed < *min ) {
+ *min = elapsed;
+ }
+ if ( elapsed > *max ) {
+ *max = elapsed;
+ }
+ }
+ *sum += elapsed;
+ *cnt += 1;
+}
+
+/*
+ * Helper function used to recursively read and print container contents
+ * for container identified by "file_id"
+ * in read context identified by "rc_id"
+ * "my_rank" used to identify the process doing the reading / printing.
+ */
+void
+print_container_contents( hid_t file_id, hid_t rc_id, int my_rank )
+{
+ herr_t ret;
+ uint64_t cv;
+ htri_t exists;
+ char path_to_object[1024];
+ char name[30];
+ int i, d;
+
+ char preface[128];
+
+ /* Get the container version for the read context */
+ ret = H5RCget_version( rc_id, &cv ); ASSERT_RET;
+
+ /* Set up the preface and adding version number */
+ sprintf( preface, "APP-r%d: cv %d: ", my_rank, (int)cv );
+
+ /* Start the printing */
+ fprintf( stderr, "%s ----- Container Contents ------------\n", preface );
+
+ for ( d = 0; d < 3; d++ ) {
+ if ( d == 0 ) {
+ sprintf( path_to_object, "%s", "/DL" );
+ } else if ( d == 1 ) {
+ sprintf( path_to_object, "%s", "/DP" );
+ } else if ( d == 2 ) {
+ sprintf( path_to_object, "%s", "/DS" );
+ }
+
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+
+ if ( exists ) {
+ hid_t dset_id;
+ hid_t space_id;
+ int nDims;
+ hsize_t current_size[2];
+ hsize_t max_size[2];
+ hsize_t totalSize;
+ hid_t dxpl_id;
+ uint64_t *data;
+ uint64_t u, e;
+
+ dset_id = H5Dopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( dset_id >= 0 );
+
+ space_id = H5Dget_space( dset_id ); assert ( space_id >= 0 );
+ nDims = H5Sget_simple_extent_dims( space_id, current_size, max_size );
+ assert( nDims == 2 );
+
+ totalSize = current_size[0] * current_size[1];
+ data = (uint64_t *)calloc( totalSize, sizeof(uint64_t) ); assert( data != NULL );
+
+ dxpl_id = H5Pcreate( H5P_DATASET_XFER );
+ if ( enable_checksums ) {
+ ret = H5Pset_rawdata_integrity_scope( dxpl_id, H5_CHECKSUM_ALL ); ASSERT_RET;
+ } else {
+ ret = H5Pset_rawdata_integrity_scope( dxpl_id, H5_CHECKSUM_NONE ); ASSERT_RET;
+ }
+
+ ret = H5Dread_ff( dset_id, H5T_NATIVE_UINT64, space_id, space_id, dxpl_id, data, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+
+ /* note that by printing piece-by-piece, other output may be interspersed. trade-off since won't know line size */
+ fprintf( stderr, "%s %s data:\n", preface, path_to_object );
+ e = 0;
+ for ( i = 0; i < totalSize; i++ ) {
+ if ( e == 0 ) fprintf( stderr, "%s\t", preface );
+ fprintf( stderr, "%09lu ", data[i] );
+ if ( ++e == cols_per_row ) {
+ fprintf( stderr, "\n" );
+ e = 0;
+ }
+ }
+
+ free( data );
+ sleep( 1 ); /* give output a chance to be printed before diag messages from lower layers on close */
+ ret = H5Pclose( dxpl_id ); ASSERT_RET;
+ ret = H5Sclose( space_id ); ASSERT_RET;
+ ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ }
+ }
+
+ /* End printing */
+ fprintf( stderr, "%s -----------------\n", preface );
+
+ return;
+}
+
+/*
+ * parse_options - parse command line options
+ */
+int
+parse_options( int argc, char** argv, int my_rank, int comm_size ) {
+ int i, n;
+ char* app = argv[0];
+
+ while ( --argc ) {
+ if ( **(++argv) != '-' ) {
+ break;
+ } else {
+ switch( *(*argv+1) ) {
+ case 'c':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "Error: No number specified after -c option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ cols_per_row = atoi( *argv );
+ }
+ break;
+ case 'w':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "Error: No number specified after -w option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ rows_per_write = atoi( *argv );
+ }
+ break;
+ case 'r':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "Error: No number specified after -r option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ rows_per_read = atoi( *argv );
+ }
+ break;
+ case 'b':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "Error: No number specified after -b option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ rows_per_rank = atoi( *argv );
+ }
+ break;
+ case 'i':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "APP-r0: Error: No number specified after -i option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ num_iterations = atoi( *argv );
+ }
+ break;
+ case 'p':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "APP-r0: Error: No number specified after -p option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ persist_rate = atoi( *argv );
+ }
+ break;
+ case 'e':
+ enable_checksums = 1;
+ break;
+ case 'L':
+ logged_dset = 1;
+ break;
+ case 'P':
+ prefetched_dset = 1;
+ break;
+ case 'S':
+ stored_dset = 1;
+ break;
+ case 'd':
+ detailed_timing = 1;
+ break;
+ case 'v':
+ verbose = 1;
+ break;
+ default:
+ if ( my_rank == 0 ) {
+ usage( app );
+ }
+ return( 1 );
+ }
+ }
+ }
+ return( 0 );
+}
+
+/*
+ * usage - display usage message
+ */
+void
+usage( const char* app ) {
+ printf( "Usage: %s [-c cols_per_row] [-w rows_per_write] [-r rows_per_read] [-b rows_per_rank] [-i num_iterations] [-p persist_rate] [-e] [-L] [-P] [-S] [-d] [-v] \n", app );
+ printf( "\tc: number of columns per row (each cell is 8 bytes)\n" );
+ printf( "\tw: number of rows per write per rank\n" );
+ printf( "\tr: number of rows per read per rank\n" );
+ printf( "\tb: number of rows in each rank's block (if not multiple of w or r, may not be fully written or read)\n" );
+ printf( "\ti: number of iterations to do writes/commit/[persist/evict/prefetch/reads/evict]\n" );
+ printf( "\tp: persist rate; for /DP and /DS, also impacts evict/prefetch/read/evict\n" );
+ printf( "\te: enable checksums on raw data in H5Datasets\n" );
+ printf( "\tL: create /DL dataset - data will not be evicted (created by default if no /DP nor /DS)\n" );
+ printf( "\tP: create /DP dataset - data will be evicted, replica prefetched, reads from replica\n" );
+ printf( "\tS: create /DS dataset - data will be evicted, reads from storage (DAOS)\n" );
+ printf( "\td: print detailed timing information for each read and write operation\n" );
+ printf( "\tv: verbose output of data values - intended for use only with small arrays and few iterations\n" );
+}
diff --git a/examples/h5ff_client_timings.c b/examples/h5ff_client_timings.c
index a30bc0d..89dda02 100644
--- a/examples/h5ff_client_timings.c
+++ b/examples/h5ff_client_timings.c
@@ -1,1116 +1,1116 @@
-/*
- * h5ff_timings.c: Do some timing of various write / persist / prefetch / read
- */
-/*
- *
- * Creates 3 groups, each with a dataset & a map
- * In /G-logged, objects will read.
- * In /G-prefetched, objects will be evicted & prefetched & read & evicted
- * In /G-stored, objects will be evicted & read
- * D = dataset; 2d with uint64 datatypes. size is [ (rows_per_rank*#ranks), (cols_per_row) ]
- * M = map; uint64 keys ranging from 0 to (rows_per_rank*#ranks*cols_per_row)-1
- * uint64 values, set to same as dset entries
- * each rank updates "its" dataset entries & map pairs, based on rows_per_rank variable
- *
- *
- * Notes for possible future modifications:
- * 1) Make Datasets 1D, with entries 2D vectors to mimic ACG Edge List representation
- * 2) Make Maps have values that are vectors of uint64, to mimic ACG Adjacency list representation (step 1)
- * 3) Make Maps have values that variable-length vectors of uint64, to mimic ACG Adjaceny list represenations (really)
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include <sys/time.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-/* define filename for this app, and max size after username prepended */
-#define FILENAME_APP "eff_timings.h5"
-#define FILENAME_SIZE 256
-
-/* define default sizing parameters - can be changed via command line arguments */
-#define DEFAULT_ROWS_PER_RANK 10
-#define DEFAULT_COLS_PER_ROW 5
-
-/* macros related to error reporting */
-#define STATUS (ret >= 0) ? " " : " - FAILED"
-#define ASSERT_RET assert( ret >= 0 )
-
-/* option flags */
-int cols_per_row = DEFAULT_COLS_PER_ROW;
-int rows_per_rank = DEFAULT_ROWS_PER_RANK;
-int num_iterations = 1; /* Number of times to perform the commit/persist/evict/prefetch/read/evict cycle */
-int verbose = 0; /* Verbose output - defaults to no */
-int time_reads = 0; /* Time reads after container re-open */
-
-/* global variables and macros used to make timing easier */
-struct timeval tv_start;
-struct timeval tv_end;
-#define START_TIME gettimeofday( &tv_start, NULL )
-#define END_TIME gettimeofday( &tv_end, NULL )
-#define ELAPSED_TIME (ulong)( (tv_end.tv_usec + 1000000*tv_end.tv_sec) - (tv_start.tv_usec + 1000000*tv_start.tv_sec) )
-
-/* function prototypes */
-void print_container_contents( hid_t, hid_t, const char*, int );
-int parse_options( int, char**, int, int );
-void usage( const char* );
-
-/**************************************************************************************************************/
-int main( int argc, char **argv ) {
-
- /* MPI */
- int my_rank, comm_size;
- int provided;
-
- /* Container */
- char *user_name; /* We'll prepend username to make filename unique */
- char file_name[FILENAME_SIZE]; /* Actual filename to be opened will be user_name + FILENAME_APP */
- hid_t fapl_id;
- hid_t file_id;
-
- /* Groups */
- hid_t grp_l_id;
- hid_t grp_p_id;
- hid_t grp_s_id;
-
- /* Dataspaces */
- hid_t space_l_id;
- hid_t space_p_id;
- hid_t space_s_id;
-
- /* Datasets */
- hid_t dset_l_id;
- hid_t dset_p_id;
- hid_t dset_s_id;
- hsize_t dset_size[2];
- hid_t space_id;
-
- /* Maps */
- hid_t map_l_id;
- hid_t map_p_id;
- hid_t map_s_id;
-
- /* Replicas and related properties */
- hrpl_t dset_p_replica;
- hid_t dxpl_p_id;
- hrpl_t map_p_replica;
- hid_t mxpl_p_id;
-
- /* Container Version & Read Contexts */
- uint64_t version, versionH;
- hid_t rc_id, rc_idH;
-
- /* Transactions */
- uint64_t tr_num;
- hid_t tr_id;
- hid_t trspl_id;
- int num_tr_leaders;
-
- /* Memory buffer - sized to hold complete DS or Map Values; sometimes partially used */
- uint64_t *mbuf;
-
- /* Sizing, dataspace, hyperslab parameters set per-rank */
- hsize_t rank_mbuf_size[1];
- hsize_t rank_offset[2], rank_count[2];
- hid_t rank_space_id;
-
- /* Variables used for reading data written by a neighbor rank */
- int neighbor_rank;
- hid_t neighbor_space_l_id;
- hid_t neighbor_space_p_id;
- hid_t neighbor_space_s_id;
- hsize_t neighbor_offset[2], neighbor_count[2];
-
- /* Misc */
- herr_t ret;
- uint64_t u, r, c;
- uint64_t key;
- int bytesPerCell;
- int i;
- int iteration;
-
- /****
- * Initialize and create container
- ****/
-
- /* Confirm needed MPI functionality is available */
- MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
- if ( MPI_THREAD_MULTIPLE != provided ) {
- fprintf( stderr, "APP: ERROR: MPI does not have MPI_THREAD_MULTIPLE support\n" );
- exit( 1 );
- }
-
- /* Find MPI rank & communicator size */
- MPI_Comm_rank( MPI_COMM_WORLD, &my_rank );
- MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
- if ( my_rank == 0 ) {
- fprintf( stderr, "APP-r%d: Number of MPI processes = %d\n", my_rank, comm_size );
- }
-
- /* Parse command-line options controlling behavior */
- if ( parse_options( argc, argv, my_rank, comm_size ) != 0 ) {
- exit( 1 );
- }
- if ( my_rank == 0 ) {
- fprintf( stderr, "APP-r0: Number of MPI processes = %d\n", comm_size );
- fprintf( stderr, "APP-r0: Datasets will have %d rows (%d per rank).\n", (rows_per_rank*comm_size), rows_per_rank );
- fprintf( stderr, "APP-r0: Datasets will have %d columns.\n", cols_per_row );
- fprintf( stderr, "APP-r0: There will be %d iterations.\n", num_iterations );
- }
-
- /* Initialize the EFF stack, starting FS client, registering HDF5 VOL calls, requesting IOD init */
- fprintf( stderr, "APP-r%d: Initialize EFF stack\n", my_rank );
- EFF_init( MPI_COMM_WORLD, MPI_INFO_NULL );
-
- /* Specify the IOD VOL plugin should be used and create H5File (EFF container), with user name prepended */
- user_name = getenv( "USER" );
- snprintf( file_name, FILENAME_SIZE, "%s_%s", user_name, FILENAME_APP );
- fprintf( stderr, "APP-r%d: Create %s\n", my_rank, file_name );
- fapl_id = H5Pcreate( H5P_FILE_ACCESS ); assert( fapl_id >= 0 );
- ret = H5Pset_fapl_iod( fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL ); ASSERT_RET;
- file_id = H5Fcreate_ff( file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL ); assert( file_id >= 0 );
-
- /* Create the memory buffer that will be used in many places - both for writing and reading */
- bytesPerCell = sizeof( uint64_t );
- mbuf = (uint64_t *) calloc( (rows_per_rank * comm_size * cols_per_row), bytesPerCell ); assert( mbuf != NULL );
-
- /****
- * Transaction 2: Rank 0 creates H5Objects in the container
- ****/
-
- if ( my_rank == 0 ) {
-
- /* Acquire read context for CV 1 */
- version = 1;
- rc_id = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); assert( rc_id >= 0); assert( version == 1 );
- fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
-
- /* Start a transaction with a single leader (the default) */
- tr_num = 2;
- fprintf( stderr, "APP-r%d: tr %lu - Start\n", my_rank, tr_num );
- tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
- ret = H5TRstart( tr_id, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Add updates to the transaction for Group creates */
- fprintf( stderr, "APP-r%d: tr %d - Create /G-logged /G-prefetched /G-stored\n", my_rank, tr_num );
-
- grp_l_id = H5Gcreate_ff( file_id, "G-logged", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- assert( grp_l_id >= 0 );
-
- grp_p_id = H5Gcreate_ff( file_id, "G-prefetched", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- assert( grp_p_id >= 0 );
-
- grp_s_id = H5Gcreate_ff( file_id, "G-stored", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- assert( grp_s_id >= 0 );
-
- /* Create Dataspace for Datasets */
- dset_size[0] = rows_per_rank * comm_size;
- dset_size[1] = cols_per_row;
- space_id = H5Screate_simple( 2, dset_size, dset_size ); assert( space_id >= 0 );
-
- /* Add updates to the transaction for Dataset creates */
- fprintf( stderr, "APP-r%d: tr %d - Create /G-logged/D /G-prefetched/D /G-stored/D\n", my_rank, tr_num );
-
- dset_l_id = H5Dcreate_ff( grp_l_id, "D", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL ); assert( dset_l_id >= 0 );
-
- dset_p_id = H5Dcreate_ff( grp_p_id, "D", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL ); assert( dset_p_id >= 0 );
-
- dset_s_id = H5Dcreate_ff( grp_s_id, "D", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL ); assert( dset_s_id >= 0 );
-
- ret = H5Sclose( space_id ); ASSERT_RET;
-
- /* Add updates to the transaction for Map creates */
- fprintf( stderr, "APP-r%d: tr %d - Create /G-logged/M /G-prefetched/M /G-stored/M\n", my_rank, tr_num );
-
- map_l_id = H5Mcreate_ff( grp_l_id, "M", H5T_NATIVE_UINT, H5T_NATIVE_UINT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL ); assert( map_l_id >= 0 );
-
- map_p_id = H5Mcreate_ff( grp_p_id, "M", H5T_NATIVE_UINT, H5T_NATIVE_UINT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL ); assert( map_p_id >= 0 );
-
- map_s_id = H5Mcreate_ff( grp_s_id, "M", H5T_NATIVE_UINT, H5T_NATIVE_UINT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
- H5_EVENT_STACK_NULL ); assert( map_s_id >= 0 );
-
- /* Finish and commit TR and get RC for it */
- ret = H5TRfinish( tr_id, H5P_DEFAULT, &rc_idH, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5TRclose( tr_id ); ASSERT_RET;
- fprintf( stderr, "APP-r%d: tr %lu - Finished\n", my_rank, tr_num );
-
- assert( rc_idH >= 0 );
- versionH = tr_num;
- fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, versionH );
-
- /* Release the read handle and close read context on earlier CV */
- ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
- ret = H5RCclose( rc_id ); ASSERT_RET;
- fprintf( stderr, "APP-r%d: rc %lu - Released and Closed\n", my_rank, version );
-
- /* The just-acquired RC is now the primary rc_id we'll work with (for awhile) */
- rc_id = rc_idH;
- version = versionH;
- }
-
- /* Rank 0 notifies other ranks that transaction 2 committed and RC acquired */
- MPI_Bcast( &version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
-
- /****
- * Read Context 2: Other ranks create RC and open HObjects created by rank 0.
- ****/
- if ( my_rank != 0 ) {
- rc_id = H5RCcreate( file_id, version );
- assert( rc_id >= 0 ); assert ( version == 2 );
- fprintf( stderr, "APP-r%d: rc %lu - Created for rank\n", my_rank, version );
-
- grp_l_id = H5Gopen_ff( file_id, "G-logged", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( grp_l_id >= 0 );
- grp_p_id = H5Gopen_ff( file_id, "G-prefetched", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( grp_p_id >= 0 );
- grp_s_id = H5Gopen_ff( file_id, "G-stored", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( grp_s_id >= 0 );
-
- dset_l_id = H5Dopen_ff( grp_l_id, "D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_l_id >= 0 );
- dset_p_id = H5Dopen_ff( grp_p_id, "D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_p_id >= 0 );
- dset_s_id = H5Dopen_ff( grp_s_id, "D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_s_id >= 0 );
-
- map_l_id = H5Mopen_ff( grp_l_id, "M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( map_l_id >= 0 );
- map_p_id = H5Mopen_ff( grp_p_id, "M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( map_p_id >= 0 );
- map_s_id = H5Mopen_ff( grp_s_id, "M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( map_s_id >= 0 );
- } else {
- if ( verbose ) print_container_contents( file_id, rc_id, "/", my_rank );
- }
-
- /* Keep track of my_rank's neighbor */
- neighbor_rank = my_rank + 1;
- if ( neighbor_rank == comm_size ) {
- neighbor_rank = 0;
- }
-
- /* Get the dataspaces for the three datasets and make a copy of each */
- space_l_id = H5Dget_space( dset_l_id ); assert ( space_l_id >= 0 );
- space_p_id = H5Dget_space( dset_p_id ); assert ( space_p_id >= 0 );
- space_s_id = H5Dget_space( dset_s_id ); assert ( space_s_id >= 0 );
-
- neighbor_space_l_id = H5Scopy( space_l_id );
- neighbor_space_p_id = H5Scopy( space_p_id );
- neighbor_space_s_id = H5Scopy( space_s_id );
-
- /* Select the hyperslab that this rank will update and the hyperslab for reading neighbor's data */
- rank_offset[0] = my_rank * rows_per_rank;
- rank_offset[1] = 0;
- rank_count[0] = rows_per_rank;
- rank_count[1] = cols_per_row;
-
- neighbor_offset[0] = neighbor_rank * rows_per_rank;
- neighbor_offset[1] = 0;
- neighbor_count[0] = rows_per_rank;
- neighbor_count[1] = cols_per_row;
-
- ret = H5Sselect_hyperslab( space_l_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( space_p_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( space_s_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
-
- ret = H5Sselect_hyperslab( neighbor_space_l_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( neighbor_space_p_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( neighbor_space_s_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL ); ASSERT_RET;
-
- /* Create the memory dataspace for a rank - always first part of memory buffer */
- rank_mbuf_size[0] = rows_per_rank * cols_per_row;
- rank_space_id = H5Screate_simple( 1, rank_mbuf_size, rank_mbuf_size ); assert( rank_space_id >= 0 );
-
- /* Create property lists that will be used */
- dxpl_p_id = H5Pcreate( H5P_DATASET_XFER );
- mxpl_p_id = H5Pcreate( H5P_DATASET_XFER );
-
- for ( iteration = 0; iteration < num_iterations; iteration++ ) {
-
- /****
- * Cycle of:
- * Tranasction N Start / Update / Commit: All ranks update Dataset and Map objects in the 3 Groups then commit
- * CV N Persist: Rank 0
- * Prefetch: Rank 0
- *
- ****/
- tr_num = tr_num + 1;
- num_tr_leaders = comm_size;
-
- /* Create a transaction and start it. */
- fprintf( stderr, "APP-r%d: iter %d tr %lu - Transaction Start with %d leaders\n",
- my_rank, iteration, tr_num, num_tr_leaders );
- tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
- trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
- ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
- ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Set cell values (in memory) for this rank based on Row, Rank, Column, Transaction */
- u = 0;
- for ( r = 0; r < rows_per_rank; r++ ) {
- for ( c = 0; c < cols_per_row; c++ ) {
- mbuf[u] = (my_rank*rows_per_rank+r)*1000000 + c*100 + tr_num;
- u++;
- }
- }
-
- fprintf( stderr, "APP-r%d: iter %d tr %lu - Add updates to transaction.\n", my_rank, iteration, tr_num );
-
- /* Add dataset updates to transaction */
- START_TIME;
- ret = H5Dwrite_ff( dset_l_id, H5T_NATIVE_UINT64, rank_space_id, space_l_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to add updates (%lu bytes) for /G-logged/D: %lu usec\n",
- my_rank, iteration, u*bytesPerCell, ELAPSED_TIME );
-
- START_TIME;
- ret = H5Dwrite_ff( dset_p_id, H5T_NATIVE_UINT64, rank_space_id, space_p_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to add updates (%lu bytes) for /G-prefetched/D: %lu usec\n",
- my_rank, iteration, u*bytesPerCell, ELAPSED_TIME );
-
- START_TIME;
- ret = H5Dwrite_ff( dset_s_id, H5T_NATIVE_UINT64, rank_space_id, space_s_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to add updates (%lu bytes) for /G-stored/D: %lu usec\n",
- my_rank, iteration, u*bytesPerCell, ELAPSED_TIME );
-
- /* Add map updates to transaction; do it map-by-map so we can do timing */
- key = rows_per_rank * cols_per_row * my_rank;
- START_TIME;
- for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
- ret = H5Mset_ff( map_l_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- key++;
- }
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to add updates ( %lu K/V pairs - %lu bytes) for /G-logged/M: %lu usec\n",
- my_rank, iteration, u, u*bytesPerCell*2, ELAPSED_TIME );
-
- key = rows_per_rank * cols_per_row * my_rank;
- START_TIME;
- for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
- ret = H5Mset_ff( map_p_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- key++;
- }
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to add updates ( %lu K/V pairs - %lu bytes) for /G-prefetched/M: %lu usec\n",
- my_rank, iteration, u, u*bytesPerCell*2, ELAPSED_TIME );
-
- key = rows_per_rank * cols_per_row * my_rank;
- START_TIME;
- for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
- ret = H5Mset_ff( map_s_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- key++;
- }
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to add updates ( %lu K/V pairs - %lu bytes) for /G-stored/M: %lu usec\n",
- my_rank, iteration, u, u*bytesPerCell*2, ELAPSED_TIME );
-
- /* Finish, (commit), and close transaction */
- fprintf( stderr, "APP-r%d: iter %d tr %lu - Finish and Commit\n", my_rank, iteration, tr_num );
- ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5TRclose( tr_id ); ASSERT_RET;
- ret = H5Pclose( trspl_id ); ASSERT_RET;
-
- /* After commit completes on all ranks, acquire RC on rank 0 and create local handle for RC on other ranks */
- MPI_Barrier( MPI_COMM_WORLD );
- versionH = tr_num;
- if ( my_rank == 0 ) {
- rc_idH = H5RCacquire( file_id, &versionH, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- assert( rc_idH >= 0 ); assert ( versionH == tr_num );
- fprintf( stderr, "APP-r%d: iter %d rc %lu - Acquired\n", my_rank, iteration, versionH );
- } else {
- rc_idH = H5RCcreate( file_id, versionH );
- assert( rc_idH >= 0 ); assert ( versionH == tr_num );
- fprintf( stderr, "APP-r%d: iter %d rc %lu - Created\n", my_rank, iteration, versionH );
- }
- MPI_Barrier( MPI_COMM_WORLD ); /* Wait to make sure rank 0 has acquired from IOD before proceeding */
-
- /* Release previous Read Context */
- if ( my_rank == 0 ) {
- ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
- fprintf( stderr, "APP-r%d: iter %d rc %lu - Released\n", my_rank, iteration, version );
- }
- ret = H5RCclose( rc_id ); ASSERT_RET;
- fprintf( stderr, "APP-r%d: iter %d rc %lu - Closed\n", my_rank, iteration, version );
-
- /* The just-acquired RC is now the primary rc_id we'll work with (for awhile) */
- rc_id = rc_idH;
- version = versionH;
-
- /* Rank 0:
- - persists the container
- - evicts D and M under /G-prefetched and /G-stored
- - prefetches D and M under /G-prefetched
- */
- if ( my_rank == 0 ) {
- if ( verbose ) print_container_contents( file_id, rc_id, "/", my_rank );
-
- START_TIME;
- ret = H5RCpersist( rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to Persist container: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
-
- START_TIME;
- ret = H5Devict_ff( dset_p_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to Evict /G-prefetched/D: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
-
- START_TIME;
- ret = H5Devict_ff( dset_s_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to Evict /G-stored/D: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
-
- START_TIME;
- ret = H5Mevict_ff( map_p_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to Evict /G-prefetched/M (currently no-op for IOD): %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
-
- START_TIME;
- ret = H5Mevict_ff( map_s_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to Evict /G-stored/M: (currently no-op for IOD) %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
-
- START_TIME;
- ret = H5Dprefetch_ff( dset_p_id, rc_id, &dset_p_replica, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to Prefetch /G-prefetched/D: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
-
- START_TIME;
- ret = H5Mprefetch_ff( map_p_id, rc_id, &map_p_replica, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to Prefetch /G-prefetched/M: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
- }
-
- /* This Bcast will make sure no nodes go ahead to use the RC before rank 0 has acquired it */
- MPI_Bcast( &dset_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
- ret = H5Pset_dxpl_replica( dxpl_p_id, dset_p_replica ); ASSERT_RET;
-
- MPI_Bcast( &map_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
- ret = H5Pset_dxpl_replica( mxpl_p_id, map_p_replica ); ASSERT_RET;
-
- MPI_Barrier( MPI_COMM_WORLD ); /* Make sure all are here before continuing */
-
- /*
- * All ranks read the same values they updated in last transaction
- */
-
- /* Datasets */
- START_TIME;
- ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, rank_space_id, space_l_id, H5P_DEFAULT, mbuf, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-logged/D: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: my first updated value in /G-logged/D: %09lu\n", my_rank, mbuf[0] );
- }
-
- START_TIME;
- ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, rank_space_id, space_p_id, dxpl_p_id, mbuf, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-prefetched/D: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: my first updated value in /G-prefetched/D: %09lu\n", my_rank, mbuf[0] );
- }
-
- START_TIME;
- ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, rank_space_id, space_s_id, H5P_DEFAULT, mbuf, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-stored/D: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: my first updated value in /G-stored/D: %09lu\n", my_rank, mbuf[0] );
- }
-
- /* Maps */
- key = rows_per_rank * cols_per_row * my_rank;
- START_TIME;
- for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
- ret = H5Mget_ff( map_l_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- key++;
- }
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-logged/M: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: my first updated value in /G-logged/M: %09lu\n", my_rank, mbuf[0] );
- }
-
- key = rows_per_rank * cols_per_row * my_rank;
- START_TIME;
- for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
- ret = H5Mget_ff( map_p_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- key++;
- }
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-prefetched/M: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose )
- { fprintf( stderr, "APP-r%d: my first updated value in /G-prefetched/M: %09lu\n", my_rank, mbuf[0] );
- }
-
- key = rows_per_rank * cols_per_row * my_rank;
- START_TIME;
- for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
- ret = H5Mget_ff( map_s_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- key++;
- }
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-stored/M: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: my first updated value in /G-stored/M: %09lu\n", my_rank, mbuf[0] );
- }
-
- /*
- * All ranks read the values "next rank" neighbor updated in last transaction
- */
-
- /* Datasets */
- START_TIME;
- ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, rank_space_id, neighbor_space_l_id, H5P_DEFAULT, mbuf, rc_id,
- H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-logged/D: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-logged/D: %09lu\n", my_rank, mbuf[0] );
- }
-
- START_TIME;
- ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, rank_space_id, neighbor_space_p_id, dxpl_p_id, mbuf, rc_id,
- H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-prefetched/D: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-prefetched/D: %09lu\n", my_rank, mbuf[0]);
- }
-
- START_TIME;
- ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, rank_space_id, neighbor_space_s_id, H5P_DEFAULT, mbuf, rc_id,
- H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-stored/D: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-stored/D: %09lu\n", my_rank, mbuf[0] );
- }
-
- /* Maps */
- key = rows_per_rank * cols_per_row * neighbor_rank;
- START_TIME;
- for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
- ret = H5Mget_ff( map_l_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- key++;
- }
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-logged/M: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-logged/M: %09lu\n", my_rank, mbuf[0] );
- }
-
- key = rows_per_rank * cols_per_row * neighbor_rank;
- START_TIME;
- for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
- ret = H5Mget_ff( map_p_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- key++;
- }
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-prefetched/M: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-prefetched/M: %09lu\n", my_rank, mbuf[0] );
- }
-
- key = rows_per_rank * cols_per_row * neighbor_rank;
- START_TIME;
- for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
- ret = H5Mget_ff( map_s_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- key++;
- }
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-stored/M: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-stored/M: %09lu\n", my_rank, mbuf[0] );
- }
-
- /*
- * All ranks read all the values
- */
-
- /* Datasets */
- START_TIME;
- ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, mbuf, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-logged/D: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: first value in /G-logged/D: %09lu\n", my_rank, mbuf[0] );
- }
-
- START_TIME;
- ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, dxpl_p_id, mbuf, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-prefetched/D: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: first value in /G-prefetched/D: %09lu\n", my_rank, mbuf[0] );
- }
-
- START_TIME;
- ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, mbuf, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-stored/D: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: first value in /G-stored/D: %09lu\n", my_rank, mbuf[0] );
- }
-
- /* Maps */
- START_TIME;
- for ( u = 0; u < rows_per_rank * cols_per_row * comm_size; u++ ) {
- ret = H5Mget_ff( map_l_id, H5T_NATIVE_UINT64, &u, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- }
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-logged/M: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: first value in /G-logged/M: %09lu\n", my_rank, mbuf[0] );
- }
-
- START_TIME;
- for ( u = 0; u < rows_per_rank * cols_per_row * comm_size; u++ ) {
- ret = H5Mget_ff( map_p_id, H5T_NATIVE_UINT64, &u, H5T_NATIVE_UINT64, &mbuf[u], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- }
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-prefetched/M: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: first value in /G-prefetched/M: %09lu\n", my_rank, mbuf[0] );
- }
-
- START_TIME;
- for ( u = 0; u < rows_per_rank * cols_per_row * comm_size; u++ ) {
- ret = H5Mget_ff( map_s_id, H5T_NATIVE_UINT64, &u, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- }
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-stored/M: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- if ( verbose ) {
- fprintf( stderr, "APP-r%d: first value in /G-stored/M: %09lu\n", my_rank, mbuf[0] );
- }
-
-
- /* Rank 0 evicts the replicas */
- MPI_Barrier( MPI_COMM_WORLD ); /* Make sure all ranks done reading replicas before evict */
- if ( my_rank == 0 ) {
- START_TIME;
- ret = H5Devict_ff( dset_p_id, version, dxpl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to Evict Replica of /G-prefetched/D: %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
-
- START_TIME;
- ret = H5Mevict_ff( map_p_id, version, mxpl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- END_TIME;
- fprintf( stderr, "APP-r%d: iter %d - Time to Evict Replica of /G-prefetched/M (currently no-op for IOD): %lu usec\n",
- my_rank, iteration, ELAPSED_TIME );
- }
-
- }
-
- /*
- * Wrap-up
- */
-
- /* Print (if verbose) the final state */
- if ( my_rank == 0 ) {
- if ( verbose ) print_container_contents( file_id, rc_id, "/", my_rank );
- }
-
- /* Close the dataset and map property lists */
- ret = H5Pclose( dxpl_p_id ); ASSERT_RET;
- ret = H5Pclose( mxpl_p_id ); ASSERT_RET;
-
- /* Close the memory dataspace for the rank */
- ret = H5Sclose( rank_space_id ); ASSERT_RET;
-
- fprintf( stderr, "APP-r%d: Closing H5Objects\n", my_rank );
-
- ret = H5Dclose_ff( dset_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Dclose_ff( dset_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Dclose_ff( dset_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- ret = H5Sclose( space_l_id ); ASSERT_RET;
- ret = H5Sclose( space_p_id ); ASSERT_RET;
- ret = H5Sclose( space_s_id ); ASSERT_RET;
-
- ret = H5Mclose_ff( map_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Mclose_ff( map_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Mclose_ff( map_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- ret = H5Gclose_ff( grp_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Gclose_ff( grp_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Gclose_ff( grp_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Release the read handle and close read context */
- MPI_Barrier( MPI_COMM_WORLD ); /* Make sure all ranks done with RC */
- if ( my_rank == 0 ) {
- ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
- fprintf( stderr, "APP-r%d: rc %lu - Released\n", my_rank, version );
- }
- ret = H5RCclose( rc_id ); ASSERT_RET;
- fprintf( stderr, "APP-r%d: rc %lu - Closed\n", my_rank, version );
-
- /* Close H5 Objects that are still open */
- fprintf( stderr, "APP-r%d: Closing %s\n", my_rank, file_name );
- MPI_Barrier( MPI_COMM_WORLD );
- ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Pclose( fapl_id ); ASSERT_RET;
-
- /* Perform wrap-up operations */
- fprintf( stderr, "APP-r%d: Finalize EFF stack\n", my_rank );
- MPI_Barrier( MPI_COMM_WORLD );
- EFF_finalize();
- MPI_Finalize();
-
- return 0;
-}
-
-
-/*
- * Helper function used to recursively read and print container contents
- * for container identified by "file_id"
- * in read context identified by "rc_id"
- * with path to current level in "grp_path"
- * and "my_rank" used to identify the process doing the reading / printing.
- */
-void
-print_container_contents( hid_t file_id, hid_t rc_id, const char* grp_path, int my_rank )
-{
- herr_t ret;
- uint64_t cv;
- htri_t exists;
- char path_to_object[1024];
- char name[30];
- int i;
-
- static int lvl = 0; /* level in recursion - used to format printing */
- char preface[128];
-
- /* Get the container version for the read context */
- ret = H5RCget_version( rc_id, &cv ); ASSERT_RET;
-
- /* Set up the preface and adding version number */
- sprintf( preface, "APP-r%d: cv %d: ", my_rank, (int)cv );
-
- /* Start the printing */
- if ( lvl == 0 ) {
- fprintf( stderr, "%s ----- Container Contents ------------\n", preface );
- }
-
- /* Datasets */
- sprintf( path_to_object, "%s%s", grp_path, "D" );
- ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
-
- if ( exists ) {
- hid_t dset_id;
- hid_t space_id;
- int nDims;
- hsize_t current_size[2];
- hsize_t max_size[2];
- hsize_t totalSize;
- uint64_t *data;
- uint64_t u, e;
-
- dset_id = H5Dopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( dset_id >= 0 );
-
- space_id = H5Dget_space( dset_id ); assert ( space_id >= 0 );
- nDims = H5Sget_simple_extent_dims( space_id, current_size, max_size );
- assert( nDims == 2 );
-
- totalSize = current_size[0] * current_size[1];
- data = (uint64_t *)calloc( totalSize, sizeof(uint64_t) ); assert( data != NULL );
-
- ret = H5Dread_ff( dset_id, H5T_NATIVE_UINT64, space_id, space_id, H5P_DEFAULT, data, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* note that by printing piece-by-piece, other output may be interspersed. trade-off since won't know line size */
- fprintf( stderr, "%s %s data:\n", preface, path_to_object );
- e = 0;
- for ( i = 0; i < totalSize; i++ ) {
- if ( e == 0 ) fprintf( stderr, "%s\t", preface );
- fprintf( stderr, "%09lu ", data[i] );
- if ( ++e == cols_per_row ) {
- fprintf( stderr, "\n" );
- e = 0;
- }
- }
-
- free( data );
- sleep( 1 ); /* give output a chance to be printed before diag messages from lower layers on close */
- ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5Sclose( space_id ); ASSERT_RET;
- }
-
- /* Maps */
- sprintf( path_to_object, "%s%s", grp_path, "M" );
- ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
-
- if ( exists ) {
- hid_t map_id;
- hsize_t totalCount;
- uint64_t *value;
- uint64_t u, e;
-
- map_id = H5Mopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- assert( map_id >= 0 );
-
- ret = H5Mget_count_ff( map_id, &totalCount, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- value = (uint64_t *)calloc( totalCount, sizeof(uint64_t) ); assert( value != NULL );
-
- for ( u = 0; u < totalCount; u++ ) {
- ret = H5Mget_ff( map_id, H5T_NATIVE_UINT64, &u, H5T_NATIVE_UINT64, &value[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- }
-
- /* note that by printing piece-by-piece, other output may be interspersed. trade-off since won't know line size */
- fprintf( stderr, "%s %s key/values:\n", preface, path_to_object );
- e = 0;
- for ( u = 0; u < totalCount; u++ ) {
- if ( e == 0 ) fprintf( stderr, "%s\t", preface );
- fprintf( stderr, "%lu/%09lu ", u, value[u] );
- if ( ++e == cols_per_row ) {
- fprintf( stderr, "\n" );
- e = 0;
- }
- }
-
- free( value );
- sleep( 1 ); /* give output a chance to be printed before diag messages from lower layers on close */
- ret = H5Mclose_ff( map_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
- }
-
- /* Groups - if found, descend */
- for ( i = 1; i < 4; i++ ) {
- if ( i == 1 ) {
- strcpy( name, "G-logged" );
- } else if ( i == 2 ) {
- strcpy( name, "G-prefetched" );
- } else if ( i == 3 ) {
- strcpy( name, "G-stored" );
- }
-
- sprintf( path_to_object, "%s%s/", grp_path, name );
- ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
- if ( exists ) {
- fprintf( stderr, "%s %s\n", preface, path_to_object );
- lvl++;
- print_container_contents( file_id, rc_id, path_to_object, my_rank );
- lvl--;
- }
- }
-
- /* End printing */
- if ( lvl == 0 ) {
- fprintf( stderr, "%s -----------------\n", preface );
- }
-
- return;
-}
-
-/*
- * parse_options - parse command line options
- */
-int
-parse_options( int argc, char** argv, int my_rank, int comm_size ) {
- int i, n;
- char* app = argv[0];
-
- while ( --argc ) {
- if ( **(++argv) != '-' ) {
- break;
- } else {
- switch( *(*argv+1) ) {
- case 'c':
- if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
- if ( my_rank == 0 ) {
- printf( "Error: No number specified after -c option.\n" );
- usage( app );
- }
- return( 1 );
- } else {
- ++argv;
- cols_per_row = atoi( *argv );
- }
- break;
- case 'r':
- if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
- if ( my_rank == 0 ) {
- printf( "Error: No number specified after -r option.\n" );
- usage( app );
- }
- return( 1 );
- } else {
- ++argv;
- rows_per_rank = atoi( *argv );
- }
- break;
- case 'i':
- if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
- if ( my_rank == 0 ) {
- printf( "APP-r0: Error: No number specified after -i option.\n" );
- usage( app );
- }
- return( 1 );
- } else {
- ++argv;
- num_iterations = atoi( *argv );
- }
- break;
- case 'v':
- verbose = 1;
- break;
- default:
- if ( my_rank == 0 ) {
- usage( app );
- }
- return( 1 );
- }
- }
- }
- return( 0 );
-}
-
-/*
- * usage - display usage message
- */
-void
-usage( const char* app ) {
- printf( "Usage: %s [-c cols_per_row] [-r rows_per_rank] [-i num_iterations] [-v] [-w]\n", app );
- printf( "\tc: number of columns in each row of created H5Datasets\n" );
- printf( "\tr: number of rows each rank writes in created H5Datasets (total rows = this * # of ranks)\n" );
- printf( "\ti: number of iterations to do update/commit/persist/evict/prefetch/read/evict\n" );
- printf( "\tv: verbose output\n" );
-}
-
-
-
-
-#ifdef THIS_CODE_WONT_COMPILE_BUT_KEEP_IT_AROUND
-/*
- * This codeblock needs work to reintegrate back into main() after the iterate loop of complete overwrites.
- * Didn't want to loose it in case I decide to add back in later.
- * It allows ranks to overwrite only part of the dataset & map obects in a transaction.
- * With more work, can be used to demonstrate / time reading from 'layers' of updates & persisting only partial objects
- * for a container (where previous layers alreay persisted -- Ruth
- */
-
- /****
- * Transaction X - All ranks update parts of the Dataset and Map objects in the 3 Groups
- ****/
- mbuf_size[0] = cols_per_row;
- space_id = H5Screate_simple( 1, mbuf_size, mbuf_size ); assert( space_id >= 0 );
-
- offset[0] = my_rank * rows_per_rank;
- offset[1] = 0;
- count[0] = 1;
- count[1] = cols_per_row;
-
- ret = H5Sselect_hyperslab( space_l_id, H5S_SELECT_SET, offset, NULL, count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( space_p_id, H5S_SELECT_SET, offset, NULL, count, NULL ); ASSERT_RET;
- ret = H5Sselect_hyperslab( space_s_id, H5S_SELECT_SET, offset, NULL, count, NULL ); ASSERT_RET;
-
- tr_num = X;
- num_tr_leaders = comm_size;
-
- /* Create a local transaction for transaction number and start it. */
- fprintf( stderr, "APP-r%d: tr %lu - Start with %d leaders\n", my_rank, tr_num, num_tr_leaders );
- tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
- trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
- ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
- ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
-
- /* Set cell values (in memory) based on Column, Row, Rank, Transaction */
- u = 0;
- for ( r = 0; r < 1; r++ ) {
- for ( c = 0; c < cols_per_row; c++ ) {
- mbuf[u] = (my_rank*rows_per_rank+r)*1000000 + c*100 + tr_num;
- u++;
- }
- }
-
- /* Add dataset updates to transaction */
- ret = H5Dwrite_ff( dset_l_id, H5T_NATIVE_UINT64, space_id, space_l_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- ret = H5Dwrite_ff( dset_p_id, H5T_NATIVE_UINT64, space_id, space_p_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- ret = H5Dwrite_ff( dset_s_id, H5T_NATIVE_UINT64, space_id, space_s_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
-
- /* Add map updates to transaction */
- /* Set cell values (in memory) based on Column, Row, Rank, Transaction */
- key = rows_per_rank * cols_per_row * my_rank;
- for ( u = 0; u < cols_per_row; u++ ) {
- ret = H5Mset_ff( map_l_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- ret = H5Mset_ff( map_p_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- ret = H5Mset_ff( map_s_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
- ASSERT_RET;
- key++;
- }
-
- ret = H5Sclose( space_id ); ASSERT_RET; // RUTH - likely keep this around
- free( mbuf ); // RUTH - likely keep this around
-
- /* Finish and close transaction */
- fprintf( stderr, "APP-r%d: tr %lu - Finish\n", my_rank, tr_num );
- ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
- ret = H5TRclose( tr_id ); ASSERT_RET;
- ret = H5Pclose( trspl_id ); ASSERT_RET;
-
- /* NEEDS more work if actually used at some point */
- MPI_Barrier( MPI_COMM_WORLD );
- if ( my_rank == 0 ) {
- version = tr_num;
- rc_idH = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
- assert( rc_idH >= 0 ); assert ( version == tr_num );
- fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
- if ( verbose ) print_container_contents( file_id, rc_idH, "/", my_rank );
- ret = H5RCrelease( rc_idH, H5_EVENT_STACK_NULL); ASSERT_RET;
- ret = H5RCclose( rc_idH ); ASSERT_RET;
- version = 2;
- }
- MPI_Barrier( MPI_COMM_WORLD );
-
-#endif
+/*
+ * h5ff_timings.c: Do some timing of various write / persist / prefetch / read
+ */
+/*
+ *
+ * Creates 3 groups, each with a dataset & a map
+ * In /G-logged, objects will read.
+ * In /G-prefetched, objects will be evicted & prefetched & read & evicted
+ * In /G-stored, objects will be evicted & read
+ * D = dataset; 2d with uint64 datatypes. size is [ (rows_per_rank*#ranks), (cols_per_row) ]
+ * M = map; uint64 keys ranging from 0 to (rows_per_rank*#ranks*cols_per_row)-1
+ * uint64 values, set to same as dset entries
+ * each rank updates "its" dataset entries & map pairs, based on rows_per_rank variable
+ *
+ *
+ * Notes for possible future modifications:
+ * 1) Make Datasets 1D, with entries 2D vectors to mimic ACG Edge List representation
+ * 2) Make Maps have values that are vectors of uint64, to mimic ACG Adjacency list representation (step 1)
+ * 3) Make Maps have values that variable-length vectors of uint64, to mimic ACG Adjaceny list represenations (really)
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include <sys/time.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+/* define filename for this app, and max size after username prepended */
+#define FILENAME_APP "eff_timings.h5"
+#define FILENAME_SIZE 256
+
+/* define default sizing parameters - can be changed via command line arguments */
+#define DEFAULT_ROWS_PER_RANK 10
+#define DEFAULT_COLS_PER_ROW 5
+
+/* macros related to error reporting */
+#define STATUS (ret >= 0) ? " " : " - FAILED"
+#define ASSERT_RET assert( ret >= 0 )
+
+/* option flags */
+int cols_per_row = DEFAULT_COLS_PER_ROW;
+int rows_per_rank = DEFAULT_ROWS_PER_RANK;
+int num_iterations = 1; /* Number of times to perform the commit/persist/evict/prefetch/read/evict cycle */
+int verbose = 0; /* Verbose output - defaults to no */
+int time_reads = 0; /* Time reads after container re-open */
+
+/* global variables and macros used to make timing easier */
+struct timeval tv_start;
+struct timeval tv_end;
+#define START_TIME gettimeofday( &tv_start, NULL )
+#define END_TIME gettimeofday( &tv_end, NULL )
+#define ELAPSED_TIME (ulong)( (tv_end.tv_usec + 1000000*tv_end.tv_sec) - (tv_start.tv_usec + 1000000*tv_start.tv_sec) )
+
+/* function prototypes */
+void print_container_contents( hid_t, hid_t, const char*, int );
+int parse_options( int, char**, int, int );
+void usage( const char* );
+
+/**************************************************************************************************************/
+int main( int argc, char **argv ) {
+
+ /* MPI */
+ int my_rank, comm_size;
+ int provided;
+
+ /* Container */
+ char *user_name; /* We'll prepend username to make filename unique */
+ char file_name[FILENAME_SIZE]; /* Actual filename to be opened will be user_name + FILENAME_APP */
+ hid_t fapl_id;
+ hid_t file_id;
+
+ /* Groups */
+ hid_t grp_l_id;
+ hid_t grp_p_id;
+ hid_t grp_s_id;
+
+ /* Dataspaces */
+ hid_t space_l_id;
+ hid_t space_p_id;
+ hid_t space_s_id;
+
+ /* Datasets */
+ hid_t dset_l_id;
+ hid_t dset_p_id;
+ hid_t dset_s_id;
+ hsize_t dset_size[2];
+ hid_t space_id;
+
+ /* Maps */
+ hid_t map_l_id;
+ hid_t map_p_id;
+ hid_t map_s_id;
+
+ /* Replicas and related properties */
+ hrpl_t dset_p_replica;
+ hid_t dxpl_p_id;
+ hrpl_t map_p_replica;
+ hid_t mxpl_p_id;
+
+ /* Container Version & Read Contexts */
+ uint64_t version, versionH;
+ hid_t rc_id, rc_idH;
+
+ /* Transactions */
+ uint64_t tr_num;
+ hid_t tr_id;
+ hid_t trspl_id;
+ int num_tr_leaders;
+
+ /* Memory buffer - sized to hold complete DS or Map Values; sometimes partially used */
+ uint64_t *mbuf;
+
+ /* Sizing, dataspace, hyperslab parameters set per-rank */
+ hsize_t rank_mbuf_size[1];
+ hsize_t rank_offset[2], rank_count[2];
+ hid_t rank_space_id;
+
+ /* Variables used for reading data written by a neighbor rank */
+ int neighbor_rank;
+ hid_t neighbor_space_l_id;
+ hid_t neighbor_space_p_id;
+ hid_t neighbor_space_s_id;
+ hsize_t neighbor_offset[2], neighbor_count[2];
+
+ /* Misc */
+ herr_t ret;
+ uint64_t u, r, c;
+ uint64_t key;
+ int bytesPerCell;
+ int i;
+ int iteration;
+
+ /****
+ * Initialize and create container
+ ****/
+
+ /* Confirm needed MPI functionality is available */
+ MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &provided );
+ if ( MPI_THREAD_MULTIPLE != provided ) {
+ fprintf( stderr, "APP: ERROR: MPI does not have MPI_THREAD_MULTIPLE support\n" );
+ exit( 1 );
+ }
+
+ /* Find MPI rank & communicator size */
+ MPI_Comm_rank( MPI_COMM_WORLD, &my_rank );
+ MPI_Comm_size( MPI_COMM_WORLD, &comm_size );
+ if ( my_rank == 0 ) {
+ fprintf( stderr, "APP-r%d: Number of MPI processes = %d\n", my_rank, comm_size );
+ }
+
+ /* Parse command-line options controlling behavior */
+ if ( parse_options( argc, argv, my_rank, comm_size ) != 0 ) {
+ exit( 1 );
+ }
+ if ( my_rank == 0 ) {
+ fprintf( stderr, "APP-r0: Number of MPI processes = %d\n", comm_size );
+ fprintf( stderr, "APP-r0: Datasets will have %d rows (%d per rank).\n", (rows_per_rank*comm_size), rows_per_rank );
+ fprintf( stderr, "APP-r0: Datasets will have %d columns.\n", cols_per_row );
+ fprintf( stderr, "APP-r0: There will be %d iterations.\n", num_iterations );
+ }
+
+ /* Initialize the EFF stack, starting FS client, registering HDF5 VOL calls, requesting IOD init */
+ fprintf( stderr, "APP-r%d: Initialize EFF stack\n", my_rank );
+ EFF_init( MPI_COMM_WORLD, MPI_INFO_NULL );
+
+ /* Specify the IOD VOL plugin should be used and create H5File (EFF container), with user name prepended */
+ user_name = getenv( "USER" );
+ snprintf( file_name, FILENAME_SIZE, "%s_%s", user_name, FILENAME_APP );
+ fprintf( stderr, "APP-r%d: Create %s\n", my_rank, file_name );
+ fapl_id = H5Pcreate( H5P_FILE_ACCESS ); assert( fapl_id >= 0 );
+ ret = H5Pset_fapl_iod( fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL ); ASSERT_RET;
+ file_id = H5Fcreate_ff( file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id, H5_EVENT_STACK_NULL ); assert( file_id >= 0 );
+
+ /* Create the memory buffer that will be used in many places - both for writing and reading */
+ bytesPerCell = sizeof( uint64_t );
+ mbuf = (uint64_t *) calloc( (rows_per_rank * comm_size * cols_per_row), bytesPerCell ); assert( mbuf != NULL );
+
+ /****
+ * Transaction 2: Rank 0 creates H5Objects in the container
+ ****/
+
+ if ( my_rank == 0 ) {
+
+ /* Acquire read context for CV 1 */
+ version = 1;
+ rc_id = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); assert( rc_id >= 0); assert( version == 1 );
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
+
+ /* Start a transaction with a single leader (the default) */
+ tr_num = 2;
+ fprintf( stderr, "APP-r%d: tr %lu - Start\n", my_rank, tr_num );
+ tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
+ ret = H5TRstart( tr_id, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Add updates to the transaction for Group creates */
+ fprintf( stderr, "APP-r%d: tr %d - Create /G-logged /G-prefetched /G-stored\n", my_rank, tr_num );
+
+ grp_l_id = H5Gcreate_ff( file_id, "G-logged", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ assert( grp_l_id >= 0 );
+
+ grp_p_id = H5Gcreate_ff( file_id, "G-prefetched", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ assert( grp_p_id >= 0 );
+
+ grp_s_id = H5Gcreate_ff( file_id, "G-stored", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ assert( grp_s_id >= 0 );
+
+ /* Create Dataspace for Datasets */
+ dset_size[0] = rows_per_rank * comm_size;
+ dset_size[1] = cols_per_row;
+ space_id = H5Screate_simple( 2, dset_size, dset_size ); assert( space_id >= 0 );
+
+ /* Add updates to the transaction for Dataset creates */
+ fprintf( stderr, "APP-r%d: tr %d - Create /G-logged/D /G-prefetched/D /G-stored/D\n", my_rank, tr_num );
+
+ dset_l_id = H5Dcreate_ff( grp_l_id, "D", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( dset_l_id >= 0 );
+
+ dset_p_id = H5Dcreate_ff( grp_p_id, "D", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( dset_p_id >= 0 );
+
+ dset_s_id = H5Dcreate_ff( grp_s_id, "D", H5T_NATIVE_UINT64, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( dset_s_id >= 0 );
+
+ ret = H5Sclose( space_id ); ASSERT_RET;
+
+ /* Add updates to the transaction for Map creates */
+ fprintf( stderr, "APP-r%d: tr %d - Create /G-logged/M /G-prefetched/M /G-stored/M\n", my_rank, tr_num );
+
+ map_l_id = H5Mcreate_ff( grp_l_id, "M", H5T_NATIVE_UINT, H5T_NATIVE_UINT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( map_l_id >= 0 );
+
+ map_p_id = H5Mcreate_ff( grp_p_id, "M", H5T_NATIVE_UINT, H5T_NATIVE_UINT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( map_p_id >= 0 );
+
+ map_s_id = H5Mcreate_ff( grp_s_id, "M", H5T_NATIVE_UINT, H5T_NATIVE_UINT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tr_id,
+ H5_EVENT_STACK_NULL ); assert( map_s_id >= 0 );
+
+ /* Finish and commit TR and get RC for it */
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, &rc_idH, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: tr %lu - Finished\n", my_rank, tr_num );
+
+ assert( rc_idH >= 0 );
+ versionH = tr_num;
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, versionH );
+
+ /* Release the read handle and close read context on earlier CV */
+ ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
+ ret = H5RCclose( rc_id ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc %lu - Released and Closed\n", my_rank, version );
+
+ /* The just-acquired RC is now the primary rc_id we'll work with (for awhile) */
+ rc_id = rc_idH;
+ version = versionH;
+ }
+
+ /* Rank 0 notifies other ranks that transaction 2 committed and RC acquired */
+ MPI_Bcast( &version, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+
+ /****
+ * Read Context 2: Other ranks create RC and open HObjects created by rank 0.
+ ****/
+ if ( my_rank != 0 ) {
+ rc_id = H5RCcreate( file_id, version );
+ assert( rc_id >= 0 ); assert ( version == 2 );
+ fprintf( stderr, "APP-r%d: rc %lu - Created for rank\n", my_rank, version );
+
+ grp_l_id = H5Gopen_ff( file_id, "G-logged", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( grp_l_id >= 0 );
+ grp_p_id = H5Gopen_ff( file_id, "G-prefetched", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( grp_p_id >= 0 );
+ grp_s_id = H5Gopen_ff( file_id, "G-stored", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( grp_s_id >= 0 );
+
+ dset_l_id = H5Dopen_ff( grp_l_id, "D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_l_id >= 0 );
+ dset_p_id = H5Dopen_ff( grp_p_id, "D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_p_id >= 0 );
+ dset_s_id = H5Dopen_ff( grp_s_id, "D", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( dset_s_id >= 0 );
+
+ map_l_id = H5Mopen_ff( grp_l_id, "M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( map_l_id >= 0 );
+ map_p_id = H5Mopen_ff( grp_p_id, "M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( map_p_id >= 0 );
+ map_s_id = H5Mopen_ff( grp_s_id, "M", H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL ); assert( map_s_id >= 0 );
+ } else {
+ if ( verbose ) print_container_contents( file_id, rc_id, "/", my_rank );
+ }
+
+ /* Keep track of my_rank's neighbor */
+ neighbor_rank = my_rank + 1;
+ if ( neighbor_rank == comm_size ) {
+ neighbor_rank = 0;
+ }
+
+ /* Get the dataspaces for the three datasets and make a copy of each */
+ space_l_id = H5Dget_space( dset_l_id ); assert ( space_l_id >= 0 );
+ space_p_id = H5Dget_space( dset_p_id ); assert ( space_p_id >= 0 );
+ space_s_id = H5Dget_space( dset_s_id ); assert ( space_s_id >= 0 );
+
+ neighbor_space_l_id = H5Scopy( space_l_id );
+ neighbor_space_p_id = H5Scopy( space_p_id );
+ neighbor_space_s_id = H5Scopy( space_s_id );
+
+ /* Select the hyperslab that this rank will update and the hyperslab for reading neighbor's data */
+ rank_offset[0] = my_rank * rows_per_rank;
+ rank_offset[1] = 0;
+ rank_count[0] = rows_per_rank;
+ rank_count[1] = cols_per_row;
+
+ neighbor_offset[0] = neighbor_rank * rows_per_rank;
+ neighbor_offset[1] = 0;
+ neighbor_count[0] = rows_per_rank;
+ neighbor_count[1] = cols_per_row;
+
+ ret = H5Sselect_hyperslab( space_l_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_p_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_s_id, H5S_SELECT_SET, rank_offset, NULL, rank_count, NULL ); ASSERT_RET;
+
+ ret = H5Sselect_hyperslab( neighbor_space_l_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( neighbor_space_p_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( neighbor_space_s_id, H5S_SELECT_SET, neighbor_offset, NULL, neighbor_count, NULL ); ASSERT_RET;
+
+ /* Create the memory dataspace for a rank - always first part of memory buffer */
+ rank_mbuf_size[0] = rows_per_rank * cols_per_row;
+ rank_space_id = H5Screate_simple( 1, rank_mbuf_size, rank_mbuf_size ); assert( rank_space_id >= 0 );
+
+ /* Create property lists that will be used */
+ dxpl_p_id = H5Pcreate( H5P_DATASET_XFER );
+ mxpl_p_id = H5Pcreate( H5P_DATASET_XFER );
+
+ for ( iteration = 0; iteration < num_iterations; iteration++ ) {
+
+ /****
+ * Cycle of:
+ * Tranasction N Start / Update / Commit: All ranks update Dataset and Map objects in the 3 Groups then commit
+ * CV N Persist: Rank 0
+ * Prefetch: Rank 0
+ *
+ ****/
+ tr_num = tr_num + 1;
+ num_tr_leaders = comm_size;
+
+ /* Create a transaction and start it. */
+ fprintf( stderr, "APP-r%d: iter %d tr %lu - Transaction Start with %d leaders\n",
+ my_rank, iteration, tr_num, num_tr_leaders );
+ tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
+ trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
+ ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
+ ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Set cell values (in memory) for this rank based on Row, Rank, Column, Transaction */
+ u = 0;
+ for ( r = 0; r < rows_per_rank; r++ ) {
+ for ( c = 0; c < cols_per_row; c++ ) {
+ mbuf[u] = (my_rank*rows_per_rank+r)*1000000 + c*100 + tr_num;
+ u++;
+ }
+ }
+
+ fprintf( stderr, "APP-r%d: iter %d tr %lu - Add updates to transaction.\n", my_rank, iteration, tr_num );
+
+ /* Add dataset updates to transaction */
+ START_TIME;
+ ret = H5Dwrite_ff( dset_l_id, H5T_NATIVE_UINT64, rank_space_id, space_l_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to add updates (%lu bytes) for /G-logged/D: %lu usec\n",
+ my_rank, iteration, u*bytesPerCell, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Dwrite_ff( dset_p_id, H5T_NATIVE_UINT64, rank_space_id, space_p_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to add updates (%lu bytes) for /G-prefetched/D: %lu usec\n",
+ my_rank, iteration, u*bytesPerCell, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Dwrite_ff( dset_s_id, H5T_NATIVE_UINT64, rank_space_id, space_s_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to add updates (%lu bytes) for /G-stored/D: %lu usec\n",
+ my_rank, iteration, u*bytesPerCell, ELAPSED_TIME );
+
+ /* Add map updates to transaction; do it map-by-map so we can do timing */
+ key = rows_per_rank * cols_per_row * my_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mset_ff( map_l_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to add updates ( %lu K/V pairs - %lu bytes) for /G-logged/M: %lu usec\n",
+ my_rank, iteration, u, u*bytesPerCell*2, ELAPSED_TIME );
+
+ key = rows_per_rank * cols_per_row * my_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mset_ff( map_p_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to add updates ( %lu K/V pairs - %lu bytes) for /G-prefetched/M: %lu usec\n",
+ my_rank, iteration, u, u*bytesPerCell*2, ELAPSED_TIME );
+
+ key = rows_per_rank * cols_per_row * my_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mset_ff( map_s_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to add updates ( %lu K/V pairs - %lu bytes) for /G-stored/M: %lu usec\n",
+ my_rank, iteration, u, u*bytesPerCell*2, ELAPSED_TIME );
+
+ /* Finish, (commit), and close transaction */
+ fprintf( stderr, "APP-r%d: iter %d tr %lu - Finish and Commit\n", my_rank, iteration, tr_num );
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ ret = H5Pclose( trspl_id ); ASSERT_RET;
+
+ /* After commit completes on all ranks, acquire RC on rank 0 and create local handle for RC on other ranks */
+ MPI_Barrier( MPI_COMM_WORLD );
+ versionH = tr_num;
+ if ( my_rank == 0 ) {
+ rc_idH = H5RCacquire( file_id, &versionH, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ assert( rc_idH >= 0 ); assert ( versionH == tr_num );
+ fprintf( stderr, "APP-r%d: iter %d rc %lu - Acquired\n", my_rank, iteration, versionH );
+ } else {
+ rc_idH = H5RCcreate( file_id, versionH );
+ assert( rc_idH >= 0 ); assert ( versionH == tr_num );
+ fprintf( stderr, "APP-r%d: iter %d rc %lu - Created\n", my_rank, iteration, versionH );
+ }
+ MPI_Barrier( MPI_COMM_WORLD ); /* Wait to make sure rank 0 has acquired from IOD before proceeding */
+
+ /* Release previous Read Context */
+ if ( my_rank == 0 ) {
+ ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: iter %d rc %lu - Released\n", my_rank, iteration, version );
+ }
+ ret = H5RCclose( rc_id ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: iter %d rc %lu - Closed\n", my_rank, iteration, version );
+
+ /* The just-acquired RC is now the primary rc_id we'll work with (for awhile) */
+ rc_id = rc_idH;
+ version = versionH;
+
+ /* Rank 0:
+ - persists the container
+ - evicts D and M under /G-prefetched and /G-stored
+ - prefetches D and M under /G-prefetched
+ */
+ if ( my_rank == 0 ) {
+ if ( verbose ) print_container_contents( file_id, rc_id, "/", my_rank );
+
+ START_TIME;
+ ret = H5RCpersist( rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Persist container: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Devict_ff( dset_p_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Evict /G-prefetched/D: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Devict_ff( dset_s_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Evict /G-stored/D: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Mevict_ff( map_p_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Evict /G-prefetched/M (currently no-op for IOD): %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Mevict_ff( map_s_id, version, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Evict /G-stored/M: (currently no-op for IOD) %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Dprefetch_ff( dset_p_id, rc_id, &dset_p_replica, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Prefetch /G-prefetched/D: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Mprefetch_ff( map_p_id, rc_id, &map_p_replica, H5P_DEFAULT, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Prefetch /G-prefetched/M: %lu usec\n", my_rank, iteration, ELAPSED_TIME );
+ }
+
+ /* This Bcast will make sure no nodes go ahead to use the RC before rank 0 has acquired it */
+ MPI_Bcast( &dset_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+ ret = H5Pset_dxpl_replica( dxpl_p_id, dset_p_replica ); ASSERT_RET;
+
+ MPI_Bcast( &map_p_replica, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD );
+ ret = H5Pset_dxpl_replica( mxpl_p_id, map_p_replica ); ASSERT_RET;
+
+ MPI_Barrier( MPI_COMM_WORLD ); /* Make sure all are here before continuing */
+
+ /*
+ * All ranks read the same values they updated in last transaction
+ */
+
+ /* Datasets */
+ START_TIME;
+ ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, rank_space_id, space_l_id, H5P_DEFAULT, mbuf, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-logged/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: my first updated value in /G-logged/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, rank_space_id, space_p_id, dxpl_p_id, mbuf, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-prefetched/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: my first updated value in /G-prefetched/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, rank_space_id, space_s_id, H5P_DEFAULT, mbuf, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-stored/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: my first updated value in /G-stored/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ /* Maps */
+ key = rows_per_rank * cols_per_row * my_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mget_ff( map_l_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-logged/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: my first updated value in /G-logged/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ key = rows_per_rank * cols_per_row * my_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mget_ff( map_p_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-prefetched/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose )
+ { fprintf( stderr, "APP-r%d: my first updated value in /G-prefetched/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ key = rows_per_rank * cols_per_row * my_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mget_ff( map_s_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for my rank entries for /G-stored/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: my first updated value in /G-stored/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ /*
+ * All ranks read the values "next rank" neighbor updated in last transaction
+ */
+
+ /* Datasets */
+ START_TIME;
+ ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, rank_space_id, neighbor_space_l_id, H5P_DEFAULT, mbuf, rc_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-logged/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-logged/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, rank_space_id, neighbor_space_p_id, dxpl_p_id, mbuf, rc_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-prefetched/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-prefetched/D: %09lu\n", my_rank, mbuf[0]);
+ }
+
+ START_TIME;
+ ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, rank_space_id, neighbor_space_s_id, H5P_DEFAULT, mbuf, rc_id,
+ H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-stored/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-stored/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ /* Maps */
+ key = rows_per_rank * cols_per_row * neighbor_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mget_ff( map_l_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-logged/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-logged/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ key = rows_per_rank * cols_per_row * neighbor_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mget_ff( map_p_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-prefetched/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-prefetched/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ key = rows_per_rank * cols_per_row * neighbor_rank;
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row; u++ ) {
+ ret = H5Mget_ff( map_s_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for neighbor's entries for /G-stored/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: neighbor's first updated value in /G-stored/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ /*
+ * All ranks read all the values
+ */
+
+ /* Datasets */
+ START_TIME;
+ ret = H5Dread_ff( dset_l_id, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, mbuf, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-logged/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: first value in /G-logged/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ ret = H5Dread_ff( dset_p_id, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, dxpl_p_id, mbuf, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-prefetched/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: first value in /G-prefetched/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ ret = H5Dread_ff( dset_s_id, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, mbuf, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-stored/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: first value in /G-stored/D: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ /* Maps */
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row * comm_size; u++ ) {
+ ret = H5Mget_ff( map_l_id, H5T_NATIVE_UINT64, &u, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-logged/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: first value in /G-logged/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row * comm_size; u++ ) {
+ ret = H5Mget_ff( map_p_id, H5T_NATIVE_UINT64, &u, H5T_NATIVE_UINT64, &mbuf[u], mxpl_p_id, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-prefetched/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: first value in /G-prefetched/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+ START_TIME;
+ for ( u = 0; u < rows_per_rank * cols_per_row * comm_size; u++ ) {
+ ret = H5Mget_ff( map_s_id, H5T_NATIVE_UINT64, &u, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Read Time for all entries for /G-stored/M: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ if ( verbose ) {
+ fprintf( stderr, "APP-r%d: first value in /G-stored/M: %09lu\n", my_rank, mbuf[0] );
+ }
+
+
+ /* Rank 0 evicts the replicas */
+ MPI_Barrier( MPI_COMM_WORLD ); /* Make sure all ranks done reading replicas before evict */
+ if ( my_rank == 0 ) {
+ START_TIME;
+ ret = H5Devict_ff( dset_p_id, version, dxpl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Evict Replica of /G-prefetched/D: %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+
+ START_TIME;
+ ret = H5Mevict_ff( map_p_id, version, mxpl_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ END_TIME;
+ fprintf( stderr, "APP-r%d: iter %d - Time to Evict Replica of /G-prefetched/M (currently no-op for IOD): %lu usec\n",
+ my_rank, iteration, ELAPSED_TIME );
+ }
+
+ }
+
+ /*
+ * Wrap-up
+ */
+
+ /* Print (if verbose) the final state */
+ if ( my_rank == 0 ) {
+ if ( verbose ) print_container_contents( file_id, rc_id, "/", my_rank );
+ }
+
+ /* Close the dataset and map property lists */
+ ret = H5Pclose( dxpl_p_id ); ASSERT_RET;
+ ret = H5Pclose( mxpl_p_id ); ASSERT_RET;
+
+ /* Close the memory dataspace for the rank */
+ ret = H5Sclose( rank_space_id ); ASSERT_RET;
+
+ fprintf( stderr, "APP-r%d: Closing H5Objects\n", my_rank );
+
+ ret = H5Dclose_ff( dset_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Dclose_ff( dset_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Dclose_ff( dset_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ ret = H5Sclose( space_l_id ); ASSERT_RET;
+ ret = H5Sclose( space_p_id ); ASSERT_RET;
+ ret = H5Sclose( space_s_id ); ASSERT_RET;
+
+ ret = H5Mclose_ff( map_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Mclose_ff( map_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Mclose_ff( map_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ ret = H5Gclose_ff( grp_l_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Gclose_ff( grp_p_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Gclose_ff( grp_s_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Release the read handle and close read context */
+ MPI_Barrier( MPI_COMM_WORLD ); /* Make sure all ranks done with RC */
+ if ( my_rank == 0 ) {
+ ret = H5RCrelease( rc_id, H5_EVENT_STACK_NULL); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc %lu - Released\n", my_rank, version );
+ }
+ ret = H5RCclose( rc_id ); ASSERT_RET;
+ fprintf( stderr, "APP-r%d: rc %lu - Closed\n", my_rank, version );
+
+ /* Close H5 Objects that are still open */
+ fprintf( stderr, "APP-r%d: Closing %s\n", my_rank, file_name );
+ MPI_Barrier( MPI_COMM_WORLD );
+ ret = H5Fclose_ff( file_id, 1, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Pclose( fapl_id ); ASSERT_RET;
+
+ /* Perform wrap-up operations */
+ fprintf( stderr, "APP-r%d: Finalize EFF stack\n", my_rank );
+ MPI_Barrier( MPI_COMM_WORLD );
+ EFF_finalize();
+ MPI_Finalize();
+
+ return 0;
+}
+
+
+/*
+ * Helper function used to recursively read and print container contents
+ * for container identified by "file_id"
+ * in read context identified by "rc_id"
+ * with path to current level in "grp_path"
+ * and "my_rank" used to identify the process doing the reading / printing.
+ */
+void
+print_container_contents( hid_t file_id, hid_t rc_id, const char* grp_path, int my_rank )
+{
+ herr_t ret;
+ uint64_t cv;
+ htri_t exists;
+ char path_to_object[1024];
+ char name[30];
+ int i;
+
+ static int lvl = 0; /* level in recursion - used to format printing */
+ char preface[128];
+
+ /* Get the container version for the read context */
+ ret = H5RCget_version( rc_id, &cv ); ASSERT_RET;
+
+ /* Set up the preface and adding version number */
+ sprintf( preface, "APP-r%d: cv %d: ", my_rank, (int)cv );
+
+ /* Start the printing */
+ if ( lvl == 0 ) {
+ fprintf( stderr, "%s ----- Container Contents ------------\n", preface );
+ }
+
+ /* Datasets */
+ sprintf( path_to_object, "%s%s", grp_path, "D" );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+
+ if ( exists ) {
+ hid_t dset_id;
+ hid_t space_id;
+ int nDims;
+ hsize_t current_size[2];
+ hsize_t max_size[2];
+ hsize_t totalSize;
+ uint64_t *data;
+ uint64_t u, e;
+
+ dset_id = H5Dopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( dset_id >= 0 );
+
+ space_id = H5Dget_space( dset_id ); assert ( space_id >= 0 );
+ nDims = H5Sget_simple_extent_dims( space_id, current_size, max_size );
+ assert( nDims == 2 );
+
+ totalSize = current_size[0] * current_size[1];
+ data = (uint64_t *)calloc( totalSize, sizeof(uint64_t) ); assert( data != NULL );
+
+ ret = H5Dread_ff( dset_id, H5T_NATIVE_UINT64, space_id, space_id, H5P_DEFAULT, data, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* note that by printing piece-by-piece, other output may be interspersed. trade-off since won't know line size */
+ fprintf( stderr, "%s %s data:\n", preface, path_to_object );
+ e = 0;
+ for ( i = 0; i < totalSize; i++ ) {
+ if ( e == 0 ) fprintf( stderr, "%s\t", preface );
+ fprintf( stderr, "%09lu ", data[i] );
+ if ( ++e == cols_per_row ) {
+ fprintf( stderr, "\n" );
+ e = 0;
+ }
+ }
+
+ free( data );
+ sleep( 1 ); /* give output a chance to be printed before diag messages from lower layers on close */
+ ret = H5Dclose_ff( dset_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5Sclose( space_id ); ASSERT_RET;
+ }
+
+ /* Maps */
+ sprintf( path_to_object, "%s%s", grp_path, "M" );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+
+ if ( exists ) {
+ hid_t map_id;
+ hsize_t totalCount;
+ uint64_t *value;
+ uint64_t u, e;
+
+ map_id = H5Mopen_ff( file_id, path_to_object, H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ assert( map_id >= 0 );
+
+ ret = H5Mget_count_ff( map_id, &totalCount, rc_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ value = (uint64_t *)calloc( totalCount, sizeof(uint64_t) ); assert( value != NULL );
+
+ for ( u = 0; u < totalCount; u++ ) {
+ ret = H5Mget_ff( map_id, H5T_NATIVE_UINT64, &u, H5T_NATIVE_UINT64, &value[u], H5P_DEFAULT, rc_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ }
+
+ /* note that by printing piece-by-piece, other output may be interspersed. trade-off since won't know line size */
+ fprintf( stderr, "%s %s key/values:\n", preface, path_to_object );
+ e = 0;
+ for ( u = 0; u < totalCount; u++ ) {
+ if ( e == 0 ) fprintf( stderr, "%s\t", preface );
+ fprintf( stderr, "%lu/%09lu ", u, value[u] );
+ if ( ++e == cols_per_row ) {
+ fprintf( stderr, "\n" );
+ e = 0;
+ }
+ }
+
+ free( value );
+ sleep( 1 ); /* give output a chance to be printed before diag messages from lower layers on close */
+ ret = H5Mclose_ff( map_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ }
+
+ /* Groups - if found, descend */
+ for ( i = 1; i < 4; i++ ) {
+ if ( i == 1 ) {
+ strcpy( name, "G-logged" );
+ } else if ( i == 2 ) {
+ strcpy( name, "G-prefetched" );
+ } else if ( i == 3 ) {
+ strcpy( name, "G-stored" );
+ }
+
+ sprintf( path_to_object, "%s%s/", grp_path, name );
+ ret = H5Lexists_ff( file_id, path_to_object, H5P_DEFAULT, &exists, rc_id, H5_EVENT_STACK_NULL );
+ if ( exists ) {
+ fprintf( stderr, "%s %s\n", preface, path_to_object );
+ lvl++;
+ print_container_contents( file_id, rc_id, path_to_object, my_rank );
+ lvl--;
+ }
+ }
+
+ /* End printing */
+ if ( lvl == 0 ) {
+ fprintf( stderr, "%s -----------------\n", preface );
+ }
+
+ return;
+}
+
+/*
+ * parse_options - parse command line options
+ */
+int
+parse_options( int argc, char** argv, int my_rank, int comm_size ) {
+ int i, n;
+ char* app = argv[0];
+
+ while ( --argc ) {
+ if ( **(++argv) != '-' ) {
+ break;
+ } else {
+ switch( *(*argv+1) ) {
+ case 'c':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "Error: No number specified after -c option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ cols_per_row = atoi( *argv );
+ }
+ break;
+ case 'r':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "Error: No number specified after -r option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ rows_per_rank = atoi( *argv );
+ }
+ break;
+ case 'i':
+ if ( ( --argc == 0 ) || ( **(argv+1) == '-' ) ) {
+ if ( my_rank == 0 ) {
+ printf( "APP-r0: Error: No number specified after -i option.\n" );
+ usage( app );
+ }
+ return( 1 );
+ } else {
+ ++argv;
+ num_iterations = atoi( *argv );
+ }
+ break;
+ case 'v':
+ verbose = 1;
+ break;
+ default:
+ if ( my_rank == 0 ) {
+ usage( app );
+ }
+ return( 1 );
+ }
+ }
+ }
+ return( 0 );
+}
+
+/*
+ * usage - display usage message
+ */
+void
+usage( const char* app ) {
+ printf( "Usage: %s [-c cols_per_row] [-r rows_per_rank] [-i num_iterations] [-v] [-w]\n", app );
+ printf( "\tc: number of columns in each row of created H5Datasets\n" );
+ printf( "\tr: number of rows each rank writes in created H5Datasets (total rows = this * # of ranks)\n" );
+ printf( "\ti: number of iterations to do update/commit/persist/evict/prefetch/read/evict\n" );
+ printf( "\tv: verbose output\n" );
+}
+
+
+
+
+#ifdef THIS_CODE_WONT_COMPILE_BUT_KEEP_IT_AROUND
+/*
+ * This codeblock needs work to reintegrate back into main() after the iterate loop of complete overwrites.
+ * Didn't want to loose it in case I decide to add back in later.
+ * It allows ranks to overwrite only part of the dataset & map obects in a transaction.
+ * With more work, can be used to demonstrate / time reading from 'layers' of updates & persisting only partial objects
+ * for a container (where previous layers alreay persisted -- Ruth
+ */
+
+ /****
+ * Transaction X - All ranks update parts of the Dataset and Map objects in the 3 Groups
+ ****/
+ mbuf_size[0] = cols_per_row;
+ space_id = H5Screate_simple( 1, mbuf_size, mbuf_size ); assert( space_id >= 0 );
+
+ offset[0] = my_rank * rows_per_rank;
+ offset[1] = 0;
+ count[0] = 1;
+ count[1] = cols_per_row;
+
+ ret = H5Sselect_hyperslab( space_l_id, H5S_SELECT_SET, offset, NULL, count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_p_id, H5S_SELECT_SET, offset, NULL, count, NULL ); ASSERT_RET;
+ ret = H5Sselect_hyperslab( space_s_id, H5S_SELECT_SET, offset, NULL, count, NULL ); ASSERT_RET;
+
+ tr_num = X;
+ num_tr_leaders = comm_size;
+
+ /* Create a local transaction for transaction number and start it. */
+ fprintf( stderr, "APP-r%d: tr %lu - Start with %d leaders\n", my_rank, tr_num, num_tr_leaders );
+ tr_id = H5TRcreate( file_id, rc_id, tr_num ); assert( tr_id >= 0 );
+ trspl_id = H5Pcreate( H5P_TR_START ); assert( trspl_id >= 0 );
+ ret = H5Pset_trspl_num_peers( trspl_id, num_tr_leaders ); ASSERT_RET;
+ ret = H5TRstart( tr_id, trspl_id, H5_EVENT_STACK_NULL ); ASSERT_RET;
+
+ /* Set cell values (in memory) based on Column, Row, Rank, Transaction */
+ u = 0;
+ for ( r = 0; r < 1; r++ ) {
+ for ( c = 0; c < cols_per_row; c++ ) {
+ mbuf[u] = (my_rank*rows_per_rank+r)*1000000 + c*100 + tr_num;
+ u++;
+ }
+ }
+
+ /* Add dataset updates to transaction */
+ ret = H5Dwrite_ff( dset_l_id, H5T_NATIVE_UINT64, space_id, space_l_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ ret = H5Dwrite_ff( dset_p_id, H5T_NATIVE_UINT64, space_id, space_p_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ ret = H5Dwrite_ff( dset_s_id, H5T_NATIVE_UINT64, space_id, space_s_id, H5P_DEFAULT, mbuf, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+
+ /* Add map updates to transaction */
+ /* Set cell values (in memory) based on Column, Row, Rank, Transaction */
+ key = rows_per_rank * cols_per_row * my_rank;
+ for ( u = 0; u < cols_per_row; u++ ) {
+ ret = H5Mset_ff( map_l_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ ret = H5Mset_ff( map_p_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ ret = H5Mset_ff( map_s_id, H5T_NATIVE_UINT64, &key, H5T_NATIVE_UINT64, &mbuf[u], H5P_DEFAULT, tr_id, H5_EVENT_STACK_NULL );
+ ASSERT_RET;
+ key++;
+ }
+
+ ret = H5Sclose( space_id ); ASSERT_RET; // RUTH - likely keep this around
+ free( mbuf ); // RUTH - likely keep this around
+
+ /* Finish and close transaction */
+ fprintf( stderr, "APP-r%d: tr %lu - Finish\n", my_rank, tr_num );
+ ret = H5TRfinish( tr_id, H5P_DEFAULT, NULL, H5_EVENT_STACK_NULL ); ASSERT_RET;
+ ret = H5TRclose( tr_id ); ASSERT_RET;
+ ret = H5Pclose( trspl_id ); ASSERT_RET;
+
+ /* NEEDS more work if actually used at some point */
+ MPI_Barrier( MPI_COMM_WORLD );
+ if ( my_rank == 0 ) {
+ version = tr_num;
+ rc_idH = H5RCacquire( file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL );
+ assert( rc_idH >= 0 ); assert ( version == tr_num );
+ fprintf( stderr, "APP-r%d: rc %lu - Acquired\n", my_rank, version );
+ if ( verbose ) print_container_contents( file_id, rc_idH, "/", my_rank );
+ ret = H5RCrelease( rc_idH, H5_EVENT_STACK_NULL); ASSERT_RET;
+ ret = H5RCclose( rc_idH ); ASSERT_RET;
+ version = 2;
+ }
+ MPI_Barrier( MPI_COMM_WORLD );
+
+#endif
diff --git a/examples/h5ff_client_trans.c b/examples/h5ff_client_trans.c
index 4f08507..d627b84 100644
--- a/examples/h5ff_client_trans.c
+++ b/examples/h5ff_client_trans.c
@@ -1,190 +1,190 @@
-/*
- * test_client.c: Client side of Milestone 4.2 Asynchronous I/O and initial
- * IOD VOL plugin demonstration. This is, in effect, the application program that
- * would run on one or more compute nodes and make calls to the HDF5 API.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-int main(int argc, char **argv) {
- char file_name[]="eff_file.h5";
- hid_t file_id;
- hid_t fapl_id, trspl_id;
- hid_t tid1, tid2, tid3;
- hid_t rid1, rid2, rid3;
- uint64_t version;
- int my_rank, my_size;
- int provided;
- hid_t e_stack;
- H5ES_status_t status;
- size_t num_events = 0;
- herr_t ret;
-
- MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
- if(MPI_THREAD_MULTIPLE != provided) {
- fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
- exit(1);
- }
-
- fprintf(stderr, "\n*****************************************************************************************************************\n");
- fprintf(stderr, "Initialize EFF stack\n");
- fprintf(stderr, "*****************************************************************************************************************\n");
-
- /* Call EFF_init to initialize the EFF stack.
- As a result of this call, the Function Shipper client is started,
- and HDF5 VOL calls are registered with the function shipper.
- An "IOD init" call is forwarded from the FS client to the FS server
- which should already be running. */
- EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
-
- MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
- MPI_Comm_size(MPI_COMM_WORLD, &my_size);
- fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
-
- fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
- /* Choose the IOD VOL plugin to use with this file.
- First we create a file access property list. Then we call a new routine to set
- the IOD plugin to use with this fapl */
- fapl_id = H5Pcreate (H5P_FILE_ACCESS);
- H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
-
- e_stack = H5EScreate();
- assert(e_stack);
-
- /* create the file */
- file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
- assert(file_id);
-
- version = 0;
- /* acquire container version 0 - EXACT */
- rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
-
- /* create transactions object (does not start transactions). Local call. */
- tid2 = H5TRcreate(file_id, rid1, (uint64_t)555);
- assert(tid2);
-
- if(my_rank == 0) {
- hid_t gid1, gid2;
-
- /* create transactions object (does not start transactions). Local call. */
- tid1 = H5TRcreate(file_id, rid1, (uint64_t)1);
- assert(tid1);
-
- /* start transaction 1 with default Leader/Delegate model. Leader
- which is rank 0 here starts the transaction. */
- ret = H5TRstart(tid1, H5P_DEFAULT, e_stack);
- assert(0 == ret);
-
- gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
- assert(gid1);
- gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
- assert(gid2);
-
- assert(H5Gclose_ff(gid1, e_stack) == 0);
- assert(H5Gclose_ff(gid2, e_stack) == 0);
-
- /* finish transaction 1. This is asynchronous, but has a
- dependency on H5TRstart() of tid1, and the operations
- pending on it. */
- ret = H5TRfinish(tid1, H5P_DEFAULT, NULL, e_stack);
- assert(0 == ret);
-
- /* Local op */
- ret = H5TRclose(tid1);
- assert(0 == ret);
- }
-
- /* skip transactions 2 till 554. This is asynchronous. */
- ret = H5TRskip(file_id, (uint64_t)2, (uint64_t)553, e_stack);
- assert(0 == ret);
-
- /* Start transaction 555 with Multiple Leader - No Delegate Model. */
- trspl_id = H5Pcreate (H5P_TR_START);
- ret = H5Pset_trspl_num_peers(trspl_id, my_size);
- assert(0 == ret);
- ret = H5TRstart(tid2, trspl_id, e_stack);
- assert(0 == ret);
- ret = H5Pclose(trspl_id);
- assert(0 == ret);
-
- /* set dependency from transaction 555 on 1.
- This is asynchronous but has a dependency on H5TRstart() of tid2. */
- ret = H5TRset_dependency(tid2, (uint64_t)1, e_stack);
- assert(0 == ret);
-
- /* finish transaction 555 and acquire a read context for it */
- ret = H5TRfinish(tid2, H5P_DEFAULT, &rid2, e_stack);
- assert(0 == ret);
- ret = H5TRclose(tid2);
- assert(0 == ret);
-
- /* release container version 0. This is async. */
- ret = H5RCrelease(rid1, e_stack);
- assert(0 == ret);
-
- ret = H5RCclose(rid1);
- assert(0 == ret);
-
- /* wait on all requests in event queue */
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- if(my_rank == 0) {
- /* Start transaction 556 from read context 555 */
- tid3 = H5TRcreate(file_id, rid2, (uint64_t)556);
- assert(tid1);
-
- ret = H5TRstart(tid3, H5P_DEFAULT, e_stack);
- assert(0 == ret);
-
- /* abort transaction 556 */
- ret = H5TRabort(tid3, e_stack);
- assert(tid1);
-
- ret = H5TRclose(tid3);
- assert(0 == ret);
- }
-
- /* persist version 555 */
- ret = H5RCpersist(rid2, e_stack);
- assert(0 == ret);
-
- /* snapshot version 555 */
- ret = H5RCsnapshot(rid2, "container_555", e_stack);
- assert(0 == ret);
-
- ret = H5RCclose(rid2);
- assert(0 == ret);
-
- ret = H5Fclose(file_id);
- assert(0 == ret);
-
- /* wait on all requests and print completion status */
- H5ESget_count(e_stack, &num_events);
- H5ESwait_all(e_stack, &status);
- H5ESclear(e_stack);
- printf("%d events in event stack. Completion status = %d\n", num_events, status);
-
- H5ESclose(e_stack);
- H5Pclose(fapl_id);
-
- fprintf(stderr, "\n*****************************************************************************************************************\n");
- fprintf(stderr, "Finalize EFF stack\n");
- fprintf(stderr, "*****************************************************************************************************************\n");
-
- /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
- and shutsdown the FS server (when all clients send the terminate request)
- and client */
- EFF_finalize();
-
- MPI_Finalize();
- return 0;
-}
+/*
+ * test_client.c: Client side of Milestone 4.2 Asynchronous I/O and initial
+ * IOD VOL plugin demonstration. This is, in effect, the application program that
+ * would run on one or more compute nodes and make calls to the HDF5 API.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+int main(int argc, char **argv) {
+ char file_name[]="eff_file.h5";
+ hid_t file_id;
+ hid_t fapl_id, trspl_id;
+ hid_t tid1, tid2, tid3;
+ hid_t rid1, rid2, rid3;
+ uint64_t version;
+ int my_rank, my_size;
+ int provided;
+ hid_t e_stack;
+ H5ES_status_t status;
+ size_t num_events = 0;
+ herr_t ret;
+
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+ if(MPI_THREAD_MULTIPLE != provided) {
+ fprintf(stderr, "MPI does not have MPI_THREAD_MULTIPLE support\n");
+ exit(1);
+ }
+
+ fprintf(stderr, "\n*****************************************************************************************************************\n");
+ fprintf(stderr, "Initialize EFF stack\n");
+ fprintf(stderr, "*****************************************************************************************************************\n");
+
+ /* Call EFF_init to initialize the EFF stack.
+ As a result of this call, the Function Shipper client is started,
+ and HDF5 VOL calls are registered with the function shipper.
+ An "IOD init" call is forwarded from the FS client to the FS server
+ which should already be running. */
+ EFF_init(MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &my_size);
+ fprintf(stderr, "APP processes = %d, my rank is %d\n", my_size, my_rank);
+
+ fprintf(stderr, "Create the FAPL to set the IOD VOL plugin and create the file\n");
+ /* Choose the IOD VOL plugin to use with this file.
+ First we create a file access property list. Then we call a new routine to set
+ the IOD plugin to use with this fapl */
+ fapl_id = H5Pcreate (H5P_FILE_ACCESS);
+ H5Pset_fapl_iod(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
+
+ e_stack = H5EScreate();
+ assert(e_stack);
+
+ /* create the file */
+ file_id = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id);
+ assert(file_id);
+
+ version = 0;
+ /* acquire container version 0 - EXACT */
+ rid1 = H5RCacquire(file_id, &version, H5P_DEFAULT, H5_EVENT_STACK_NULL);
+
+ /* create transactions object (does not start transactions). Local call. */
+ tid2 = H5TRcreate(file_id, rid1, (uint64_t)555);
+ assert(tid2);
+
+ if(my_rank == 0) {
+ hid_t gid1, gid2;
+
+ /* create transactions object (does not start transactions). Local call. */
+ tid1 = H5TRcreate(file_id, rid1, (uint64_t)1);
+ assert(tid1);
+
+ /* start transaction 1 with default Leader/Delegate model. Leader
+ which is rank 0 here starts the transaction. */
+ ret = H5TRstart(tid1, H5P_DEFAULT, e_stack);
+ assert(0 == ret);
+
+ gid1 = H5Gcreate_ff(file_id, "G1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+ assert(gid1);
+ gid2 = H5Gcreate_ff(gid1, "G2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, tid1, e_stack);
+ assert(gid2);
+
+ assert(H5Gclose_ff(gid1, e_stack) == 0);
+ assert(H5Gclose_ff(gid2, e_stack) == 0);
+
+ /* finish transaction 1. This is asynchronous, but has a
+ dependency on H5TRstart() of tid1, and the operations
+ pending on it. */
+ ret = H5TRfinish(tid1, H5P_DEFAULT, NULL, e_stack);
+ assert(0 == ret);
+
+ /* Local op */
+ ret = H5TRclose(tid1);
+ assert(0 == ret);
+ }
+
+ /* skip transactions 2 till 554. This is asynchronous. */
+ ret = H5TRskip(file_id, (uint64_t)2, (uint64_t)553, e_stack);
+ assert(0 == ret);
+
+ /* Start transaction 555 with Multiple Leader - No Delegate Model. */
+ trspl_id = H5Pcreate (H5P_TR_START);
+ ret = H5Pset_trspl_num_peers(trspl_id, my_size);
+ assert(0 == ret);
+ ret = H5TRstart(tid2, trspl_id, e_stack);
+ assert(0 == ret);
+ ret = H5Pclose(trspl_id);
+ assert(0 == ret);
+
+ /* set dependency from transaction 555 on 1.
+ This is asynchronous but has a dependency on H5TRstart() of tid2. */
+ ret = H5TRset_dependency(tid2, (uint64_t)1, e_stack);
+ assert(0 == ret);
+
+ /* finish transaction 555 and acquire a read context for it */
+ ret = H5TRfinish(tid2, H5P_DEFAULT, &rid2, e_stack);
+ assert(0 == ret);
+ ret = H5TRclose(tid2);
+ assert(0 == ret);
+
+ /* release container version 0. This is async. */
+ ret = H5RCrelease(rid1, e_stack);
+ assert(0 == ret);
+
+ ret = H5RCclose(rid1);
+ assert(0 == ret);
+
+ /* wait on all requests in event queue */
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ if(my_rank == 0) {
+ /* Start transaction 556 from read context 555 */
+ tid3 = H5TRcreate(file_id, rid2, (uint64_t)556);
+ assert(tid1);
+
+ ret = H5TRstart(tid3, H5P_DEFAULT, e_stack);
+ assert(0 == ret);
+
+ /* abort transaction 556 */
+ ret = H5TRabort(tid3, e_stack);
+ assert(tid1);
+
+ ret = H5TRclose(tid3);
+ assert(0 == ret);
+ }
+
+ /* persist version 555 */
+ ret = H5RCpersist(rid2, e_stack);
+ assert(0 == ret);
+
+ /* snapshot version 555 */
+ ret = H5RCsnapshot(rid2, "container_555", e_stack);
+ assert(0 == ret);
+
+ ret = H5RCclose(rid2);
+ assert(0 == ret);
+
+ ret = H5Fclose(file_id);
+ assert(0 == ret);
+
+ /* wait on all requests and print completion status */
+ H5ESget_count(e_stack, &num_events);
+ H5ESwait_all(e_stack, &status);
+ H5ESclear(e_stack);
+ printf("%d events in event stack. Completion status = %d\n", num_events, status);
+
+ H5ESclose(e_stack);
+ H5Pclose(fapl_id);
+
+ fprintf(stderr, "\n*****************************************************************************************************************\n");
+ fprintf(stderr, "Finalize EFF stack\n");
+ fprintf(stderr, "*****************************************************************************************************************\n");
+
+ /* This finalizes the EFF stack. ships a terminate and IOD finalize to the server
+ and shutsdown the FS server (when all clients send the terminate request)
+ and client */
+ EFF_finalize();
+
+ MPI_Finalize();
+ return 0;
+}
diff --git a/examples/h5ff_server.c b/examples/h5ff_server.c
index 690b0f1..3da8fff 100644
--- a/examples/h5ff_server.c
+++ b/examples/h5ff_server.c
@@ -1,48 +1,48 @@
-/*
- * test_server.c: Server side of Milestone 3.3 Asynchronous I/O and initial
- * IOD VOL plugin demonstration.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-#include "mpi.h"
-#include "hdf5.h"
-
-int main(int argc, char **argv) {
- int my_size, my_rank;
- int provided;
-
- MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
- if(MPI_THREAD_MULTIPLE != provided) {
- printf("MPI does not have MPI_THREAD_MULTIPLE support\n");
- exit(1);
- }
-
- MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
- MPI_Comm_size(MPI_COMM_WORLD, &my_size);
- printf("Number of server processes = %d, my rank is %d\n", my_size, my_rank);
-
- //H5open();
- /* This call initiliazes the FS for the server processes (create metadata and
- bulk data handles). It also registers with the Function shipper the
- HDF5 VOL server routines that will be executed when the clients ship the VOL
- routines. Then it executes a loop to receive requests from clients that map
- to one of the registered callbacks.
-
- Whenever a request is received, the corresponding callback is called which
- inserts the operation into the AXE (the Asynchronous eXecution Engine) and
- returns to receive another request. The AXE schedules that operation to
- execute by assigning a thread to it. After the operation is complete, the
- result is returned to the client through the function shipper complete call.
-
- Finally, when all clients send a terminate call, the function shipper interface
- is finalized the operation returns. */
- EFF_start_server(MPI_COMM_WORLD, MPI_INFO_NULL);
- //H5close();
- MPI_Finalize();
-
- return 0;
-}
-
+/*
+ * test_server.c: Server side of Milestone 3.3 Asynchronous I/O and initial
+ * IOD VOL plugin demonstration.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include "mpi.h"
+#include "hdf5.h"
+
+int main(int argc, char **argv) {
+ int my_size, my_rank;
+ int provided;
+
+ MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+ if(MPI_THREAD_MULTIPLE != provided) {
+ printf("MPI does not have MPI_THREAD_MULTIPLE support\n");
+ exit(1);
+ }
+
+ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+ MPI_Comm_size(MPI_COMM_WORLD, &my_size);
+ printf("Number of server processes = %d, my rank is %d\n", my_size, my_rank);
+
+ //H5open();
+ /* This call initiliazes the FS for the server processes (create metadata and
+ bulk data handles). It also registers with the Function shipper the
+ HDF5 VOL server routines that will be executed when the clients ship the VOL
+ routines. Then it executes a loop to receive requests from clients that map
+ to one of the registered callbacks.
+
+ Whenever a request is received, the corresponding callback is called which
+ inserts the operation into the AXE (the Asynchronous eXecution Engine) and
+ returns to receive another request. The AXE schedules that operation to
+ execute by assigning a thread to it. After the operation is complete, the
+ result is returned to the client through the function shipper complete call.
+
+ Finally, when all clients send a terminate call, the function shipper interface
+ is finalized the operation returns. */
+ EFF_start_server(MPI_COMM_WORLD, MPI_INFO_NULL);
+ //H5close();
+ MPI_Finalize();
+
+ return 0;
+}
+
generated by cgit v0.12 at 2024-10-29 19:07:58 (GMT)