/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by the Board of Trustees of the University of Illinois. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * * terms governing use, modification, and redistribution, is contained in * * the files COPYING and Copyright.html. COPYING can be found at the root * * of the source code distribution tree; Copyright.html can be found at the * * root level of an installed copy of the electronic HDF5 document set and * * is linked from the top-level documents page. It can also be found at * * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* Private header files */ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error Handling */ #include "H5Oprivate.h" /* Object Headers */ #include "H5TBprivate.h" /* Threaded, Balanced, Binary Trees */ #ifdef H5_HAVE_FPHDF5 #include "H5FPprivate.h" /* Flexible Parallel Functions */ /* Pablo mask */ #define PABLO_MASK H5FP_mask /* Interface initialization */ #define INTERFACE_INIT NULL static int interface_initialize_g = 0; MPI_Datatype H5FP_request_t; /* MPI datatype for the H5FP_request obj*/ MPI_Datatype H5FP_reply_t; /* MPI datatype for the H5FP_reply obj */ MPI_Datatype H5FP_read_t; /* MPI datatype for the H5FP_read obj */ /* SAP specific variables */ MPI_Comm H5FP_SAP_COMM; /* Comm we use: Supplied by user */ MPI_Comm H5FP_SAP_BARRIER_COMM; /* Comm if you want to do a barrier */ unsigned H5FP_sap_rank; /* The rank of the SAP: Supplied by user*/ unsigned H5FP_capt_rank; /* The rank which tells SAP of opens */ unsigned H5FP_my_rank; /* Rank of this process in the COMM */ int H5FP_comm_size; /* Size of the COMM */ /* local functions */ static herr_t H5FP_commit_sap_datatypes(void); static herr_t H5FP_request_sap_stop(void); /* *===----------------------------------------------------------------------=== * Public (API) Functions *===----------------------------------------------------------------------=== */ /* * Function: H5FPinit * Purpose: Initialize the SAP environment: duplicate the COMM the user * supplies to us, set aside the SAP_RANK as the SAP. * Return: Success: SUCCEED * Failure: FAIL * Programmer: Bill Wendling, 26. July, 2002 * Modifications: */ herr_t H5FPinit(MPI_Comm comm, int sap_rank) { MPI_Group sap_group = MPI_GROUP_NULL, sap_barrier_group = MPI_GROUP_NULL; herr_t ret_value = SUCCEED; FUNC_ENTER_API(H5FPinit, FAIL); H5TRACE2("e","McIs",comm,sap_rank); /* initialize to NULL so that we can release if an error occurs */ H5FP_request_t = MPI_DATATYPE_NULL; H5FP_reply_t = MPI_DATATYPE_NULL; H5FP_read_t = MPI_DATATYPE_NULL; H5FP_SAP_COMM = MPI_COMM_NULL; H5FP_SAP_BARRIER_COMM = MPI_COMM_NULL; /* Set the global variable to track the SAP's rank */ H5FP_sap_rank = sap_rank; /* Make a private copy of the communicator passed to us */ if (MPI_Comm_dup(comm, &H5FP_SAP_COMM) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_dup failed"); if (MPI_Comm_group(H5FP_SAP_COMM, &sap_group) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_group failed"); /* Exclude the SAP from the barrier group group */ if (MPI_Group_excl(sap_group, 1, (int *)&H5FP_sap_rank, &sap_barrier_group) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Group_excl failed"); /* Create communicator for barrier group (all processes except the SAP) */ if (MPI_Comm_create(H5FP_SAP_COMM, sap_barrier_group, &H5FP_SAP_BARRIER_COMM) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_create failed"); /* Get the size of all the processes (including the SAP) */ if (MPI_Comm_size(H5FP_SAP_COMM, &H5FP_comm_size) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_size failed"); /* * We assign the process right after the sap_rank as the one which * will tell the SAP that files have been opened or closed. We mod * it so that we don't go over the size of the communicator. */ H5FP_capt_rank = (H5FP_sap_rank + 1) % H5FP_comm_size; /* Get this processes rank */ if (MPI_Comm_rank(H5FP_SAP_COMM, (int *)&H5FP_my_rank) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_rank failed"); /* Create the MPI types used for communicating with the SAP */ if (H5FP_commit_sap_datatypes() != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "H5FP_commit_sap_datatypes failed"); /* Go loop, if we are the SAP */ if (H5FP_my_rank == H5FP_sap_rank) H5FP_sap_receive_loop(); /* Fall through and return to user, if not SAP */ done: if (ret_value == FAIL) { /* we've encountered an error...clean up */ if (H5FP_request_t != MPI_DATATYPE_NULL) if (MPI_Type_free(&H5FP_request_t) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed"); if (H5FP_reply_t != MPI_DATATYPE_NULL) if (MPI_Type_free(&H5FP_reply_t) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed"); if (H5FP_read_t != MPI_DATATYPE_NULL) if (MPI_Type_free(&H5FP_read_t) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed"); if (H5FP_SAP_BARRIER_COMM != MPI_COMM_NULL) /* this comm will be NULL for the SAP */ if (MPI_Comm_free(&H5FP_SAP_BARRIER_COMM) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_free failed"); if (H5FP_SAP_COMM != MPI_COMM_NULL) if (MPI_Comm_free(&H5FP_SAP_COMM) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_free failed"); } if (sap_group != MPI_GROUP_NULL) MPI_Group_free(&sap_group); if (sap_barrier_group != MPI_GROUP_NULL) MPI_Group_free(&sap_barrier_group); FUNC_LEAVE_API(ret_value); } /* * Function: H5FPfinalize * Purpose: Get rid of the initilized environment we setup with H5FPinit. * Mostly just freeing the duplicated COMM object and committed * datatypes. * Return: Success: SUCCEED * Failure: FAIL * Programmer: Bill Wendling, 26. July, 2002 * Modifications: */ herr_t H5FPfinalize(void) { herr_t ret_value = SUCCEED; FUNC_ENTER_API(H5FPfinalize, FAIL); H5TRACE0("e",""); /* Stop the SAP */ if (H5FP_my_rank != H5FP_sap_rank) if (H5FP_request_sap_stop() < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Error stopping the SAP"); /* Release the MPI types we created */ if (MPI_Type_free(&H5FP_request_t) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed"); if (MPI_Type_free(&H5FP_reply_t) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed"); if (MPI_Type_free(&H5FP_read_t) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed"); /* Release the barrier communicator */ if (H5FP_SAP_BARRIER_COMM != MPI_COMM_NULL) /* this comm will be NULL for the SAP */ if (MPI_Comm_free(&H5FP_SAP_BARRIER_COMM) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_free failed"); /* Release the FPH5 communicator */ if (MPI_Comm_free(&H5FP_SAP_COMM) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_free failed"); done: FUNC_LEAVE_API(ret_value); } /* *===----------------------------------------------------------------------=== * Public Library (non-API) Functions *===----------------------------------------------------------------------=== */ /* * Function: H5FP_send_metadata * Purpose: Send a string of metadata to a process. * * NOTE: You should never call this function directly!! * There's special setup for sending a string to a processor * which needs to occur first. The H5FP_request_* and * H5FP_reply_* functions take care of this for you. * * Return: Success: SUCCEED * Failure: FAIL * Programmer: Bill Wendling, 30. July, 2002 * Modifications: */ herr_t H5FP_send_metadata(const char *mdata, int len, int rank) { herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI(H5FP_send_metadata, FAIL); assert(mdata); assert(len); /* casts the CONST away: Okay */ if (MPI_Send((void *)mdata, len, MPI_BYTE, rank, H5FP_TAG_METADATA, H5FP_SAP_COMM) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Send failed"); done: FUNC_LEAVE_NOAPI(ret_value); } /* *===----------------------------------------------------------------------=== * Private Functions *===----------------------------------------------------------------------=== */ /* * Function: H5FP_commit_sap_datatypes * Purpose: Commit the H5FP_request_t, H5FP_reply_t, and H5FP_read_t * structure datatypes to MPI. * Return: Success: SUCCEED * Failure: FAIL * Programmer: Bill Wendling, 26. July, 2002 * Modifications: */ static herr_t H5FP_commit_sap_datatypes(void) { int block_length[3]; MPI_Aint displs[3]; MPI_Datatype old_types[3]; H5FP_request req; H5FP_read sap_read; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI(H5FP_commit_sap_datatypes, FAIL); /* Commit the H5FP_request_t datatype */ block_length[0] = 8; block_length[1] = 1; block_length[2] = sizeof(req.oid); old_types[0] = MPI_INT; old_types[1] = HADDR_AS_MPI_TYPE; old_types[2] = MPI_UNSIGNED_CHAR; MPI_Address(&req.req_type, &displs[0]); MPI_Address(&req.addr, &displs[1]); MPI_Address(&req.oid, &displs[2]); displs[2] -= displs[1]; displs[1] -= displs[0]; displs[0] -= displs[0]; if (MPI_Type_struct(3, block_length, displs, old_types, &H5FP_request_t) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_struct failed"); if (MPI_Type_commit(&H5FP_request_t) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_commit failed"); /* Commit the H5FP_reply_t datatype */ block_length[0] = 4; displs[0] = 0; old_types[0] = MPI_INT; if (MPI_Type_struct(1, block_length, displs, old_types, &H5FP_reply_t) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_struct failed"); if (MPI_Type_commit(&H5FP_reply_t) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_commit failed"); /* Commit the H5FP_read_t datatype */ block_length[0] = 6; block_length[1] = 1; block_length[2] = 1; old_types[0] = MPI_INT; old_types[1] = HADDR_AS_MPI_TYPE; old_types[2] = MPI_LONG_LONG; MPI_Address(&sap_read.req_id, &displs[0]); MPI_Address(&sap_read.addr, &displs[1]); MPI_Address(&sap_read.size, &displs[2]); displs[2] -= displs[1]; displs[1] -= displs[0]; displs[0] -= displs[0]; if (MPI_Type_struct(3, block_length, displs, old_types, &H5FP_read_t) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_struct failed"); if (MPI_Type_commit(&H5FP_read_t) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_commit failed"); done: FUNC_LEAVE_NOAPI(ret_value); } /* * Function: H5FP_request_sap_stop * Purpose: Request that the SAP stop it's loop processing. Each * process should send this to the SAP. * Return: Success: SUCCEED * Failure: FAIL * Programmer: Bill Wendling, 02. August, 2002 * Modifications: */ static herr_t H5FP_request_sap_stop(void) { H5FP_request req; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI(H5FP_request_sap_stop, FAIL); HDmemset(&req, 0, sizeof(req)); req.req_type = H5FP_REQ_STOP; req.req_id = 0; req.proc_rank = H5FP_my_rank; if (MPI_Send(&req, 1, H5FP_request_t, (int)H5FP_sap_rank, H5FP_TAG_REQUEST, H5FP_SAP_COMM) != MPI_SUCCESS) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Send failed"); done: FUNC_LEAVE_NOAPI(ret_value); } #endif /* H5_HAVE_FPHDF5 */