path: root/src/H5FP.c

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 * Copyright by the Board of Trustees of the University of Illinois.         *
 * All rights reserved.                                                      *
 *                                                                           *
 * This file is part of HDF5.  The full HDF5 copyright notice, including     *
 * terms governing use, modification, and redistribution, is contained in    *
 * the files COPYING and Copyright.html.  COPYING can be found at the root   *
 * of the source code distribution tree; Copyright.html can be found at the  *
 * root level of an installed copy of the electronic HDF5 document set and   *
 * is linked from the top-level documents page.  It can also be found at     *
 * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html.  If you do not have     *
 * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

/* Private header files */
#include "H5private.h"          /* Generic Functions                    */
#include "H5Eprivate.h"         /* Error Handling                       */
#include "H5Oprivate.h"         /* Object Headers                       */
#include "H5TBprivate.h"        /* Threaded, Balanced, Binary Trees     */

#ifdef H5_HAVE_FPHDF5

#include "H5FPprivate.h"        /* Flexible Parallel Functions          */

/* Pablo mask */
#define PABLO_MASK          H5FP_mask

/* Interface initialization */
#define INTERFACE_INIT      NULL
static int interface_initialize_g = 0;

MPI_Datatype H5FP_request_t;    /* MPI datatype for the H5FP_request obj*/
MPI_Datatype H5FP_reply_t;      /* MPI datatype for the H5FP_reply obj  */
MPI_Datatype H5FP_read_t;       /* MPI datatype for the H5FP_read obj   */

/* SAP specific variables */
MPI_Comm H5FP_SAP_COMM;         /* Comm we use: Supplied by user        */
MPI_Comm H5FP_SAP_BARRIER_COMM; /* Comm if you want to do a barrier     */

unsigned H5FP_sap_rank;         /* The rank of the SAP: Supplied by user*/
unsigned H5FP_capt_rank;        /* The rank which tells SAP of opens    */
unsigned H5FP_my_rank;          /* Rank of this process in the COMM     */
int H5FP_comm_size;             /* Size of the COMM                     */

/* local functions */
static herr_t H5FP_commit_sap_datatypes(void);
static herr_t H5FP_request_sap_stop(void);

/*
 *===----------------------------------------------------------------------===
 *                          Public (API) Functions
 *===----------------------------------------------------------------------===
 */

/*
 * Function:    H5FPinit
 * Purpose:     Initialize the SAP environment: duplicate the COMM the user
 *              supplies to us, set aside the SAP_RANK as the SAP.
 * Return:      Success:    SUCCEED
 *              Failure:    FAIL
 * Programmer:  Bill Wendling, 26. July, 2002
 * Modifications:
 */
herr_t
H5FPinit(MPI_Comm comm, int sap_rank)
{
    MPI_Group sap_group = MPI_GROUP_NULL, sap_barrier_group = MPI_GROUP_NULL;
    herr_t ret_value = SUCCEED;

    FUNC_ENTER_API(H5FPinit, FAIL);
    H5TRACE2("e","McIs",comm,sap_rank);

    /* initialize to NULL so that we can release if an error occurs */
    H5FP_request_t = MPI_DATATYPE_NULL;
    H5FP_reply_t = MPI_DATATYPE_NULL;
    H5FP_read_t = MPI_DATATYPE_NULL;

    H5FP_SAP_COMM = MPI_COMM_NULL;
    H5FP_SAP_BARRIER_COMM = MPI_COMM_NULL;

    /* Set the global variable to track the SAP's rank */
    H5FP_sap_rank = sap_rank;

    /* Make a private copy of the communicator passed to us */
    if (MPI_Comm_dup(comm, &H5FP_SAP_COMM) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_dup failed");

    if (MPI_Comm_group(H5FP_SAP_COMM, &sap_group) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_group failed");

    /* Exclude the SAP from the barrier group group */
    if (MPI_Group_excl(sap_group, 1, (int *)&H5FP_sap_rank,
                       &sap_barrier_group) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Group_excl failed");

    /* Create communicator for barrier group (all processes except the SAP) */
    if (MPI_Comm_create(H5FP_SAP_COMM, sap_barrier_group,
                        &H5FP_SAP_BARRIER_COMM) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_create failed");

    /* Get the size of all the processes (including the SAP) */
    if (MPI_Comm_size(H5FP_SAP_COMM, &H5FP_comm_size) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_size failed");

    /*
     * We assign the process right after the sap_rank as the one which
     * will tell the SAP that files have been opened or closed. We mod
     * it so that we don't go over the size of the communicator.
     */
    H5FP_capt_rank = (H5FP_sap_rank + 1) % H5FP_comm_size;

    /* Get this processes rank */
    if (MPI_Comm_rank(H5FP_SAP_COMM, (int *)&H5FP_my_rank) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_rank failed");

    /* Create the MPI types used for communicating with the SAP */
    if (H5FP_commit_sap_datatypes() != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "H5FP_commit_sap_datatypes failed");

    /* Go loop, if we are the SAP */
    if (H5FP_my_rank == H5FP_sap_rank)
        H5FP_sap_receive_loop();

    /* Fall through and return to user, if not SAP */

done:
    if (ret_value == FAIL) {
        /* we've encountered an error...clean up */
        if (H5FP_request_t != MPI_DATATYPE_NULL)
            if (MPI_Type_free(&H5FP_request_t) != MPI_SUCCESS)
                HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed");

        if (H5FP_reply_t != MPI_DATATYPE_NULL)
            if (MPI_Type_free(&H5FP_reply_t) != MPI_SUCCESS)
                HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed");

        if (H5FP_read_t != MPI_DATATYPE_NULL)
            if (MPI_Type_free(&H5FP_read_t) != MPI_SUCCESS)
                HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed");

        if (H5FP_SAP_BARRIER_COMM != MPI_COMM_NULL)
            /* this comm will be NULL for the SAP */
            if (MPI_Comm_free(&H5FP_SAP_BARRIER_COMM) != MPI_SUCCESS)
                HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_free failed");

        if (H5FP_SAP_COMM != MPI_COMM_NULL)
            if (MPI_Comm_free(&H5FP_SAP_COMM) != MPI_SUCCESS)
                HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_free failed");
    }

    if (sap_group != MPI_GROUP_NULL)
        MPI_Group_free(&sap_group);

    if (sap_barrier_group != MPI_GROUP_NULL)
        MPI_Group_free(&sap_barrier_group);

    FUNC_LEAVE_API(ret_value);
}

/*
 * Function:    H5FPfinalize
 * Purpose:     Get rid of the initilized environment we setup with H5FPinit.
 *              Mostly just freeing the duplicated COMM object and committed
 *              datatypes.
 * Return:      Success:    SUCCEED
 *              Failure:    FAIL
 * Programmer:  Bill Wendling, 26. July, 2002
 * Modifications:
 */
herr_t
H5FPfinalize(void)
{
    herr_t ret_value = SUCCEED;

    FUNC_ENTER_API(H5FPfinalize, FAIL);
    H5TRACE0("e","");

    /* Stop the SAP */
    if (H5FP_my_rank != H5FP_sap_rank)
        if (H5FP_request_sap_stop() < 0)
            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Error stopping the SAP");

    /* Release the MPI types we created */
    if (MPI_Type_free(&H5FP_request_t) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed");

    if (MPI_Type_free(&H5FP_reply_t) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed");

    if (MPI_Type_free(&H5FP_read_t) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed");

    /* Release the barrier communicator */
    if (H5FP_SAP_BARRIER_COMM != MPI_COMM_NULL)
        /* this comm will be NULL for the SAP */
        if (MPI_Comm_free(&H5FP_SAP_BARRIER_COMM) != MPI_SUCCESS)
            HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_free failed");

    /* Release the FPH5 communicator */
    if (MPI_Comm_free(&H5FP_SAP_COMM) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_free failed");

done:
    FUNC_LEAVE_API(ret_value);
}

/*
 *===----------------------------------------------------------------------===
 *                    Public Library (non-API) Functions
 *===----------------------------------------------------------------------===
 */

/*
 * Function:    H5FP_send_metadata
 * Purpose:     Send a string of metadata to a process.
 *
 *              NOTE: You should never call this function directly!!
 *              There's special setup for sending a string to a processor
 *              which needs to occur first. The H5FP_request_* and
 *              H5FP_reply_* functions take care of this for you.
 *
 * Return:      Success:    SUCCEED
 *              Failure:    FAIL
 * Programmer:  Bill Wendling, 30. July, 2002
 * Modifications:
 */
herr_t
H5FP_send_metadata(const char *mdata, int len, int rank)
{
    herr_t ret_value = SUCCEED;

    FUNC_ENTER_NOAPI(H5FP_send_metadata, FAIL);

    assert(mdata);
    assert(len);

    /* casts the CONST away: Okay */
    if (MPI_Send((void *)mdata, len, MPI_BYTE, rank, H5FP_TAG_METADATA, H5FP_SAP_COMM)
            != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Send failed");

done:
    FUNC_LEAVE_NOAPI(ret_value);
}

/*
 *===----------------------------------------------------------------------===
 *                            Private Functions
 *===----------------------------------------------------------------------===
 */

/*
 * Function:    H5FP_commit_sap_datatypes
 * Purpose:     Commit the H5FP_request_t, H5FP_reply_t, and H5FP_read_t
 *              structure datatypes to MPI.
 * Return:      Success:    SUCCEED
 *              Failure:    FAIL
 * Programmer:  Bill Wendling, 26. July, 2002
 * Modifications:
 */
static herr_t
H5FP_commit_sap_datatypes(void)
{
    int block_length[3];
    MPI_Aint displs[3];
    MPI_Datatype old_types[3];
    H5FP_request req;
    H5FP_read sap_read;
    herr_t ret_value = SUCCEED;

    FUNC_ENTER_NOAPI(H5FP_commit_sap_datatypes, FAIL);

    /* Commit the H5FP_request_t datatype */
    block_length[0] = 8;
    block_length[1] = 1;
    block_length[2] = sizeof(req.oid);
    old_types[0] = MPI_INT;
    old_types[1] = HADDR_AS_MPI_TYPE;
    old_types[2] = MPI_UNSIGNED_CHAR;
    MPI_Address(&req.req_type, &displs[0]);
    MPI_Address(&req.addr, &displs[1]);
    MPI_Address(&req.oid, &displs[2]);
    displs[2] -= displs[1];
    displs[1] -= displs[0];
    displs[0] -= displs[0];

    if (MPI_Type_struct(3, block_length, displs, old_types,
                        &H5FP_request_t) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_struct failed");

    if (MPI_Type_commit(&H5FP_request_t) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_commit failed");

    /* Commit the H5FP_reply_t datatype */
    block_length[0] = 4;
    displs[0] = 0;
    old_types[0] = MPI_INT;

    if (MPI_Type_struct(1, block_length, displs, old_types,
                        &H5FP_reply_t) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_struct failed");

    if (MPI_Type_commit(&H5FP_reply_t) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_commit failed");

    /* Commit the H5FP_read_t datatype */
    block_length[0] = 6;
    block_length[1] = 1;
    block_length[2] = 1;
    old_types[0] = MPI_INT;
    old_types[1] = HADDR_AS_MPI_TYPE;
    old_types[2] = MPI_LONG_LONG;
    MPI_Address(&sap_read.req_id, &displs[0]);
    MPI_Address(&sap_read.addr, &displs[1]);
    MPI_Address(&sap_read.size, &displs[2]);
    displs[2] -= displs[1];
    displs[1] -= displs[0];
    displs[0] -= displs[0];

    if (MPI_Type_struct(3, block_length, displs, old_types,
                        &H5FP_read_t) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_struct failed");

    if (MPI_Type_commit(&H5FP_read_t) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_commit failed");

done:
    FUNC_LEAVE_NOAPI(ret_value);
}

/*
 * Function:    H5FP_request_sap_stop
 * Purpose:     Request that the SAP stop it's loop processing. Each
 *              process should send this to the SAP.
 * Return:      Success:    SUCCEED
 *              Failure:    FAIL
 * Programmer:  Bill Wendling, 02. August, 2002
 * Modifications:
 */
static herr_t
H5FP_request_sap_stop(void)
{
    H5FP_request req;
    herr_t ret_value = SUCCEED;

    FUNC_ENTER_NOAPI(H5FP_request_sap_stop, FAIL);

    HDmemset(&req, 0, sizeof(req));
    req.req_type = H5FP_REQ_STOP;
    req.req_id = 0;
    req.proc_rank = H5FP_my_rank;

    if (MPI_Send(&req, 1, H5FP_request_t, (int)H5FP_sap_rank,
                 H5FP_TAG_REQUEST, H5FP_SAP_COMM) != MPI_SUCCESS)
        HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Send failed");

done:
    FUNC_LEAVE_NOAPI(ret_value);
}

#endif  /* H5_HAVE_FPHDF5 */