/*============================================================================ * Base functions for parallelism *============================================================================*/ /* This file is part of the "Finite Volume Mesh" library, intended to provide finite volume mesh and associated fields I/O and manipulation services. Copyright (C) 2004-2006 EDF This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ /*---------------------------------------------------------------------------- * Standard C library headers *----------------------------------------------------------------------------*/ #include #include /*---------------------------------------------------------------------------- * BFT library headers *----------------------------------------------------------------------------*/ #include /*---------------------------------------------------------------------------- * Local headers *----------------------------------------------------------------------------*/ #include #include /*---------------------------------------------------------------------------- * Header for the current file *----------------------------------------------------------------------------*/ #include "fvm_parall.h" /*----------------------------------------------------------------------------*/ #ifdef __cplusplus extern "C" { #if 0 } /* Fake brace to force back Emacs auto-indentation back to column 0 */ #endif #endif /* __cplusplus */ /*============================================================================ * Local structure definitions *============================================================================*/ /*============================================================================ * Global variables *============================================================================*/ #if defined(FVM_HAVE_MPI) /* Basic communicator info */ MPI_Comm _fvm_mpi_parall_comm = MPI_COMM_NULL; /* Intra-communicator */ int _fvm_mpi_parall_size = 1; int _fvm_mpi_parall_rank = 0; /* MPI Datatypes associated with fvm datatypes */ MPI_Datatype fvm_datatype_to_mpi[] = {MPI_DATATYPE_NULL, MPI_DOUBLE, MPI_FLOAT, MPI_INT, /* FVM_INT32 */ MPI_LONG_INT, /* FVM_INT64 */ MPI_UNSIGNED, /* FVM_UINT32 */ MPI_UNSIGNED_LONG}; /* FVM_UINT64 */ /* Minimum recommended scatter/gather buffer size */ static size_t _fvm_parall_min_coll_buf_size = 1024*1024*8; static _Bool _fvm_parall_safe_gather_mode = false; #endif /*============================================================================= * Private function definitions *============================================================================*/ /*============================================================================= * Public function definitions *============================================================================*/ #if defined(FVM_HAVE_MPI) /*---------------------------------------------------------------------------- * Return default MPI communicator for FVM library functions. * * returns: * handle to MPI communicator *----------------------------------------------------------------------------*/ MPI_Comm fvm_parall_get_mpi_comm(void) { return _fvm_mpi_parall_comm; } /*---------------------------------------------------------------------------- * Set default MPI communicator for FVM library functions. * * parameters: * comm <-- handle to MPI communicator *----------------------------------------------------------------------------*/ void fvm_parall_set_mpi_comm(const MPI_Comm comm) { int mpi_flag; MPI_Initialized(&mpi_flag); /* Set communicator */ _fvm_mpi_parall_comm = comm; if (mpi_flag != 0 && _fvm_mpi_parall_comm != MPI_COMM_NULL) { MPI_Comm_size(_fvm_mpi_parall_comm, &_fvm_mpi_parall_size); MPI_Comm_rank(_fvm_mpi_parall_comm, &_fvm_mpi_parall_rank); } else { _fvm_mpi_parall_size = 1; _fvm_mpi_parall_rank = 0; } /* Check (correct) fvm_datatype_to_mpi values */ if (mpi_flag != 0) { int size_short, size_int, size_long, size_long_long; MPI_Type_size(MPI_SHORT, &size_short); MPI_Type_size(MPI_INT, &size_int); MPI_Type_size(MPI_LONG, &size_long); #if defined(MPI_LONG_LONG) MPI_Type_size(MPI_LONG_LONG, &size_long_long); #else size_long_long = 0; #endif if (size_int == 4) { fvm_datatype_to_mpi[FVM_INT32] = MPI_INT; fvm_datatype_to_mpi[FVM_UINT32] = MPI_UNSIGNED; } else if (size_short == 4) { fvm_datatype_to_mpi[FVM_INT32] = MPI_SHORT; fvm_datatype_to_mpi[FVM_UINT32] = MPI_UNSIGNED_SHORT; } else if (size_long == 4) { fvm_datatype_to_mpi[FVM_INT32] = MPI_LONG; fvm_datatype_to_mpi[FVM_UINT32] = MPI_UNSIGNED_LONG; } if (size_int == 8) { fvm_datatype_to_mpi[FVM_INT64] = MPI_INT; fvm_datatype_to_mpi[FVM_UINT64] = MPI_UNSIGNED; } else if (size_long == 8) { fvm_datatype_to_mpi[FVM_INT64] = MPI_LONG; fvm_datatype_to_mpi[FVM_UINT64] = MPI_UNSIGNED_LONG; } #if defined(MPI_LONG_LONG) else if (size_long_long == 8) { fvm_datatype_to_mpi[FVM_INT64] = MPI_LONG_LONG; #if defined(MPI_UNSIGNED_LONG_LONG) fvm_datatype_to_mpi[FVM_UINT64] = MPI_UNSIGNED_LONG_LONG; #else fvm_datatype_to_mpi[FVM_UINT64] = MPI_LONG_LONG; #endif } #endif } } #endif /*---------------------------------------------------------------------------- * Return rank of current process among associated program processes. * * returns: * rank of current process in current communicator, or 0 in scalar mode *----------------------------------------------------------------------------*/ int fvm_parall_get_rank(void) { #if defined(FVM_HAVE_MPI) return _fvm_mpi_parall_rank; #else return 0; #endif } /*---------------------------------------------------------------------------- * Return number of processes associated with the current program. * * returns: * number of processes in current communicator, or 1 in scalar mode *----------------------------------------------------------------------------*/ int fvm_parall_get_size(void) { #if defined(FVM_HAVE_MPI) return _fvm_mpi_parall_size; #else return 1; #endif } #if defined(FVM_HAVE_MPI) /*---------------------------------------------------------------------------- * Sum counters on all FVM default communicator processes. * * parameters: * cpt <-> local counter value input, global counter value output (array) * n <-- number of counter array values *----------------------------------------------------------------------------*/ void fvm_parall_counter(fvm_gnum_t cpt[], const int n) { if (_fvm_mpi_parall_size > 1) { int i; fvm_gnum_t *sum; fvm_gnum_t _sum[64]; if (n > 64) BFT_MALLOC(sum, n, fvm_gnum_t); else sum = _sum; MPI_Allreduce(cpt, sum, n, FVM_MPI_GNUM, MPI_SUM, _fvm_mpi_parall_comm); for (i = 0; i < n ; i++) cpt[i] = sum[i]; if (sum != _sum) BFT_FREE(sum); } } /*---------------------------------------------------------------------------- * Maximum values of a counter on all FVM default communicator processes. * * parameters: * cpt <-> local counter value input, global counter value output (array) * n <-- number of counter array values *----------------------------------------------------------------------------*/ void fvm_parall_counter_max(fvm_lnum_t cpt[], const int n) { if (_fvm_mpi_parall_size > 1) { int i; fvm_lnum_t *maxval; fvm_lnum_t _maxval[64]; if (n > 64) BFT_MALLOC(maxval, n, fvm_lnum_t); else maxval = _maxval; MPI_Allreduce(cpt, maxval, n, FVM_MPI_LNUM, MPI_MAX, _fvm_mpi_parall_comm); for (i = 0; i < n ; i++) cpt[i] = maxval[i]; if (maxval != _maxval) BFT_FREE(maxval); } } #endif /* defined(FVM_HAVE_MPI) */ /*---------------------------------------------------------------------------- * Return minimum recommended scatter or gather buffer size. * * This is used by FVM's internal strided and indexed array scatter/gather * algorithms, for non MPI-IO Input/output. * * returns: * minimum recommended gather buffer size (in bytes) *----------------------------------------------------------------------------*/ size_t fvm_parall_get_min_coll_buf_size(void) { #if defined(FVM_HAVE_MPI) return _fvm_parall_min_coll_buf_size; #else return 0; #endif } /*---------------------------------------------------------------------------- * Define minimum recommended gather buffer size. * * This is used by FVM's internal strided and indexed array scatter/gather * algorithms, for non MPI-IO Input/output. * * parameters: * minimum recommended gather buffer size (in bytes) *----------------------------------------------------------------------------*/ void fvm_parall_set_min_coll_buf_size(size_t buffer_size) { #if defined(FVM_HAVE_MPI) _fvm_parall_min_coll_buf_size = buffer_size; #endif } /*---------------------------------------------------------------------------- * Indicates if safe synchronization gather mode is set. * * This is used by FVM's internal strided and indexed array gather * algorithms, for non MPI-IO Input/output. * * Using this mode (unset by default), rank 0 sends a message to other * ranks prior to receiving data from them, and data size is indicated * by an extra message rather than probing the incoming messages. * This may be slightly slower, but should avoid buffer allocation issues * with high processor counts. * * returns: * 1 if safe mode is set, 0 if unset. *----------------------------------------------------------------------------*/ int fvm_parall_get_safe_gather_mode(void) { #if defined(FVM_HAVE_MPI) return _fvm_parall_safe_gather_mode; #else return 0; #endif } /*---------------------------------------------------------------------------- * Set or unset safe synchronization gather mode. * * This is used by FVM's internal strided and indexed array gather * algorithms, for non MPI-IO Input/output. * * Using this mode (unset by default), rank 0 sends a message to other * ranks prior to receiving data from them, and data size is indicated * by an extra message rather than probing the incoming messages. * This may be slightly slower, but should avoid buffer allocation issues * with high processor counts. * * parameters: * 1 to set safe mode, 0 to unset. *----------------------------------------------------------------------------*/ void fvm_parall_set_safe_gather_mode(int safe_mode) { #if defined(FVM_HAVE_MPI) _fvm_parall_safe_gather_mode = safe_mode; #endif } /*----------------------------------------------------------------------------*/ #ifdef __cplusplus } #endif /* __cplusplus */