Line data    Source code

       1             : #ifndef AMREX_PARALLELCONTEXT_H
       2             : #define AMREX_PARALLELCONTEXT_H
       3             : #include <AMReX_Config.H>
       4             : 
       5             : #include <AMReX_Extension.H>
       6             : #include <AMReX_Vector.H>
       7             : #include <AMReX_ccse-mpi.H>
       8             : 
       9             : #include <iosfwd>
      10             : #include <memory>
      11             : 
      12             : namespace amrex::ParallelContext {
      13             : 
      14             : class Frame
      15             : {
      16             : public:
      17             : 
      18             :     Frame (MPI_Comm c, int id, int io_rank);
      19             :     explicit Frame (MPI_Comm c);
      20             :     Frame (Frame const&) = delete;
      21             :     Frame& operator= (Frame const&) = delete;
      22             :     Frame& operator= (Frame &&) = delete;
      23             :     Frame (Frame && rhs) noexcept;
      24             :     ~Frame ();
      25             : 
      26             :     [[nodiscard]] int MyID   () const noexcept { return m_id; }
      27       14017 :     [[nodiscard]] int MyProc () const noexcept { return m_rank_me; }
      28     1230500 :     [[nodiscard]] int NProcs () const noexcept { return m_nranks; }
      29        6235 :     [[nodiscard]] int IOProc () const noexcept { return m_io_rank; }
      30             :     static int local_to_global_rank (int lrank);
      31             :     static void local_to_global_rank (int* global, const int* local, int n);
      32             :     static void global_to_local_rank (int* local, const int* global, int n);
      33             :     static int global_to_local_rank (int grank);
      34             :     int get_inc_mpi_tag ();
      35             :     void set_ofs_name (std::string filename);
      36             :     std::ofstream * get_ofs_ptr ();
      37             : 
      38             :     //! sub-communicator associated with frame
      39             :     MPI_Comm comm = MPI_COMM_NULL;
      40             :     MPI_Group group = MPI_GROUP_NULL; //!< to avoid repeatedly creating groups in rank translation
      41             : 
      42             : private:
      43             :     int m_id      = -1; //!< frame ID
      44             :     int m_rank_me =  0; //!< local rank
      45             :     int m_nranks  =  1; //!< local # of ranks
      46             :     int m_mpi_tag = -1;
      47             :     int m_io_rank = -1;
      48             :     std::string m_out_filename;
      49             :     std::unique_ptr<std::ofstream> m_out;
      50             : };
      51             : 
      52             : extern AMREX_EXPORT Vector<Frame> frames; //!< stack of communicator frames
      53             : 
      54             : //! world communicator
      55             : inline MPI_Comm CommunicatorAll () noexcept { return frames[0].comm; }
      56             : //! world group
      57             : inline MPI_Group GroupAll () noexcept { return frames[0].group; }
      58             : //! number of ranks in world communicator
      59         908 : inline int NProcsAll () noexcept { return frames[0].NProcs(); }
      60             : //! my rank in world communicator
      61        6756 : inline int MyProcAll () noexcept { return frames[0].MyProc(); }
      62             : //! IO rank in world communicator
      63             : inline int IOProcessorNumberAll () noexcept { return frames[0].IOProc(); }
      64             : //! Am IO processor for world communicator?
      65             : inline bool IOProcessorAll () noexcept { return MyProcAll() == IOProcessorNumberAll(); }
      66             : //! Pointer to ofstream
      67             : inline std::ofstream * OFSPtrAll () noexcept { return frames[0].get_ofs_ptr(); }
      68             : 
      69             : //! sub-communicator for current frame
      70        9313 : inline MPI_Comm CommunicatorSub () noexcept { return frames.back().comm; }
      71             : //! sub-group for current frame
      72             : inline MPI_Group GroupSub () noexcept { return frames.back().group; }
      73             : //! number of ranks in current frame
      74     1229600 : inline int NProcsSub () noexcept { return frames.back().NProcs(); }
      75             : //! my sub-rank in current frame
      76        7261 : inline int MyProcSub () noexcept { return frames.back().MyProc(); }
      77             : //! IO sub-rank in current frame
      78        6235 : inline int IOProcessorNumberSub () noexcept { return frames.back().IOProc(); }
      79             : //! Am IO processor for current frame?
      80             : inline bool IOProcessorSub () noexcept { return MyProcSub() == IOProcessorNumberSub(); }
      81             : //! Pointer to ofstream
      82        7261 : inline std::ofstream * OFSPtrSub () noexcept { return frames.back().get_ofs_ptr(); }
      83             : 
      84             : #ifdef BL_USE_MPI
      85             : inline void BarrierSub () noexcept { MPI_Barrier(CommunicatorSub()); }
      86             : inline void BarrierAll () noexcept { MPI_Barrier(CommunicatorAll()); }
      87             : #else
      88             : inline void BarrierSub () noexcept { }
      89             : inline void BarrierAll () noexcept { }
      90             : #endif
      91             : 
      92             : //! get and increment mpi tag in current frame
      93           0 : inline int get_inc_mpi_tag () noexcept { return frames.back().get_inc_mpi_tag(); }
      94             : //! translate between local rank and global rank
      95             : inline int local_to_global_rank (int rank) noexcept { return Frame::local_to_global_rank(rank); }
      96             : inline void local_to_global_rank (int* global, const int* local, int n) noexcept
      97             :     { Frame::local_to_global_rank(global, local, n); }
      98           0 : inline int global_to_local_rank (int rank) noexcept { return Frame::global_to_local_rank(rank); }
      99             : inline void global_to_local_rank (int* local, const int* global, int n) noexcept
     100             :     { Frame::global_to_local_rank(local, global, n); }
     101             : 
     102             : inline void push (MPI_Comm c) { frames.emplace_back(c); }
     103             : inline void push (MPI_Comm c, int id, int io_rank) { frames.emplace_back(c, id, io_rank); }
     104             : inline void set_last_frame_ofs (const std::string & filename) {
     105             :     frames.back().set_ofs_name(filename);
     106             : }
     107             : //! Note that it's the user's responsibility to free the MPI_Comm
     108             : inline void pop () { frames.pop_back(); }
     109             : 
     110             : }
     111             : 
     112             : #endif // AMREX_PARALLELCONTEXT_H