Skip to content

Commit

Permalink
Merge MPI Reduction Calls (ECP-WarpX#4051)
Browse files Browse the repository at this point in the history
  • Loading branch information
@WeiqunZhang
WeiqunZhang authored Jul 24, 2023
1 parent b0abf41 commit 87091b3
Show file tree
Hide file tree
Showing 5 changed files with 17 additions and 48 deletions.
4 changes: 1 addition & 3 deletions Source/Diagnostics/ReducedDiags/FieldMomentum.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,9 +182,7 @@ void FieldMomentum::ComputeDiags (int step)
amrex::Real ExB_x = amrex::get<0>(r);
amrex::Real ExB_y = amrex::get<1>(r);
amrex::Real ExB_z = amrex::get<2>(r);
amrex::ParallelDescriptor::ReduceRealSum(ExB_x);
amrex::ParallelDescriptor::ReduceRealSum(ExB_y);
amrex::ParallelDescriptor::ReduceRealSum(ExB_z);
amrex::ParallelDescriptor::ReduceRealSum({ExB_x,ExB_y,ExB_z});

// Get cell size
amrex::Geometry const & geom = warpx.Geom(lev);
Expand Down
3 changes: 1 addition & 2 deletions Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,7 @@ void ParticleEnergy::ComputeDiags (int step)
}

// Reduced sum over MPI ranks
ParallelDescriptor::ReduceRealSum(Etot, ParallelDescriptor::IOProcessorNumber());
ParallelDescriptor::ReduceRealSum(Ws , ParallelDescriptor::IOProcessorNumber());
ParallelDescriptor::ReduceRealSum({Etot,Ws}, ParallelDescriptor::IOProcessorNumber());

// Accumulate sum of weights over all species (must come after MPI reduction of Ws)
Wtot += Ws;
Expand Down
49 changes: 13 additions & 36 deletions Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,114 +197,92 @@ void ParticleExtrema::ComputeDiags (int step)
Real xmin = ReduceMin( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.pos(0)*std::cos(p.rdata(PIdx::theta)); });
ParallelDescriptor::ReduceRealMin(xmin);
#elif (defined WARPX_DIM_1D_Z)
const Real xmin = 0.0_rt;
Real xmin = 0.0_rt;
#else
Real xmin = ReduceMin( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.pos(0); });
ParallelDescriptor::ReduceRealMin(xmin);
#endif

// xmax
#if (defined WARPX_DIM_RZ)
Real t_xmax = ReduceMax( myspc,
Real xmax = ReduceMax( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.pos(0)*std::cos(p.rdata(PIdx::theta)); });
ParallelDescriptor::ReduceRealMax(t_xmax);
const Real xmax = t_xmax;
#elif (defined WARPX_DIM_1D_Z)
const Real xmax = 0.0_rt;
Real xmax = 0.0_rt;
#else
Real t_xmax = ReduceMax( myspc,
Real xmax = ReduceMax( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.pos(0); });
ParallelDescriptor::ReduceRealMax(t_xmax);
const Real xmax = t_xmax;
#endif

// ymin
#if (defined WARPX_DIM_RZ)
Real t_ymin = ReduceMin( myspc,
Real ymin = ReduceMin( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.pos(0)*std::sin(p.rdata(PIdx::theta)); });
ParallelDescriptor::ReduceRealMin(t_ymin);
const Real ymin = t_ymin;
#elif (defined WARPX_DIM_XZ || WARPX_DIM_1D_Z)
const Real ymin = 0.0_rt;
Real ymin = 0.0_rt;
#else
Real t_ymin = ReduceMin( myspc,
Real ymin = ReduceMin( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.pos(1); });
ParallelDescriptor::ReduceRealMin(t_ymin);
const Real ymin = t_ymin;
#endif

// ymax
#if (defined WARPX_DIM_RZ)
Real t_ymax = ReduceMax( myspc,
Real ymax = ReduceMax( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.pos(0)*std::sin(p.rdata(PIdx::theta)); });
ParallelDescriptor::ReduceRealMax(t_ymax);
const Real ymax = t_ymax;
#elif (defined WARPX_DIM_XZ || WARPX_DIM_1D_Z)
const Real ymax = 0.0_rt;
Real ymax = 0.0_rt;
#else
Real t_ymax = ReduceMax( myspc,
Real ymax = ReduceMax( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.pos(1); });
ParallelDescriptor::ReduceRealMax(t_ymax);
const Real ymax = t_ymax;
#endif

// zmin
Real zmin = ReduceMin( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.pos(index_z); });
ParallelDescriptor::ReduceRealMin(zmin);

// zmax
Real zmax = ReduceMax( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.pos(index_z); });
ParallelDescriptor::ReduceRealMax(zmax);

// uxmin
Real uxmin = ReduceMin( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.rdata(PIdx::ux); });
ParallelDescriptor::ReduceRealMin(uxmin);

// uxmax
Real uxmax = ReduceMax( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.rdata(PIdx::ux); });
ParallelDescriptor::ReduceRealMax(uxmax);

// uymin
Real uymin = ReduceMin( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.rdata(PIdx::uy); });
ParallelDescriptor::ReduceRealMin(uymin);

// uymax
Real uymax = ReduceMax( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.rdata(PIdx::uy); });
ParallelDescriptor::ReduceRealMax(uymax);

// uzmin
Real uzmin = ReduceMin( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.rdata(PIdx::uz); });
ParallelDescriptor::ReduceRealMin(uzmin);

// uzmax
Real uzmax = ReduceMax( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.rdata(PIdx::uz); });
ParallelDescriptor::ReduceRealMax(uzmax);

// gmin
Real gmin = 0.0_rt;
Expand All @@ -329,7 +307,6 @@ void ParticleExtrema::ComputeDiags (int step)
return std::sqrt(1.0_rt + us*inv_c2);
});
}
ParallelDescriptor::ReduceRealMin(gmin);

// gmax
Real gmax = 0.0_rt;
Expand All @@ -354,19 +331,19 @@ void ParticleExtrema::ComputeDiags (int step)
return std::sqrt(1.0_rt + us*inv_c2);
});
}
ParallelDescriptor::ReduceRealMax(gmax);

// wmin
Real wmin = ReduceMin( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.rdata(PIdx::w); });
ParallelDescriptor::ReduceRealMin(wmin);

// wmax
Real wmax = ReduceMax( myspc,
[=] AMREX_GPU_HOST_DEVICE (const PType& p)
{ return p.rdata(PIdx::w); });
ParallelDescriptor::ReduceRealMax(wmax);

ParallelDescriptor::ReduceRealMin({xmin,ymin,zmin,uxmin,uymin,uzmin,gmin,wmin});
ParallelDescriptor::ReduceRealMax({xmax,ymax,zmax,uxmax,uymax,uzmax,gmax,wmax});

#if (defined WARPX_QED)
// get number of level (int)
Expand Down
5 changes: 1 addition & 4 deletions Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,10 +167,7 @@ void ParticleMomentum::ComputeDiags (int step)
amrex::Real Ws = amrex::get<3>(r);

// Reduced sum over MPI ranks
ParallelDescriptor::ReduceRealSum(Px, ParallelDescriptor::IOProcessorNumber());
ParallelDescriptor::ReduceRealSum(Py, ParallelDescriptor::IOProcessorNumber());
ParallelDescriptor::ReduceRealSum(Pz, ParallelDescriptor::IOProcessorNumber());
ParallelDescriptor::ReduceRealSum(Ws, ParallelDescriptor::IOProcessorNumber());
ParallelDescriptor::ReduceRealSum({Px,Py,Pz,Ws}, ParallelDescriptor::IOProcessorNumber());

// Accumulate sum of weights over all species (must come after MPI reduction of Ws)
Wtot += Ws;
Expand Down
4 changes: 1 addition & 3 deletions Source/Particles/WarpXParticleContainer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1223,9 +1223,7 @@ std::array<ParticleReal, 3> WarpXParticleContainer::meanParticleVelocity(bool lo
}

if (local == false) {
ParallelDescriptor::ReduceRealSum(vx_total);
ParallelDescriptor::ReduceRealSum(vy_total);
ParallelDescriptor::ReduceRealSum(vz_total);
ParallelDescriptor::ReduceRealSum<ParticleReal>({vx_total,vy_total,vz_total});
ParallelDescriptor::ReduceLongSum(np_total);
}

Expand Down

0 comments on commit 87091b3

Please sign in to comment.