Merge pull request #953 from lkotipal/dccrg-raii

Dccrg RAII
fmihpc · Nov 12, 2024 · 636e623 · 636e623
2 parents 52786e1 + 50147f7
commit 636e623
Show file tree

Hide file tree

Showing 7 changed files with 22 additions and 21 deletions.
diff --git a/common.h b/common.h
@@ -553,6 +553,5 @@ namespace physicalconstants {
 }
 
 const std::vector<CellID>& getLocalCells();
-void recalculateLocalCellsCache();
 
 #endif
diff --git a/grid.cpp b/grid.cpp
@@ -141,12 +141,12 @@ void initializeGrids(
 
    phiprof::Timer refineTimer {"Refine spatial cells"};
    // We need this first as well
-   recalculateLocalCellsCache();
+   recalculateLocalCellsCache(mpiGrid);
    if (!P::isRestart) {
       // Note call to project.refineSpatialCells below
       if (P::amrMaxSpatialRefLevel > 0 && project.refineSpatialCells(mpiGrid)) {
          mpiGrid.balance_load();
-         recalculateLocalCellsCache();
+         recalculateLocalCellsCache(mpiGrid);
          mapRefinement(mpiGrid, technicalGrid);
       }
    } else {
@@ -155,7 +155,7 @@ void initializeGrids(
       if (myRank == MASTER_RANK) logFile << "        ...done." << endl << writeVerbose;
       if (restartSuccess) {
          mpiGrid.balance_load();
-         recalculateLocalCellsCache();
+         recalculateLocalCellsCache(mpiGrid);
          mapRefinement(mpiGrid, technicalGrid);
       }
    }
@@ -171,7 +171,7 @@ void initializeGrids(
    phiprof::Timer initialLBTimer {"Initial load-balancing"};
    if (myRank == MASTER_RANK) logFile << "(INIT): Starting initial load balance." << endl << writeVerbose;
    mpiGrid.balance_load(); // Direct DCCRG call, recalculate cache afterwards
-   recalculateLocalCellsCache();
+   recalculateLocalCellsCache(mpiGrid);
 
    SpatialCell::set_mpi_transfer_type(Transfer::VEL_BLOCK_DATA);
    mpiGrid.update_copies_of_remote_neighbors(NEAREST_NEIGHBORHOOD_ID);
@@ -650,7 +650,7 @@ void balanceLoad(dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry>& mpiGrid, S
    finishLBTimer.stop();
 
    //Make sure transfers are enabled for all cells
-   recalculateLocalCellsCache();
+   recalculateLocalCellsCache(mpiGrid);
    #pragma omp parallel for
    for (uint i=0; i<cells.size(); ++i) {
       mpiGrid[cells[i]]->set_mpi_transfer_enabled(true);
@@ -1674,7 +1674,7 @@ bool adaptRefinement(dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry>& mpiGri
 
    memory_purge(); // Purge jemalloc allocator to actually release memory
 
-   recalculateLocalCellsCache();
+   recalculateLocalCellsCache(mpiGrid);
    initSpatialCellCoordinates(mpiGrid);
 
    SpatialCell::set_mpi_transfer_type(Transfer::CELL_DIMENSIONS);
@@ -1714,3 +1714,10 @@ bool adaptRefinement(dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry>& mpiGri
    prepareAMRLists(mpiGrid);
    return true;
 }
+
+void recalculateLocalCellsCache(const dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry>& mpiGrid) {
+   // Clear-and-minimize idiom for minimizing capacity
+   // TODO: consider shrink_to_fit() or alternatively benchmark just copy assigning
+   std::vector<CellID>().swap(Parameters::localCells);
+   Parameters::localCells = mpiGrid.get_cells();
+}
diff --git a/grid.h b/grid.h
@@ -136,4 +136,6 @@ void mapRefinement(dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry>& mpiGrid,
  */
 bool adaptRefinement(dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry>& mpiGrid, FsGrid<fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, Project& project, int useStatic = -1);
 
+void recalculateLocalCellsCache(const dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry>& mpiGrid);
+
 #endif
diff --git a/ioread.cpp b/ioread.cpp
@@ -38,6 +38,7 @@
 #include "vlsv_reader_parallel.h"
 #include "vlasovmover.h"
 #include "object_wrapper.h"
+#include "grid.h"
 
 using namespace std;
 using namespace phiprof;
@@ -1236,7 +1237,7 @@ bool exec_readGrid(dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry>& mpiGrid,
    mpiGrid.balance_load(false);
 
    //update list of local gridcells
-   recalculateLocalCellsCache();
+   recalculateLocalCellsCache(mpiGrid);
 
    //get new list of local gridcells
    const vector<CellID>& gridCells = getLocalCells();

diff --git a/mini-apps/ionosphereSolverTests/main.cpp b/mini-apps/ionosphereSolverTests/main.cpp
@@ -26,9 +26,8 @@ ObjectWrapper& getObjectWrapper() {
 std::vector<CellID> localCellDummy;
 const std::vector<CellID>& getLocalCells() { return localCellDummy; }
 void deallocateRemoteCellBlocks(dccrg::Dccrg<spatial_cell::SpatialCell, dccrg::Cartesian_Geometry, std::tuple<>, std::tuple<> >&) {};
-void updateRemoteVelocityBlockLists(dccrg::Dccrg<spatial_cell::SpatialCell, dccrg::Cartesian_Geometry, std::tuple<>, std::tuple<> >&, unsigned int, unsigned int) {
-};
-void recalculateLocalCellsCache() {}
+void updateRemoteVelocityBlockLists(dccrg::Dccrg<spatial_cell::SpatialCell, dccrg::Cartesian_Geometry, std::tuple<>, std::tuple<> >&, unsigned int, unsigned int) {};
+void recalculateLocalCellsCache(const dccrg::Dccrg<spatial_cell::SpatialCell, dccrg::Cartesian_Geometry, std::tuple<>, std::tuple<> >&) {};
 SysBoundary::SysBoundary() {}
 SysBoundary::~SysBoundary() {}
 

diff --git a/submodules/dccrg b/submodules/dccrg
diff --git a/vlasiator.cpp b/vlasiator.cpp
@@ -71,7 +71,6 @@ void fpehandler(int sig_num)
 #include "phiprof.hpp"
 
 Logger logFile, diagnostic;
-static dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry> mpiGrid;
 
 using namespace std;
 
@@ -281,14 +280,6 @@ const std::vector<CellID>& getLocalCells() {
    return Parameters::localCells;
 }
 
-void recalculateLocalCellsCache() {
-     {
-        vector<CellID> dummy;
-        dummy.swap(Parameters::localCells);
-     }
-   Parameters::localCells = mpiGrid.get_cells();
-}
-
 int simulate(int argn,char* args[]) {
    int myRank, doBailout=0;
    const creal DT_EPSILON=1e-12;
@@ -473,6 +464,8 @@ int simulate(int argn,char* args[]) {
    // FULL_NEIGHBORHOOD. Block lists up to date for
    // VLASOV_SOLVER_NEIGHBORHOOD (but dist function has not been communicated)
    phiprof::Timer initGridsTimer {"Init grids"};
+   dccrg::Dccrg<SpatialCell,dccrg::Cartesian_Geometry> mpiGrid;
+
    initializeGrids(
       argn,
       args,