Resource Integration

examples/CMakeLists.txt

@@ -28,7 +28,22 @@ if (CHAI_ENABLE_CUDA OR CHAI_ENABLE_HIP)
     NAME chai-example.exe
     SOURCES example.cpp
     DEPENDS_ON ${chai_umpire_example_depends})
-
+  blt_add_executable(
+    NAME resource-depends.exe
+    SOURCES resource-depends.cpp
+    DEPENDS_ON ${chai_umpire_example_depends})
+  blt_add_executable(
+    NAME resource-simple.exe
+    SOURCES resource-simple.cpp
+    DEPENDS_ON ${chai_umpire_example_depends})
+  blt_add_executable(
+    NAME resource-not-managed.exe
+    SOURCES resource-not-managed.cpp
+    DEPENDS_ON ${chai_umpire_example_depends})
+  blt_add_executable(
+    NAME resource-multi-array.exe
+    SOURCES resource-multi-array.cpp
+    DEPENDS_ON ${chai_umpire_example_depends})
   blt_add_executable(
     NAME pinned.exe
     SOURCES pinned.cpp

examples/resource-depends.cpp

@@ -0,0 +1,52 @@
+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-20, Lawrence Livermore National Security, LLC and CHAI
+// project contributors. See the COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//////////////////////////////////////////////////////////////////////////////
+#include "camp/resource.hpp"
+#include "chai/ManagedArray.hpp"
+
+#include "../src/util/forall.hpp"
+#include "../src/util/gpu_clock.hpp"
+
+#include <vector>
+#include <utility>
+
+
+int main()
+{
+  constexpr std::size_t ARRAY_SIZE{1000};
+  int clockrate{get_clockrate()}; 
+
+  camp::resources::Resource dev1{camp::resources::Cuda{}};
+  camp::resources::Resource dev2{camp::resources::Cuda{}};
+  camp::resources::Resource host{camp::resources::Host{}};
+
+  chai::ManagedArray<double> array1(ARRAY_SIZE);
+  chai::ManagedArray<double> array2(ARRAY_SIZE);
+
+  auto e1 = forall(&dev1, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      array1[i] = i;
+      gpu_time_wait_for(10, clockrate);
+  });
+
+  auto e2 = forall(&dev2, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      array2[i] = -1;
+      gpu_time_wait_for(20, clockrate);
+  });
+
+  dev1.wait_for(&e2);
+
+  forall(&dev1, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      array1[i] *= array2[i];
+      gpu_time_wait_for(10, clockrate);
+  });
+
+  array1.move(chai::CPU, &dev1);
+
+  forall(&host, 0, 10, [=] CHAI_HOST_DEVICE (int i) {
+      printf("%f ", array1[i]);
+  });
+  printf("\n");
+}

examples/resource-multi-array.cpp

@@ -0,0 +1,50 @@
+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-20, Lawrence Livermore National Security, LLC and CHAI
+// project contributors. See the COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//////////////////////////////////////////////////////////////////////////////
+#include "camp/resource.hpp"
+#include "chai/ManagedArray.hpp"
+
+#include "../src/util/forall.hpp"
+#include "../src/util/gpu_clock.hpp"
+
+#include <vector>
+#include <utility>
+
+
+int main()
+{
+  constexpr std::size_t ARRAY_SIZE{1000};
+  int clockrate{get_clockrate()}; 
+
+  chai::ManagedArray<double> array1(ARRAY_SIZE);
+
+  camp::resources::Resource dev1{camp::resources::Cuda{}};
+  camp::resources::Resource dev2{camp::resources::Cuda{}};
+
+
+  auto e1 = forall(&dev1, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      if (i % 2 == 0) {
+        array1[i] = i;
+        gpu_time_wait_for(10, clockrate);
+      }
+  });
+
+  auto e2 = forall(&dev2, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      if (i % 2 == 1) {
+        gpu_time_wait_for(20, clockrate);
+        array1[i] = i;
+      }
+  });
+
+  array1.move(chai::CPU, &dev1);
+
+  camp::resources::Resource host{camp::resources::Host{}};
+
+  forall(&host, 0, 10, [=] CHAI_HOST_DEVICE (int i) {
+      printf("%f ", array1[i]);
+  });
+  printf("\n");
+}

examples/resource-not-managed.cpp

@@ -0,0 +1,52 @@
+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-20, Lawrence Livermore National Security, LLC and CHAI
+// project contributors. See the COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//////////////////////////////////////////////////////////////////////////////
+#include "camp/resource.hpp"
+
+#include "../src/util/forall.hpp"
+#include "../src/util/gpu_clock.hpp"
+
+#include <vector>
+#include <utility>
+
+
+int main()
+{
+  constexpr std::size_t ARRAY_SIZE{1000};
+  int clockrate{get_clockrate()};
+
+  camp::resources::Resource dev1{camp::resources::Cuda{}};
+  camp::resources::Resource dev2{camp::resources::Cuda{}};
+  camp::resources::Resource host{camp::resources::Host{}};
+
+  float * d_array1 = dev1.allocate<float>(1000);
+  float * d_array2 = dev2.allocate<float>(1000);
+  float * h_array1 = host.allocate<float>(1000);
+
+  auto e1 = forall(&dev1, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      d_array1[i] = i;
+      gpu_time_wait_for(10, clockrate);
+  });
+
+  auto e2 = forall(&dev2, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      d_array2[i] = -1;
+      gpu_time_wait_for(20, clockrate);
+  });
+
+  dev1.wait_for(&e2);
+
+  forall(&dev1, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+      d_array1[i] *= d_array2[i];
+      gpu_time_wait_for(10, clockrate);
+  });
+
+  dev1.memcpy(h_array1, d_array1, sizeof(float) * 1000);
+
+  forall(&host, 0, 10, [=] CHAI_HOST_DEVICE (int i) {
+      printf("%f ", h_array1[i]);
+  });
+  printf("\n");
+}

examples/resource-simple.cpp

@@ -0,0 +1,57 @@
+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-20, Lawrence Livermore National Security, LLC and CHAI
+// project contributors. See the COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//////////////////////////////////////////////////////////////////////////////
+#include "../src/util/forall.hpp"
+#include "../src/util/gpu_clock.hpp"
+
+#include "chai/ManagedArray.hpp"
+#include "camp/resource.hpp"
+
+#include <vector>
+#include <utility>
+
+
+int main()
+{
+
+  constexpr int NUM_ARRAYS = 16;
+  constexpr std::size_t ARRAY_SIZE{100};
+
+  std::vector<chai::ManagedArray<double>> arrays;
+  camp::resources::Resource host{camp::resources::Host{}}; 
+
+
+  int clockrate{get_clockrate()};
+
+  for (std::size_t i = 0; i < NUM_ARRAYS; ++i) {
+    arrays.push_back(chai::ManagedArray<double>(ARRAY_SIZE));
+  }
+
+  for (auto array : arrays) {
+    forall(&host, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+        array[i] = i;
+    }); 
+  }
+
+  for (auto array : arrays) {
+    camp::resources::Resource resource{camp::resources::Cuda{}}; 
+
+    forall(&resource, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+        array[i] = array[i] * 2.0;
+        gpu_time_wait_for(20, clockrate);
+    });
+
+    array.move(chai::CPU, &resource);
+  }
+
+  for (auto array : arrays) {
+    forall(&host, 0, ARRAY_SIZE, [=] CHAI_HOST_DEVICE (int i) {
+        if (i == 25) {
+          printf("array[%d] = %f \n", i, array[i]);
+        }
+    }); 
+  }
+}

scripts/spack_packages/chai/package.py

@@ -74,9 +74,12 @@ class Chai(CMakePackage, CudaPackage, ROCmPackage):
             multi=False, description='Tests to run')
 
     depends_on('umpire')
-    depends_on('raja', when="+raja")
-
     depends_on('umpire@main', when='@main')
+
+    depends_on('camp')
+    depends_on('camp@main', when='@main')
+
+    depends_on('raja', when="+raja")
     depends_on('raja@main', when="@main+raja")
 
     depends_on('[email protected]:', type='build')

src/chai/ActiveResourceManager.hpp

@@ -0,0 +1,77 @@
+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-19, Lawrence Livermore National Security, LLC and CHAI
+// project contributors. See the COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//////////////////////////////////////////////////////////////////////////////
+#ifndef CHAI_ActiveResourceManager_HPP
+#define CHAI_ActiveResourceManager_HPP
+
+#include "camp/resource.hpp"
+
+#include <array>
+#include <vector>
+
+namespace chai
+{
+
+/*!
+ * \Class to store list of Resource pointers. Holds data on the stack 
+ * until a certain threshold, then uses heap memory.
+ */
+class ActiveResourceManager {
+
+  /*!
+   * Size of array on the stack.
+   */
+  static constexpr int BASE_SIZE = 16;
+
+  /*!
+   * Base array on the stack.
+   */
+  std::array<camp::resources::Resource*, BASE_SIZE> m_res_base;
+
+  /*!
+   * Heap container for extra resources if more than BASE_SIZE pushed.
+   */
+  std::vector<camp::resources::Resource*> m_res_overflow;
+
+  /*!
+   * Current number of active resources in the list.
+   */
+  int m_size = 0;
+
+public:
+  ActiveResourceManager() = default;
+
+  /*!
+   * Retrun current size of the resource list. 
+   */
+  int size();
+
+  /*!
+   * Push a new resource onto the list. 
+   */
+  void push_back(camp::resources::Resource* res);
+
+  /*!
+   * Clear all values on the heap and set m_size to 0. 
+   */
+  void clear();
+
+  /*!
+   * Check if empty.
+   */
+  bool is_empty() const;
+
+  /*!
+   * Get resource at given index. 
+   */
+  camp::resources::Resource* operator [](int i) const;
+};
+
+}  // end of namespace chai
+
+#include "chai/ActiveResourceManager.inl"
+
+#endif  // CHAI_ActiveResourceManager_HPP

src/chai/ActiveResourceManager.inl

@@ -0,0 +1,59 @@
+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-19, Lawrence Livermore National Security, LLC and CHAI
+// project contributors. See the COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: BSD-3-Clause
+//////////////////////////////////////////////////////////////////////////////
+#ifndef CHAI_ActiveResourceManager_INL
+#define CHAI_ActiveResourceManager_INL
+
+#include "ActiveResourceManager.hpp"
+
+namespace chai
+{
+
+CHAI_INLINE
+int ActiveResourceManager::size() {
+  return m_size;
+}
+
+
+CHAI_INLINE
+void ActiveResourceManager::push_back(camp::resources::Resource * res) {
+  if (m_size < BASE_SIZE) {
+    m_res_base[m_size] = res;
+  }
+  else {
+    m_res_overflow.push_back(res); 
+  }
+
+  m_size++;
+}
+
+
+CHAI_INLINE
+void ActiveResourceManager::clear() {
+  m_res_overflow.clear();
+  m_size = 0;
+}
+
+
+CHAI_INLINE
+bool ActiveResourceManager::is_empty() const {
+  return m_size == 0;
+}
+
+
+CHAI_INLINE
+camp::resources::Resource* ActiveResourceManager::operator[](int i) const {
+  if (i < 0 || i >= m_size) {
+    return nullptr;
+  }
+  else {
+    return i < BASE_SIZE ? m_res_base[i] : m_res_overflow[i - BASE_SIZE];
+  }
+}
+
+} //end of namespace chai
+
+#endif // CHAI_ActiveResourceManager_INL