From 23142ce81d5892869e3df6df7580645a9f50355e Mon Sep 17 00:00:00 2001 From: Yuuichi Asahi Date: Mon, 22 Jan 2024 23:51:36 +0900 Subject: [PATCH 1/7] Compare the input and output View extents if plans are reused --- common/src/KokkosFFT_utils.hpp | 13 +++++++++ common/unit_test/Test_Utils.cpp | 40 +++++++++++++++++++++++++ fft/src/KokkosFFT_Plans.hpp | 30 +++++++++++++++---- fft/src/KokkosFFT_Transform.hpp | 12 ++++---- fft/unit_test/Test_Transform.cpp | 50 ++++++++++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 11 deletions(-) diff --git a/common/src/KokkosFFT_utils.hpp b/common/src/KokkosFFT_utils.hpp index d3be71df..45961150 100644 --- a/common/src/KokkosFFT_utils.hpp +++ b/common/src/KokkosFFT_utils.hpp @@ -186,6 +186,19 @@ void conjugate(const ExecutionSpace& exec_space, const InViewType& in, out_data[i] = Kokkos::conj(in_data[i]); }); } + +template +auto extract_extents(const ViewType& view) { + static_assert(Kokkos::is_view::value, + "KokkosFFT::Impl::extract_extents: ViewType is not a Kokkos::View."); + constexpr std::size_t rank = ViewType::rank(); + std::array extents; + for(std::size_t i=0; i; + using View2Dtype = Kokkos::View; + using View3Dtype = Kokkos::View; + using View4Dtype = Kokkos::View; + using View5Dtype = Kokkos::View; + using View6Dtype = Kokkos::View; + using View7Dtype = Kokkos::View; + using View8Dtype = Kokkos::View; + + std::size_t n1 = 1, n2 = 1, n3 = 2, n4 = 3, n5= 5, n6 = 8, n7 = 13, n8 = 21; + + std::array ref_extents1D = {n1}; + std::array ref_extents2D = {n1, n2}; + std::array ref_extents3D = {n1, n2, n3}; + std::array ref_extents4D = {n1, n2, n3, n4}; + std::array ref_extents5D = {n1, n2, n3, n4, n5}; + std::array ref_extents6D = {n1, n2, n3, n4, n5, n6}; + std::array ref_extents7D = {n1, n2, n3, n4, n5, n6, n7}; + std::array ref_extents8D = {n1, n2, n3, n4, n5, n6, n7, n8}; + + View1Dtype view1D("view1D", n1); + View2Dtype view2D("view2D", n1, n2); + View3Dtype view3D("view3D", n1, n2, n3); + View4Dtype view4D("view4D", n1, n2, n3, n4); + View5Dtype view5D("view5D", n1, n2, n3, n4, n5); + View6Dtype view6D("view6D", n1, n2, n3, n4, n5, n6); + View7Dtype view7D("view7D", n1, n2, n3, n4, n5, n6, n7); + View8Dtype view8D("view8D", n1, n2, n3, n4, n5, n6, n7, n8); + + EXPECT_EQ( KokkosFFT::Impl::extract_extents(view1D), ref_extents1D ); + EXPECT_EQ( KokkosFFT::Impl::extract_extents(view2D), ref_extents2D ); + EXPECT_EQ( KokkosFFT::Impl::extract_extents(view3D), ref_extents3D ); + EXPECT_EQ( KokkosFFT::Impl::extract_extents(view4D), ref_extents4D ); + EXPECT_EQ( KokkosFFT::Impl::extract_extents(view5D), ref_extents5D ); + EXPECT_EQ( KokkosFFT::Impl::extract_extents(view6D), ref_extents6D ); + EXPECT_EQ( KokkosFFT::Impl::extract_extents(view7D), ref_extents7D ); + EXPECT_EQ( KokkosFFT::Impl::extract_extents(view8D), ref_extents8D ); } \ No newline at end of file diff --git a/fft/src/KokkosFFT_Plans.hpp b/fft/src/KokkosFFT_Plans.hpp index e1c03fd4..23f85968 100644 --- a/fft/src/KokkosFFT_Plans.hpp +++ b/fft/src/KokkosFFT_Plans.hpp @@ -44,6 +44,7 @@ class Plan { using map_type = axis_type; using nonConstInViewType = std::remove_cv_t; using nonConstOutViewType = std::remove_cv_t; + using extents_type = shape_type; fft_plan_type m_plan; fft_size_type m_fft_size; @@ -52,6 +53,9 @@ class Plan { axis_type m_axes; KokkosFFT::Impl::Direction m_direction; + // Keep extents + extents_type m_in_extents, m_out_extents; + // Only used when transpose needed nonConstInViewType m_in_T; nonConstOutViewType m_out_T; @@ -67,6 +71,8 @@ class Plan { "KokkosFFT::Plan: OutViewType is not a Kokkos::View."); m_axes = {axis}; + m_in_extents = KokkosFFT::Impl::extract_extents(in); + m_out_extents = KokkosFFT::Impl::extract_extents(out); std::tie(m_map, m_map_inv) = KokkosFFT::Impl::get_map_axes(in, axis); m_is_transpose_needed = KokkosFFT::Impl::is_transpose_needed(m_map); m_fft_size = KokkosFFT::Impl::_create(exec_space, m_plan, in, out, @@ -85,6 +91,8 @@ class Plan { static_assert(Kokkos::is_view::value, "KokkosFFT::Plan: OutViewType is not a Kokkos::View."); + m_in_extents = KokkosFFT::Impl::extract_extents(in); + m_out_extents = KokkosFFT::Impl::extract_extents(out); std::tie(m_map, m_map_inv) = KokkosFFT::Impl::get_map_axes(in, axes); m_is_transpose_needed = KokkosFFT::Impl::is_transpose_needed(m_map); m_fft_size = @@ -95,18 +103,18 @@ class Plan { template - void good(KokkosFFT::Impl::Direction direction, axis_type axes) const { + void good(const InViewType2& in, const OutViewType2& out, KokkosFFT::Impl::Direction direction, axis_type axes) const { static_assert(std::is_same_v, - "KokkosFFT::Plan: is_good: Execution spaces for plan and " + "KokkosFFT::Impl::Plan::good: Execution spaces for plan and " "execution are inconsistent."); using nonConstInViewType2 = std::remove_cv_t; using nonConstOutViewType2 = std::remove_cv_t; static_assert(std::is_same_v, - "KokkosFFT::Plan: is_good: InViewType for plan and execution " + "KokkosFFT::Impl::Plan::good: InViewType for plan and execution " "are inconsistent."); static_assert(std::is_same_v, - "KokkosFFT::Plan: is_good: OutViewType for plan and " + "KokkosFFT::Impl::Plan::good: OutViewType for plan and " "execution are inconsistent."); if (direction != m_direction) { @@ -121,7 +129,19 @@ class Plan { "inconsistent."); } - // [TO DO] Check view extents + auto in_extents = KokkosFFT::Impl::extract_extents(in); + auto out_extents = KokkosFFT::Impl::extract_extents(out); + if (in_extents != m_in_extents) { + throw std::runtime_error( + "KokkosFFT::Impl::Plan::good: extents of input View for plan and execution are " + "inconsistent."); + } + + if (out_extents != m_out_extents) { + throw std::runtime_error( + "KokkosFFT::Impl::Plan::good: extents of output View for plan and execution are " + "inconsistent."); + } } fft_plan_type plan() const { return m_plan; } diff --git a/fft/src/KokkosFFT_Transform.hpp b/fft/src/KokkosFFT_Transform.hpp index 3c9500e6..bb4d2e59 100644 --- a/fft/src/KokkosFFT_Transform.hpp +++ b/fft/src/KokkosFFT_Transform.hpp @@ -179,7 +179,7 @@ void fft(const ExecutionSpace& exec_space, const InViewType& in, } plan.template good( - KokkosFFT::Impl::Direction::Forward, axis_type<1>{axis}); + _in, out, KokkosFFT::Impl::Direction::Forward, axis_type<1>{axis}); if (plan.is_transpose_needed()) { InViewType in_T; @@ -269,7 +269,7 @@ void ifft(const ExecutionSpace& exec_space, const InViewType& in, } plan.template good( - KokkosFFT::Impl::Direction::Backward, axis_type<1>{axis}); + _in, out, KokkosFFT::Impl::Direction::Backward, axis_type<1>{axis}); if (plan.is_transpose_needed()) { InViewType in_T; @@ -554,7 +554,7 @@ void fft2(const ExecutionSpace& exec_space, const InViewType& in, } plan.template good( - KokkosFFT::Impl::Direction::Forward, axes); + _in, out, KokkosFFT::Impl::Direction::Forward, axes); if (plan.is_transpose_needed()) { InViewType in_T; @@ -637,7 +637,7 @@ void ifft2(const ExecutionSpace& exec_space, const InViewType& in, } plan.template good( - KokkosFFT::Impl::Direction::Backward, axes); + _in, out, KokkosFFT::Impl::Direction::Backward, axes); if (plan.is_transpose_needed()) { InViewType in_T; @@ -871,7 +871,7 @@ void fftn(const ExecutionSpace& exec_space, const InViewType& in, } plan.template good( - KokkosFFT::Impl::Direction::Forward, axes); + _in, out, KokkosFFT::Impl::Direction::Forward, axes); if (plan.is_transpose_needed()) { InViewType in_T; @@ -1000,7 +1000,7 @@ void ifftn(const ExecutionSpace& exec_space, const InViewType& in, } plan.template good( - KokkosFFT::Impl::Direction::Backward, axes); + _in, out, KokkosFFT::Impl::Direction::Backward, axes); if (plan.is_transpose_needed()) { InViewType in_T; diff --git a/fft/unit_test/Test_Transform.cpp b/fft/unit_test/Test_Transform.cpp index 6a8aa90c..8c245c68 100644 --- a/fft/unit_test/Test_Transform.cpp +++ b/fft/unit_test/Test_Transform.cpp @@ -264,6 +264,56 @@ void test_fft1_identity_reuse_plan(T atol = 1.0e-12) { EXPECT_THROW(KokkosFFT::irfft(execution_space(), outr, _ar, wrong_irfft_plan, KokkosFFT::Normalization::BACKWARD, axis), std::runtime_error); + + // Check if errors are correctly raised aginst wrong extents + const int maxlen_wrong = 32 * 2; + ComplexView1DType a_wrong("a", maxlen_wrong), _a_wrong("_a", maxlen_wrong); + ComplexView1DType out_wrong("out", maxlen_wrong), outr_wrong("outr", maxlen_wrong / 2 + 1); + RealView1DType ar_wrong("ar", maxlen_wrong), _ar_wrong("_ar", maxlen_wrong); + + // fft + // With incorrect input shape + EXPECT_THROW(KokkosFFT::fft(execution_space(), a_wrong, out, fft_plan, + KokkosFFT::Normalization::BACKWARD, axis), + std::runtime_error); + + // With incorrect output shape + EXPECT_THROW(KokkosFFT::fft(execution_space(), a, out_wrong, fft_plan, + KokkosFFT::Normalization::BACKWARD, axis), + std::runtime_error); + + // ifft + // With incorrect input shape + EXPECT_THROW(KokkosFFT::ifft(execution_space(), out_wrong, _a, ifft_plan, + KokkosFFT::Normalization::BACKWARD, axis), + std::runtime_error); + + // With incorrect output shape + EXPECT_THROW(KokkosFFT::ifft(execution_space(), out, _a_wrong, ifft_plan, + KokkosFFT::Normalization::BACKWARD, axis), + std::runtime_error); + + // rfft + // With incorrect input shape + EXPECT_THROW(KokkosFFT::rfft(execution_space(), ar_wrong, outr, rfft_plan, + KokkosFFT::Normalization::BACKWARD, axis), + std::runtime_error); + + // With incorrect output shape + EXPECT_THROW(KokkosFFT::rfft(execution_space(), ar, out_wrong, rfft_plan, + KokkosFFT::Normalization::BACKWARD, axis), + std::runtime_error); + + // irfft + // With incorrect input shape + EXPECT_THROW(KokkosFFT::irfft(execution_space(), outr_wrong, _ar, irfft_plan, + KokkosFFT::Normalization::BACKWARD, axis), + std::runtime_error); + + // With incorrect output shape + EXPECT_THROW(KokkosFFT::irfft(execution_space(), outr, _ar_wrong, irfft_plan, + KokkosFFT::Normalization::BACKWARD, axis), + std::runtime_error); } template From 439e377b184216952d3f66dfeeb38ac39dddc140 Mon Sep 17 00:00:00 2001 From: Yuuichi Asahi Date: Tue, 23 Jan 2024 00:01:38 +0900 Subject: [PATCH 2/7] Getting the mirror view on the same device --- common/src/KokkosFFT_utils.hpp | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/common/src/KokkosFFT_utils.hpp b/common/src/KokkosFFT_utils.hpp index 45961150..7e2251d6 100644 --- a/common/src/KokkosFFT_utils.hpp +++ b/common/src/KokkosFFT_utils.hpp @@ -156,24 +156,18 @@ inline std::vector arange(const ElementType start, template void conjugate(const ExecutionSpace& exec_space, const InViewType& in, OutViewType& out) { + static_assert(Kokkos::is_view::value, + "KokkosFFT::Impl::conjugate: InViewType is not a Kokkos::View."); + static_assert(Kokkos::is_view::value, + "KokkosFFT::Impl::conjugate: OutViewType is not a Kokkos::View."); + using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(KokkosFFT::Impl::is_complex::value, "KokkosFFT::Impl::conjugate: OutViewType must be complex"); std::size_t size = in.size(); - - // [TO DO] is there a way to get device mirror? - if constexpr (InViewType::rank() == 1) { - out = OutViewType("out", in.extent(0)); - } else if constexpr (InViewType::rank() == 2) { - out = OutViewType("out", in.extent(0), in.extent(1)); - } else if constexpr (InViewType::rank() == 3) { - out = OutViewType("out", in.extent(0), in.extent(1), in.extent(2)); - } else if constexpr (InViewType::rank() == 4) { - out = OutViewType("out", in.extent(0), in.extent(1), in.extent(2), - in.extent(3)); - } + out = OutViewType("out", in.layout()); auto* in_data = in.data(); auto* out_data = out.data(); From 6c922b99de18f0397566e5092ebcdd308800a68d Mon Sep 17 00:00:00 2001 From: Yuuichi Asahi Date: Tue, 23 Jan 2024 22:04:15 +0900 Subject: [PATCH 3/7] Add static type check for layout --- common/src/KokkosFFT_utils.hpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/common/src/KokkosFFT_utils.hpp b/common/src/KokkosFFT_utils.hpp index 7e2251d6..b97a3c2c 100644 --- a/common/src/KokkosFFT_utils.hpp +++ b/common/src/KokkosFFT_utils.hpp @@ -28,6 +28,17 @@ struct is_complex : std::false_type {}; template struct is_complex> : std::true_type {}; +template +struct is_layout_left_or_right : std::false_type {}; + +template +struct is_layout_left_or_right || + std::is_same_v > > : std::true_type {}; + +template +inline constexpr bool is_layout_left_or_right_v = is_layout_left_or_right::value; + template = nullptr> struct complex_view_type { @@ -42,7 +53,7 @@ template auto convert_negative_axis(const ViewType& view, int _axis = -1) { static_assert( Kokkos::is_view::value, - "KokkosFFT::convert_negative_axis: ViewType is not a Kokkos::View."); + "convert_negative_axis: ViewType is not a Kokkos::View."); int rank = static_cast(ViewType::rank()); assert(abs(_axis) < rank); // axis should be in [-(rank-1), rank-1] int axis = _axis < 0 ? rank + _axis : _axis; @@ -53,7 +64,7 @@ template auto convert_negative_shift(const ViewType& view, int _shift, int _axis) { static_assert( Kokkos::is_view::value, - "KokkosFFT::convert_negative_shift: ViewType is not a Kokkos::View."); + "convert_negative_shift: ViewType is not a Kokkos::View."); int axis = convert_negative_axis(view, _axis); int extent = view.extent(axis); int shift0 = 0, shift1 = 0, shift2 = extent / 2; @@ -157,15 +168,15 @@ template void conjugate(const ExecutionSpace& exec_space, const InViewType& in, OutViewType& out) { static_assert(Kokkos::is_view::value, - "KokkosFFT::Impl::conjugate: InViewType is not a Kokkos::View."); + "conjugate: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::Impl::conjugate: OutViewType is not a Kokkos::View."); + "conjugate: OutViewType is not a Kokkos::View."); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::Impl::conjugate: OutViewType must be complex"); + "conjugate: OutViewType must be complex"); std::size_t size = in.size(); out = OutViewType("out", in.layout()); @@ -184,7 +195,7 @@ void conjugate(const ExecutionSpace& exec_space, const InViewType& in, template auto extract_extents(const ViewType& view) { static_assert(Kokkos::is_view::value, - "KokkosFFT::Impl::extract_extents: ViewType is not a Kokkos::View."); + "extract_extents: ViewType is not a Kokkos::View."); constexpr std::size_t rank = ViewType::rank(); std::array extents; for(std::size_t i=0; i Date: Tue, 23 Jan 2024 22:06:27 +0900 Subject: [PATCH 4/7] More static assertions for APIs --- fft/src/KokkosFFT_Helpers.hpp | 32 +- fft/src/KokkosFFT_Plans.hpp | 85 +++- fft/src/KokkosFFT_Transform.hpp | 848 +++++++++++++++++++++++++++----- 3 files changed, 828 insertions(+), 137 deletions(-) diff --git a/fft/src/KokkosFFT_Helpers.hpp b/fft/src/KokkosFFT_Helpers.hpp index df0ac35f..ae53b667 100644 --- a/fft/src/KokkosFFT_Helpers.hpp +++ b/fft/src/KokkosFFT_Helpers.hpp @@ -11,7 +11,7 @@ template auto _get_shift(const ViewType& inout, axis_type _axes, int direction = 1) { static_assert(DIM > 0, - "KokkosFFT::Impl::_get_shift: Rank of shift axes must be " + "_get_shift: Rank of shift axes must be " "larger than or equal to 1."); // Convert the input axes to be in the range of [0, rank-1] @@ -38,7 +38,7 @@ template void _roll(const ExecutionSpace& exec_space, ViewType& inout, axis_type<1> shift, axis_type<1> axes) { static_assert(ViewType::rank() == 1, - "KokkosFFT::Impl::_roll: Rank of View must be 1."); + "_roll: Rank of View must be 1."); std::size_t n0 = inout.extent(0); ViewType tmp("tmp", n0); @@ -69,7 +69,7 @@ void _roll(const ExecutionSpace& exec_space, ViewType& inout, axis_type<2> shift, axis_type axes) { constexpr std::size_t DIM0 = 2; static_assert(ViewType::rank() == DIM0, - "KokkosFFT::Impl::_roll: Rank of View must be 2."); + "_roll: Rank of View must be 2."); int n0 = inout.extent(0), n1 = inout.extent(1); ViewType tmp("tmp", n0, n1); @@ -129,8 +129,17 @@ void _roll(const ExecutionSpace& exec_space, ViewType& inout, template void _fftshift(const ExecutionSpace& exec_space, ViewType& inout, axis_type axes) { + static_assert(Kokkos::is_view::value, + "_fftshift: ViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "_fftshift: ViewType must be either LayoutLeft or LayoutRight."); + static_assert( + Kokkos::SpaceAccessibility::accessible, + "_fftshift: execution_space cannot access data in ViewType"); + static_assert(ViewType::rank() >= DIM, - "KokkosFFT::Impl::_fftshift: Rank of View must be larger thane " + "_fftshift: Rank of View must be larger thane " "or equal to the Rank of shift axes."); auto shift = _get_shift(inout, axes); _roll(exec_space, inout, shift, axes); @@ -139,8 +148,17 @@ void _fftshift(const ExecutionSpace& exec_space, ViewType& inout, template void _ifftshift(const ExecutionSpace& exec_space, ViewType& inout, axis_type axes) { + static_assert(Kokkos::is_view::value, + "_ifftshift: ViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "_ifftshift: ViewType must be either LayoutLeft or LayoutRight."); + static_assert( + Kokkos::SpaceAccessibility::accessible, + "_ifftshift: execution_space cannot access data in ViewType"); + static_assert(ViewType::rank() >= DIM, - "KokkosFFT::Impl::_ifftshift: Rank of View must be larger " + "_ifftshift: Rank of View must be larger " "thane or equal to the Rank of shift axes."); auto shift = _get_shift(inout, axes, -1); _roll(exec_space, inout, shift, axes); @@ -153,7 +171,7 @@ template auto fftfreq(const ExecutionSpace& exec_space, const std::size_t n, const RealType d = 1.0) { static_assert(std::is_floating_point::value, - "KokkosFFT::fftfreq: d must be real"); + "fftfreq: d must be float or double"); using ViewType = Kokkos::View; ViewType freq("freq", n); @@ -181,7 +199,7 @@ template auto rfftfreq(const ExecutionSpace& exec_space, const std::size_t n, const RealType d = 1.0) { static_assert(std::is_floating_point::value, - "KokkosFFT::fftfreq: d must be real"); + "fftfreq: d must be float or double"); using ViewType = Kokkos::View; RealType val = 1.0 / (static_cast(n) * d); diff --git a/fft/src/KokkosFFT_Plans.hpp b/fft/src/KokkosFFT_Plans.hpp index 23f85968..219bf5d5 100644 --- a/fft/src/KokkosFFT_Plans.hpp +++ b/fft/src/KokkosFFT_Plans.hpp @@ -66,9 +66,29 @@ class Plan { int axis) : m_fft_size(1), m_is_transpose_needed(false), m_direction(direction) { static_assert(Kokkos::is_view::value, - "KokkosFFT::Plan: InViewType is not a Kokkos::View."); + "Plan::Plan: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::Plan: OutViewType is not a Kokkos::View."); + "Plan::Plan: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "Plan::Plan: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "Plan::Plan: OutViewType must be either LayoutLeft or LayoutRight."); + + static_assert(InViewType::rank() == OutViewType::rank(), + "Plan::Plan: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "Plan::Plan: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Plan::Plan: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Plan::Plan: execution_space cannot access data in OutViewType"); m_axes = {axis}; m_in_extents = KokkosFFT::Impl::extract_extents(in); @@ -87,9 +107,30 @@ class Plan { m_direction(direction), m_axes(axes) { static_assert(Kokkos::is_view::value, - "KokkosFFT::Plan: InViewType is not a Kokkos::View."); + "Plan::Plan: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::Plan: OutViewType is not a Kokkos::View."); + "Plan::Plan: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "Plan::Plan: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "Plan::Plan: OutViewType must be either LayoutLeft or LayoutRight."); + + static_assert(InViewType::rank() == OutViewType::rank(), + "Plan::Plan: InViewType and OutViewType must have " + "the same rank."); + + static_assert(std::is_same_v, + "Plan::Plan: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Plan::Plan: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Plan::Plan: execution_space cannot access data in OutViewType"); m_in_extents = KokkosFFT::Impl::extract_extents(in); m_out_extents = KokkosFFT::Impl::extract_extents(out); @@ -105,42 +146,50 @@ class Plan { typename OutViewType2> void good(const InViewType2& in, const OutViewType2& out, KokkosFFT::Impl::Direction direction, axis_type axes) const { static_assert(std::is_same_v, - "KokkosFFT::Impl::Plan::good: Execution spaces for plan and " - "execution are inconsistent."); + "Plan::good: Execution spaces for plan and " + "execution are not identical."); + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Plan::good: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility::accessible, + "Plan::good: execution_space cannot access data in OutViewType"); using nonConstInViewType2 = std::remove_cv_t; using nonConstOutViewType2 = std::remove_cv_t; static_assert(std::is_same_v, - "KokkosFFT::Impl::Plan::good: InViewType for plan and execution " - "are inconsistent."); + "Plan::good: InViewType for plan and execution " + "are not identical."); static_assert(std::is_same_v, - "KokkosFFT::Impl::Plan::good: OutViewType for plan and " - "execution are inconsistent."); + "Plan::good: OutViewType for plan and " + "execution are not identical."); if (direction != m_direction) { throw std::runtime_error( - "KokkosFFT::Impl::Plan::good: direction for plan and execution are " - "inconsistent."); + "Plan::good: directions for plan and execution are " + "not identical."); } if (axes != m_axes) { throw std::runtime_error( - "KokkosFFT::Impl::Plan::good: axes for plan and execution are " - "inconsistent."); + "Plan::good: axes for plan and execution are " + "not identical."); } auto in_extents = KokkosFFT::Impl::extract_extents(in); auto out_extents = KokkosFFT::Impl::extract_extents(out); if (in_extents != m_in_extents) { throw std::runtime_error( - "KokkosFFT::Impl::Plan::good: extents of input View for plan and execution are " - "inconsistent."); + "Plan::good: extents of input View for plan and execution are " + "not identical."); } if (out_extents != m_out_extents) { throw std::runtime_error( - "KokkosFFT::Impl::Plan::good: extents of output View for plan and execution are " - "inconsistent."); + "Plan::good: extents of output View for plan and execution are " + "not identical."); } } diff --git a/fft/src/KokkosFFT_Transform.hpp b/fft/src/KokkosFFT_Transform.hpp index bb4d2e59..ed96966a 100644 --- a/fft/src/KokkosFFT_Transform.hpp +++ b/fft/src/KokkosFFT_Transform.hpp @@ -41,9 +41,29 @@ void _fft(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, OutViewType& out, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD) { static_assert(Kokkos::is_view::value, - "KokkosFFT::_fft: InViewType is not a Kokkos::View."); + "_fft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::_fft: OutViewType is not a Kokkos::View."); + "_fft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "_fft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "_fft: OutViewType must be either LayoutLeft or LayoutRight."); + + static_assert(InViewType::rank() == OutViewType::rank(), + "_fft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "_fft: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "_fft: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility::accessible, + "_fft: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; @@ -67,9 +87,29 @@ void _ifft(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, OutViewType& out, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD) { static_assert(Kokkos::is_view::value, - "KokkosFFT::_ifft: InViewType is not a Kokkos::View."); + "_ifft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::_ifft: OutViewType is not a Kokkos::View."); + "_ifft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "_ifft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "_ifft: OutViewType must be either LayoutLeft or LayoutRight."); + + static_assert(InViewType::rank() == OutViewType::rank(), + "_ifft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "_ifft: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "_ifft: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility::accessible, + "_ifft: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; @@ -97,19 +137,28 @@ void fft(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, int axis = -1, std::optional n = std::nullopt) { static_assert(Kokkos::is_view::value, - "KokkosFFT::fft: InViewType is not a Kokkos::View."); + "fft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::fft: OutViewType is not a Kokkos::View."); + "fft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fft: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "fft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "fft: InViewType and OutViewType must have " + "the same Layout."); static_assert( Kokkos::SpaceAccessibility::accessible, - "KokkosFFT::fft: execution_space cannot access data in InViewType"); - + "fft: execution_space cannot access data in InViewType"); static_assert( Kokkos::SpaceAccessibility< ExecutionSpace, typename OutViewType::memory_space>::accessible, - "KokkosFFT::fft: execution_space cannot access data in OutViewType"); + "fft: execution_space cannot access data in OutViewType"); InViewType _in; if (n) { @@ -150,19 +199,28 @@ void fft(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, int axis = -1, std::optional n = std::nullopt) { static_assert(Kokkos::is_view::value, - "KokkosFFT::fft: InViewType is not a Kokkos::View."); + "fft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::fft: OutViewType is not a Kokkos::View."); + "fft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fft: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "fft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "fft: InViewType and OutViewType must have " + "the same Layout."); static_assert( Kokkos::SpaceAccessibility::accessible, - "KokkosFFT::fft: execution_space cannot access data in InViewType"); - + "fft: execution_space cannot access data in InViewType"); static_assert( Kokkos::SpaceAccessibility< ExecutionSpace, typename OutViewType::memory_space>::accessible, - "KokkosFFT::fft: execution_space cannot access data in OutViewType"); + "fft: execution_space cannot access data in OutViewType"); InViewType _in; if (n) { @@ -203,9 +261,28 @@ void ifft(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, int axis = -1, std::optional n = std::nullopt) { static_assert(Kokkos::is_view::value, - "KokkosFFT::ifft: InViewType is not a Kokkos::View."); + "ifft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::ifft: OutViewType is not a Kokkos::View."); + "ifft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifft: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "ifft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "ifft: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "ifft: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "ifft: execution_space cannot access data in OutViewType"); InViewType _in; // [TO DO] Modify crop_or_pad to perform the following lines @@ -248,9 +325,28 @@ void ifft(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, int axis = -1, std::optional n = std::nullopt) { static_assert(Kokkos::is_view::value, - "KokkosFFT::ifft: InViewType is not a Kokkos::View."); + "ifft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::ifft: OutViewType is not a Kokkos::View."); + "ifft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifft: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "ifft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "ifft: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "ifft: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "ifft: execution_space cannot access data in OutViewType"); InViewType _in; // [TO DO] Modify crop_or_pad to perform the following lines @@ -293,17 +389,36 @@ void rfft(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, int axis = -1, std::optional n = std::nullopt) { static_assert(Kokkos::is_view::value, - "KokkosFFT::rfft: InViewType is not a Kokkos::View."); + "rfft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::rfft: OutViewType is not a Kokkos::View."); + "rfft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfft: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "rfft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "rfft: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "rfft: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "rfft: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(std::is_floating_point::value, - "KokkosFFT::rfft: InViewType must be real"); + "rfft: InViewType must be real"); static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfft: OutViewType must be complex"); + "rfft: OutViewType must be complex"); fft(exec_space, in, out, norm, axis, n); } @@ -315,17 +430,36 @@ void rfft(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, int axis = -1, std::optional n = std::nullopt) { static_assert(Kokkos::is_view::value, - "KokkosFFT::rfft: InViewType is not a Kokkos::View."); + "rfft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::rfft: OutViewType is not a Kokkos::View."); + "rfft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfft: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "rfft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "rfft: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "rfft: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "rfft: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(std::is_floating_point::value, - "KokkosFFT::rfft: InViewType must be real"); + "rfft: InViewType must be real"); static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfft: OutViewType must be complex"); + "rfft: OutViewType must be complex"); fft(exec_space, in, out, plan, norm, axis, n); } @@ -336,17 +470,36 @@ void irfft(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, int axis = -1, std::optional n = std::nullopt) { static_assert(Kokkos::is_view::value, - "KokkosFFT::irfft: InViewType is not a Kokkos::View."); + "irfft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::irfft: OutViewType is not a Kokkos::View."); + "irfft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "irfft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rifft: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "irfft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "irfft: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "irfft: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "irfft: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfft: InViewType must be complex"); + "irfft: InViewType must be complex"); static_assert(std::is_floating_point::value, - "KokkosFFT::irfft: OutViewType must be real"); + "irfft: OutViewType must be real"); if (n) { std::size_t _n = n.value() / 2 + 1; ifft(exec_space, in, out, norm, axis, _n); @@ -362,17 +515,32 @@ void irfft(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, int axis = -1, std::optional n = std::nullopt) { static_assert(Kokkos::is_view::value, - "KokkosFFT::irfft: InViewType is not a Kokkos::View."); + "irfft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::irfft: OutViewType is not a Kokkos::View."); + "irfft: OutViewType is not a Kokkos::View."); + static_assert(InViewType::rank() == OutViewType::rank(), + "irfft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "irfft: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "irfft: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "irfft: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfft: InViewType must be complex"); + "irfft: InViewType must be complex"); static_assert(std::is_floating_point::value, - "KokkosFFT::irfft: OutViewType must be real"); + "irfft: OutViewType must be real"); if (n) { std::size_t _n = n.value() / 2 + 1; ifft(exec_space, in, out, plan, norm, axis, _n); @@ -387,9 +555,28 @@ void hfft(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, int axis = -1, std::optional n = std::nullopt) { static_assert(Kokkos::is_view::value, - "KokkosFFT::hfft: InViewType is not a Kokkos::View."); + "hfft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::hfft: OutViewType is not a Kokkos::View."); + "hfft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "hfft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "hfft: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "hfft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "hfft: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "hfft: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "hfft: execution_space cannot access data in OutViewType"); // [TO DO] // allow real type as input, need to obtain complex view type from in view @@ -397,9 +584,9 @@ void hfft(const ExecutionSpace& exec_space, const InViewType& in, using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::hfft: InViewType must be complex"); + "hfft: InViewType must be complex"); static_assert(std::is_floating_point::value, - "KokkosFFT::hfft: OutViewType must be real"); + "hfft: OutViewType must be real"); auto new_norm = KokkosFFT::Impl::swap_direction(norm); // using ComplexViewType = typename // KokkosFFT::Impl::complex_view_type::type; @@ -416,9 +603,28 @@ void hfft(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, int axis = -1, std::optional n = std::nullopt) { static_assert(Kokkos::is_view::value, - "KokkosFFT::hfft: InViewType is not a Kokkos::View."); + "hfft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::hfft: OutViewType is not a Kokkos::View."); + "hfft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "hfft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "hfft: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "hfft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "hfft: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "hfft: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "hfft: execution_space cannot access data in OutViewType"); // [TO DO] // allow real type as input, need to obtain complex view type from in view @@ -426,9 +632,9 @@ void hfft(const ExecutionSpace& exec_space, const InViewType& in, using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::hfft: InViewType must be complex"); + "hfft: InViewType must be complex"); static_assert(std::is_floating_point::value, - "KokkosFFT::hfft: OutViewType must be real"); + "hfft: OutViewType must be real"); auto new_norm = KokkosFFT::Impl::swap_direction(norm); // using ComplexViewType = typename // KokkosFFT::Impl::complex_view_type::type; @@ -444,16 +650,35 @@ void ihfft(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, int axis = -1, std::optional n = std::nullopt) { static_assert(Kokkos::is_view::value, - "KokkosFFT::ihfft: InViewType is not a Kokkos::View."); + "ihfft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::ihfft: OutViewType is not a Kokkos::View."); + "ihfft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ihfft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ihfft: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "ihfft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "ihfft: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "ihfft: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "ihfft: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(std::is_floating_point::value, - "KokkosFFT::rfft: InViewType must be real"); + "ihfft: InViewType must be real"); static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::ihfft: OutViewType must be complex"); + "ihfft: OutViewType must be complex"); auto new_norm = KokkosFFT::Impl::swap_direction(norm); OutViewType out_conj; @@ -469,16 +694,35 @@ void ihfft(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, int axis = -1, std::optional n = std::nullopt) { static_assert(Kokkos::is_view::value, - "KokkosFFT::ihfft: InViewType is not a Kokkos::View."); + "ihfft: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::ihfft: OutViewType is not a Kokkos::View."); + "ihfft: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ihfft: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ihfft: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "ihfft: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "ihfft: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "ihfft: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "ihfft: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(std::is_floating_point::value, - "KokkosFFT::rfft: InViewType must be real"); + "ihfft: InViewType must be real"); static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::ihfft: OutViewType must be complex"); + "ihfft: OutViewType must be complex"); auto new_norm = KokkosFFT::Impl::swap_direction(norm); OutViewType out_conj; @@ -487,6 +731,7 @@ void ihfft(const ExecutionSpace& exec_space, const InViewType& in, out = out_conj; } +// 2D FFT template void fft2(const ExecutionSpace& exec_space, const InViewType& in, @@ -494,9 +739,28 @@ void fft2(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, axis_type<2> axes = {-2, -1}, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::fft2: InViewType is not a Kokkos::View."); + "fft2: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::fft2: OutViewType is not a Kokkos::View."); + "fft2: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fft2: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fft2: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "fft2: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "fft2: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "fft2: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "fft2: execution_space cannot access data in OutViewType"); InViewType _in; shape_type zeros = {0}; // default shape means no crop or pad @@ -528,7 +792,6 @@ void fft2(const ExecutionSpace& exec_space, const InViewType& in, } } -// 2D FFT template void fft2(const ExecutionSpace& exec_space, const InViewType& in, @@ -536,9 +799,28 @@ void fft2(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, axis_type<2> axes = {-2, -1}, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::fft2: InViewType is not a Kokkos::View."); + "fft2: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::fft2: OutViewType is not a Kokkos::View."); + "fft2: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fft2: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fft2: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "fft2: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "fft2: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "fft2: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "fft2: execution_space cannot access data in OutViewType"); InViewType _in; shape_type zeros = {0}; // default shape means no crop or pad @@ -578,9 +860,28 @@ void ifft2(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, axis_type<2> axes = {-2, -1}, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::ifft2: InViewType is not a Kokkos::View."); + "ifft2: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::ifft2: OutViewType is not a Kokkos::View."); + "ifft2: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifft2: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifft2: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "ifft2: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "ifft2: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "ifft2: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "ifft2: execution_space cannot access data in OutViewType"); InViewType _in; shape_type zeros = {0}; // default shape means no crop or pad @@ -619,9 +920,28 @@ void ifft2(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, axis_type<2> axes = {-2, -1}, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::ifft2: InViewType is not a Kokkos::View."); + "ifft2: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::ifft2: OutViewType is not a Kokkos::View."); + "ifft2: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifft2: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifft2: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "ifft2: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "ifft2: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "ifft2: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "ifft2: execution_space cannot access data in OutViewType"); InViewType _in; shape_type zeros = {0}; // default shape means no crop or pad @@ -661,17 +981,36 @@ void rfft2(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, axis_type<2> axes = {-2, -1}, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::rfft2: InViewType is not a Kokkos::View."); + "rfft2: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::rfft2: OutViewType is not a Kokkos::View."); + "rfft2: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfft2: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfft2: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "rfft2: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "rfft2: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "rfft2: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "rfft2: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(std::is_floating_point::value, - "KokkosFFT::rfft2: InViewType must be real"); + "rfft2: InViewType must be real"); static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfft2: OutViewType must be complex"); + "rfft2: OutViewType must be complex"); fft2(exec_space, in, out, norm, axes, s); } @@ -683,17 +1022,36 @@ void rfft2(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, axis_type<2> axes = {-2, -1}, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::rfft2: InViewType is not a Kokkos::View."); + "rfft2: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::rfft2: OutViewType is not a Kokkos::View."); + "rfft2: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfft2: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfft2: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "rfft2: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "rfft2: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "rfft2: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "rfft2: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(std::is_floating_point::value, - "KokkosFFT::rfft2: InViewType must be real"); + "rfft2: InViewType must be real"); static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfft2: OutViewType must be complex"); + "rfft2: OutViewType must be complex"); fft2(exec_space, in, out, plan, norm, axes, s); } @@ -705,17 +1063,36 @@ void irfft2(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, axis_type<2> axes = {-2, -1}, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::irfft2: InViewType is not a Kokkos::View."); + "irfft2: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::irfft2: OutViewType is not a Kokkos::View."); + "irfft2: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "irfft2: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "irfft2: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "irfft2: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "irfft2: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "irfft2: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "irfft2: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfft2: InViewType must be complex"); + "irfft2: InViewType must be complex"); static_assert(std::is_floating_point::value, - "KokkosFFT::irfft2: OutViewType must be real"); + "irfft2: OutViewType must be real"); shape_type zeros = {0}; // default shape means no crop or pad shape_type _s = {0}; @@ -735,17 +1112,36 @@ void irfft2(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, axis_type<2> axes = {-2, -1}, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::irfft2: InViewType is not a Kokkos::View."); + "irfft2: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::irfft2: OutViewType is not a Kokkos::View."); + "irfft2: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "irfft2: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "irfft2: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "irfft2: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "irfft2: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "irfft2: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "irfft2: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfft2: InViewType must be complex"); + "irfft2: InViewType must be complex"); static_assert(std::is_floating_point::value, - "KokkosFFT::irfft2: OutViewType must be real"); + "irfft2: OutViewType must be real"); shape_type zeros = {0}; // default shape means no crop or pad shape_type _s = {0}; @@ -766,9 +1162,28 @@ void fftn(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::fftn: InViewType is not a Kokkos::View."); + "fftn: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::fftn: OutViewType is not a Kokkos::View."); + "fftn: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fftn: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "fftn: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "fftn: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "fftn: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "fftn: execution_space cannot access data in OutViewType"); // Create a default sequence of axes {-rank, -(rank-1), ..., -1} constexpr std::size_t rank = InViewType::rank(); @@ -812,9 +1227,28 @@ void fftn(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::fftn: InViewType is not a Kokkos::View."); + "fftn: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::fftn: OutViewType is not a Kokkos::View."); + "fftn: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fftn: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "fftn: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "fftn: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "fftn: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "fftn: execution_space cannot access data in OutViewType"); InViewType _in; shape_type zeros = {0}; // default shape means no crop or pad @@ -853,9 +1287,28 @@ void fftn(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::fftn: InViewType is not a Kokkos::View."); + "fftn: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::fftn: OutViewType is not a Kokkos::View."); + "fftn: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fftn: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "fftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "fftn: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "fftn: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "fftn: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "fftn: execution_space cannot access data in OutViewType"); InViewType _in; shape_type zeros = {0}; // default shape means no crop or pad @@ -895,9 +1348,28 @@ void ifftn(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::ifftn: InViewType is not a Kokkos::View."); + "ifftn: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::ifftn: OutViewType is not a Kokkos::View."); + "ifftn: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifftn: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "ifftn: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "ifftn: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "ifftn: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "ifftn: execution_space cannot access data in OutViewType"); // Create a default sequence of axes {-rank, -(rank-1), ..., -1} constexpr std::size_t rank = InViewType::rank(); @@ -941,9 +1413,28 @@ void ifftn(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::ifftn: InViewType is not a Kokkos::View."); + "ifftn: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::ifftn: OutViewType is not a Kokkos::View."); + "ifftn: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifftn: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "ifftn: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "ifftn: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "ifftn: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "ifftn: execution_space cannot access data in OutViewType"); InViewType _in; shape_type zeros = {0}; // default shape means no crop or pad @@ -982,9 +1473,28 @@ void ifftn(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::ifftn: InViewType is not a Kokkos::View."); + "ifftn: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::ifftn: OutViewType is not a Kokkos::View."); + "ifftn: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifftn: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "ifftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "ifftn: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "ifftn: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "ifftn: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "ifftn: execution_space cannot access data in OutViewType"); InViewType _in; shape_type zeros = {0}; // default shape means no crop or pad @@ -1024,17 +1534,36 @@ void rfftn(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::rfftn: InViewType is not a Kokkos::View."); + "rfftn: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::rfftn: OutViewType is not a Kokkos::View."); + "rfftn: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfftn: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "rfftn: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "rfftn: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "rfftn: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "rfftn: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(std::is_floating_point::value, - "KokkosFFT::rfftn: InViewType must be real"); + "rfftn: InViewType must be real"); static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfftn: OutViewType must be complex"); + "rfftn: OutViewType must be complex"); fftn(exec_space, in, out, norm, s); } @@ -1046,17 +1575,36 @@ void rfftn(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::rfftn: InViewType is not a Kokkos::View."); + "rfftn: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::rfftn: OutViewType is not a Kokkos::View."); + "rfftn: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfftn: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "rfftn: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "rfftn: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "rfftn: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "rfftn: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(std::is_floating_point::value, - "KokkosFFT::rfftn: InViewType must be real"); + "rfftn: InViewType must be real"); static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfftn: OutViewType must be complex"); + "rfftn: OutViewType must be complex"); fftn(exec_space, in, out, plan, axes, norm, s); } @@ -1068,17 +1616,36 @@ void rfftn(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::rfftn: InViewType is not a Kokkos::View."); + "rfftn: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::rfftn: OutViewType is not a Kokkos::View."); + "rfftn: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfftn: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "rfftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "rfftn: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "rfftn: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "rfftn: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "rfftn: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(std::is_floating_point::value, - "KokkosFFT::rfftn: InViewType must be real"); + "rfftn: InViewType must be real"); static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfftn: OutViewType must be complex"); + "rfftn: OutViewType must be complex"); fftn(exec_space, in, out, axes, norm, s); } @@ -1090,17 +1657,36 @@ void irfftn(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::irfftn: InViewType is not a Kokkos::View."); + "irfftn: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::irfftn: OutViewType is not a Kokkos::View."); + "irfftn: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "irfftn: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "irfftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "irfftn: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "irfftn: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "irfftn: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "irfftn: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfftn: InViewType must be complex"); + "irfftn: InViewType must be complex"); static_assert(std::is_floating_point::value, - "KokkosFFT::irfftn: OutViewType must be real"); + "irfftn: OutViewType must be real"); shape_type zeros = {0}; // default shape means no crop or pad shape_type _s = {0}; @@ -1120,17 +1706,36 @@ void irfftn(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::irfftn: InViewType is not a Kokkos::View."); + "irfftn: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::irfftn: OutViewType is not a Kokkos::View."); + "irfftn: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "irfftn: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "irfftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "irfftn: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "irfftn: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "irfftn: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "irfftn: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfftn: InViewType must be complex"); + "irfftn: InViewType must be complex"); static_assert(std::is_floating_point::value, - "KokkosFFT::irfftn: OutViewType must be real"); + "irfftn: OutViewType must be real"); shape_type zeros = {0}; // default shape means no crop or pad shape_type _s = {0}; @@ -1150,17 +1755,36 @@ void irfftn(const ExecutionSpace& exec_space, const InViewType& in, KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, shape_type s = {0}) { static_assert(Kokkos::is_view::value, - "KokkosFFT::irfftn: InViewType is not a Kokkos::View."); + "irfftn: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, - "KokkosFFT::irfftn: OutViewType is not a Kokkos::View."); + "irfftn: OutViewType is not a Kokkos::View."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "irfftn: InViewType must be either LayoutLeft or LayoutRight."); + static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, + "irfftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert(InViewType::rank() == OutViewType::rank(), + "irfftn: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "irfftn: InViewType and OutViewType must have " + "the same Layout."); + + static_assert( + Kokkos::SpaceAccessibility::accessible, + "ifftn: execution_space cannot access data in InViewType"); + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "ifftn: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; using out_value_type = typename OutViewType::non_const_value_type; static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfftn: InViewType must be complex"); + "irfftn: InViewType must be complex"); static_assert(std::is_floating_point::value, - "KokkosFFT::irfftn: OutViewType must be real"); + "irfftn: OutViewType must be real"); shape_type zeros = {0}; // default shape means no crop or pad shape_type _s = {0}; @@ -1172,6 +1796,6 @@ void irfftn(const ExecutionSpace& exec_space, const InViewType& in, ifftn(exec_space, in, out, plan, axes, norm, _s); } -}; // namespace KokkosFFT +} // namespace KokkosFFT #endif \ No newline at end of file From be2d4ee8672329a3414892ce58b11201094ab617 Mon Sep 17 00:00:00 2001 From: Yuuichi Asahi Date: Tue, 23 Jan 2024 22:12:27 +0900 Subject: [PATCH 5/7] Bind streams to plans for CUDA and HIP backends --- fft/src/KokkosFFT_Cuda_plans.hpp | 15 +++++++++++++++ fft/src/KokkosFFT_HIP_plans.hpp | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/fft/src/KokkosFFT_Cuda_plans.hpp b/fft/src/KokkosFFT_Cuda_plans.hpp index e78dd43c..53709683 100644 --- a/fft/src/KokkosFFT_Cuda_plans.hpp +++ b/fft/src/KokkosFFT_Cuda_plans.hpp @@ -26,6 +26,9 @@ auto _create(const ExecutionSpace& exec_space, PlanType& plan, cufftResult cufft_rt = cufftCreate(&plan); if (cufft_rt != CUFFT_SUCCESS) throw std::runtime_error("cufftCreate failed"); + cudaStream_t stream = exec_space.cuda_stream(); + cufftSetStream(plan, stream); + const int batch = 1; const int axis = 0; @@ -61,6 +64,9 @@ auto _create(const ExecutionSpace& exec_space, PlanType& plan, cufftResult cufft_rt = cufftCreate(&plan); if (cufft_rt != CUFFT_SUCCESS) throw std::runtime_error("cufftCreate failed"); + cudaStream_t stream = exec_space.cuda_stream(); + cufftSetStream(plan, stream); + const int axis = 0; auto type = KokkosFFT::Impl::transform_type::type(); @@ -94,6 +100,9 @@ auto _create(const ExecutionSpace& exec_space, PlanType& plan, cufftResult cufft_rt = cufftCreate(&plan); if (cufft_rt != CUFFT_SUCCESS) throw std::runtime_error("cufftCreate failed"); + cudaStream_t stream = exec_space.cuda_stream(); + cufftSetStream(plan, stream); + const int axis = 0; auto type = KokkosFFT::Impl::transform_type::type(); @@ -99,6 +105,9 @@ auto _create(const ExecutionSpace& exec_space, PlanType& plan, if (hipfft_rt != HIPFFT_SUCCESS) throw std::runtime_error("hipfftCreate failed"); + hipStream_t stream = exec_space.hip_stream(); + hipfftSetStream(plan, stream); + const int batch = 1; const int axis = 0; @@ -138,6 +147,9 @@ auto _create(const ExecutionSpace& exec_space, PlanType& plan, if (hipfft_rt != HIPFFT_SUCCESS) throw std::runtime_error("hipfftCreate failed"); + hipStream_t stream = exec_space.hip_stream(); + hipfftSetStream(plan, stream); + const int rank = InViewType::rank(); const int batch = 1; const int axis = 0; @@ -199,6 +211,9 @@ auto _create(const ExecutionSpace& exec_space, PlanType& plan, if (hipfft_rt != HIPFFT_SUCCESS) throw std::runtime_error("hipfftCreate failed"); + hipStream_t stream = exec_space.hip_stream(); + hipfftSetStream(plan, stream); + hipfft_rt = hipfftPlanMany(&plan, rank, fft_extents.data(), in_extents.data(), istride, idist, out_extents.data(), ostride, odist, type, howmany); From 8c7555ba4baf87ba4e89020523d7d9a2a9a3a18b Mon Sep 17 00:00:00 2001 From: Yuuichi Asahi Date: Tue, 23 Jan 2024 22:21:19 +0900 Subject: [PATCH 6/7] formatting --- common/src/KokkosFFT_utils.hpp | 26 +++--- fft/src/KokkosFFT_Helpers.hpp | 16 ++-- fft/src/KokkosFFT_Plans.hpp | 83 +++++++++++--------- fft/src/KokkosFFT_Transform.hpp | 135 ++++++++++++++++++++------------ 4 files changed, 154 insertions(+), 106 deletions(-) diff --git a/common/src/KokkosFFT_utils.hpp b/common/src/KokkosFFT_utils.hpp index b97a3c2c..40b43795 100644 --- a/common/src/KokkosFFT_utils.hpp +++ b/common/src/KokkosFFT_utils.hpp @@ -32,12 +32,16 @@ template struct is_layout_left_or_right : std::false_type {}; template -struct is_layout_left_or_right || - std::is_same_v > > : std::true_type {}; +struct is_layout_left_or_right< + ViewType, + std::enable_if_t< + std::is_same_v || + std::is_same_v>> + : std::true_type {}; template -inline constexpr bool is_layout_left_or_right_v = is_layout_left_or_right::value; +inline constexpr bool is_layout_left_or_right_v = + is_layout_left_or_right::value; template = nullptr> @@ -51,9 +55,8 @@ struct complex_view_type { template auto convert_negative_axis(const ViewType& view, int _axis = -1) { - static_assert( - Kokkos::is_view::value, - "convert_negative_axis: ViewType is not a Kokkos::View."); + static_assert(Kokkos::is_view::value, + "convert_negative_axis: ViewType is not a Kokkos::View."); int rank = static_cast(ViewType::rank()); assert(abs(_axis) < rank); // axis should be in [-(rank-1), rank-1] int axis = _axis < 0 ? rank + _axis : _axis; @@ -62,9 +65,8 @@ auto convert_negative_axis(const ViewType& view, int _axis = -1) { template auto convert_negative_shift(const ViewType& view, int _shift, int _axis) { - static_assert( - Kokkos::is_view::value, - "convert_negative_shift: ViewType is not a Kokkos::View."); + static_assert(Kokkos::is_view::value, + "convert_negative_shift: ViewType is not a Kokkos::View."); int axis = convert_negative_axis(view, _axis); int extent = view.extent(axis); int shift0 = 0, shift1 = 0, shift2 = extent / 2; @@ -178,7 +180,7 @@ void conjugate(const ExecutionSpace& exec_space, const InViewType& in, static_assert(KokkosFFT::Impl::is_complex::value, "conjugate: OutViewType must be complex"); std::size_t size = in.size(); - out = OutViewType("out", in.layout()); + out = OutViewType("out", in.layout()); auto* in_data = in.data(); auto* out_data = out.data(); @@ -198,7 +200,7 @@ auto extract_extents(const ViewType& view) { "extract_extents: ViewType is not a Kokkos::View."); constexpr std::size_t rank = ViewType::rank(); std::array extents; - for(std::size_t i=0; i _axes, template void _roll(const ExecutionSpace& exec_space, ViewType& inout, axis_type<1> shift, axis_type<1> axes) { - static_assert(ViewType::rank() == 1, - "_roll: Rank of View must be 1."); + static_assert(ViewType::rank() == 1, "_roll: Rank of View must be 1."); std::size_t n0 = inout.extent(0); ViewType tmp("tmp", n0); @@ -68,8 +67,7 @@ template void _roll(const ExecutionSpace& exec_space, ViewType& inout, axis_type<2> shift, axis_type axes) { constexpr std::size_t DIM0 = 2; - static_assert(ViewType::rank() == DIM0, - "_roll: Rank of View must be 2."); + static_assert(ViewType::rank() == DIM0, "_roll: Rank of View must be 2."); int n0 = inout.extent(0), n1 = inout.extent(1); ViewType tmp("tmp", n0, n1); @@ -131,8 +129,9 @@ void _fftshift(const ExecutionSpace& exec_space, ViewType& inout, axis_type axes) { static_assert(Kokkos::is_view::value, "_fftshift: ViewType is not a Kokkos::View."); - static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, - "_fftshift: ViewType must be either LayoutLeft or LayoutRight."); + static_assert( + KokkosFFT::Impl::is_layout_left_or_right_v, + "_fftshift: ViewType must be either LayoutLeft or LayoutRight."); static_assert( Kokkos::SpaceAccessibility::accessible, @@ -150,8 +149,9 @@ void _ifftshift(const ExecutionSpace& exec_space, ViewType& inout, axis_type axes) { static_assert(Kokkos::is_view::value, "_ifftshift: ViewType is not a Kokkos::View."); - static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, - "_ifftshift: ViewType must be either LayoutLeft or LayoutRight."); + static_assert( + KokkosFFT::Impl::is_layout_left_or_right_v, + "_ifftshift: ViewType must be either LayoutLeft or LayoutRight."); static_assert( Kokkos::SpaceAccessibility::accessible, diff --git a/fft/src/KokkosFFT_Plans.hpp b/fft/src/KokkosFFT_Plans.hpp index 219bf5d5..c106a2d4 100644 --- a/fft/src/KokkosFFT_Plans.hpp +++ b/fft/src/KokkosFFT_Plans.hpp @@ -44,7 +44,7 @@ class Plan { using map_type = axis_type; using nonConstInViewType = std::remove_cv_t; using nonConstOutViewType = std::remove_cv_t; - using extents_type = shape_type; + using extents_type = shape_type; fft_plan_type m_plan; fft_size_type m_fft_size; @@ -69,26 +69,29 @@ class Plan { "Plan::Plan: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, "Plan::Plan: OutViewType is not a Kokkos::View."); - static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, - "Plan::Plan: InViewType must be either LayoutLeft or LayoutRight."); - static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, - "Plan::Plan: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert( + KokkosFFT::Impl::is_layout_left_or_right_v, + "Plan::Plan: InViewType must be either LayoutLeft or LayoutRight."); + static_assert( + KokkosFFT::Impl::is_layout_left_or_right_v, + "Plan::Plan: OutViewType must be either LayoutLeft or LayoutRight."); static_assert(InViewType::rank() == OutViewType::rank(), - "Plan::Plan: InViewType and OutViewType must have " - "the same rank."); - static_assert(std::is_same_v, - "Plan::Plan: InViewType and OutViewType must have " - "the same Layout."); + "Plan::Plan: InViewType and OutViewType must have " + "the same rank."); + static_assert(std::is_same_v, + "Plan::Plan: InViewType and OutViewType must have " + "the same Layout."); static_assert( - Kokkos::SpaceAccessibility::accessible, - "Plan::Plan: execution_space cannot access data in InViewType"); + Kokkos::SpaceAccessibility< + ExecutionSpace, typename InViewType::memory_space>::accessible, + "Plan::Plan: execution_space cannot access data in InViewType"); static_assert( - Kokkos::SpaceAccessibility::accessible, - "Plan::Plan: execution_space cannot access data in OutViewType"); + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "Plan::Plan: execution_space cannot access data in OutViewType"); m_axes = {axis}; m_in_extents = KokkosFFT::Impl::extract_extents(in); @@ -110,27 +113,30 @@ class Plan { "Plan::Plan: InViewType is not a Kokkos::View."); static_assert(Kokkos::is_view::value, "Plan::Plan: OutViewType is not a Kokkos::View."); - static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, - "Plan::Plan: InViewType must be either LayoutLeft or LayoutRight."); - static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, - "Plan::Plan: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert( + KokkosFFT::Impl::is_layout_left_or_right_v, + "Plan::Plan: InViewType must be either LayoutLeft or LayoutRight."); + static_assert( + KokkosFFT::Impl::is_layout_left_or_right_v, + "Plan::Plan: OutViewType must be either LayoutLeft or LayoutRight."); static_assert(InViewType::rank() == OutViewType::rank(), - "Plan::Plan: InViewType and OutViewType must have " - "the same rank."); + "Plan::Plan: InViewType and OutViewType must have " + "the same rank."); - static_assert(std::is_same_v, - "Plan::Plan: InViewType and OutViewType must have " - "the same Layout."); + static_assert(std::is_same_v, + "Plan::Plan: InViewType and OutViewType must have " + "the same Layout."); static_assert( - Kokkos::SpaceAccessibility::accessible, - "Plan::Plan: execution_space cannot access data in InViewType"); + Kokkos::SpaceAccessibility< + ExecutionSpace, typename InViewType::memory_space>::accessible, + "Plan::Plan: execution_space cannot access data in InViewType"); static_assert( - Kokkos::SpaceAccessibility::accessible, - "Plan::Plan: execution_space cannot access data in OutViewType"); + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "Plan::Plan: execution_space cannot access data in OutViewType"); m_in_extents = KokkosFFT::Impl::extract_extents(in); m_out_extents = KokkosFFT::Impl::extract_extents(out); @@ -144,18 +150,19 @@ class Plan { template - void good(const InViewType2& in, const OutViewType2& out, KokkosFFT::Impl::Direction direction, axis_type axes) const { + void good(const InViewType2& in, const OutViewType2& out, + KokkosFFT::Impl::Direction direction, axis_type axes) const { static_assert(std::is_same_v, "Plan::good: Execution spaces for plan and " "execution are not identical."); static_assert( - Kokkos::SpaceAccessibility::accessible, - "Plan::good: execution_space cannot access data in InViewType"); + Kokkos::SpaceAccessibility< + ExecutionSpace2, typename InViewType2::memory_space>::accessible, + "Plan::good: execution_space cannot access data in InViewType"); static_assert( - Kokkos::SpaceAccessibility::accessible, - "Plan::good: execution_space cannot access data in OutViewType"); + Kokkos::SpaceAccessibility< + ExecutionSpace2, typename OutViewType2::memory_space>::accessible, + "Plan::good: execution_space cannot access data in OutViewType"); using nonConstInViewType2 = std::remove_cv_t; using nonConstOutViewType2 = std::remove_cv_t; diff --git a/fft/src/KokkosFFT_Transform.hpp b/fft/src/KokkosFFT_Transform.hpp index ed96966a..45fc7ef3 100644 --- a/fft/src/KokkosFFT_Transform.hpp +++ b/fft/src/KokkosFFT_Transform.hpp @@ -52,7 +52,8 @@ void _fft(const ExecutionSpace& exec_space, PlanType& plan, static_assert(InViewType::rank() == OutViewType::rank(), "_fft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "_fft: InViewType and OutViewType must have " "the same Layout."); @@ -61,8 +62,8 @@ void _fft(const ExecutionSpace& exec_space, PlanType& plan, typename InViewType::memory_space>::accessible, "_fft: execution_space cannot access data in InViewType"); static_assert( - Kokkos::SpaceAccessibility::accessible, + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, "_fft: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; @@ -98,7 +99,8 @@ void _ifft(const ExecutionSpace& exec_space, PlanType& plan, static_assert(InViewType::rank() == OutViewType::rank(), "_ifft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "_ifft: InViewType and OutViewType must have " "the same Layout."); @@ -107,8 +109,8 @@ void _ifft(const ExecutionSpace& exec_space, PlanType& plan, typename InViewType::memory_space>::accessible, "_ifft: execution_space cannot access data in InViewType"); static_assert( - Kokkos::SpaceAccessibility::accessible, + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, "_ifft: execution_space cannot access data in OutViewType"); using in_value_type = typename InViewType::non_const_value_type; @@ -147,7 +149,8 @@ void fft(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "fft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "fft: InViewType and OutViewType must have " "the same Layout."); @@ -209,7 +212,8 @@ void fft(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "fft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "fft: InViewType and OutViewType must have " "the same Layout."); @@ -271,7 +275,8 @@ void ifft(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "ifft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "ifft: InViewType and OutViewType must have " "the same Layout."); @@ -335,7 +340,8 @@ void ifft(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "ifft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "ifft: InViewType and OutViewType must have " "the same Layout."); @@ -399,7 +405,8 @@ void rfft(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "rfft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "rfft: InViewType and OutViewType must have " "the same Layout."); @@ -440,7 +447,8 @@ void rfft(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "rfft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "rfft: InViewType and OutViewType must have " "the same Layout."); @@ -480,7 +488,8 @@ void irfft(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "irfft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "irfft: InViewType and OutViewType must have " "the same Layout."); @@ -521,7 +530,8 @@ void irfft(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "irfft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "irfft: InViewType and OutViewType must have " "the same Layout."); @@ -565,7 +575,8 @@ void hfft(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "hfft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "hfft: InViewType and OutViewType must have " "the same Layout."); @@ -613,7 +624,8 @@ void hfft(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "hfft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "hfft: InViewType and OutViewType must have " "the same Layout."); @@ -660,7 +672,8 @@ void ihfft(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "ihfft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "ihfft: InViewType and OutViewType must have " "the same Layout."); @@ -704,7 +717,8 @@ void ihfft(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "ihfft: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "ihfft: InViewType and OutViewType must have " "the same Layout."); @@ -749,7 +763,8 @@ void fft2(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "fft2: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "fft2: InViewType and OutViewType must have " "the same Layout."); @@ -809,7 +824,8 @@ void fft2(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "fft2: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "fft2: InViewType and OutViewType must have " "the same Layout."); @@ -870,7 +886,8 @@ void ifft2(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "ifft2: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "ifft2: InViewType and OutViewType must have " "the same Layout."); @@ -930,7 +947,8 @@ void ifft2(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "ifft2: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "ifft2: InViewType and OutViewType must have " "the same Layout."); @@ -991,7 +1009,8 @@ void rfft2(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "rfft2: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "rfft2: InViewType and OutViewType must have " "the same Layout."); @@ -1032,7 +1051,8 @@ void rfft2(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "rfft2: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "rfft2: InViewType and OutViewType must have " "the same Layout."); @@ -1068,12 +1088,14 @@ void irfft2(const ExecutionSpace& exec_space, const InViewType& in, "irfft2: OutViewType is not a Kokkos::View."); static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, "irfft2: InViewType must be either LayoutLeft or LayoutRight."); - static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, - "irfft2: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert( + KokkosFFT::Impl::is_layout_left_or_right_v, + "irfft2: OutViewType must be either LayoutLeft or LayoutRight."); static_assert(InViewType::rank() == OutViewType::rank(), "irfft2: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "irfft2: InViewType and OutViewType must have " "the same Layout."); @@ -1117,12 +1139,14 @@ void irfft2(const ExecutionSpace& exec_space, const InViewType& in, "irfft2: OutViewType is not a Kokkos::View."); static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, "irfft2: InViewType must be either LayoutLeft or LayoutRight."); - static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, - "irfft2: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert( + KokkosFFT::Impl::is_layout_left_or_right_v, + "irfft2: OutViewType must be either LayoutLeft or LayoutRight."); static_assert(InViewType::rank() == OutViewType::rank(), "irfft2: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "irfft2: InViewType and OutViewType must have " "the same Layout."); @@ -1172,7 +1196,8 @@ void fftn(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "fftn: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "fftn: InViewType and OutViewType must have " "the same Layout."); @@ -1237,7 +1262,8 @@ void fftn(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "fftn: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "fftn: InViewType and OutViewType must have " "the same Layout."); @@ -1297,7 +1323,8 @@ void fftn(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "fftn: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "fftn: InViewType and OutViewType must have " "the same Layout."); @@ -1358,7 +1385,8 @@ void ifftn(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "ifftn: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "ifftn: InViewType and OutViewType must have " "the same Layout."); @@ -1423,7 +1451,8 @@ void ifftn(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "ifftn: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "ifftn: InViewType and OutViewType must have " "the same Layout."); @@ -1483,7 +1512,8 @@ void ifftn(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "ifftn: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "ifftn: InViewType and OutViewType must have " "the same Layout."); @@ -1544,7 +1574,8 @@ void rfftn(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "rfftn: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "rfftn: InViewType and OutViewType must have " "the same Layout."); @@ -1585,7 +1616,8 @@ void rfftn(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "rfftn: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "rfftn: InViewType and OutViewType must have " "the same Layout."); @@ -1626,7 +1658,8 @@ void rfftn(const ExecutionSpace& exec_space, const InViewType& in, static_assert(InViewType::rank() == OutViewType::rank(), "rfftn: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "rfftn: InViewType and OutViewType must have " "the same Layout."); @@ -1662,12 +1695,14 @@ void irfftn(const ExecutionSpace& exec_space, const InViewType& in, "irfftn: OutViewType is not a Kokkos::View."); static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, "irfftn: InViewType must be either LayoutLeft or LayoutRight."); - static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, - "irfftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert( + KokkosFFT::Impl::is_layout_left_or_right_v, + "irfftn: OutViewType must be either LayoutLeft or LayoutRight."); static_assert(InViewType::rank() == OutViewType::rank(), "irfftn: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "irfftn: InViewType and OutViewType must have " "the same Layout."); @@ -1711,12 +1746,14 @@ void irfftn(const ExecutionSpace& exec_space, const InViewType& in, "irfftn: OutViewType is not a Kokkos::View."); static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, "irfftn: InViewType must be either LayoutLeft or LayoutRight."); - static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, - "irfftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert( + KokkosFFT::Impl::is_layout_left_or_right_v, + "irfftn: OutViewType must be either LayoutLeft or LayoutRight."); static_assert(InViewType::rank() == OutViewType::rank(), "irfftn: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "irfftn: InViewType and OutViewType must have " "the same Layout."); @@ -1760,12 +1797,14 @@ void irfftn(const ExecutionSpace& exec_space, const InViewType& in, "irfftn: OutViewType is not a Kokkos::View."); static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, "irfftn: InViewType must be either LayoutLeft or LayoutRight."); - static_assert(KokkosFFT::Impl::is_layout_left_or_right_v, - "irfftn: OutViewType must be either LayoutLeft or LayoutRight."); + static_assert( + KokkosFFT::Impl::is_layout_left_or_right_v, + "irfftn: OutViewType must be either LayoutLeft or LayoutRight."); static_assert(InViewType::rank() == OutViewType::rank(), "irfftn: InViewType and OutViewType must have " "the same rank."); - static_assert(std::is_same_v, + static_assert(std::is_same_v, "irfftn: InViewType and OutViewType must have " "the same Layout."); From 189031aaa10c5edef9b50fdcc158e09418bcfdb3 Mon Sep 17 00:00:00 2001 From: Yuuichi Asahi Date: Tue, 23 Jan 2024 22:27:02 +0900 Subject: [PATCH 7/7] Formatting tests --- common/unit_test/Test_Utils.cpp | 18 +++++++++--------- fft/unit_test/Test_Transform.cpp | 19 ++++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/common/unit_test/Test_Utils.cpp b/common/unit_test/Test_Utils.cpp index 42b5ef71..bef3d2b9 100644 --- a/common/unit_test/Test_Utils.cpp +++ b/common/unit_test/Test_Utils.cpp @@ -264,7 +264,7 @@ TEST(ExtractExtents, 1Dto8D) { using View7Dtype = Kokkos::View; using View8Dtype = Kokkos::View; - std::size_t n1 = 1, n2 = 1, n3 = 2, n4 = 3, n5= 5, n6 = 8, n7 = 13, n8 = 21; + std::size_t n1 = 1, n2 = 1, n3 = 2, n4 = 3, n5 = 5, n6 = 8, n7 = 13, n8 = 21; std::array ref_extents1D = {n1}; std::array ref_extents2D = {n1, n2}; @@ -284,12 +284,12 @@ TEST(ExtractExtents, 1Dto8D) { View7Dtype view7D("view7D", n1, n2, n3, n4, n5, n6, n7); View8Dtype view8D("view8D", n1, n2, n3, n4, n5, n6, n7, n8); - EXPECT_EQ( KokkosFFT::Impl::extract_extents(view1D), ref_extents1D ); - EXPECT_EQ( KokkosFFT::Impl::extract_extents(view2D), ref_extents2D ); - EXPECT_EQ( KokkosFFT::Impl::extract_extents(view3D), ref_extents3D ); - EXPECT_EQ( KokkosFFT::Impl::extract_extents(view4D), ref_extents4D ); - EXPECT_EQ( KokkosFFT::Impl::extract_extents(view5D), ref_extents5D ); - EXPECT_EQ( KokkosFFT::Impl::extract_extents(view6D), ref_extents6D ); - EXPECT_EQ( KokkosFFT::Impl::extract_extents(view7D), ref_extents7D ); - EXPECT_EQ( KokkosFFT::Impl::extract_extents(view8D), ref_extents8D ); + EXPECT_EQ(KokkosFFT::Impl::extract_extents(view1D), ref_extents1D); + EXPECT_EQ(KokkosFFT::Impl::extract_extents(view2D), ref_extents2D); + EXPECT_EQ(KokkosFFT::Impl::extract_extents(view3D), ref_extents3D); + EXPECT_EQ(KokkosFFT::Impl::extract_extents(view4D), ref_extents4D); + EXPECT_EQ(KokkosFFT::Impl::extract_extents(view5D), ref_extents5D); + EXPECT_EQ(KokkosFFT::Impl::extract_extents(view6D), ref_extents6D); + EXPECT_EQ(KokkosFFT::Impl::extract_extents(view7D), ref_extents7D); + EXPECT_EQ(KokkosFFT::Impl::extract_extents(view8D), ref_extents8D); } \ No newline at end of file diff --git a/fft/unit_test/Test_Transform.cpp b/fft/unit_test/Test_Transform.cpp index 8c245c68..e77d17c3 100644 --- a/fft/unit_test/Test_Transform.cpp +++ b/fft/unit_test/Test_Transform.cpp @@ -268,52 +268,53 @@ void test_fft1_identity_reuse_plan(T atol = 1.0e-12) { // Check if errors are correctly raised aginst wrong extents const int maxlen_wrong = 32 * 2; ComplexView1DType a_wrong("a", maxlen_wrong), _a_wrong("_a", maxlen_wrong); - ComplexView1DType out_wrong("out", maxlen_wrong), outr_wrong("outr", maxlen_wrong / 2 + 1); + ComplexView1DType out_wrong("out", maxlen_wrong), + outr_wrong("outr", maxlen_wrong / 2 + 1); RealView1DType ar_wrong("ar", maxlen_wrong), _ar_wrong("_ar", maxlen_wrong); // fft // With incorrect input shape EXPECT_THROW(KokkosFFT::fft(execution_space(), a_wrong, out, fft_plan, KokkosFFT::Normalization::BACKWARD, axis), - std::runtime_error); + std::runtime_error); // With incorrect output shape EXPECT_THROW(KokkosFFT::fft(execution_space(), a, out_wrong, fft_plan, KokkosFFT::Normalization::BACKWARD, axis), - std::runtime_error); + std::runtime_error); // ifft // With incorrect input shape EXPECT_THROW(KokkosFFT::ifft(execution_space(), out_wrong, _a, ifft_plan, KokkosFFT::Normalization::BACKWARD, axis), - std::runtime_error); + std::runtime_error); // With incorrect output shape EXPECT_THROW(KokkosFFT::ifft(execution_space(), out, _a_wrong, ifft_plan, KokkosFFT::Normalization::BACKWARD, axis), - std::runtime_error); + std::runtime_error); // rfft // With incorrect input shape EXPECT_THROW(KokkosFFT::rfft(execution_space(), ar_wrong, outr, rfft_plan, KokkosFFT::Normalization::BACKWARD, axis), - std::runtime_error); + std::runtime_error); // With incorrect output shape EXPECT_THROW(KokkosFFT::rfft(execution_space(), ar, out_wrong, rfft_plan, KokkosFFT::Normalization::BACKWARD, axis), - std::runtime_error); + std::runtime_error); // irfft // With incorrect input shape EXPECT_THROW(KokkosFFT::irfft(execution_space(), outr_wrong, _ar, irfft_plan, KokkosFFT::Normalization::BACKWARD, axis), - std::runtime_error); + std::runtime_error); // With incorrect output shape EXPECT_THROW(KokkosFFT::irfft(execution_space(), outr, _ar_wrong, irfft_plan, KokkosFFT::Normalization::BACKWARD, axis), - std::runtime_error); + std::runtime_error); } template