From 55bd18098168746ba3ba3d621089f5d3c0f73f24 Mon Sep 17 00:00:00 2001 From: Yuuichi Asahi Date: Mon, 8 Jan 2024 19:17:03 +0900 Subject: [PATCH 1/5] .clang-format file from kokkos repo --- .clang-format | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..6048a76d --- /dev/null +++ b/.clang-format @@ -0,0 +1,5 @@ +BasedOnStyle: google +SortIncludes: false +AlignConsecutiveAssignments: true +AllowShortCaseLabelsOnASingleLine: true +AllowShortIfStatementsOnASingleLine: true \ No newline at end of file From 242ac8daf3416705e17b4e5a3adb04b0fbd05495 Mon Sep 17 00:00:00 2001 From: Yuuichi Asahi Date: Mon, 8 Jan 2024 19:17:30 +0900 Subject: [PATCH 2/5] Apply format to common/ --- common/src/KokkosFFT_Cuda_types.hpp | 338 +++++++------ common/src/KokkosFFT_HIP_types.hpp | 338 +++++++------ common/src/KokkosFFT_OpenMP_types.hpp | 148 +++--- common/src/KokkosFFT_default_types.hpp | 70 +-- common/src/KokkosFFT_layouts.hpp | 310 ++++++------ common/src/KokkosFFT_normalization.hpp | 87 ++-- common/src/KokkosFFT_padding.hpp | 265 +++++----- common/src/KokkosFFT_transpose.hpp | 390 ++++++++------- common/src/KokkosFFT_utils.hpp | 328 +++++++------ common/unit_test/Test_Layouts.cpp | 623 +++++++++++++----------- common/unit_test/Test_Main.cpp | 6 +- common/unit_test/Test_Normalization.cpp | 120 +++-- common/unit_test/Test_Padding.cpp | 546 ++++++++++++--------- common/unit_test/Test_Transpose.cpp | 404 ++++++++------- common/unit_test/Test_Types.hpp | 21 +- common/unit_test/Test_Utils.cpp | 220 +++++---- common/unit_test/Test_Utils.hpp | 40 +- 17 files changed, 2299 insertions(+), 1955 deletions(-) diff --git a/common/src/KokkosFFT_Cuda_types.hpp b/common/src/KokkosFFT_Cuda_types.hpp index 034674c4..7234a791 100644 --- a/common/src/KokkosFFT_Cuda_types.hpp +++ b/common/src/KokkosFFT_Cuda_types.hpp @@ -11,165 +11,197 @@ static_assert(sizeof(cufftDoubleComplex) == sizeof(Kokkos::complex)); static_assert(alignof(cufftDoubleComplex) <= alignof(Kokkos::complex)); #ifdef ENABLE_HOST_AND_DEVICE - #include - #include "KokkosFFT_utils.hpp" - static_assert(sizeof(fftwf_complex) == sizeof(Kokkos::complex)); - static_assert(alignof(fftwf_complex) <= alignof(Kokkos::complex)); +#include +#include "KokkosFFT_utils.hpp" +static_assert(sizeof(fftwf_complex) == sizeof(Kokkos::complex)); +static_assert(alignof(fftwf_complex) <= alignof(Kokkos::complex)); - static_assert(sizeof(fftw_complex) == sizeof(Kokkos::complex)); - static_assert(alignof(fftw_complex) <= alignof(Kokkos::complex)); +static_assert(sizeof(fftw_complex) == sizeof(Kokkos::complex)); +static_assert(alignof(fftw_complex) <= alignof(Kokkos::complex)); #endif namespace KokkosFFT { namespace Impl { - enum class Direction { - Forward, - Backward, - }; - - using FFTDirectionType = int; - - #ifdef ENABLE_HOST_AND_DEVICE - enum class FFTWTransformType { - R2C, - D2Z, - C2R, - Z2D, - C2C, - Z2Z - }; - - template - struct FFTDataType { - using float32 = std::conditional_t, cufftReal, float>; - using float64 = std::conditional_t, cufftDoubleReal, double>; - using complex64 = std::conditional_t, cufftComplex, fftwf_complex>; - using complex128 = std::conditional_t, cufftDoubleComplex, fftw_complex>; - }; - - template - struct FFTPlanType { - using fftwHandle = std::conditional_t, float>, fftwf_plan, fftw_plan>; - using type = std::conditional_t, cufftHandle, fftwHandle>; - }; - - template - using TransformType = std::conditional_t, cufftType, FFTWTransformType>; - - template - struct transform_type { - static_assert(std::is_same_v, "Real to real transform is unavailable"); - }; - - template - struct transform_type> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - - static constexpr _TransformType m_cuda_type = std::is_same_v ? CUFFT_R2C : CUFFT_D2Z; - static constexpr _TransformType m_cpu_type = std::is_same_v ? FFTWTransformType::R2C : FFTWTransformType::D2Z; - - static constexpr _TransformType type() { - if constexpr(std::is_same_v) { - return m_cuda_type; - } else { - return m_cpu_type; - } - } - }; - - template - struct transform_type, T2> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - - static constexpr _TransformType m_cuda_type = std::is_same_v ? CUFFT_C2R : CUFFT_Z2D; - static constexpr _TransformType m_cpu_type = std::is_same_v ? FFTWTransformType::C2R : FFTWTransformType::Z2D; - - static constexpr _TransformType type() { - if constexpr(std::is_same_v) { - return m_cuda_type; - } else { - return m_cpu_type; - } - } - }; - - template - struct transform_type, Kokkos::complex> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - - static constexpr _TransformType m_cuda_type = std::is_same_v ? CUFFT_C2C : CUFFT_Z2Z; - static constexpr _TransformType m_cpu_type = std::is_same_v ? FFTWTransformType::C2C : FFTWTransformType::Z2Z; - - static constexpr _TransformType type() { - if constexpr(std::is_same_v) { - return m_cuda_type; - } else { - return m_cpu_type; - } - } - }; - - template - auto direction_type(Direction direction) { - static constexpr FFTDirectionType _FORWARD = std::is_same_v ? CUFFT_FORWARD : FFTW_FORWARD; - static constexpr FFTDirectionType _BACKWARD = std::is_same_v ? CUFFT_INVERSE : FFTW_BACKWARD; - return direction==Direction::Forward ? _FORWARD : _BACKWARD; +enum class Direction { + Forward, + Backward, +}; + +using FFTDirectionType = int; + +#ifdef ENABLE_HOST_AND_DEVICE +enum class FFTWTransformType { R2C, D2Z, C2R, Z2D, C2C, Z2Z }; + +template +struct FFTDataType { + using float32 = + std::conditional_t, + cufftReal, float>; + using float64 = + std::conditional_t, + cufftDoubleReal, double>; + using complex64 = + std::conditional_t, + cufftComplex, fftwf_complex>; + using complex128 = + std::conditional_t, + cufftDoubleComplex, fftw_complex>; +}; + +template +struct FFTPlanType { + using fftwHandle = + std::conditional_t, float>, + fftwf_plan, fftw_plan>; + using type = std::conditional_t, + cufftHandle, fftwHandle>; +}; + +template +using TransformType = + std::conditional_t, cufftType, + FFTWTransformType>; + +template +struct transform_type { + static_assert(std::is_same_v, + "Real to real transform is unavailable"); +}; + +template +struct transform_type> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + + static constexpr _TransformType m_cuda_type = + std::is_same_v ? CUFFT_R2C : CUFFT_D2Z; + static constexpr _TransformType m_cpu_type = std::is_same_v + ? FFTWTransformType::R2C + : FFTWTransformType::D2Z; + + static constexpr _TransformType type() { + if constexpr (std::is_same_v) { + return m_cuda_type; + } else { + return m_cpu_type; } - #else - template - struct FFTDataType { - using float32 = cufftReal; - using float64 = cufftDoubleReal; - using complex64 = cufftComplex; - using complex128 = cufftDoubleComplex; - }; - - template - struct FFTPlanType { - using type = cufftHandle; - }; - - template - using TransformType = cufftType; - - template - struct transform_type { - static_assert(std::is_same_v, "Real to real transform is unavailable"); - }; - - template - struct transform_type> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - static constexpr _TransformType m_type = std::is_same_v ? CUFFT_R2C : CUFFT_D2Z; - static constexpr _TransformType type() { return m_type; }; - }; - - template - struct transform_type, T2> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - static constexpr _TransformType m_type = std::is_same_v ? CUFFT_C2R : CUFFT_Z2D; - static constexpr _TransformType type() { return m_type; }; - }; - - template - struct transform_type, Kokkos::complex> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - static constexpr _TransformType m_type = std::is_same_v ? CUFFT_C2C : CUFFT_Z2Z; - static constexpr _TransformType type() { return m_type; }; - }; - - template - auto direction_type(Direction direction) { - return direction==Direction::Forward ? CUFFT_FORWARD : CUFFT_INVERSE; + } +}; + +template +struct transform_type, T2> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + + static constexpr _TransformType m_cuda_type = + std::is_same_v ? CUFFT_C2R : CUFFT_Z2D; + static constexpr _TransformType m_cpu_type = std::is_same_v + ? FFTWTransformType::C2R + : FFTWTransformType::Z2D; + + static constexpr _TransformType type() { + if constexpr (std::is_same_v) { + return m_cuda_type; + } else { + return m_cpu_type; } - #endif -} // namespace Impl -} // namespace KokkosFFT + } +}; + +template +struct transform_type, + Kokkos::complex> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + + static constexpr _TransformType m_cuda_type = + std::is_same_v ? CUFFT_C2C : CUFFT_Z2Z; + static constexpr _TransformType m_cpu_type = std::is_same_v + ? FFTWTransformType::C2C + : FFTWTransformType::Z2Z; + + static constexpr _TransformType type() { + if constexpr (std::is_same_v) { + return m_cuda_type; + } else { + return m_cpu_type; + } + } +}; + +template +auto direction_type(Direction direction) { + static constexpr FFTDirectionType _FORWARD = + std::is_same_v ? CUFFT_FORWARD + : FFTW_FORWARD; + static constexpr FFTDirectionType _BACKWARD = + std::is_same_v ? CUFFT_INVERSE + : FFTW_BACKWARD; + return direction == Direction::Forward ? _FORWARD : _BACKWARD; +} +#else +template +struct FFTDataType { + using float32 = cufftReal; + using float64 = cufftDoubleReal; + using complex64 = cufftComplex; + using complex128 = cufftDoubleComplex; +}; + +template +struct FFTPlanType { + using type = cufftHandle; +}; + +template +using TransformType = cufftType; + +template +struct transform_type { + static_assert(std::is_same_v, + "Real to real transform is unavailable"); +}; + +template +struct transform_type> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + static constexpr _TransformType m_type = + std::is_same_v ? CUFFT_R2C : CUFFT_D2Z; + static constexpr _TransformType type() { return m_type; }; +}; + +template +struct transform_type, T2> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + static constexpr _TransformType m_type = + std::is_same_v ? CUFFT_C2R : CUFFT_Z2D; + static constexpr _TransformType type() { return m_type; }; +}; + +template +struct transform_type, + Kokkos::complex> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + static constexpr _TransformType m_type = + std::is_same_v ? CUFFT_C2C : CUFFT_Z2Z; + static constexpr _TransformType type() { return m_type; }; +}; + +template +auto direction_type(Direction direction) { + return direction == Direction::Forward ? CUFFT_FORWARD : CUFFT_INVERSE; +} +#endif +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/common/src/KokkosFFT_HIP_types.hpp b/common/src/KokkosFFT_HIP_types.hpp index 0a717e42..8729f1e7 100644 --- a/common/src/KokkosFFT_HIP_types.hpp +++ b/common/src/KokkosFFT_HIP_types.hpp @@ -11,165 +11,197 @@ static_assert(sizeof(hipfftDoubleComplex) == sizeof(Kokkos::complex)); static_assert(alignof(hipfftDoubleComplex) <= alignof(Kokkos::complex)); #ifdef ENABLE_HOST_AND_DEVICE - #include - #include "KokkosFFT_utils.hpp" - static_assert(sizeof(fftwf_complex) == sizeof(Kokkos::complex)); - static_assert(alignof(fftwf_complex) <= alignof(Kokkos::complex)); +#include +#include "KokkosFFT_utils.hpp" +static_assert(sizeof(fftwf_complex) == sizeof(Kokkos::complex)); +static_assert(alignof(fftwf_complex) <= alignof(Kokkos::complex)); - static_assert(sizeof(fftw_complex) == sizeof(Kokkos::complex)); - static_assert(alignof(fftw_complex) <= alignof(Kokkos::complex)); +static_assert(sizeof(fftw_complex) == sizeof(Kokkos::complex)); +static_assert(alignof(fftw_complex) <= alignof(Kokkos::complex)); #endif namespace KokkosFFT { namespace Impl { - enum class Direction { - Forward, - Backward, - }; - - using FFTDirectionType = int; - - #ifdef ENABLE_HOST_AND_DEVICE - enum class FFTWTransformType { - R2C, - D2Z, - C2R, - Z2D, - C2C, - Z2Z - }; - - template - struct FFTDataType { - using float32 = std::conditional_t, hipfftReal, float>; - using float64 = std::conditional_t, hipfftDoubleReal, double>; - using complex64 = std::conditional_t, hipfftComplex, fftwf_complex>; - using complex128 = std::conditional_t, hipfftDoubleComplex, fftw_complex>; - }; - - template - struct FFTPlanType { - using fftwHandle = std::conditional_t, float>, fftwf_plan, fftw_plan>; - using type = std::conditional_t, hipfftHandle, fftwHandle>; - }; - - template - using TransformType = std::conditional_t, hipfftType, FFTWTransformType>; - - template - struct transform_type { - static_assert(std::is_same_v, "Real to real transform is unavailable"); - }; - - template - struct transform_type> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - - static constexpr _TransformType m_cuda_type = std::is_same_v ? HIPFFT_R2C : HIPFFT_D2Z; - static constexpr _TransformType m_cpu_type = std::is_same_v ? FFTWTransformType::R2C : FFTWTransformType::D2Z; - - static constexpr _TransformType type() { - if constexpr(std::is_same_v) { - return m_cuda_type; - } else { - return m_cpu_type; - } - } - }; - - template - struct transform_type, T2> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - - static constexpr _TransformType m_cuda_type = std::is_same_v ? HIPFFT_C2R : HIPFFT_Z2D; - static constexpr _TransformType m_cpu_type = std::is_same_v ? FFTWTransformType::C2R : FFTWTransformType::Z2D; - - static constexpr _TransformType type() { - if constexpr(std::is_same_v) { - return m_cuda_type; - } else { - return m_cpu_type; - } - } - }; - - template - struct transform_type, Kokkos::complex> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - - static constexpr _TransformType m_cuda_type = std::is_same_v ? HIPFFT_C2C : HIPFFT_Z2Z; - static constexpr _TransformType m_cpu_type = std::is_same_v ? FFTWTransformType::C2C : FFTWTransformType::Z2Z; - - static constexpr _TransformType type() { - if constexpr(std::is_same_v) { - return m_cuda_type; - } else { - return m_cpu_type; - } - } - }; - - template - auto direction_type(Direction direction) { - static constexpr FFTDirectionType _FORWARD = std::is_same_v ? HIPFFT_FORWARD : FFTW_FORWARD; - static constexpr FFTDirectionType _BACKWARD = std::is_same_v ? HIPFFT_BACKWARD : FFTW_BACKWARD; - return direction==Direction::Forward ? _FORWARD : _BACKWARD; +enum class Direction { + Forward, + Backward, +}; + +using FFTDirectionType = int; + +#ifdef ENABLE_HOST_AND_DEVICE +enum class FFTWTransformType { R2C, D2Z, C2R, Z2D, C2C, Z2Z }; + +template +struct FFTDataType { + using float32 = + std::conditional_t, + hipfftReal, float>; + using float64 = + std::conditional_t, + hipfftDoubleReal, double>; + using complex64 = + std::conditional_t, + hipfftComplex, fftwf_complex>; + using complex128 = + std::conditional_t, + hipfftDoubleComplex, fftw_complex>; +}; + +template +struct FFTPlanType { + using fftwHandle = + std::conditional_t, float>, + fftwf_plan, fftw_plan>; + using type = std::conditional_t, + hipfftHandle, fftwHandle>; +}; + +template +using TransformType = + std::conditional_t, hipfftType, + FFTWTransformType>; + +template +struct transform_type { + static_assert(std::is_same_v, + "Real to real transform is unavailable"); +}; + +template +struct transform_type> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + + static constexpr _TransformType m_cuda_type = + std::is_same_v ? HIPFFT_R2C : HIPFFT_D2Z; + static constexpr _TransformType m_cpu_type = std::is_same_v + ? FFTWTransformType::R2C + : FFTWTransformType::D2Z; + + static constexpr _TransformType type() { + if constexpr (std::is_same_v) { + return m_cuda_type; + } else { + return m_cpu_type; } - #else - template - struct FFTDataType { - using float32 = hipfftReal; - using float64 = hipfftDoubleReal; - using complex64 = hipfftComplex; - using complex128 = hipfftDoubleComplex; - }; - - template - struct FFTPlanType { - using type = hipfftHandle; - }; - - template - using TransformType = hipfftType; - - template - struct transform_type { - static_assert(std::is_same_v, "Real to real transform is unavailable"); - }; - - template - struct transform_type> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - static constexpr _TransformType m_type = std::is_same_v ? HIPFFT_R2C : HIPFFT_D2Z; - static constexpr _TransformType type() { return m_type; }; - }; - - template - struct transform_type, T2> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - static constexpr _TransformType m_type = std::is_same_v ? HIPFFT_C2R : HIPFFT_Z2D; - static constexpr _TransformType type() { return m_type; }; - }; - - template - struct transform_type, Kokkos::complex> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - static constexpr _TransformType m_type = std::is_same_v ? HIPFFT_C2C : HIPFFT_Z2Z; - static constexpr _TransformType type() { return m_type; }; - }; - - template - auto direction_type(Direction direction) { - return direction==Direction::Forward ? HIPFFT_FORWARD : HIPFFT_BACKWARD; + } +}; + +template +struct transform_type, T2> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + + static constexpr _TransformType m_cuda_type = + std::is_same_v ? HIPFFT_C2R : HIPFFT_Z2D; + static constexpr _TransformType m_cpu_type = std::is_same_v + ? FFTWTransformType::C2R + : FFTWTransformType::Z2D; + + static constexpr _TransformType type() { + if constexpr (std::is_same_v) { + return m_cuda_type; + } else { + return m_cpu_type; } - #endif -} // namespace Impl -} // namespace KokkosFFT + } +}; + +template +struct transform_type, + Kokkos::complex> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + + static constexpr _TransformType m_cuda_type = + std::is_same_v ? HIPFFT_C2C : HIPFFT_Z2Z; + static constexpr _TransformType m_cpu_type = std::is_same_v + ? FFTWTransformType::C2C + : FFTWTransformType::Z2Z; + + static constexpr _TransformType type() { + if constexpr (std::is_same_v) { + return m_cuda_type; + } else { + return m_cpu_type; + } + } +}; + +template +auto direction_type(Direction direction) { + static constexpr FFTDirectionType _FORWARD = + std::is_same_v ? HIPFFT_FORWARD + : FFTW_FORWARD; + static constexpr FFTDirectionType _BACKWARD = + std::is_same_v ? HIPFFT_BACKWARD + : FFTW_BACKWARD; + return direction == Direction::Forward ? _FORWARD : _BACKWARD; +} +#else +template +struct FFTDataType { + using float32 = hipfftReal; + using float64 = hipfftDoubleReal; + using complex64 = hipfftComplex; + using complex128 = hipfftDoubleComplex; +}; + +template +struct FFTPlanType { + using type = hipfftHandle; +}; + +template +using TransformType = hipfftType; + +template +struct transform_type { + static_assert(std::is_same_v, + "Real to real transform is unavailable"); +}; + +template +struct transform_type> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + static constexpr _TransformType m_type = + std::is_same_v ? HIPFFT_R2C : HIPFFT_D2Z; + static constexpr _TransformType type() { return m_type; }; +}; + +template +struct transform_type, T2> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + static constexpr _TransformType m_type = + std::is_same_v ? HIPFFT_C2R : HIPFFT_Z2D; + static constexpr _TransformType type() { return m_type; }; +}; + +template +struct transform_type, + Kokkos::complex> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + static constexpr _TransformType m_type = + std::is_same_v ? HIPFFT_C2C : HIPFFT_Z2Z; + static constexpr _TransformType type() { return m_type; }; +}; + +template +auto direction_type(Direction direction) { + return direction == Direction::Forward ? HIPFFT_FORWARD : HIPFFT_BACKWARD; +} +#endif +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/common/src/KokkosFFT_OpenMP_types.hpp b/common/src/KokkosFFT_OpenMP_types.hpp index 1767852b..e97edf79 100644 --- a/common/src/KokkosFFT_OpenMP_types.hpp +++ b/common/src/KokkosFFT_OpenMP_types.hpp @@ -13,76 +13,82 @@ static_assert(alignof(fftw_complex) <= alignof(Kokkos::complex)); namespace KokkosFFT { namespace Impl { - enum class Direction { - Forward, - Backward, - }; - - enum class FFTWTransformType { - R2C, - D2Z, - C2R, - Z2D, - C2C, - Z2Z - }; - - template - struct FFTDataType { - using float32 = float; - using float64 = double; - using complex64 = fftwf_complex; - using complex128 = fftw_complex; - }; - - template - struct FFTPlanType { - using type = std::conditional_t, float>, fftwf_plan, fftw_plan>; - }; - - template - using TransformType = FFTWTransformType; - - using FFTDirectionType = int; - - // Define fft transform types - template - struct transform_type { - static_assert(std::is_same_v, "Real to real transform is unavailable"); - }; - - template - struct transform_type> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - - static constexpr _TransformType m_type = std::is_same_v ? FFTWTransformType::R2C : FFTWTransformType::D2Z; - static constexpr _TransformType type() { return m_type; }; - }; - - template - struct transform_type, T2> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - - static constexpr _TransformType m_type = std::is_same_v ? FFTWTransformType::C2R : FFTWTransformType::Z2D; - static constexpr _TransformType type() { return m_type; }; - }; - - template - struct transform_type, Kokkos::complex> { - static_assert(std::is_same_v, "T1 and T2 should have the same precision"); - using _TransformType = TransformType; - - static constexpr _TransformType m_type = std::is_same_v ? FFTWTransformType::C2C : FFTWTransformType::Z2Z; - static constexpr _TransformType type() { return m_type; }; - }; - - template - auto direction_type(Direction direction) { - return direction==Direction::Forward ? FFTW_FORWARD : FFTW_BACKWARD; - } -} // namespace Impl -} // namespace KokkosFFT +enum class Direction { + Forward, + Backward, +}; + +enum class FFTWTransformType { R2C, D2Z, C2R, Z2D, C2C, Z2Z }; + +template +struct FFTDataType { + using float32 = float; + using float64 = double; + using complex64 = fftwf_complex; + using complex128 = fftw_complex; +}; + +template +struct FFTPlanType { + using type = + std::conditional_t, float>, + fftwf_plan, fftw_plan>; +}; + +template +using TransformType = FFTWTransformType; + +using FFTDirectionType = int; + +// Define fft transform types +template +struct transform_type { + static_assert(std::is_same_v, + "Real to real transform is unavailable"); +}; + +template +struct transform_type> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + + static constexpr _TransformType m_type = std::is_same_v + ? FFTWTransformType::R2C + : FFTWTransformType::D2Z; + static constexpr _TransformType type() { return m_type; }; +}; + +template +struct transform_type, T2> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + + static constexpr _TransformType m_type = std::is_same_v + ? FFTWTransformType::C2R + : FFTWTransformType::Z2D; + static constexpr _TransformType type() { return m_type; }; +}; + +template +struct transform_type, + Kokkos::complex> { + static_assert(std::is_same_v, + "T1 and T2 should have the same precision"); + using _TransformType = TransformType; + + static constexpr _TransformType m_type = std::is_same_v + ? FFTWTransformType::C2C + : FFTWTransformType::Z2Z; + static constexpr _TransformType type() { return m_type; }; +}; + +template +auto direction_type(Direction direction) { + return direction == Direction::Forward ? FFTW_FORWARD : FFTW_BACKWARD; +} +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/common/src/KokkosFFT_default_types.hpp b/common/src/KokkosFFT_default_types.hpp index 8925ca12..0cba34e1 100644 --- a/common/src/KokkosFFT_default_types.hpp +++ b/common/src/KokkosFFT_default_types.hpp @@ -4,53 +4,55 @@ #include #if defined(KOKKOS_ENABLE_CUDA) - using default_device = Kokkos::Cuda; - #include "KokkosFFT_Cuda_types.hpp" +using default_device = Kokkos::Cuda; +#include "KokkosFFT_Cuda_types.hpp" #elif defined(KOKKOS_ENABLE_HIP) - using default_device = Kokkos::HIP; - #include "KokkosFFT_HIP_types.hpp" +using default_device = Kokkos::HIP; +#include "KokkosFFT_HIP_types.hpp" #elif defined(KOKKOS_ENABLE_OPENMP) - using default_device = Kokkos::OpenMP; - #include "KokkosFFT_OpenMP_types.hpp" +using default_device = Kokkos::OpenMP; +#include "KokkosFFT_OpenMP_types.hpp" #elif defined(KOKKOS_ENABLE_THREADS) - using default_device = Kokkos::Threads; - #include "KokkosFFT_OpenMP_types.hpp" +using default_device = Kokkos::Threads; +#include "KokkosFFT_OpenMP_types.hpp" #else - using default_device = Kokkos::Serial; - #include "KokkosFFT_OpenMP_types.hpp" +using default_device = Kokkos::Serial; +#include "KokkosFFT_OpenMP_types.hpp" #endif #include "KokkosFFT_utils.hpp" namespace KokkosFFT { - // Define type to specify transform axis - template - using axis_type = std::array; +// Define type to specify transform axis +template +using axis_type = std::array; - // Define type to specify new shape - template - using shape_type = std::array; +// Define type to specify new shape +template +using shape_type = std::array; - enum class Normalization { - FORWARD, - BACKWARD, - ORTHO - }; -} // namespace KokkosFFT +enum class Normalization { FORWARD, BACKWARD, ORTHO }; +} // namespace KokkosFFT namespace KokkosFFT { namespace Impl { - // Define fft data types - template - struct fft_data_type { - using type = std::conditional_t, typename KokkosFFT::Impl::FFTDataType::float32, typename KokkosFFT::Impl::FFTDataType::float64>; - }; - - template - struct fft_data_type> { - using type = std::conditional_t, typename KokkosFFT::Impl::FFTDataType::complex64, typename KokkosFFT::Impl::FFTDataType::complex128>; - }; -} // namespace Impl -} // namespace KokkosFFT +// Define fft data types +template +struct fft_data_type { + using type = std::conditional_t< + std::is_same_v, + typename KokkosFFT::Impl::FFTDataType::float32, + typename KokkosFFT::Impl::FFTDataType::float64>; +}; + +template +struct fft_data_type> { + using type = std::conditional_t< + std::is_same_v, + typename KokkosFFT::Impl::FFTDataType::complex64, + typename KokkosFFT::Impl::FFTDataType::complex128>; +}; +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/common/src/KokkosFFT_layouts.hpp b/common/src/KokkosFFT_layouts.hpp index c7e98b26..4e474431 100644 --- a/common/src/KokkosFFT_layouts.hpp +++ b/common/src/KokkosFFT_layouts.hpp @@ -1,7 +1,6 @@ #ifndef KOKKOSFFT_LAYOUTS_HPP #define KOKKOSFFT_LAYOUTS_HPP - #include #include #include @@ -12,164 +11,183 @@ namespace KokkosFFT { namespace Impl { - /* Input and output extents exposed to the fft library - i.e extents are converted into Layout Right - */ - template - auto get_extents(InViewType& in, OutViewType& out, axis_type _axes) { - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - using array_layout_type = typename InViewType::array_layout; - - // index map after transpose over axis - auto [map, map_inv] = KokkosFFT::Impl::get_map_axes(in, _axes); - - constexpr std::size_t rank = InViewType::rank; - int inner_most_axis = std::is_same_v ? 0 : rank - 1; - - std::vector in_extents, out_extents, fft_extents; - for(std::size_t i=0; i::value) { - // Then R2C - if(is_complex::value) { - assert( out_extents.at(inner_most_axis) == in.extent(inner_most_axis)/2+1 ); - } else { - throw std::runtime_error("If the input type is real, the output type should be complex"); - } - } - - if(std::is_floating_point::value) { - // Then C2R - if(is_complex::value) { - assert( in_extents.at(inner_most_axis) == out.extent(inner_most_axis)/2+1 ); - } else { - throw std::runtime_error("If the output type is real, the input type should be complex"); - } - } - - if(std::is_same::value) { - std::reverse(in_extents.begin(), in_extents.end()); - std::reverse(out_extents.begin(), out_extents.end()); - std::reverse(fft_extents.begin(), fft_extents.end()); - } - - return std::tuple< std::vector, std::vector, std::vector >({in_extents, out_extents, fft_extents}); +/* Input and output extents exposed to the fft library + i.e extents are converted into Layout Right +*/ +template +auto get_extents(InViewType& in, OutViewType& out, axis_type _axes) { + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + using array_layout_type = typename InViewType::array_layout; + + // index map after transpose over axis + auto [map, map_inv] = KokkosFFT::Impl::get_map_axes(in, _axes); + + constexpr std::size_t rank = InViewType::rank; + int inner_most_axis = + std::is_same_v ? 0 + : rank - 1; + + std::vector in_extents, out_extents, fft_extents; + for (std::size_t i = 0; i < rank; i++) { + auto _idx = map.at(i); + in_extents.push_back(in.extent(_idx)); + out_extents.push_back(out.extent(_idx)); + + // The extent for transform is always equal to the extent + // of the extent of real type (R2C or C2R) + // For C2C, the in and out extents are the same. + // In the end, we can just use the largest extent among in and out extents. + auto fft_extent = std::max(in.extent(_idx), out.extent(_idx)); + fft_extents.push_back(fft_extent); } - /* Input and output extents exposed to the fft library - i.e extents are converted into Layout Right - */ - template - auto get_extents_batched(InViewType& in, OutViewType& out, axis_type _axes) { - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - using array_layout_type = typename InViewType::array_layout; - - // index map after transpose over axis - auto [map, map_inv] = KokkosFFT::Impl::get_map_axes(in, _axes); - - static_assert(InViewType::rank() >= DIM, - "KokkosFFT::get_map_axes: Rank of View must be larger thane or equal to the Rank of FFT axes."); - static_assert(DIM > 0, - "KokkosFFT::get_map_axes: Rank of FFT axes must be larger than or equal to 1."); - - constexpr std::size_t rank = InViewType::rank; - int inner_most_axis = rank - 1; - - std::vector _in_extents, _out_extents, _fft_extents; - - // Get extents for the inner most axes in LayoutRight - // If we allow the FFT on the layoutLeft, this part should be modified - for(std::size_t i=0; i::value) { - std::reverse(_in_extents.begin(), _in_extents.end()); - std::reverse(_out_extents.begin(), _out_extents.end()); - std::reverse(_fft_extents.begin(), _fft_extents.end()); - } - - // Define subvectors starting from last - DIM - // Dimensions relevant to FFTs - std::vector in_extents(_in_extents.end()-DIM, _in_extents.end()); - std::vector out_extents(_out_extents.end()-DIM, _out_extents.end()); - std::vector fft_extents(_fft_extents.end()-DIM, _fft_extents.end()); - - int total_fft_size = std::accumulate(_fft_extents.begin(), _fft_extents.end(), 1, std::multiplies<>()); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - int howmany = total_fft_size / fft_size; - - if(std::is_floating_point::value) { - // Then R2C - if(is_complex::value) { - assert( out_extents.at(inner_most_axis) == in.extent(inner_most_axis)/2+1 ); - } else { - throw std::runtime_error("If the input type is real, the output type should be complex"); - } + if (std::is_floating_point::value) { + // Then R2C + if (is_complex::value) { + assert(out_extents.at(inner_most_axis) == + in.extent(inner_most_axis) / 2 + 1); + } else { + throw std::runtime_error( + "If the input type is real, the output type should be complex"); } + } - if(std::is_floating_point::value) { - // Then C2R - if(is_complex::value) { - assert( in_extents.at(inner_most_axis) == out.extent(inner_most_axis)/2+1 ); - } else { - throw std::runtime_error("If the output type is real, the input type should be complex"); - } + if (std::is_floating_point::value) { + // Then C2R + if (is_complex::value) { + assert(in_extents.at(inner_most_axis) == + out.extent(inner_most_axis) / 2 + 1); + } else { + throw std::runtime_error( + "If the output type is real, the input type should be complex"); } + } - return std::tuple< std::vector, std::vector, std::vector, int >({in_extents, out_extents, fft_extents, howmany}); + if (std::is_same::value) { + std::reverse(in_extents.begin(), in_extents.end()); + std::reverse(out_extents.begin(), out_extents.end()); + std::reverse(fft_extents.begin(), fft_extents.end()); } - /* Input and output extents exposed to the fft library - i.e extents are converted into Layout Right - */ - template - auto get_extents(InViewType& in, OutViewType& out) { - using array_layout_type = typename InViewType::array_layout; + return std::tuple, std::vector, std::vector >( + {in_extents, out_extents, fft_extents}); +} + +/* Input and output extents exposed to the fft library + i.e extents are converted into Layout Right +*/ +template +auto get_extents_batched(InViewType& in, OutViewType& out, + axis_type _axes) { + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + using array_layout_type = typename InViewType::array_layout; + + // index map after transpose over axis + auto [map, map_inv] = KokkosFFT::Impl::get_map_axes(in, _axes); + + static_assert(InViewType::rank() >= DIM, + "KokkosFFT::get_map_axes: Rank of View must be larger thane or " + "equal to the Rank of FFT axes."); + static_assert(DIM > 0, + "KokkosFFT::get_map_axes: Rank of FFT axes must be larger than " + "or equal to 1."); + + constexpr std::size_t rank = InViewType::rank; + int inner_most_axis = rank - 1; + + std::vector _in_extents, _out_extents, _fft_extents; + + // Get extents for the inner most axes in LayoutRight + // If we allow the FFT on the layoutLeft, this part should be modified + for (std::size_t i = 0; i < rank; i++) { + auto _idx = map.at(i); + _in_extents.push_back(in.extent(_idx)); + _out_extents.push_back(out.extent(_idx)); + + // The extent for transform is always equal to the extent + // of the extent of real type (R2C or C2R) + // For C2C, the in and out extents are the same. + // In the end, we can just use the largest extent among in and out extents. + auto fft_extent = std::max(in.extent(_idx), out.extent(_idx)); + _fft_extents.push_back(fft_extent); + } - constexpr std::size_t rank = InViewType::rank; - int inner_most_axis = std::is_same_v ? 0 : rank - 1; - return get_extents(in, out, axis_type<1>{inner_most_axis}); + if (std::is_same::value) { + std::reverse(_in_extents.begin(), _in_extents.end()); + std::reverse(_out_extents.begin(), _out_extents.end()); + std::reverse(_fft_extents.begin(), _fft_extents.end()); } - /* Input and output extents exposed to the fft library - i.e extents are converted into Layout Right - */ - template - auto get_extents(InViewType& in, OutViewType& out, int _axis) { - return get_extents(in, out, axis_type<1>{_axis}); + // Define subvectors starting from last - DIM + // Dimensions relevant to FFTs + std::vector in_extents(_in_extents.end() - DIM, _in_extents.end()); + std::vector out_extents(_out_extents.end() - DIM, _out_extents.end()); + std::vector fft_extents(_fft_extents.end() - DIM, _fft_extents.end()); + + int total_fft_size = std::accumulate(_fft_extents.begin(), _fft_extents.end(), + 1, std::multiplies<>()); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + int howmany = total_fft_size / fft_size; + + if (std::is_floating_point::value) { + // Then R2C + if (is_complex::value) { + assert(out_extents.at(inner_most_axis) == + in.extent(inner_most_axis) / 2 + 1); + } else { + throw std::runtime_error( + "If the input type is real, the output type should be complex"); + } } - /* Input and output extents exposed to the fft library - i.e extents are converted into Layout Right - */ - template - auto get_extents_batched(InViewType& in, OutViewType& out, int _axis) { - return get_extents_batched(in, out, axis_type<1>{_axis}); + if (std::is_floating_point::value) { + // Then C2R + if (is_complex::value) { + assert(in_extents.at(inner_most_axis) == + out.extent(inner_most_axis) / 2 + 1); + } else { + throw std::runtime_error( + "If the output type is real, the input type should be complex"); + } } -} // namespace Impl -}; // namespace KokkosFFT + + return std::tuple, std::vector, std::vector, int>( + {in_extents, out_extents, fft_extents, howmany}); +} + +/* Input and output extents exposed to the fft library + i.e extents are converted into Layout Right +*/ +template +auto get_extents(InViewType& in, OutViewType& out) { + using array_layout_type = typename InViewType::array_layout; + + constexpr std::size_t rank = InViewType::rank; + int inner_most_axis = + std::is_same_v ? 0 + : rank - 1; + return get_extents(in, out, axis_type<1>{inner_most_axis}); +} + +/* Input and output extents exposed to the fft library + i.e extents are converted into Layout Right +*/ +template +auto get_extents(InViewType& in, OutViewType& out, int _axis) { + return get_extents(in, out, axis_type<1>{_axis}); +} + +/* Input and output extents exposed to the fft library + i.e extents are converted into Layout Right +*/ +template +auto get_extents_batched(InViewType& in, OutViewType& out, int _axis) { + return get_extents_batched(in, out, axis_type<1>{_axis}); +} +} // namespace Impl +}; // namespace KokkosFFT #endif \ No newline at end of file diff --git a/common/src/KokkosFFT_normalization.hpp b/common/src/KokkosFFT_normalization.hpp index 82468ef7..28a90024 100644 --- a/common/src/KokkosFFT_normalization.hpp +++ b/common/src/KokkosFFT_normalization.hpp @@ -7,69 +7,70 @@ namespace KokkosFFT { namespace Impl { - template - void _normalize(const ExecutionSpace& exec_space, ViewType& inout, const T coef) { - std::size_t size = inout.size(); - auto* data = inout.data(); +template +void _normalize(const ExecutionSpace& exec_space, ViewType& inout, + const T coef) { + std::size_t size = inout.size(); + auto* data = inout.data(); - Kokkos::parallel_for( - Kokkos::RangePolicy>(exec_space, 0, size), - KOKKOS_LAMBDA(const int& i) { data[i] *= coef; } - ); - } + Kokkos::parallel_for( + Kokkos::RangePolicy>( + exec_space, 0, size), + KOKKOS_LAMBDA(const int& i) { data[i] *= coef; }); +} - template - auto _coefficients(const ViewType& inout, Direction direction, Normalization normalization, std::size_t fft_size) { - using value_type = KokkosFFT::Impl::real_type_t; - value_type coef = 1; - bool to_normalize = false; +template +auto _coefficients(const ViewType& inout, Direction direction, + Normalization normalization, std::size_t fft_size) { + using value_type = + KokkosFFT::Impl::real_type_t; + value_type coef = 1; + bool to_normalize = false; - switch (normalization) { + switch (normalization) { case Normalization::FORWARD: - if(direction == Direction::Forward) { + if (direction == Direction::Forward) { coef = static_cast(1) / static_cast(fft_size); to_normalize = true; } break; case Normalization::BACKWARD: - if(direction == Direction::Backward) { + if (direction == Direction::Backward) { coef = static_cast(1) / static_cast(fft_size); to_normalize = true; } break; case Normalization::ORTHO: - coef = static_cast(1) / Kokkos::sqrt(static_cast(fft_size)); + coef = static_cast(1) / + Kokkos::sqrt(static_cast(fft_size)); to_normalize = true; break; - }; - return std::tuple ({coef, to_normalize}); - } + }; + return std::tuple({coef, to_normalize}); +} - template - void normalize(const ExecutionSpace& exec_space, ViewType& inout, Direction direction, Normalization normalization, std::size_t fft_size) { - auto [coef, to_normalize] = _coefficients(inout, direction, normalization, fft_size); - if(to_normalize) _normalize(exec_space, inout, coef); - } +template +void normalize(const ExecutionSpace& exec_space, ViewType& inout, + Direction direction, Normalization normalization, + std::size_t fft_size) { + auto [coef, to_normalize] = + _coefficients(inout, direction, normalization, fft_size); + if (to_normalize) _normalize(exec_space, inout, coef); +} - auto swap_direction(Normalization normalization) { - Normalization new_direction = Normalization::FORWARD; - switch (normalization) { - case Normalization::FORWARD: - new_direction = Normalization::BACKWARD; - break; - case Normalization::BACKWARD: - new_direction = Normalization::FORWARD; - break; - case Normalization::ORTHO: - new_direction = Normalization::ORTHO; - break; - }; - return new_direction; - } -} // namespace Impl -} // namespace KokkosFFT +auto swap_direction(Normalization normalization) { + Normalization new_direction = Normalization::FORWARD; + switch (normalization) { + case Normalization::FORWARD: new_direction = Normalization::BACKWARD; break; + case Normalization::BACKWARD: new_direction = Normalization::FORWARD; break; + case Normalization::ORTHO: new_direction = Normalization::ORTHO; break; + }; + return new_direction; +} +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/common/src/KokkosFFT_padding.hpp b/common/src/KokkosFFT_padding.hpp index 044b9d45..334c4f28 100644 --- a/common/src/KokkosFFT_padding.hpp +++ b/common/src/KokkosFFT_padding.hpp @@ -7,147 +7,142 @@ namespace KokkosFFT { namespace Impl { - template - auto get_modified_shape(const ViewType& view, shape_type shape) { - static_assert(ViewType::rank() >= DIM, - "KokkosFFT::get_modified_shape: Rank of View must be larger than or equal to the Rank of new shape"); - static_assert(DIM > 0, - "KokkosFFT::get_modified_shape: Rank of FFT axes must be larger than or equal to 1"); - - // [TO DO] Add a is_C2R arg. If is_C2R is true, then shape should be shape/2+1 - constexpr int rank = static_cast( ViewType::rank() ); - - using full_shape_type = shape_type; - full_shape_type modified_shape; - for(int i=0; i 0); - modified_shape.at(i) = shape.at(i); - } - - // [TO DO] may return, is_modification_needed if modified_shape is not equal view.extents() - // May be implement other function. is_crop_or_pad_needed(view, shape); - return modified_shape; - } - - template - auto is_crop_or_pad_needed(const ViewType& view, const shape_type& modified_shape) { - static_assert(ViewType::rank() == DIM, - "KokkosFFT::_crop_or_pad: Rank of View must be equal to Rank of extended shape."); - - // [TO DO] Add a is_C2R arg. If is_C2R is true, then shape should be shape/2+1 - constexpr int rank = static_cast( ViewType::rank() ); - - bool not_same = false; - for(int i=0; i +auto get_modified_shape(const ViewType& view, shape_type shape) { + static_assert(ViewType::rank() >= DIM, + "KokkosFFT::get_modified_shape: Rank of View must be larger " + "than or equal to the Rank of new shape"); + static_assert(DIM > 0, + "KokkosFFT::get_modified_shape: Rank of FFT axes must be " + "larger than or equal to 1"); + + // [TO DO] Add a is_C2R arg. If is_C2R is true, then shape should be shape/2+1 + constexpr int rank = static_cast(ViewType::rank()); + + using full_shape_type = shape_type; + full_shape_type modified_shape; + for (int i = 0; i < rank; i++) { + modified_shape.at(i) = view.extent(i); } - template - void _crop_or_pad(const ExecutionSpace& exec_space, - const ViewType& in, - ViewType& out, - shape_type<1> s) { - constexpr std::size_t DIM = 1; - static_assert(ViewType::rank() == DIM, - "KokkosFFT::_crop_or_pad: Rank of View must be equal to Rank of extended shape."); - - auto _n0 = s.at(0); - out = ViewType("out", _n0); - - auto n0 = std::min(_n0, in.extent(0)); - - Kokkos::parallel_for( - Kokkos::RangePolicy>(exec_space, 0, n0), - KOKKOS_LAMBDA (int i0) { - out(i0) = in(i0); - } - ); + // Update shapes based on newly given shape + for (int i = 0; i < DIM; i++) { + assert(shape.at(i) > 0); + modified_shape.at(i) = shape.at(i); } - template - void _crop_or_pad(const ExecutionSpace& exec_space, - const ViewType& in, - ViewType& out, - shape_type<2> s) { - constexpr std::size_t DIM = 2; - static_assert(ViewType::rank() == DIM, - "KokkosFFT::_crop_or_pad: Rank of View must be equal to Rank of extended shape."); - - auto [_n0, _n1] = s; - out = ViewType("out", _n0, _n1); - - int n0 = std::min(_n0, in.extent(0)); - int n1 = std::min(_n1, in.extent(1)); - - using range_type = Kokkos::MDRangePolicy >; - using tile_type = typename range_type::tile_type; - using point_type = typename range_type::point_type; - - range_type range( - point_type{{0, 0}}, - point_type{{n0, n1}}, - tile_type{{4, 4}} // [TO DO] Choose optimal tile sizes for each device - ); - - Kokkos::parallel_for(range, - KOKKOS_LAMBDA (int i0, int i1) { - out(i0, i1) = in(i0, i1); - } - ); + // [TO DO] may return, is_modification_needed if modified_shape is not equal + // view.extents() May be implement other function. is_crop_or_pad_needed(view, + // shape); + return modified_shape; +} + +template +auto is_crop_or_pad_needed(const ViewType& view, + const shape_type& modified_shape) { + static_assert(ViewType::rank() == DIM, + "KokkosFFT::_crop_or_pad: Rank of View must be equal to Rank " + "of extended shape."); + + // [TO DO] Add a is_C2R arg. If is_C2R is true, then shape should be shape/2+1 + constexpr int rank = static_cast(ViewType::rank()); + + bool not_same = false; + for (int i = 0; i < rank; i++) { + if (modified_shape.at(i) != view.extent(i)) { + not_same = true; + break; + } } - template - void _crop_or_pad(const ExecutionSpace& exec_space, - const ViewType& in, - ViewType& out, - shape_type<3> s) { - constexpr std::size_t DIM = 3; - static_assert(ViewType::rank() == DIM, - "KokkosFFT::_crop_or_pad: Rank of View must be equal to Rank of extended shape."); - - auto [_n0, _n1, _n2] = s; - out = ViewType("out", _n0, _n1, _n2); - - int n0 = std::min(_n0, in.extent(0)); - int n1 = std::min(_n1, in.extent(1)); - int n2 = std::min(_n2, in.extent(2)); - - using range_type = Kokkos::MDRangePolicy >; - using tile_type = typename range_type::tile_type; - using point_type = typename range_type::point_type; - - range_type range( - point_type{{0, 0, 0}}, - point_type{{n0, n1, n2}}, - tile_type{{4, 4, 4}} // [TO DO] Choose optimal tile sizes for each device - ); - - Kokkos::parallel_for(range, - KOKKOS_LAMBDA (int i0, int i1, int i2) { + return not_same; +} + +template +void _crop_or_pad(const ExecutionSpace& exec_space, const ViewType& in, + ViewType& out, shape_type<1> s) { + constexpr std::size_t DIM = 1; + static_assert(ViewType::rank() == DIM, + "KokkosFFT::_crop_or_pad: Rank of View must be equal to Rank " + "of extended shape."); + + auto _n0 = s.at(0); + out = ViewType("out", _n0); + + auto n0 = std::min(_n0, in.extent(0)); + + Kokkos::parallel_for( + Kokkos::RangePolicy>( + exec_space, 0, n0), + KOKKOS_LAMBDA(int i0) { out(i0) = in(i0); }); +} + +template +void _crop_or_pad(const ExecutionSpace& exec_space, const ViewType& in, + ViewType& out, shape_type<2> s) { + constexpr std::size_t DIM = 2; + static_assert(ViewType::rank() == DIM, + "KokkosFFT::_crop_or_pad: Rank of View must be equal to Rank " + "of extended shape."); + + auto [_n0, _n1] = s; + out = ViewType("out", _n0, _n1); + + int n0 = std::min(_n0, in.extent(0)); + int n1 = std::min(_n1, in.extent(1)); + + using range_type = Kokkos::MDRangePolicy< + ExecutionSpace, + Kokkos::Rank<2, Kokkos::Iterate::Default, Kokkos::Iterate::Default>>; + using tile_type = typename range_type::tile_type; + using point_type = typename range_type::point_type; + + range_type range(point_type{{0, 0}}, point_type{{n0, n1}}, tile_type{{4, 4}} + // [TO DO] Choose optimal tile sizes for each device + ); + + Kokkos::parallel_for( + range, KOKKOS_LAMBDA(int i0, int i1) { out(i0, i1) = in(i0, i1); }); +} + +template +void _crop_or_pad(const ExecutionSpace& exec_space, const ViewType& in, + ViewType& out, shape_type<3> s) { + constexpr std::size_t DIM = 3; + static_assert(ViewType::rank() == DIM, + "KokkosFFT::_crop_or_pad: Rank of View must be equal to Rank " + "of extended shape."); + + auto [_n0, _n1, _n2] = s; + out = ViewType("out", _n0, _n1, _n2); + + int n0 = std::min(_n0, in.extent(0)); + int n1 = std::min(_n1, in.extent(1)); + int n2 = std::min(_n2, in.extent(2)); + + using range_type = Kokkos::MDRangePolicy< + ExecutionSpace, + Kokkos::Rank<3, Kokkos::Iterate::Default, Kokkos::Iterate::Default>>; + using tile_type = typename range_type::tile_type; + using point_type = typename range_type::point_type; + + range_type range( + point_type{{0, 0, 0}}, point_type{{n0, n1, n2}}, tile_type{{4, 4, 4}} + // [TO DO] Choose optimal tile sizes for each device + ); + + Kokkos::parallel_for( + range, KOKKOS_LAMBDA(int i0, int i1, int i2) { out(i0, i1, i2) = in(i0, i1, i2); - } - ); - } - - template - void crop_or_pad(const ExecutionSpace& exec_space, - const ViewType& in, - ViewType& out, - shape_type s) { - _crop_or_pad(exec_space, in, out, s); - } -} // namespace Impl -} // namespace KokkosFFT + }); +} + +template +void crop_or_pad(const ExecutionSpace& exec_space, const ViewType& in, + ViewType& out, shape_type s) { + _crop_or_pad(exec_space, in, out, s); +} +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/common/src/KokkosFFT_transpose.hpp b/common/src/KokkosFFT_transpose.hpp index 2ad2f117..772d0a6d 100644 --- a/common/src/KokkosFFT_transpose.hpp +++ b/common/src/KokkosFFT_transpose.hpp @@ -8,217 +8,231 @@ namespace KokkosFFT { namespace Impl { - template - auto get_map_axes(const ViewType& view, axis_type _axes) { - static_assert(ViewType::rank() >= DIM, - "KokkosFFT::get_map_axes: Rank of View must be larger thane or equal to the Rank of FFT axes."); - static_assert(DIM > 0, - "KokkosFFT::get_map_axes: Rank of FFT axes must be larger than or equal to 1."); - - constexpr int rank = static_cast( ViewType::rank() ); - using array_layout_type = typename ViewType::array_layout; - - // Convert the input axes to be in the range of [0, rank-1] - std::vector axes; - for(std::size_t i=0; i +auto get_map_axes(const ViewType& view, axis_type _axes) { + static_assert(ViewType::rank() >= DIM, + "KokkosFFT::get_map_axes: Rank of View must be larger thane or " + "equal to the Rank of FFT axes."); + static_assert(DIM > 0, + "KokkosFFT::get_map_axes: Rank of FFT axes must be larger than " + "or equal to 1."); + + constexpr int rank = static_cast(ViewType::rank()); + using array_layout_type = typename ViewType::array_layout; + + // Convert the input axes to be in the range of [0, rank-1] + std::vector axes; + for (std::size_t i = 0; i < DIM; i++) { + int axis = KokkosFFT::Impl::convert_negative_axis(view, _axes.at(i)); + axes.push_back(axis); + } - // how indices are map - // For 5D View and axes are (2,3), map would be (0, 1, 4, 2, 3) - std::vector map, map_inv; - map.reserve(rank); - map_inv.reserve(rank); + // Assert if the elements are overlapped + assert(!KokkosFFT::Impl::has_duplicate_values(axes)); - if(std::is_same::value) { - // Stack axes not specified by axes (0, 1, 4) - for(int i=0; i map, map_inv; + map.reserve(rank); + map_inv.reserve(rank); - // Then stack remaining axes - for(int i=0; i::value) { + // Stack axes not specified by axes (0, 1, 4) + for (int i = 0; i < rank; i++) { + if (!is_found(axes, i)) { + map.push_back(i); } } - // Construct inverse map - for(int i=0; i; - full_axis_type array_map, array_map_inv; - std::copy(map.begin(), map.end(), array_map.begin()); - std::copy(map_inv.begin(), map_inv.end(), array_map_inv.begin()); - - return std::tuple< full_axis_type, full_axis_type >({array_map, array_map_inv}); - } - - template - auto get_map_axes(const ViewType& view, int axis) { - return get_map_axes(view, axis_type<1>({axis})); + // Then stack remaining axes + for (int i = 0; i < rank; i++) { + if (!is_found(axes, i)) { + map.push_back(i); + } + } } - template = nullptr> - void _transpose(const ExecutionSpace& exec_space, InViewType& in, OutViewType& out, [[maybe_unused]] axis_type<1> _map) { + // Construct inverse map + for (int i = 0; i < rank; i++) { + map_inv.push_back(get_index(map, i)); } - template - void _transpose(const ExecutionSpace& exec_space, InViewType& in, OutViewType& out, axis_type<2> _map) { - constexpr std::size_t DIM = 2; - static_assert(InViewType::rank() == DIM, - "KokkosFFT::_transpose: Rank of View must be equal to Rank of transpose axes."); - - constexpr std::size_t rank = InViewType::rank(); - using array_layout_type = typename InViewType::array_layout; - - using range_type = Kokkos::MDRangePolicy >; - using tile_type = typename range_type::tile_type; - using point_type = typename range_type::point_type; - - int n0 = in.extent(0), n1 = in.extent(1); - - range_type range( - point_type{{0, 0}}, - point_type{{n0, n1}}, - tile_type{{4, 4}} // [TO DO] Choose optimal tile sizes for each device - ); - - bool is_out_view_ready = true; - std::array out_extents; - for(int i=0; i; + full_axis_type array_map, array_map_inv; + std::copy(map.begin(), map.end(), array_map.begin()); + std::copy(map_inv.begin(), map_inv.end(), array_map_inv.begin()); + + return std::tuple({array_map, array_map_inv}); +} + +template +auto get_map_axes(const ViewType& view, int axis) { + return get_map_axes(view, axis_type<1>({axis})); +} + +template = nullptr> +void _transpose(const ExecutionSpace& exec_space, InViewType& in, + OutViewType& out, [[maybe_unused]] axis_type<1> _map) {} + +template +void _transpose(const ExecutionSpace& exec_space, InViewType& in, + OutViewType& out, axis_type<2> _map) { + constexpr std::size_t DIM = 2; + static_assert(InViewType::rank() == DIM, + "KokkosFFT::_transpose: Rank of View must be equal to Rank of " + "transpose axes."); + + constexpr std::size_t rank = InViewType::rank(); + using array_layout_type = typename InViewType::array_layout; + + using range_type = Kokkos::MDRangePolicy< + ExecutionSpace, + Kokkos::Rank >; + using tile_type = typename range_type::tile_type; + using point_type = typename range_type::point_type; + + int n0 = in.extent(0), n1 = in.extent(1); + + range_type range(point_type{{0, 0}}, point_type{{n0, n1}}, tile_type{{4, 4}} + // [TO DO] Choose optimal tile sizes for each device + ); + + bool is_out_view_ready = true; + std::array out_extents; + for (int i = 0; i < rank; i++) { + out_extents.at(i) = in.extent(_map.at(i)); + if (out_extents.at(i) != out.extent(i)) { + is_out_view_ready = false; } + } - Kokkos::parallel_for(range, - KOKKOS_LAMBDA (int i0, int i1) { - out(i1, i0) = in(i0, i1); - } - ); + if (!is_out_view_ready) { + auto [_n0, _n1] = out_extents; + out = OutViewType("out", _n0, _n1); } - template - void _transpose(const ExecutionSpace& exec_space, InViewType& in, OutViewType& out, axis_type<3> _map) { - constexpr std::size_t DIM = 3; - static_assert(InViewType::rank() == DIM, - "KokkosFFT::_transpose: Rank of View must be equal to Rank of transpose axes."); - constexpr std::size_t rank = InViewType::rank(); - using array_layout_type = typename InViewType::array_layout; - - using range_type = Kokkos::MDRangePolicy >; - using tile_type = typename range_type::tile_type; - using point_type = typename range_type::point_type; - - int n0 = in.extent(0), n1 = in.extent(1), n2 = in.extent(2); - - range_type range( - point_type{{0, 0, 0}}, - point_type{{n0, n1, n2}}, - tile_type{{4, 4, 4}} // [TO DO] Choose optimal tile sizes for each device - ); - - // Assign a View if not a shallow copy - bool is_out_view_ready = true; - std::array out_extents; - for(int i=0; i +void _transpose(const ExecutionSpace& exec_space, InViewType& in, + OutViewType& out, axis_type<3> _map) { + constexpr std::size_t DIM = 3; + static_assert(InViewType::rank() == DIM, + "KokkosFFT::_transpose: Rank of View must be equal to Rank of " + "transpose axes."); + constexpr std::size_t rank = InViewType::rank(); + using array_layout_type = typename InViewType::array_layout; + + using range_type = Kokkos::MDRangePolicy< + ExecutionSpace, + Kokkos::Rank >; + using tile_type = typename range_type::tile_type; + using point_type = typename range_type::point_type; + + int n0 = in.extent(0), n1 = in.extent(1), n2 = in.extent(2); + + range_type range( + point_type{{0, 0, 0}}, point_type{{n0, n1, n2}}, tile_type{{4, 4, 4}} + // [TO DO] Choose optimal tile sizes for each device + ); + + // Assign a View if not a shallow copy + bool is_out_view_ready = true; + std::array out_extents; + for (int i = 0; i < rank; i++) { + out_extents.at(i) = in.extent(_map.at(i)); + if (out_extents.at(i) != out.extent(i)) { + is_out_view_ready = false; } + } - if (! is_out_view_ready ) { - auto [_n0, _n1, _n2] = out_extents; - out = OutViewType("out", _n0, _n1, _n2); - } - Kokkos::parallel_for(range, - KOKKOS_LAMBDA (int i0, int i1, int i2) { + if (!is_out_view_ready) { + auto [_n0, _n1, _n2] = out_extents; + out = OutViewType("out", _n0, _n1, _n2); + } + Kokkos::parallel_for( + range, KOKKOS_LAMBDA(int i0, int i1, int i2) { int _i0 = i0, _i1 = i1, _i2 = i2; - if(_map[0] == 0 && _map[1] == 2 && _map[2] == 1) { - _i1 = i2; _i2 = i1; - } else if(_map[0] == 1 && _map[1] == 0 && _map[2] == 2) { - _i0 = i1; _i1 = i0; - } else if(_map[0] == 1 && _map[1] == 2 && _map[2] == 0) { - _i0 = i1; _i1 = i2; _i2 = i0; - } else if(_map[0] == 2 && _map[1] == 1 && _map[2] == 0) { - _i0 = i2; _i2 = i0; - } else if(_map[0] == 2 && _map[1] == 0 && _map[2] == 1) { - _i0 = i2; _i1 = i0; _i2 = i1; + if (_map[0] == 0 && _map[1] == 2 && _map[2] == 1) { + _i1 = i2; + _i2 = i1; + } else if (_map[0] == 1 && _map[1] == 0 && _map[2] == 2) { + _i0 = i1; + _i1 = i0; + } else if (_map[0] == 1 && _map[1] == 2 && _map[2] == 0) { + _i0 = i1; + _i1 = i2; + _i2 = i0; + } else if (_map[0] == 2 && _map[1] == 1 && _map[2] == 0) { + _i0 = i2; + _i2 = i0; + } else if (_map[0] == 2 && _map[1] == 0 && _map[2] == 1) { + _i0 = i2; + _i1 = i0; + _i2 = i1; } out(_i0, _i1, _i2) = in(i0, i1, i2); }); +} + +/* Make the axis direction to the inner most direction + axis should be the range in [-(rank-1), rank-1], where + negative number is interpreted as rank + axis. + E.g. axis = -1 for rank 3 matrix is interpreted as axis = 2 + + * E.g. + Layout Left + A (3, 4, 2) and axis = 1 -> A' (4, 3, 2) + B (2, 4, 3, 5) and axis = 2 -> B' (3, 2, 4, 5) + C (8, 6, 3) and axis = 0 -> C' (8, 6, 3) + D (7, 5) and axis = -1 -> D' (5, 7) + + Layout Right + A (3, 4, 2) and axis = 1 -> A' (3, 2, 4) + B (2, 4, 3, 5) and axis = 2 -> B' (2, 4, 5, 3) + C (8, 6, 3) and axis = 0 -> C' (6, 3, 8) + D (5, 7) and axis = -1 -> D' (5, 7) + * +*/ +template +void transpose(const ExecutionSpace& exec_space, InViewType& in, + OutViewType& out, axis_type _map) { + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + using array_layout_type = typename InViewType::array_layout; + + static_assert(Kokkos::is_view::value, + "KokkosFFT::transpose: InViewType is not a Kokkos::View."); + static_assert(Kokkos::is_view::value, + "KokkosFFT::transpose: OutViewType is not a Kokkos::View."); + + static_assert(InViewType::rank() == OutViewType::rank(), + "KokkosFFT::transpose: InViewType and OutViewType must have " + "the same rank."); + + if (!KokkosFFT::Impl::is_transpose_needed(_map)) { + throw std::runtime_error("KokkosFFT::transpose: transpose not necessary"); } - /* Make the axis direction to the inner most direction - axis should be the range in [-(rank-1), rank-1], where - negative number is interpreted as rank + axis. - E.g. axis = -1 for rank 3 matrix is interpreted as axis = 2 - - * E.g. - Layout Left - A (3, 4, 2) and axis = 1 -> A' (4, 3, 2) - B (2, 4, 3, 5) and axis = 2 -> B' (3, 2, 4, 5) - C (8, 6, 3) and axis = 0 -> C' (8, 6, 3) - D (7, 5) and axis = -1 -> D' (5, 7) - - Layout Right - A (3, 4, 2) and axis = 1 -> A' (3, 2, 4) - B (2, 4, 3, 5) and axis = 2 -> B' (2, 4, 5, 3) - C (8, 6, 3) and axis = 0 -> C' (6, 3, 8) - D (5, 7) and axis = -1 -> D' (5, 7) - * - */ - template - void transpose(const ExecutionSpace& exec_space, InViewType& in, OutViewType& out, axis_type _map) { - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - using array_layout_type = typename InViewType::array_layout; - - static_assert(Kokkos::is_view::value, - "KokkosFFT::transpose: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, - "KokkosFFT::transpose: OutViewType is not a Kokkos::View."); - - static_assert(InViewType::rank() == OutViewType::rank(), - "KokkosFFT::transpose: InViewType and OutViewType must have the same rank."); - - if(!KokkosFFT::Impl::is_transpose_needed(_map)) { - throw std::runtime_error("KokkosFFT::transpose: transpose not necessary"); - } - - _transpose(exec_space, in, out, _map); - } -} // namespace Impl -} // namespace KokkosFFT + _transpose(exec_space, in, out, _map); +} +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/common/src/KokkosFFT_utils.hpp b/common/src/KokkosFFT_utils.hpp index f138d0fa..d3be71df 100644 --- a/common/src/KokkosFFT_utils.hpp +++ b/common/src/KokkosFFT_utils.hpp @@ -9,180 +9,184 @@ namespace KokkosFFT { namespace Impl { - template - struct real_type { - using type = T; - }; - - template - struct real_type >{ - using type = T; - }; - - template - using real_type_t = typename real_type::type; - - template - struct is_complex : std::false_type {}; - - template - struct is_complex> : std::true_type {}; - - template = nullptr> - struct complex_view_type { - using value_type = typename ViewType::non_const_value_type; - using float_type = KokkosFFT::Impl::real_type_t; - using complex_type = Kokkos::complex; - using array_layout_type = typename ViewType::array_layout; - using type = Kokkos::View; - }; - - template - auto convert_negative_axis(const ViewType& view, int _axis=-1) { - static_assert(Kokkos::is_view::value, - "KokkosFFT::convert_negative_axis: ViewType is not a Kokkos::View."); - int rank = static_cast( ViewType::rank() ); - assert( abs(_axis) < rank ); // axis should be in [-(rank-1), rank-1] - int axis = _axis < 0 ? rank + _axis : _axis; - return axis; +template +struct real_type { + using type = T; +}; + +template +struct real_type> { + using type = T; +}; + +template +using real_type_t = typename real_type::type; + +template +struct is_complex : std::false_type {}; + +template +struct is_complex> : std::true_type {}; + +template = nullptr> +struct complex_view_type { + using value_type = typename ViewType::non_const_value_type; + using float_type = KokkosFFT::Impl::real_type_t; + using complex_type = Kokkos::complex; + using array_layout_type = typename ViewType::array_layout; + using type = Kokkos::View; +}; + +template +auto convert_negative_axis(const ViewType& view, int _axis = -1) { + static_assert( + Kokkos::is_view::value, + "KokkosFFT::convert_negative_axis: ViewType is not a Kokkos::View."); + int rank = static_cast(ViewType::rank()); + assert(abs(_axis) < rank); // axis should be in [-(rank-1), rank-1] + int axis = _axis < 0 ? rank + _axis : _axis; + return axis; +} + +template +auto convert_negative_shift(const ViewType& view, int _shift, int _axis) { + static_assert( + Kokkos::is_view::value, + "KokkosFFT::convert_negative_shift: ViewType is not a Kokkos::View."); + int axis = convert_negative_axis(view, _axis); + int extent = view.extent(axis); + int shift0 = 0, shift1 = 0, shift2 = extent / 2; + + if (_shift < 0) { + shift0 = -_shift + extent % 2; // add 1 for odd case + shift1 = -_shift; + shift2 = 0; + } else if (_shift > 0) { + shift0 = _shift; + shift1 = _shift + extent % 2; // add 1 for odd case + shift2 = 0; } - template - auto convert_negative_shift(const ViewType& view, int _shift, int _axis) { - static_assert(Kokkos::is_view::value, - "KokkosFFT::convert_negative_shift: ViewType is not a Kokkos::View."); - int axis = convert_negative_axis(view, _axis); - int extent = view.extent(axis); - int shift0 = 0, shift1 = 0, shift2 = extent/2; - - if(_shift < 0) { - shift0 = -_shift + extent%2; // add 1 for odd case - shift1 = -_shift; - shift2 = 0; - } else if(_shift>0){ - shift0 = _shift; - shift1 = _shift + extent%2; // add 1 for odd case - shift2 = 0; - } - - return std::tuple({shift0, shift1, shift2}); - } - - template - void permute(std::vector& values, const std::vector& indices) { - std::vector out; - out.reserve(indices.size()); - for(auto index: indices) { - out.push_back(values.at(index)); - } - values = std::move(out); - } - - template - bool is_found(std::vector& values, const T& key) { - return std::find(values.begin(), values.end(), key) != values.end(); - } - - template - bool has_duplicate_values(const std::vector& values) { - std::set set_values(values.begin(), values.end()); - return set_values.size() < values.size(); - } + return std::tuple({shift0, shift1, shift2}); +} - template , std::nullptr_t> = nullptr> - bool is_out_of_range_value_included(const std::array& values, IntType max) { - bool is_included = false; - for(auto value: values) { - is_included = value >= max; - } - return is_included; +template +void permute(std::vector& values, const std::vector& indices) { + std::vector out; + out.reserve(indices.size()); + for (auto index : indices) { + out.push_back(values.at(index)); } - - template - bool is_transpose_needed(std::array map) { - std::array contiguous_map; - std::iota(contiguous_map.begin(), contiguous_map.end(), 0); - return map != contiguous_map; + values = std::move(out); +} + +template +bool is_found(std::vector& values, const T& key) { + return std::find(values.begin(), values.end(), key) != values.end(); +} + +template +bool has_duplicate_values(const std::vector& values) { + std::set set_values(values.begin(), values.end()); + return set_values.size() < values.size(); +} + +template < + typename IntType, std::size_t DIM = 1, + std::enable_if_t, std::nullptr_t> = nullptr> +bool is_out_of_range_value_included(const std::array& values, + IntType max) { + bool is_included = false; + for (auto value : values) { + is_included = value >= max; } - - template - std::size_t get_index(std::vector& values, const T& key) { - auto it = find(values.begin(), values.end(), key); - std::size_t index = 0; - if(it != values.end()) { - index = it - values.begin(); - } else { - throw std::runtime_error("key is not included in values"); - } - - return index; + return is_included; +} + +template +bool is_transpose_needed(std::array map) { + std::array contiguous_map; + std::iota(contiguous_map.begin(), contiguous_map.end(), 0); + return map != contiguous_map; +} + +template +std::size_t get_index(std::vector& values, const T& key) { + auto it = find(values.begin(), values.end(), key); + std::size_t index = 0; + if (it != values.end()) { + index = it - values.begin(); + } else { + throw std::runtime_error("key is not included in values"); } - template - std::array make_sequence_array(std::index_sequence) { - return std::array{ {I...} }; + return index; +} + +template +std::array make_sequence_array(std::index_sequence) { + return std::array{{I...}}; +} + +template +std::array index_sequence(T const& start) { + auto sequence = make_sequence_array(std::make_index_sequence()); + std::transform(sequence.begin(), sequence.end(), sequence.begin(), + [=](const T sequence) -> T { return start + sequence; }); + return sequence; +} + +template +inline std::vector arange(const ElementType start, + const ElementType stop, + const ElementType step = 1) { + const size_t length = ceil((stop - start) / step); + std::vector result; + ElementType delta = (stop - start) / length; + + // thrust::sequence + for (auto i = 0; i < length; i++) { + ElementType value = start + delta * i; + result.push_back(value); } - - template - std::array index_sequence(T const& start) { - auto sequence = make_sequence_array(std::make_index_sequence()); - std::transform(sequence.begin(), sequence.end(), sequence.begin(), - [=](const T sequence) -> T {return start + sequence;}); - return sequence; + return result; +} + +template +void conjugate(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out) { + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::Impl::conjugate: OutViewType must be complex"); + std::size_t size = in.size(); + + // [TO DO] is there a way to get device mirror? + if constexpr (InViewType::rank() == 1) { + out = OutViewType("out", in.extent(0)); + } else if constexpr (InViewType::rank() == 2) { + out = OutViewType("out", in.extent(0), in.extent(1)); + } else if constexpr (InViewType::rank() == 3) { + out = OutViewType("out", in.extent(0), in.extent(1), in.extent(2)); + } else if constexpr (InViewType::rank() == 4) { + out = OutViewType("out", in.extent(0), in.extent(1), in.extent(2), + in.extent(3)); } - template - inline std::vector arange(const ElementType start, - const ElementType stop, - const ElementType step=1 - ) { - const size_t length = ceil((stop - start) / step); - std::vector result; - ElementType delta = (stop - start) / length; - - // thrust::sequence - for(auto i=0; i - void conjugate(const ExecutionSpace& exec_space, const InViewType& in, OutViewType& out) { - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::Impl::conjugate: OutViewType must be complex"); - std::size_t size = in.size(); - - // [TO DO] is there a way to get device mirror? - if constexpr(InViewType::rank() == 1) { - out = OutViewType("out", in.extent(0)); - } else if constexpr(InViewType::rank() == 2) { - out = OutViewType("out", in.extent(0), in.extent(1)); - } else if constexpr(InViewType::rank() == 3) { - out = OutViewType("out", in.extent(0), in.extent(1), in.extent(2)); - } else if constexpr(InViewType::rank() == 4) { - out = OutViewType("out", in.extent(0), in.extent(1), in.extent(2), in.extent(3)); - } - - auto* in_data = in.data(); - auto* out_data = out.data(); - - Kokkos::parallel_for( - Kokkos::RangePolicy>(exec_space, 0, size), + Kokkos::parallel_for( + Kokkos::RangePolicy>( + exec_space, 0, size), KOKKOS_LAMBDA(const int& i) { out_value_type tmp = in_data[i]; - out_data[i] = Kokkos::conj(in_data[i]); - } - ); - } -} // namespace Impl -} // namespace KokkosFFT + out_data[i] = Kokkos::conj(in_data[i]); + }); +} +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/common/unit_test/Test_Layouts.cpp b/common/unit_test/Test_Layouts.cpp index 6b347899..bef36435 100644 --- a/common/unit_test/Test_Layouts.cpp +++ b/common/unit_test/Test_Layouts.cpp @@ -5,10 +5,7 @@ #include "Test_Types.hpp" #include "Test_Utils.hpp" -using test_types = ::testing::Types< - Kokkos::LayoutLeft, - Kokkos::LayoutRight ->; +using test_types = ::testing::Types; // Basically the same fixtures, used for labeling tests template @@ -27,190 +24,230 @@ TYPED_TEST_SUITE(Layouts2D, test_types); // Tests for 1D FFT template void test_layouts_1d() { - const int n0 = 6; + const int n0 = 6; using RealView1Dtype = Kokkos::View; - using ComplexView1Dtype = Kokkos::View*, LayoutType, execution_space>; + using ComplexView1Dtype = + Kokkos::View*, LayoutType, execution_space>; RealView1Dtype xr("xr", n0); - ComplexView1Dtype xc("xc", n0/2+1); + ComplexView1Dtype xc("xc", n0 / 2 + 1); ComplexView1Dtype xcin("xcin", n0), xcout("xcout", n0); // R2C - std::vector ref_in_extents_r2c(1), ref_out_extents_r2c(1), ref_fft_extents_r2c(1); - ref_in_extents_r2c.at(0) = n0; - ref_out_extents_r2c.at(0) = n0/2+1; + std::vector ref_in_extents_r2c(1), ref_out_extents_r2c(1), + ref_fft_extents_r2c(1); + ref_in_extents_r2c.at(0) = n0; + ref_out_extents_r2c.at(0) = n0 / 2 + 1; ref_fft_extents_r2c.at(0) = n0; - auto [in_extents_r2c, out_extents_r2c, fft_extents_r2c] = KokkosFFT::Impl::get_extents(xr, xc, 0); - EXPECT_TRUE( in_extents_r2c == ref_in_extents_r2c ); - EXPECT_TRUE( out_extents_r2c == ref_out_extents_r2c ); - EXPECT_TRUE( fft_extents_r2c == ref_fft_extents_r2c ); + auto [in_extents_r2c, out_extents_r2c, fft_extents_r2c] = + KokkosFFT::Impl::get_extents(xr, xc, 0); + EXPECT_TRUE(in_extents_r2c == ref_in_extents_r2c); + EXPECT_TRUE(out_extents_r2c == ref_out_extents_r2c); + EXPECT_TRUE(fft_extents_r2c == ref_fft_extents_r2c); // C2R - std::vector ref_in_extents_c2r(1), ref_out_extents_c2r(1), ref_fft_extents_c2r(1); - ref_in_extents_c2r.at(0) = n0/2+1; + std::vector ref_in_extents_c2r(1), ref_out_extents_c2r(1), + ref_fft_extents_c2r(1); + ref_in_extents_c2r.at(0) = n0 / 2 + 1; ref_out_extents_c2r.at(0) = n0; ref_fft_extents_c2r.at(0) = n0; - auto [in_extents_c2r, out_extents_c2r, fft_extents_c2r] = KokkosFFT::Impl::get_extents(xc, xr, 0); - EXPECT_TRUE( in_extents_c2r == ref_in_extents_c2r ); - EXPECT_TRUE( out_extents_c2r == ref_out_extents_c2r ); - EXPECT_TRUE( fft_extents_c2r == ref_fft_extents_c2r ); + auto [in_extents_c2r, out_extents_c2r, fft_extents_c2r] = + KokkosFFT::Impl::get_extents(xc, xr, 0); + EXPECT_TRUE(in_extents_c2r == ref_in_extents_c2r); + EXPECT_TRUE(out_extents_c2r == ref_out_extents_c2r); + EXPECT_TRUE(fft_extents_c2r == ref_fft_extents_c2r); // C2C - std::vector ref_in_extents_c2c(1), ref_out_extents_c2c(1), ref_fft_extents_c2c(1); - ref_in_extents_c2c.at(0) = n0; + std::vector ref_in_extents_c2c(1), ref_out_extents_c2c(1), + ref_fft_extents_c2c(1); + ref_in_extents_c2c.at(0) = n0; ref_out_extents_c2c.at(0) = n0; ref_fft_extents_c2c.at(0) = n0; - auto [in_extents_c2c, out_extents_c2c, fft_extents_c2c] = KokkosFFT::Impl::get_extents(xcin, xcout, 0); - EXPECT_TRUE( in_extents_c2c == ref_in_extents_c2c ); - EXPECT_TRUE( out_extents_c2c == ref_out_extents_c2c ); - EXPECT_TRUE( fft_extents_c2c == ref_fft_extents_c2c ); + auto [in_extents_c2c, out_extents_c2c, fft_extents_c2c] = + KokkosFFT::Impl::get_extents(xcin, xcout, 0); + EXPECT_TRUE(in_extents_c2c == ref_in_extents_c2c); + EXPECT_TRUE(out_extents_c2c == ref_out_extents_c2c); + EXPECT_TRUE(fft_extents_c2c == ref_fft_extents_c2c); } template void test_layouts_1d_batched_FFT_2d() { const int n0 = 6, n1 = 10; using RealView2Dtype = Kokkos::View; - using ComplexView2Dtype = Kokkos::View**, LayoutType, execution_space>; + using ComplexView2Dtype = + Kokkos::View**, LayoutType, execution_space>; RealView2Dtype xr2("xr2", n0, n1); - ComplexView2Dtype xc2_axis0("xc2_axis0", n0/2+1, n1); - ComplexView2Dtype xc2_axis1("xc2_axis1", n0, n1/2+1); + ComplexView2Dtype xc2_axis0("xc2_axis0", n0 / 2 + 1, n1); + ComplexView2Dtype xc2_axis1("xc2_axis1", n0, n1 / 2 + 1); ComplexView2Dtype xcin2("xcin2", n0, n1), xcout2("xcout2", n0, n1); // Reference shapes std::vector ref_in_extents_r2c_axis1{n1}, ref_in_extents_r2c_axis0{n0}; std::vector ref_fft_extents_r2c_axis1{n1}, ref_fft_extents_r2c_axis0{n0}; - std::vector ref_out_extents_r2c_axis1{n1/2+1}, ref_out_extents_r2c_axis0{n0/2+1}; + std::vector ref_out_extents_r2c_axis1{n1 / 2 + 1}, + ref_out_extents_r2c_axis0{n0 / 2 + 1}; int ref_howmany_r2c_axis0 = n1; int ref_howmany_r2c_axis1 = n0; // R2C - auto [in_extents_r2c_axis0, out_extents_r2c_axis0, fft_extents_r2c_axis0, howmany_r2c_axis0] = KokkosFFT::Impl::get_extents_batched(xr2, xc2_axis0, 0); - EXPECT_TRUE( in_extents_r2c_axis0 == ref_in_extents_r2c_axis0 ); - EXPECT_TRUE( fft_extents_r2c_axis0 == ref_fft_extents_r2c_axis0 ); - EXPECT_TRUE( out_extents_r2c_axis0 == ref_out_extents_r2c_axis0 ); - EXPECT_EQ( howmany_r2c_axis0, ref_howmany_r2c_axis0 ); - - auto [in_extents_r2c_axis1, out_extents_r2c_axis1, fft_extents_r2c_axis1, howmany_r2c_axis1] = KokkosFFT::Impl::get_extents_batched(xr2, xc2_axis1, 1); - EXPECT_TRUE( in_extents_r2c_axis1 == ref_in_extents_r2c_axis1 ); - EXPECT_TRUE( fft_extents_r2c_axis1 == ref_fft_extents_r2c_axis1 ); - EXPECT_TRUE( out_extents_r2c_axis1 == ref_out_extents_r2c_axis1 ); - EXPECT_EQ( howmany_r2c_axis1, ref_howmany_r2c_axis1 ); + auto [in_extents_r2c_axis0, out_extents_r2c_axis0, fft_extents_r2c_axis0, + howmany_r2c_axis0] = + KokkosFFT::Impl::get_extents_batched(xr2, xc2_axis0, 0); + EXPECT_TRUE(in_extents_r2c_axis0 == ref_in_extents_r2c_axis0); + EXPECT_TRUE(fft_extents_r2c_axis0 == ref_fft_extents_r2c_axis0); + EXPECT_TRUE(out_extents_r2c_axis0 == ref_out_extents_r2c_axis0); + EXPECT_EQ(howmany_r2c_axis0, ref_howmany_r2c_axis0); + + auto [in_extents_r2c_axis1, out_extents_r2c_axis1, fft_extents_r2c_axis1, + howmany_r2c_axis1] = + KokkosFFT::Impl::get_extents_batched(xr2, xc2_axis1, 1); + EXPECT_TRUE(in_extents_r2c_axis1 == ref_in_extents_r2c_axis1); + EXPECT_TRUE(fft_extents_r2c_axis1 == ref_fft_extents_r2c_axis1); + EXPECT_TRUE(out_extents_r2c_axis1 == ref_out_extents_r2c_axis1); + EXPECT_EQ(howmany_r2c_axis1, ref_howmany_r2c_axis1); // C2R - auto [in_extents_c2r_axis0, out_extents_c2r_axis0, fft_extents_c2r_axis0, howmany_c2r_axis0] = KokkosFFT::Impl::get_extents_batched(xc2_axis0, xr2, 0); - EXPECT_TRUE( in_extents_c2r_axis0 == ref_out_extents_r2c_axis0 ); - EXPECT_TRUE( fft_extents_c2r_axis0 == ref_fft_extents_r2c_axis0 ); - EXPECT_TRUE( out_extents_c2r_axis0 == ref_in_extents_r2c_axis0 ); - EXPECT_EQ( howmany_c2r_axis0, ref_howmany_r2c_axis0 ); - - auto [in_extents_c2r_axis1, out_extents_c2r_axis1, fft_extents_c2r_axis1, howmany_c2r_axis1] = KokkosFFT::Impl::get_extents_batched(xc2_axis1, xr2, 1); - EXPECT_TRUE( in_extents_c2r_axis1 == ref_out_extents_r2c_axis1 ); - EXPECT_TRUE( fft_extents_c2r_axis1 == ref_fft_extents_r2c_axis1 ); - EXPECT_TRUE( out_extents_c2r_axis1 == ref_in_extents_r2c_axis1 ); - EXPECT_EQ( howmany_c2r_axis1, ref_howmany_r2c_axis1 ); + auto [in_extents_c2r_axis0, out_extents_c2r_axis0, fft_extents_c2r_axis0, + howmany_c2r_axis0] = + KokkosFFT::Impl::get_extents_batched(xc2_axis0, xr2, 0); + EXPECT_TRUE(in_extents_c2r_axis0 == ref_out_extents_r2c_axis0); + EXPECT_TRUE(fft_extents_c2r_axis0 == ref_fft_extents_r2c_axis0); + EXPECT_TRUE(out_extents_c2r_axis0 == ref_in_extents_r2c_axis0); + EXPECT_EQ(howmany_c2r_axis0, ref_howmany_r2c_axis0); + + auto [in_extents_c2r_axis1, out_extents_c2r_axis1, fft_extents_c2r_axis1, + howmany_c2r_axis1] = + KokkosFFT::Impl::get_extents_batched(xc2_axis1, xr2, 1); + EXPECT_TRUE(in_extents_c2r_axis1 == ref_out_extents_r2c_axis1); + EXPECT_TRUE(fft_extents_c2r_axis1 == ref_fft_extents_r2c_axis1); + EXPECT_TRUE(out_extents_c2r_axis1 == ref_in_extents_r2c_axis1); + EXPECT_EQ(howmany_c2r_axis1, ref_howmany_r2c_axis1); // C2C - auto [in_extents_c2c_axis0, out_extents_c2c_axis0, fft_extents_c2c_axis0, howmany_c2c_axis0] = KokkosFFT::Impl::get_extents_batched(xcin2, xcout2, 0); - EXPECT_TRUE( in_extents_c2c_axis0 == ref_in_extents_r2c_axis0 ); - EXPECT_TRUE( fft_extents_c2c_axis0 == ref_fft_extents_r2c_axis0 ); - EXPECT_TRUE( out_extents_c2c_axis0 == ref_in_extents_r2c_axis0 ); - EXPECT_EQ( howmany_c2c_axis0, ref_howmany_r2c_axis0 ); - - auto [in_extents_c2c_axis1, out_extents_c2c_axis1, fft_extents_c2c_axis1, howmany_c2c_axis1] = KokkosFFT::Impl::get_extents_batched(xcin2, xcout2, 1); - EXPECT_TRUE( in_extents_c2c_axis1 == ref_in_extents_r2c_axis1 ); - EXPECT_TRUE( fft_extents_c2c_axis1 == ref_fft_extents_r2c_axis1 ); - EXPECT_TRUE( out_extents_c2c_axis1 == ref_in_extents_r2c_axis1 ); - EXPECT_EQ( howmany_c2c_axis1, ref_howmany_r2c_axis1 ); + auto [in_extents_c2c_axis0, out_extents_c2c_axis0, fft_extents_c2c_axis0, + howmany_c2c_axis0] = + KokkosFFT::Impl::get_extents_batched(xcin2, xcout2, 0); + EXPECT_TRUE(in_extents_c2c_axis0 == ref_in_extents_r2c_axis0); + EXPECT_TRUE(fft_extents_c2c_axis0 == ref_fft_extents_r2c_axis0); + EXPECT_TRUE(out_extents_c2c_axis0 == ref_in_extents_r2c_axis0); + EXPECT_EQ(howmany_c2c_axis0, ref_howmany_r2c_axis0); + + auto [in_extents_c2c_axis1, out_extents_c2c_axis1, fft_extents_c2c_axis1, + howmany_c2c_axis1] = + KokkosFFT::Impl::get_extents_batched(xcin2, xcout2, 1); + EXPECT_TRUE(in_extents_c2c_axis1 == ref_in_extents_r2c_axis1); + EXPECT_TRUE(fft_extents_c2c_axis1 == ref_fft_extents_r2c_axis1); + EXPECT_TRUE(out_extents_c2c_axis1 == ref_in_extents_r2c_axis1); + EXPECT_EQ(howmany_c2c_axis1, ref_howmany_r2c_axis1); } template void test_layouts_1d_batched_FFT_3d() { const int n0 = 6, n1 = 10, n2 = 8; using RealView3Dtype = Kokkos::View; - using ComplexView3Dtype = Kokkos::View***, LayoutType, execution_space>; + using ComplexView3Dtype = + Kokkos::View***, LayoutType, execution_space>; RealView3Dtype xr3("xr3", n0, n1, n2); - ComplexView3Dtype xc3_axis0("xc3_axis0", n0/2+1, n1, n2); - ComplexView3Dtype xc3_axis1("xc3_axis1", n0, n1/2+1, n2); - ComplexView3Dtype xc3_axis2("xc3_axis2", n0, n1, n2/2+1); + ComplexView3Dtype xc3_axis0("xc3_axis0", n0 / 2 + 1, n1, n2); + ComplexView3Dtype xc3_axis1("xc3_axis1", n0, n1 / 2 + 1, n2); + ComplexView3Dtype xc3_axis2("xc3_axis2", n0, n1, n2 / 2 + 1); ComplexView3Dtype xcin3("xcin3", n0, n1, n2), xcout3("xcout3", n0, n1, n2); // Reference shapes std::vector ref_in_extents_r2c_axis0{n0}; std::vector ref_fft_extents_r2c_axis0{n0}; - std::vector ref_out_extents_r2c_axis0{n0/2+1}; + std::vector ref_out_extents_r2c_axis0{n0 / 2 + 1}; int ref_howmany_r2c_axis0 = n1 * n2; std::vector ref_in_extents_r2c_axis1{n1}; std::vector ref_fft_extents_r2c_axis1{n1}; - std::vector ref_out_extents_r2c_axis1{n1/2+1}; + std::vector ref_out_extents_r2c_axis1{n1 / 2 + 1}; int ref_howmany_r2c_axis1 = n0 * n2; std::vector ref_in_extents_r2c_axis2{n2}; std::vector ref_fft_extents_r2c_axis2{n2}; - std::vector ref_out_extents_r2c_axis2{n2/2+1}; + std::vector ref_out_extents_r2c_axis2{n2 / 2 + 1}; int ref_howmany_r2c_axis2 = n0 * n1; // R2C - auto [in_extents_r2c_axis0, out_extents_r2c_axis0, fft_extents_r2c_axis0, howmany_r2c_axis0] = KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis0, 0); - EXPECT_TRUE( in_extents_r2c_axis0 == ref_in_extents_r2c_axis0 ); - EXPECT_TRUE( fft_extents_r2c_axis0 == ref_fft_extents_r2c_axis0 ); - EXPECT_TRUE( out_extents_r2c_axis0 == ref_out_extents_r2c_axis0 ); - EXPECT_EQ( howmany_r2c_axis0, ref_howmany_r2c_axis0 ); - - auto [in_extents_r2c_axis1, out_extents_r2c_axis1, fft_extents_r2c_axis1, howmany_r2c_axis1] = KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis1, 1); - EXPECT_TRUE( in_extents_r2c_axis1 == ref_in_extents_r2c_axis1 ); - EXPECT_TRUE( fft_extents_r2c_axis1 == ref_fft_extents_r2c_axis1 ); - EXPECT_TRUE( out_extents_r2c_axis1 == ref_out_extents_r2c_axis1 ); - EXPECT_EQ( howmany_r2c_axis1, ref_howmany_r2c_axis1 ); - - auto [in_extents_r2c_axis2, out_extents_r2c_axis2, fft_extents_r2c_axis2, howmany_r2c_axis2] = KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis2, 2); - EXPECT_TRUE( in_extents_r2c_axis2 == ref_in_extents_r2c_axis2 ); - EXPECT_TRUE( fft_extents_r2c_axis2 == ref_fft_extents_r2c_axis2 ); - EXPECT_TRUE( out_extents_r2c_axis2 == ref_out_extents_r2c_axis2 ); - EXPECT_EQ( howmany_r2c_axis2, ref_howmany_r2c_axis2 ); + auto [in_extents_r2c_axis0, out_extents_r2c_axis0, fft_extents_r2c_axis0, + howmany_r2c_axis0] = + KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis0, 0); + EXPECT_TRUE(in_extents_r2c_axis0 == ref_in_extents_r2c_axis0); + EXPECT_TRUE(fft_extents_r2c_axis0 == ref_fft_extents_r2c_axis0); + EXPECT_TRUE(out_extents_r2c_axis0 == ref_out_extents_r2c_axis0); + EXPECT_EQ(howmany_r2c_axis0, ref_howmany_r2c_axis0); + + auto [in_extents_r2c_axis1, out_extents_r2c_axis1, fft_extents_r2c_axis1, + howmany_r2c_axis1] = + KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis1, 1); + EXPECT_TRUE(in_extents_r2c_axis1 == ref_in_extents_r2c_axis1); + EXPECT_TRUE(fft_extents_r2c_axis1 == ref_fft_extents_r2c_axis1); + EXPECT_TRUE(out_extents_r2c_axis1 == ref_out_extents_r2c_axis1); + EXPECT_EQ(howmany_r2c_axis1, ref_howmany_r2c_axis1); + + auto [in_extents_r2c_axis2, out_extents_r2c_axis2, fft_extents_r2c_axis2, + howmany_r2c_axis2] = + KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis2, 2); + EXPECT_TRUE(in_extents_r2c_axis2 == ref_in_extents_r2c_axis2); + EXPECT_TRUE(fft_extents_r2c_axis2 == ref_fft_extents_r2c_axis2); + EXPECT_TRUE(out_extents_r2c_axis2 == ref_out_extents_r2c_axis2); + EXPECT_EQ(howmany_r2c_axis2, ref_howmany_r2c_axis2); // C2R - auto [in_extents_c2r_axis0, out_extents_c2r_axis0, fft_extents_c2r_axis0, howmany_c2r_axis0] = KokkosFFT::Impl::get_extents_batched(xc3_axis0, xr3, 0); - EXPECT_TRUE( in_extents_c2r_axis0 == ref_out_extents_r2c_axis0 ); - EXPECT_TRUE( fft_extents_c2r_axis0 == ref_fft_extents_r2c_axis0 ); - EXPECT_TRUE( out_extents_c2r_axis0 == ref_in_extents_r2c_axis0 ); - EXPECT_EQ( howmany_c2r_axis0, ref_howmany_r2c_axis0 ); - - auto [in_extents_c2r_axis1, out_extents_c2r_axis1, fft_extents_c2r_axis1, howmany_c2r_axis1] = KokkosFFT::Impl::get_extents_batched(xc3_axis1, xr3, 1); - EXPECT_TRUE( in_extents_c2r_axis1 == ref_out_extents_r2c_axis1 ); - EXPECT_TRUE( fft_extents_c2r_axis1 == ref_fft_extents_r2c_axis1 ); - EXPECT_TRUE( out_extents_c2r_axis1 == ref_in_extents_r2c_axis1 ); - EXPECT_EQ( howmany_c2r_axis1, ref_howmany_r2c_axis1 ); - - auto [in_extents_c2r_axis2, out_extents_c2r_axis2, fft_extents_c2r_axis2, howmany_c2r_axis2] = KokkosFFT::Impl::get_extents_batched(xc3_axis2, xr3, 2); - EXPECT_TRUE( in_extents_c2r_axis2 == ref_out_extents_r2c_axis2 ); - EXPECT_TRUE( fft_extents_c2r_axis2 == ref_fft_extents_r2c_axis2 ); - EXPECT_TRUE( out_extents_c2r_axis2 == ref_in_extents_r2c_axis2 ); - EXPECT_EQ( howmany_c2r_axis2, ref_howmany_r2c_axis2 ); + auto [in_extents_c2r_axis0, out_extents_c2r_axis0, fft_extents_c2r_axis0, + howmany_c2r_axis0] = + KokkosFFT::Impl::get_extents_batched(xc3_axis0, xr3, 0); + EXPECT_TRUE(in_extents_c2r_axis0 == ref_out_extents_r2c_axis0); + EXPECT_TRUE(fft_extents_c2r_axis0 == ref_fft_extents_r2c_axis0); + EXPECT_TRUE(out_extents_c2r_axis0 == ref_in_extents_r2c_axis0); + EXPECT_EQ(howmany_c2r_axis0, ref_howmany_r2c_axis0); + + auto [in_extents_c2r_axis1, out_extents_c2r_axis1, fft_extents_c2r_axis1, + howmany_c2r_axis1] = + KokkosFFT::Impl::get_extents_batched(xc3_axis1, xr3, 1); + EXPECT_TRUE(in_extents_c2r_axis1 == ref_out_extents_r2c_axis1); + EXPECT_TRUE(fft_extents_c2r_axis1 == ref_fft_extents_r2c_axis1); + EXPECT_TRUE(out_extents_c2r_axis1 == ref_in_extents_r2c_axis1); + EXPECT_EQ(howmany_c2r_axis1, ref_howmany_r2c_axis1); + + auto [in_extents_c2r_axis2, out_extents_c2r_axis2, fft_extents_c2r_axis2, + howmany_c2r_axis2] = + KokkosFFT::Impl::get_extents_batched(xc3_axis2, xr3, 2); + EXPECT_TRUE(in_extents_c2r_axis2 == ref_out_extents_r2c_axis2); + EXPECT_TRUE(fft_extents_c2r_axis2 == ref_fft_extents_r2c_axis2); + EXPECT_TRUE(out_extents_c2r_axis2 == ref_in_extents_r2c_axis2); + EXPECT_EQ(howmany_c2r_axis2, ref_howmany_r2c_axis2); // C2C - auto [in_extents_c2c_axis0, out_extents_c2c_axis0, fft_extents_c2c_axis0, howmany_c2c_axis0] = KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, 0); - EXPECT_TRUE( in_extents_c2c_axis0 == ref_in_extents_r2c_axis0 ); - EXPECT_TRUE( fft_extents_c2c_axis0 == ref_fft_extents_r2c_axis0 ); - EXPECT_TRUE( out_extents_c2c_axis0 == ref_in_extents_r2c_axis0 ); - EXPECT_EQ( howmany_c2c_axis0, ref_howmany_r2c_axis0 ); - - auto [in_extents_c2c_axis1, out_extents_c2c_axis1, fft_extents_c2c_axis1, howmany_c2c_axis1] = KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, 1); - EXPECT_TRUE( in_extents_c2c_axis1 == ref_in_extents_r2c_axis1 ); - EXPECT_TRUE( fft_extents_c2c_axis1 == ref_fft_extents_r2c_axis1 ); - EXPECT_TRUE( out_extents_c2c_axis1 == ref_in_extents_r2c_axis1 ); - EXPECT_EQ( howmany_c2c_axis1, ref_howmany_r2c_axis1 ); - - auto [in_extents_c2c_axis2, out_extents_c2c_axis2, fft_extents_c2c_axis2, howmany_c2c_axis2] = KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, 2); - EXPECT_TRUE( in_extents_c2c_axis2 == ref_in_extents_r2c_axis2 ); - EXPECT_TRUE( fft_extents_c2c_axis2 == ref_fft_extents_r2c_axis2 ); - EXPECT_TRUE( out_extents_c2c_axis2 == ref_in_extents_r2c_axis2 ); - EXPECT_EQ( howmany_c2c_axis2, ref_howmany_r2c_axis2 ); + auto [in_extents_c2c_axis0, out_extents_c2c_axis0, fft_extents_c2c_axis0, + howmany_c2c_axis0] = + KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, 0); + EXPECT_TRUE(in_extents_c2c_axis0 == ref_in_extents_r2c_axis0); + EXPECT_TRUE(fft_extents_c2c_axis0 == ref_fft_extents_r2c_axis0); + EXPECT_TRUE(out_extents_c2c_axis0 == ref_in_extents_r2c_axis0); + EXPECT_EQ(howmany_c2c_axis0, ref_howmany_r2c_axis0); + + auto [in_extents_c2c_axis1, out_extents_c2c_axis1, fft_extents_c2c_axis1, + howmany_c2c_axis1] = + KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, 1); + EXPECT_TRUE(in_extents_c2c_axis1 == ref_in_extents_r2c_axis1); + EXPECT_TRUE(fft_extents_c2c_axis1 == ref_fft_extents_r2c_axis1); + EXPECT_TRUE(out_extents_c2c_axis1 == ref_in_extents_r2c_axis1); + EXPECT_EQ(howmany_c2c_axis1, ref_howmany_r2c_axis1); + + auto [in_extents_c2c_axis2, out_extents_c2c_axis2, fft_extents_c2c_axis2, + howmany_c2c_axis2] = + KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, 2); + EXPECT_TRUE(in_extents_c2c_axis2 == ref_in_extents_r2c_axis2); + EXPECT_TRUE(fft_extents_c2c_axis2 == ref_fft_extents_r2c_axis2); + EXPECT_TRUE(out_extents_c2c_axis2 == ref_in_extents_r2c_axis2); + EXPECT_EQ(howmany_c2c_axis2, ref_howmany_r2c_axis2); } TYPED_TEST(Layouts1D, 1DFFT_1DView) { @@ -236,214 +273,258 @@ template void test_layouts_2d() { const int n0 = 6, n1 = 10; using RealView2Dtype = Kokkos::View; - using ComplexView2Dtype = Kokkos::View**, LayoutType, execution_space>; + using ComplexView2Dtype = + Kokkos::View**, LayoutType, execution_space>; RealView2Dtype xr2("xr2", n0, n1); - ComplexView2Dtype xc2_axis0("xc2_axis0", n0/2+1, n1); - ComplexView2Dtype xc2_axis1("xc2_axis1", n0, n1/2+1); + ComplexView2Dtype xc2_axis0("xc2_axis0", n0 / 2 + 1, n1); + ComplexView2Dtype xc2_axis1("xc2_axis1", n0, n1 / 2 + 1); ComplexView2Dtype xcin2("xcin2", n0, n1), xcout2("xcout2", n0, n1); std::vector ref_in_extents_r2c_axis0{n1, n0}; std::vector ref_in_extents_r2c_axis1{n0, n1}; std::vector ref_fft_extents_r2c_axis0{n1, n0}; std::vector ref_fft_extents_r2c_axis1{n0, n1}; - std::vector ref_out_extents_r2c_axis0{n1, n0/2+1}; - std::vector ref_out_extents_r2c_axis1{n0, n1/2+1}; + std::vector ref_out_extents_r2c_axis0{n1, n0 / 2 + 1}; + std::vector ref_out_extents_r2c_axis1{n0, n1 / 2 + 1}; // R2C - auto [in_extents_r2c_axis0, out_extents_r2c_axis0, fft_extents_r2c_axis0] = KokkosFFT::Impl::get_extents(xr2, xc2_axis0, 0); - auto [in_extents_r2c_axis1, out_extents_r2c_axis1, fft_extents_r2c_axis1] = KokkosFFT::Impl::get_extents(xr2, xc2_axis1, 1); - EXPECT_TRUE( in_extents_r2c_axis0 == ref_in_extents_r2c_axis0 ); - EXPECT_TRUE( in_extents_r2c_axis1 == ref_in_extents_r2c_axis1 ); + auto [in_extents_r2c_axis0, out_extents_r2c_axis0, fft_extents_r2c_axis0] = + KokkosFFT::Impl::get_extents(xr2, xc2_axis0, 0); + auto [in_extents_r2c_axis1, out_extents_r2c_axis1, fft_extents_r2c_axis1] = + KokkosFFT::Impl::get_extents(xr2, xc2_axis1, 1); + EXPECT_TRUE(in_extents_r2c_axis0 == ref_in_extents_r2c_axis0); + EXPECT_TRUE(in_extents_r2c_axis1 == ref_in_extents_r2c_axis1); - EXPECT_TRUE( fft_extents_r2c_axis0 == ref_fft_extents_r2c_axis0 ); - EXPECT_TRUE( fft_extents_r2c_axis1 == ref_fft_extents_r2c_axis1 ); + EXPECT_TRUE(fft_extents_r2c_axis0 == ref_fft_extents_r2c_axis0); + EXPECT_TRUE(fft_extents_r2c_axis1 == ref_fft_extents_r2c_axis1); - EXPECT_TRUE( out_extents_r2c_axis0 == ref_out_extents_r2c_axis0 ); - EXPECT_TRUE( out_extents_r2c_axis1 == ref_out_extents_r2c_axis1 ); + EXPECT_TRUE(out_extents_r2c_axis0 == ref_out_extents_r2c_axis0); + EXPECT_TRUE(out_extents_r2c_axis1 == ref_out_extents_r2c_axis1); // C2R - auto [in_extents_c2r_axis0, out_extents_c2r_axis0, fft_extents_c2r_axis0] = KokkosFFT::Impl::get_extents(xc2_axis0, xr2, 0); - auto [in_extents_c2r_axis1, out_extents_c2r_axis1, fft_extents_c2r_axis1] = KokkosFFT::Impl::get_extents(xc2_axis1, xr2, 1); - EXPECT_TRUE( in_extents_c2r_axis0 == ref_out_extents_r2c_axis0 ); - EXPECT_TRUE( in_extents_c2r_axis1 == ref_out_extents_r2c_axis1 ); + auto [in_extents_c2r_axis0, out_extents_c2r_axis0, fft_extents_c2r_axis0] = + KokkosFFT::Impl::get_extents(xc2_axis0, xr2, 0); + auto [in_extents_c2r_axis1, out_extents_c2r_axis1, fft_extents_c2r_axis1] = + KokkosFFT::Impl::get_extents(xc2_axis1, xr2, 1); + EXPECT_TRUE(in_extents_c2r_axis0 == ref_out_extents_r2c_axis0); + EXPECT_TRUE(in_extents_c2r_axis1 == ref_out_extents_r2c_axis1); - EXPECT_TRUE( fft_extents_c2r_axis0 == ref_fft_extents_r2c_axis0 ); - EXPECT_TRUE( fft_extents_c2r_axis1 == ref_fft_extents_r2c_axis1 ); + EXPECT_TRUE(fft_extents_c2r_axis0 == ref_fft_extents_r2c_axis0); + EXPECT_TRUE(fft_extents_c2r_axis1 == ref_fft_extents_r2c_axis1); - EXPECT_TRUE( out_extents_c2r_axis0 == ref_in_extents_r2c_axis0 ); - EXPECT_TRUE( out_extents_c2r_axis1 == ref_in_extents_r2c_axis1 ); + EXPECT_TRUE(out_extents_c2r_axis0 == ref_in_extents_r2c_axis0); + EXPECT_TRUE(out_extents_c2r_axis1 == ref_in_extents_r2c_axis1); // C2C - auto [in_extents_c2c_axis0, out_extents_c2c_axis0, fft_extents_c2c_axis0] = KokkosFFT::Impl::get_extents(xcin2, xcout2, 0); - auto [in_extents_c2c_axis1, out_extents_c2c_axis1, fft_extents_c2c_axis1] = KokkosFFT::Impl::get_extents(xcin2, xcout2, 1); - EXPECT_TRUE( in_extents_c2c_axis0 == ref_in_extents_r2c_axis0 ); - EXPECT_TRUE( in_extents_c2c_axis1 == ref_in_extents_r2c_axis1 ); - - EXPECT_TRUE( fft_extents_c2c_axis0 == ref_fft_extents_r2c_axis0 ); - EXPECT_TRUE( fft_extents_c2c_axis1 == ref_fft_extents_r2c_axis1 ); - - EXPECT_TRUE( out_extents_c2c_axis0 == ref_in_extents_r2c_axis0 ); - EXPECT_TRUE( out_extents_c2c_axis1 == ref_in_extents_r2c_axis1 ); + auto [in_extents_c2c_axis0, out_extents_c2c_axis0, fft_extents_c2c_axis0] = + KokkosFFT::Impl::get_extents(xcin2, xcout2, 0); + auto [in_extents_c2c_axis1, out_extents_c2c_axis1, fft_extents_c2c_axis1] = + KokkosFFT::Impl::get_extents(xcin2, xcout2, 1); + EXPECT_TRUE(in_extents_c2c_axis0 == ref_in_extents_r2c_axis0); + EXPECT_TRUE(in_extents_c2c_axis1 == ref_in_extents_r2c_axis1); + + EXPECT_TRUE(fft_extents_c2c_axis0 == ref_fft_extents_r2c_axis0); + EXPECT_TRUE(fft_extents_c2c_axis1 == ref_fft_extents_r2c_axis1); + + EXPECT_TRUE(out_extents_c2c_axis0 == ref_in_extents_r2c_axis0); + EXPECT_TRUE(out_extents_c2c_axis1 == ref_in_extents_r2c_axis1); } template void test_layouts_2d_batched_FFT_3d() { const int n0 = 6, n1 = 10, n2 = 8; - using axes_type = KokkosFFT::axis_type<2>; + using axes_type = KokkosFFT::axis_type<2>; using RealView3Dtype = Kokkos::View; - using ComplexView3Dtype = Kokkos::View***, LayoutType, execution_space>; + using ComplexView3Dtype = + Kokkos::View***, LayoutType, execution_space>; RealView3Dtype xr3("xr3", n0, n1, n2); - ComplexView3Dtype xc3_axis_01("xc3_axis_01", n0, n1/2+1, n2); - ComplexView3Dtype xc3_axis_02("xc3_axis_02", n0, n1, n2/2+1); - ComplexView3Dtype xc3_axis_10("xc3_axis_10", n0/2+1, n1, n2); - ComplexView3Dtype xc3_axis_12("xc3_axis_12", n0, n1, n2/2+1); - ComplexView3Dtype xc3_axis_20("xc3_axis_20", n0/2+1, n1, n2); - ComplexView3Dtype xc3_axis_21("xc3_axis_21", n0, n1/2+1, n2); + ComplexView3Dtype xc3_axis_01("xc3_axis_01", n0, n1 / 2 + 1, n2); + ComplexView3Dtype xc3_axis_02("xc3_axis_02", n0, n1, n2 / 2 + 1); + ComplexView3Dtype xc3_axis_10("xc3_axis_10", n0 / 2 + 1, n1, n2); + ComplexView3Dtype xc3_axis_12("xc3_axis_12", n0, n1, n2 / 2 + 1); + ComplexView3Dtype xc3_axis_20("xc3_axis_20", n0 / 2 + 1, n1, n2); + ComplexView3Dtype xc3_axis_21("xc3_axis_21", n0, n1 / 2 + 1, n2); ComplexView3Dtype xcin3("xcin3", n0, n1, n2), xcout3("xcout3", n0, n1, n2); // Reference shapes std::vector ref_in_extents_r2c_axis_01{n0, n1}; std::vector ref_fft_extents_r2c_axis_01{n0, n1}; - std::vector ref_out_extents_r2c_axis_01{n0, n1/2+1}; + std::vector ref_out_extents_r2c_axis_01{n0, n1 / 2 + 1}; int ref_howmany_r2c_axis_01 = n2; std::vector ref_in_extents_r2c_axis_02{n0, n2}; std::vector ref_fft_extents_r2c_axis_02{n0, n2}; - std::vector ref_out_extents_r2c_axis_02{n0, n2/2+1}; + std::vector ref_out_extents_r2c_axis_02{n0, n2 / 2 + 1}; int ref_howmany_r2c_axis_02 = n1; std::vector ref_in_extents_r2c_axis_10{n1, n0}; std::vector ref_fft_extents_r2c_axis_10{n1, n0}; - std::vector ref_out_extents_r2c_axis_10{n1, n0/2+1}; + std::vector ref_out_extents_r2c_axis_10{n1, n0 / 2 + 1}; int ref_howmany_r2c_axis_10 = n2; std::vector ref_in_extents_r2c_axis_12{n1, n2}; std::vector ref_fft_extents_r2c_axis_12{n1, n2}; - std::vector ref_out_extents_r2c_axis_12{n1, n2/2+1}; + std::vector ref_out_extents_r2c_axis_12{n1, n2 / 2 + 1}; int ref_howmany_r2c_axis_12 = n0; std::vector ref_in_extents_r2c_axis_20{n2, n0}; std::vector ref_fft_extents_r2c_axis_20{n2, n0}; - std::vector ref_out_extents_r2c_axis_20{n2, n0/2+1}; + std::vector ref_out_extents_r2c_axis_20{n2, n0 / 2 + 1}; int ref_howmany_r2c_axis_20 = n1; std::vector ref_in_extents_r2c_axis_21{n2, n1}; std::vector ref_fft_extents_r2c_axis_21{n2, n1}; - std::vector ref_out_extents_r2c_axis_21{n2, n1/2+1}; + std::vector ref_out_extents_r2c_axis_21{n2, n1 / 2 + 1}; int ref_howmany_r2c_axis_21 = n0; // R2C - auto [in_extents_r2c_axis_01, out_extents_r2c_axis_01, fft_extents_r2c_axis_01, howmany_r2c_axis_01] = KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis_01, axes_type({0, 1})); - EXPECT_TRUE( in_extents_r2c_axis_01 == ref_in_extents_r2c_axis_01 ); - EXPECT_TRUE( fft_extents_r2c_axis_01 == ref_fft_extents_r2c_axis_01 ); - EXPECT_TRUE( out_extents_r2c_axis_01 == ref_out_extents_r2c_axis_01 ); - EXPECT_EQ( howmany_r2c_axis_01, ref_howmany_r2c_axis_01 ); - - auto [in_extents_r2c_axis_02, out_extents_r2c_axis_02, fft_extents_r2c_axis_02, howmany_r2c_axis_02] = KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis_02, axes_type({0, 2})); - EXPECT_TRUE( in_extents_r2c_axis_02 == ref_in_extents_r2c_axis_02 ); - EXPECT_TRUE( fft_extents_r2c_axis_02 == ref_fft_extents_r2c_axis_02 ); - EXPECT_TRUE( out_extents_r2c_axis_02 == ref_out_extents_r2c_axis_02 ); - EXPECT_EQ( howmany_r2c_axis_02, ref_howmany_r2c_axis_02 ); - - auto [in_extents_r2c_axis_10, out_extents_r2c_axis_10, fft_extents_r2c_axis_10, howmany_r2c_axis_10] = KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis_10, axes_type({1, 0})); - EXPECT_TRUE( in_extents_r2c_axis_10 == ref_in_extents_r2c_axis_10 ); - EXPECT_TRUE( fft_extents_r2c_axis_10 == ref_fft_extents_r2c_axis_10 ); - EXPECT_TRUE( out_extents_r2c_axis_10 == ref_out_extents_r2c_axis_10 ); - EXPECT_EQ( howmany_r2c_axis_10, ref_howmany_r2c_axis_10 ); - - auto [in_extents_r2c_axis_12, out_extents_r2c_axis_12, fft_extents_r2c_axis_12, howmany_r2c_axis_12] = KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis_12, axes_type({1, 2})); - EXPECT_TRUE( in_extents_r2c_axis_12 == ref_in_extents_r2c_axis_12 ); - EXPECT_TRUE( fft_extents_r2c_axis_12 == ref_fft_extents_r2c_axis_12 ); - EXPECT_TRUE( out_extents_r2c_axis_12 == ref_out_extents_r2c_axis_12 ); - EXPECT_EQ( howmany_r2c_axis_12, ref_howmany_r2c_axis_12 ); - - auto [in_extents_r2c_axis_20, out_extents_r2c_axis_20, fft_extents_r2c_axis_20, howmany_r2c_axis_20] = KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis_20, axes_type({2, 0})); - EXPECT_TRUE( in_extents_r2c_axis_20 == ref_in_extents_r2c_axis_20 ); - EXPECT_TRUE( fft_extents_r2c_axis_20 == ref_fft_extents_r2c_axis_20 ); - EXPECT_TRUE( out_extents_r2c_axis_20 == ref_out_extents_r2c_axis_20 ); - EXPECT_EQ( howmany_r2c_axis_20, ref_howmany_r2c_axis_20 ); - - auto [in_extents_r2c_axis_21, out_extents_r2c_axis_21, fft_extents_r2c_axis_21, howmany_r2c_axis_21] = KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis_21, axes_type({2, 1})); - EXPECT_TRUE( in_extents_r2c_axis_21 == ref_in_extents_r2c_axis_21 ); - EXPECT_TRUE( fft_extents_r2c_axis_21 == ref_fft_extents_r2c_axis_21 ); - EXPECT_TRUE( out_extents_r2c_axis_21 == ref_out_extents_r2c_axis_21 ); - EXPECT_EQ( howmany_r2c_axis_21, ref_howmany_r2c_axis_21 ); + auto [in_extents_r2c_axis_01, out_extents_r2c_axis_01, + fft_extents_r2c_axis_01, howmany_r2c_axis_01] = + KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis_01, axes_type({0, 1})); + EXPECT_TRUE(in_extents_r2c_axis_01 == ref_in_extents_r2c_axis_01); + EXPECT_TRUE(fft_extents_r2c_axis_01 == ref_fft_extents_r2c_axis_01); + EXPECT_TRUE(out_extents_r2c_axis_01 == ref_out_extents_r2c_axis_01); + EXPECT_EQ(howmany_r2c_axis_01, ref_howmany_r2c_axis_01); + + auto [in_extents_r2c_axis_02, out_extents_r2c_axis_02, + fft_extents_r2c_axis_02, howmany_r2c_axis_02] = + KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis_02, axes_type({0, 2})); + EXPECT_TRUE(in_extents_r2c_axis_02 == ref_in_extents_r2c_axis_02); + EXPECT_TRUE(fft_extents_r2c_axis_02 == ref_fft_extents_r2c_axis_02); + EXPECT_TRUE(out_extents_r2c_axis_02 == ref_out_extents_r2c_axis_02); + EXPECT_EQ(howmany_r2c_axis_02, ref_howmany_r2c_axis_02); + + auto [in_extents_r2c_axis_10, out_extents_r2c_axis_10, + fft_extents_r2c_axis_10, howmany_r2c_axis_10] = + KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis_10, axes_type({1, 0})); + EXPECT_TRUE(in_extents_r2c_axis_10 == ref_in_extents_r2c_axis_10); + EXPECT_TRUE(fft_extents_r2c_axis_10 == ref_fft_extents_r2c_axis_10); + EXPECT_TRUE(out_extents_r2c_axis_10 == ref_out_extents_r2c_axis_10); + EXPECT_EQ(howmany_r2c_axis_10, ref_howmany_r2c_axis_10); + + auto [in_extents_r2c_axis_12, out_extents_r2c_axis_12, + fft_extents_r2c_axis_12, howmany_r2c_axis_12] = + KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis_12, axes_type({1, 2})); + EXPECT_TRUE(in_extents_r2c_axis_12 == ref_in_extents_r2c_axis_12); + EXPECT_TRUE(fft_extents_r2c_axis_12 == ref_fft_extents_r2c_axis_12); + EXPECT_TRUE(out_extents_r2c_axis_12 == ref_out_extents_r2c_axis_12); + EXPECT_EQ(howmany_r2c_axis_12, ref_howmany_r2c_axis_12); + + auto [in_extents_r2c_axis_20, out_extents_r2c_axis_20, + fft_extents_r2c_axis_20, howmany_r2c_axis_20] = + KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis_20, axes_type({2, 0})); + EXPECT_TRUE(in_extents_r2c_axis_20 == ref_in_extents_r2c_axis_20); + EXPECT_TRUE(fft_extents_r2c_axis_20 == ref_fft_extents_r2c_axis_20); + EXPECT_TRUE(out_extents_r2c_axis_20 == ref_out_extents_r2c_axis_20); + EXPECT_EQ(howmany_r2c_axis_20, ref_howmany_r2c_axis_20); + + auto [in_extents_r2c_axis_21, out_extents_r2c_axis_21, + fft_extents_r2c_axis_21, howmany_r2c_axis_21] = + KokkosFFT::Impl::get_extents_batched(xr3, xc3_axis_21, axes_type({2, 1})); + EXPECT_TRUE(in_extents_r2c_axis_21 == ref_in_extents_r2c_axis_21); + EXPECT_TRUE(fft_extents_r2c_axis_21 == ref_fft_extents_r2c_axis_21); + EXPECT_TRUE(out_extents_r2c_axis_21 == ref_out_extents_r2c_axis_21); + EXPECT_EQ(howmany_r2c_axis_21, ref_howmany_r2c_axis_21); // C2R - auto [in_extents_c2r_axis_01, out_extents_c2r_axis_01, fft_extents_c2r_axis_01, howmany_c2r_axis_01] = KokkosFFT::Impl::get_extents_batched(xc3_axis_01, xr3, axes_type({0, 1})); - EXPECT_TRUE( in_extents_c2r_axis_01 == ref_out_extents_r2c_axis_01 ); - EXPECT_TRUE( fft_extents_c2r_axis_01 == ref_fft_extents_r2c_axis_01 ); - EXPECT_TRUE( out_extents_c2r_axis_01 == ref_in_extents_r2c_axis_01 ); - EXPECT_EQ( howmany_c2r_axis_01, ref_howmany_r2c_axis_01 ); - - auto [in_extents_c2r_axis_02, out_extents_c2r_axis_02, fft_extents_c2r_axis_02, howmany_c2r_axis_02] = KokkosFFT::Impl::get_extents_batched(xc3_axis_02, xr3, axes_type({0, 2})); - EXPECT_TRUE( in_extents_c2r_axis_02 == ref_out_extents_r2c_axis_02 ); - EXPECT_TRUE( fft_extents_c2r_axis_02 == ref_fft_extents_r2c_axis_02 ); - EXPECT_TRUE( out_extents_c2r_axis_02 == ref_in_extents_r2c_axis_02 ); - EXPECT_EQ( howmany_c2r_axis_02, ref_howmany_r2c_axis_02 ); - - auto [in_extents_c2r_axis_10, out_extents_c2r_axis_10, fft_extents_c2r_axis_10, howmany_c2r_axis_10] = KokkosFFT::Impl::get_extents_batched(xc3_axis_10, xr3, axes_type({1, 0})); - EXPECT_TRUE( in_extents_c2r_axis_10 == ref_out_extents_r2c_axis_10 ); - EXPECT_TRUE( fft_extents_c2r_axis_10 == ref_fft_extents_r2c_axis_10 ); - EXPECT_TRUE( out_extents_c2r_axis_10 == ref_in_extents_r2c_axis_10 ); - EXPECT_EQ( howmany_c2r_axis_10, ref_howmany_r2c_axis_10 ); - - auto [in_extents_c2r_axis_12, out_extents_c2r_axis_12, fft_extents_c2r_axis_12, howmany_c2r_axis_12] = KokkosFFT::Impl::get_extents_batched(xc3_axis_12, xr3, axes_type({1, 2})); - EXPECT_TRUE( in_extents_c2r_axis_12 == ref_out_extents_r2c_axis_12 ); - EXPECT_TRUE( fft_extents_c2r_axis_12 == ref_fft_extents_r2c_axis_12 ); - EXPECT_TRUE( out_extents_c2r_axis_12 == ref_in_extents_r2c_axis_12 ); - EXPECT_EQ( howmany_c2r_axis_12, ref_howmany_r2c_axis_12 ); - - auto [in_extents_c2r_axis_20, out_extents_c2r_axis_20, fft_extents_c2r_axis_20, howmany_c2r_axis_20] = KokkosFFT::Impl::get_extents_batched(xc3_axis_20, xr3, axes_type({2, 0})); - EXPECT_TRUE( in_extents_c2r_axis_20 == ref_out_extents_r2c_axis_20 ); - EXPECT_TRUE( fft_extents_c2r_axis_20 == ref_fft_extents_r2c_axis_20 ); - EXPECT_TRUE( out_extents_c2r_axis_20 == ref_in_extents_r2c_axis_20 ); - EXPECT_EQ( howmany_c2r_axis_20, ref_howmany_r2c_axis_20 ); - - auto [in_extents_c2r_axis_21, out_extents_c2r_axis_21, fft_extents_c2r_axis_21, howmany_c2r_axis_21] = KokkosFFT::Impl::get_extents_batched(xc3_axis_21, xr3, axes_type({2, 1})); - EXPECT_TRUE( in_extents_c2r_axis_21 == ref_out_extents_r2c_axis_21 ); - EXPECT_TRUE( fft_extents_c2r_axis_21 == ref_fft_extents_r2c_axis_21 ); - EXPECT_TRUE( out_extents_c2r_axis_21 == ref_in_extents_r2c_axis_21 ); - EXPECT_EQ( howmany_c2r_axis_21, ref_howmany_r2c_axis_21 ); + auto [in_extents_c2r_axis_01, out_extents_c2r_axis_01, + fft_extents_c2r_axis_01, howmany_c2r_axis_01] = + KokkosFFT::Impl::get_extents_batched(xc3_axis_01, xr3, axes_type({0, 1})); + EXPECT_TRUE(in_extents_c2r_axis_01 == ref_out_extents_r2c_axis_01); + EXPECT_TRUE(fft_extents_c2r_axis_01 == ref_fft_extents_r2c_axis_01); + EXPECT_TRUE(out_extents_c2r_axis_01 == ref_in_extents_r2c_axis_01); + EXPECT_EQ(howmany_c2r_axis_01, ref_howmany_r2c_axis_01); + + auto [in_extents_c2r_axis_02, out_extents_c2r_axis_02, + fft_extents_c2r_axis_02, howmany_c2r_axis_02] = + KokkosFFT::Impl::get_extents_batched(xc3_axis_02, xr3, axes_type({0, 2})); + EXPECT_TRUE(in_extents_c2r_axis_02 == ref_out_extents_r2c_axis_02); + EXPECT_TRUE(fft_extents_c2r_axis_02 == ref_fft_extents_r2c_axis_02); + EXPECT_TRUE(out_extents_c2r_axis_02 == ref_in_extents_r2c_axis_02); + EXPECT_EQ(howmany_c2r_axis_02, ref_howmany_r2c_axis_02); + + auto [in_extents_c2r_axis_10, out_extents_c2r_axis_10, + fft_extents_c2r_axis_10, howmany_c2r_axis_10] = + KokkosFFT::Impl::get_extents_batched(xc3_axis_10, xr3, axes_type({1, 0})); + EXPECT_TRUE(in_extents_c2r_axis_10 == ref_out_extents_r2c_axis_10); + EXPECT_TRUE(fft_extents_c2r_axis_10 == ref_fft_extents_r2c_axis_10); + EXPECT_TRUE(out_extents_c2r_axis_10 == ref_in_extents_r2c_axis_10); + EXPECT_EQ(howmany_c2r_axis_10, ref_howmany_r2c_axis_10); + + auto [in_extents_c2r_axis_12, out_extents_c2r_axis_12, + fft_extents_c2r_axis_12, howmany_c2r_axis_12] = + KokkosFFT::Impl::get_extents_batched(xc3_axis_12, xr3, axes_type({1, 2})); + EXPECT_TRUE(in_extents_c2r_axis_12 == ref_out_extents_r2c_axis_12); + EXPECT_TRUE(fft_extents_c2r_axis_12 == ref_fft_extents_r2c_axis_12); + EXPECT_TRUE(out_extents_c2r_axis_12 == ref_in_extents_r2c_axis_12); + EXPECT_EQ(howmany_c2r_axis_12, ref_howmany_r2c_axis_12); + + auto [in_extents_c2r_axis_20, out_extents_c2r_axis_20, + fft_extents_c2r_axis_20, howmany_c2r_axis_20] = + KokkosFFT::Impl::get_extents_batched(xc3_axis_20, xr3, axes_type({2, 0})); + EXPECT_TRUE(in_extents_c2r_axis_20 == ref_out_extents_r2c_axis_20); + EXPECT_TRUE(fft_extents_c2r_axis_20 == ref_fft_extents_r2c_axis_20); + EXPECT_TRUE(out_extents_c2r_axis_20 == ref_in_extents_r2c_axis_20); + EXPECT_EQ(howmany_c2r_axis_20, ref_howmany_r2c_axis_20); + + auto [in_extents_c2r_axis_21, out_extents_c2r_axis_21, + fft_extents_c2r_axis_21, howmany_c2r_axis_21] = + KokkosFFT::Impl::get_extents_batched(xc3_axis_21, xr3, axes_type({2, 1})); + EXPECT_TRUE(in_extents_c2r_axis_21 == ref_out_extents_r2c_axis_21); + EXPECT_TRUE(fft_extents_c2r_axis_21 == ref_fft_extents_r2c_axis_21); + EXPECT_TRUE(out_extents_c2r_axis_21 == ref_in_extents_r2c_axis_21); + EXPECT_EQ(howmany_c2r_axis_21, ref_howmany_r2c_axis_21); // C2C - auto [in_extents_c2c_axis_01, out_extents_c2c_axis_01, fft_extents_c2c_axis_01, howmany_c2c_axis_01] = KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, axes_type({0, 1})); - EXPECT_TRUE( in_extents_c2c_axis_01 == ref_in_extents_r2c_axis_01 ); - EXPECT_TRUE( fft_extents_c2c_axis_01 == ref_fft_extents_r2c_axis_01 ); - EXPECT_TRUE( out_extents_c2c_axis_01 == ref_in_extents_r2c_axis_01 ); - EXPECT_EQ( howmany_c2c_axis_01, ref_howmany_r2c_axis_01 ); - - auto [in_extents_c2c_axis_02, out_extents_c2c_axis_02, fft_extents_c2c_axis_02, howmany_c2c_axis_02] = KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, axes_type({0, 2})); - EXPECT_TRUE( in_extents_c2c_axis_02 == ref_in_extents_r2c_axis_02 ); - EXPECT_TRUE( fft_extents_c2c_axis_02 == ref_fft_extents_r2c_axis_02 ); - EXPECT_TRUE( out_extents_c2c_axis_02 == ref_in_extents_r2c_axis_02 ); - EXPECT_EQ( howmany_c2c_axis_02, ref_howmany_r2c_axis_02 ); - - auto [in_extents_c2c_axis_10, out_extents_c2c_axis_10, fft_extents_c2c_axis_10, howmany_c2c_axis_10] = KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, axes_type({1, 0})); - EXPECT_TRUE( in_extents_c2c_axis_10 == ref_in_extents_r2c_axis_10 ); - EXPECT_TRUE( fft_extents_c2c_axis_10 == ref_fft_extents_r2c_axis_10 ); - EXPECT_TRUE( out_extents_c2c_axis_10 == ref_in_extents_r2c_axis_10 ); - EXPECT_EQ( howmany_c2c_axis_10, ref_howmany_r2c_axis_10 ); - - auto [in_extents_c2c_axis_12, out_extents_c2c_axis_12, fft_extents_c2c_axis_12, howmany_c2c_axis_12] = KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, axes_type({1, 2})); - EXPECT_TRUE( in_extents_c2c_axis_12 == ref_in_extents_r2c_axis_12 ); - EXPECT_TRUE( fft_extents_c2c_axis_12 == ref_fft_extents_r2c_axis_12 ); - EXPECT_TRUE( out_extents_c2c_axis_12 == ref_in_extents_r2c_axis_12 ); - EXPECT_EQ( howmany_c2c_axis_12, ref_howmany_r2c_axis_12 ); - - auto [in_extents_c2c_axis_20, out_extents_c2c_axis_20, fft_extents_c2c_axis_20, howmany_c2c_axis_20] = KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, axes_type({2, 0})); - EXPECT_TRUE( in_extents_c2c_axis_20 == ref_in_extents_r2c_axis_20 ); - EXPECT_TRUE( fft_extents_c2c_axis_20 == ref_fft_extents_r2c_axis_20 ); - EXPECT_TRUE( out_extents_c2c_axis_20 == ref_in_extents_r2c_axis_20 ); - EXPECT_EQ( howmany_c2c_axis_20, ref_howmany_r2c_axis_20 ); - - auto [in_extents_c2c_axis_21, out_extents_c2c_axis_21, fft_extents_c2c_axis_21, howmany_c2c_axis_21] = KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, axes_type({2, 1})); - EXPECT_TRUE( in_extents_c2c_axis_21 == ref_in_extents_r2c_axis_21 ); - EXPECT_TRUE( fft_extents_c2c_axis_21 == ref_fft_extents_r2c_axis_21 ); - EXPECT_TRUE( out_extents_c2c_axis_21 == ref_in_extents_r2c_axis_21 ); - EXPECT_EQ( howmany_c2c_axis_21, ref_howmany_r2c_axis_21 ); + auto [in_extents_c2c_axis_01, out_extents_c2c_axis_01, + fft_extents_c2c_axis_01, howmany_c2c_axis_01] = + KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, axes_type({0, 1})); + EXPECT_TRUE(in_extents_c2c_axis_01 == ref_in_extents_r2c_axis_01); + EXPECT_TRUE(fft_extents_c2c_axis_01 == ref_fft_extents_r2c_axis_01); + EXPECT_TRUE(out_extents_c2c_axis_01 == ref_in_extents_r2c_axis_01); + EXPECT_EQ(howmany_c2c_axis_01, ref_howmany_r2c_axis_01); + + auto [in_extents_c2c_axis_02, out_extents_c2c_axis_02, + fft_extents_c2c_axis_02, howmany_c2c_axis_02] = + KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, axes_type({0, 2})); + EXPECT_TRUE(in_extents_c2c_axis_02 == ref_in_extents_r2c_axis_02); + EXPECT_TRUE(fft_extents_c2c_axis_02 == ref_fft_extents_r2c_axis_02); + EXPECT_TRUE(out_extents_c2c_axis_02 == ref_in_extents_r2c_axis_02); + EXPECT_EQ(howmany_c2c_axis_02, ref_howmany_r2c_axis_02); + + auto [in_extents_c2c_axis_10, out_extents_c2c_axis_10, + fft_extents_c2c_axis_10, howmany_c2c_axis_10] = + KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, axes_type({1, 0})); + EXPECT_TRUE(in_extents_c2c_axis_10 == ref_in_extents_r2c_axis_10); + EXPECT_TRUE(fft_extents_c2c_axis_10 == ref_fft_extents_r2c_axis_10); + EXPECT_TRUE(out_extents_c2c_axis_10 == ref_in_extents_r2c_axis_10); + EXPECT_EQ(howmany_c2c_axis_10, ref_howmany_r2c_axis_10); + + auto [in_extents_c2c_axis_12, out_extents_c2c_axis_12, + fft_extents_c2c_axis_12, howmany_c2c_axis_12] = + KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, axes_type({1, 2})); + EXPECT_TRUE(in_extents_c2c_axis_12 == ref_in_extents_r2c_axis_12); + EXPECT_TRUE(fft_extents_c2c_axis_12 == ref_fft_extents_r2c_axis_12); + EXPECT_TRUE(out_extents_c2c_axis_12 == ref_in_extents_r2c_axis_12); + EXPECT_EQ(howmany_c2c_axis_12, ref_howmany_r2c_axis_12); + + auto [in_extents_c2c_axis_20, out_extents_c2c_axis_20, + fft_extents_c2c_axis_20, howmany_c2c_axis_20] = + KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, axes_type({2, 0})); + EXPECT_TRUE(in_extents_c2c_axis_20 == ref_in_extents_r2c_axis_20); + EXPECT_TRUE(fft_extents_c2c_axis_20 == ref_fft_extents_r2c_axis_20); + EXPECT_TRUE(out_extents_c2c_axis_20 == ref_in_extents_r2c_axis_20); + EXPECT_EQ(howmany_c2c_axis_20, ref_howmany_r2c_axis_20); + + auto [in_extents_c2c_axis_21, out_extents_c2c_axis_21, + fft_extents_c2c_axis_21, howmany_c2c_axis_21] = + KokkosFFT::Impl::get_extents_batched(xcin3, xcout3, axes_type({2, 1})); + EXPECT_TRUE(in_extents_c2c_axis_21 == ref_in_extents_r2c_axis_21); + EXPECT_TRUE(fft_extents_c2c_axis_21 == ref_fft_extents_r2c_axis_21); + EXPECT_TRUE(out_extents_c2c_axis_21 == ref_in_extents_r2c_axis_21); + EXPECT_EQ(howmany_c2c_axis_21, ref_howmany_r2c_axis_21); } TYPED_TEST(Layouts2D, 2DFFT_2DView) { diff --git a/common/unit_test/Test_Main.cpp b/common/unit_test/Test_Main.cpp index 9beb4ecf..b4cfc19a 100644 --- a/common/unit_test/Test_Main.cpp +++ b/common/unit_test/Test_Main.cpp @@ -17,12 +17,12 @@ #include namespace testing::internal { -// accessing gtest internals is not very clean, but gtest provides no public access... +// accessing gtest internals is not very clean, but gtest provides no public +// access... extern bool g_help_flag; -} // namespace testing::internal +} // namespace testing::internal int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); int result = 0; if (::testing::GTEST_FLAG(list_tests) || ::testing::internal::g_help_flag) { diff --git a/common/unit_test/Test_Normalization.cpp b/common/unit_test/Test_Normalization.cpp index 6e14ab1e..53c872fc 100644 --- a/common/unit_test/Test_Normalization.cpp +++ b/common/unit_test/Test_Normalization.cpp @@ -5,76 +5,88 @@ #include "Test_Utils.hpp" TEST(Normalization, Forward) { - const int len = 30; - View1D x("x", len), ref_f("ref_f", len), ref_b("ref_b", len); + const int len = 30; + View1D x("x", len), ref_f("ref_f", len), ref_b("ref_b", len); - Kokkos::Random_XorShift64_Pool<> random_pool(/*seed=*/12345); - Kokkos::fill_random(x, random_pool, 1.0); + Kokkos::Random_XorShift64_Pool<> random_pool(/*seed=*/12345); + Kokkos::fill_random(x, random_pool, 1.0); - Kokkos::deep_copy(ref_f, x); - Kokkos::deep_copy(ref_b, x); + Kokkos::deep_copy(ref_f, x); + Kokkos::deep_copy(ref_b, x); - double coef = 1.0 / static_cast(len); - multiply(ref_f, coef); + double coef = 1.0 / static_cast(len); + multiply(ref_f, coef); - Kokkos::fence(); + Kokkos::fence(); - // Backward FFT with Forward Normalization -> Do nothing - KokkosFFT::Impl::normalize(execution_space(), x, KokkosFFT::Impl::Direction::Backward, KokkosFFT::Normalization::FORWARD, len); - EXPECT_TRUE( allclose(x, ref_b, 1.e-5, 1.e-12) ); + // Backward FFT with Forward Normalization -> Do nothing + KokkosFFT::Impl::normalize(execution_space(), x, + KokkosFFT::Impl::Direction::Backward, + KokkosFFT::Normalization::FORWARD, len); + EXPECT_TRUE(allclose(x, ref_b, 1.e-5, 1.e-12)); - // Forward FFT with Forward Normalization -> 1/N normalization - KokkosFFT::Impl::normalize(execution_space(), x, KokkosFFT::Impl::Direction::Forward, KokkosFFT::Normalization::FORWARD, len); - EXPECT_TRUE( allclose(x, ref_f, 1.e-5, 1.e-12) ); + // Forward FFT with Forward Normalization -> 1/N normalization + KokkosFFT::Impl::normalize(execution_space(), x, + KokkosFFT::Impl::Direction::Forward, + KokkosFFT::Normalization::FORWARD, len); + EXPECT_TRUE(allclose(x, ref_f, 1.e-5, 1.e-12)); } TEST(Normalization, Backward) { - const int len = 30; - View1D x("x", len), ref_f("ref_f", len), ref_b("ref_b", len); + const int len = 30; + View1D x("x", len), ref_f("ref_f", len), ref_b("ref_b", len); - Kokkos::Random_XorShift64_Pool<> random_pool(/*seed=*/12345); - Kokkos::fill_random(x, random_pool, 1.0); + Kokkos::Random_XorShift64_Pool<> random_pool(/*seed=*/12345); + Kokkos::fill_random(x, random_pool, 1.0); - Kokkos::deep_copy(ref_f, x); - Kokkos::deep_copy(ref_b, x); + Kokkos::deep_copy(ref_f, x); + Kokkos::deep_copy(ref_b, x); - double coef = 1.0 / static_cast(len); - multiply(ref_b, coef); + double coef = 1.0 / static_cast(len); + multiply(ref_b, coef); - Kokkos::fence(); + Kokkos::fence(); - // Forward FFT with Backward Normalization -> Do nothing - KokkosFFT::Impl::normalize(execution_space(), x, KokkosFFT::Impl::Direction::Forward, KokkosFFT::Normalization::BACKWARD, len); - EXPECT_TRUE( allclose(x, ref_f, 1.e-5, 1.e-12) ); + // Forward FFT with Backward Normalization -> Do nothing + KokkosFFT::Impl::normalize(execution_space(), x, + KokkosFFT::Impl::Direction::Forward, + KokkosFFT::Normalization::BACKWARD, len); + EXPECT_TRUE(allclose(x, ref_f, 1.e-5, 1.e-12)); - // Backward FFT with Backward Normalization -> 1/N normalization - KokkosFFT::Impl::normalize(execution_space(), x, KokkosFFT::Impl::Direction::Backward, KokkosFFT::Normalization::BACKWARD, len); - EXPECT_TRUE( allclose(x, ref_b, 1.e-5, 1.e-12) ); + // Backward FFT with Backward Normalization -> 1/N normalization + KokkosFFT::Impl::normalize(execution_space(), x, + KokkosFFT::Impl::Direction::Backward, + KokkosFFT::Normalization::BACKWARD, len); + EXPECT_TRUE(allclose(x, ref_b, 1.e-5, 1.e-12)); } TEST(Normalization, Ortho) { - const int len = 30; - View1D x_f("x_f", len), x_b("x_b", len); - View1D ref_f("ref_f", len), ref_b("ref_b", len); - - Kokkos::Random_XorShift64_Pool<> random_pool(/*seed=*/12345); - Kokkos::fill_random(x_f, random_pool, 1.0); - - Kokkos::deep_copy(x_b, x_f); - Kokkos::deep_copy(ref_f, x_f); - Kokkos::deep_copy(ref_b, x_f); - - double coef = 1.0 / Kokkos::sqrt( static_cast(len) ); - multiply(ref_f, coef); - multiply(ref_b, coef); - - Kokkos::fence(); - - // Forward FFT with Ortho Normalization -> 1 / sqrt(N) normalization - KokkosFFT::Impl::normalize(execution_space(), x_f, KokkosFFT::Impl::Direction::Forward, KokkosFFT::Normalization::ORTHO, len); - EXPECT_TRUE( allclose(x_f, ref_f, 1.e-5, 1.e-12) ); - - // Backward FFT with Ortho Normalization -> 1 / sqrt(N) normalization - KokkosFFT::Impl::normalize(execution_space(), x_b, KokkosFFT::Impl::Direction::Backward, KokkosFFT::Normalization::ORTHO, len); - EXPECT_TRUE( allclose(x_b, ref_b, 1.e-5, 1.e-12) ); + const int len = 30; + View1D x_f("x_f", len), x_b("x_b", len); + View1D ref_f("ref_f", len), ref_b("ref_b", len); + + Kokkos::Random_XorShift64_Pool<> random_pool(/*seed=*/12345); + Kokkos::fill_random(x_f, random_pool, 1.0); + + Kokkos::deep_copy(x_b, x_f); + Kokkos::deep_copy(ref_f, x_f); + Kokkos::deep_copy(ref_b, x_f); + + double coef = 1.0 / Kokkos::sqrt(static_cast(len)); + multiply(ref_f, coef); + multiply(ref_b, coef); + + Kokkos::fence(); + + // Forward FFT with Ortho Normalization -> 1 / sqrt(N) normalization + KokkosFFT::Impl::normalize(execution_space(), x_f, + KokkosFFT::Impl::Direction::Forward, + KokkosFFT::Normalization::ORTHO, len); + EXPECT_TRUE(allclose(x_f, ref_f, 1.e-5, 1.e-12)); + + // Backward FFT with Ortho Normalization -> 1 / sqrt(N) normalization + KokkosFFT::Impl::normalize(execution_space(), x_b, + KokkosFFT::Impl::Direction::Backward, + KokkosFFT::Normalization::ORTHO, len); + EXPECT_TRUE(allclose(x_b, ref_b, 1.e-5, 1.e-12)); } \ No newline at end of file diff --git a/common/unit_test/Test_Padding.cpp b/common/unit_test/Test_Padding.cpp index 3eb66e10..877e3fc1 100644 --- a/common/unit_test/Test_Padding.cpp +++ b/common/unit_test/Test_Padding.cpp @@ -13,17 +13,19 @@ TEST(ModifyShape1D, View1D) { View1D x("x", len); - auto shape = KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{len}); - auto shape_pad = KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{len_pad}); - auto shape_crop = KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{len_crop}); + auto shape = KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{len}); + auto shape_pad = + KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{len_pad}); + auto shape_crop = + KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{len_crop}); shape_type<1> ref_shape = {len}; shape_type<1> ref_shape_pad = {len_pad}; shape_type<1> ref_shape_crop = {len_crop}; - EXPECT_TRUE( shape == ref_shape ); - EXPECT_TRUE( shape_pad == ref_shape_pad ); - EXPECT_TRUE( shape_crop == ref_shape_crop ); + EXPECT_TRUE(shape == ref_shape); + EXPECT_TRUE(shape_pad == ref_shape_pad); + EXPECT_TRUE(shape_crop == ref_shape_crop); } TEST(ModifyShape1D, View2D) { @@ -32,17 +34,19 @@ TEST(ModifyShape1D, View2D) { View2D x("x", n0, n1); - auto shape = KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{n0}); - auto shape_pad = KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{n0_pad}); - auto shape_crop = KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{n0_crop}); + auto shape = KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{n0}); + auto shape_pad = + KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{n0_pad}); + auto shape_crop = + KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{n0_crop}); shape_type<2> ref_shape = {n0, n1}; shape_type<2> ref_shape_pad = {n0_pad, n1}; shape_type<2> ref_shape_crop = {n0_crop, n1}; - EXPECT_TRUE( shape == ref_shape ); - EXPECT_TRUE( shape_pad == ref_shape_pad ); - EXPECT_TRUE( shape_crop == ref_shape_crop ); + EXPECT_TRUE(shape == ref_shape); + EXPECT_TRUE(shape_pad == ref_shape_pad); + EXPECT_TRUE(shape_crop == ref_shape_crop); } TEST(ModifyShape1D, View3D) { @@ -51,17 +55,19 @@ TEST(ModifyShape1D, View3D) { View3D x("x", n0, n1, n2); - auto shape = KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{n0}); - auto shape_pad = KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{n0_pad}); - auto shape_crop = KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{n0_crop}); + auto shape = KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{n0}); + auto shape_pad = + KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{n0_pad}); + auto shape_crop = + KokkosFFT::Impl::get_modified_shape(x, shape_type<1>{n0_crop}); shape_type<3> ref_shape = {n0, n1, n2}; shape_type<3> ref_shape_pad = {n0_pad, n1, n2}; shape_type<3> ref_shape_crop = {n0_crop, n1, n2}; - EXPECT_TRUE( shape == ref_shape ); - EXPECT_TRUE( shape_pad == ref_shape_pad ); - EXPECT_TRUE( shape_crop == ref_shape_crop ); + EXPECT_TRUE(shape == ref_shape); + EXPECT_TRUE(shape_pad == ref_shape_pad); + EXPECT_TRUE(shape_crop == ref_shape_crop); } TEST(ModifyShape2D, View2D) { @@ -70,15 +76,24 @@ TEST(ModifyShape2D, View2D) { View2D x("x", n0, n1); - auto shape_n0_n1 = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0, n1}); - auto shape_n0_n1pad = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0, n1_pad}); - auto shape_n0_n1crop = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0, n1_crop}); - auto shape_n0pad_n1 = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_pad, n1}); - auto shape_n0pad_n1pad = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_pad, n1_pad}); - auto shape_n0pad_n1crop = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_pad, n1_crop}); - auto shape_n0crop_n1 = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_crop, n1}); - auto shape_n0crop_n1pad = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_crop, n1_pad}); - auto shape_n0crop_n1crop = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_crop, n1_crop}); + auto shape_n0_n1 = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0, n1}); + auto shape_n0_n1pad = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0, n1_pad}); + auto shape_n0_n1crop = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0, n1_crop}); + auto shape_n0pad_n1 = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_pad, n1}); + auto shape_n0pad_n1pad = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_pad, n1_pad}); + auto shape_n0pad_n1crop = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_pad, n1_crop}); + auto shape_n0crop_n1 = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_crop, n1}); + auto shape_n0crop_n1pad = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_crop, n1_pad}); + auto shape_n0crop_n1crop = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_crop, n1_crop}); shape_type<2> ref_shape_n0_n1 = {n0, n1}; shape_type<2> ref_shape_n0_n1pad = {n0, n1_pad}; @@ -90,15 +105,15 @@ TEST(ModifyShape2D, View2D) { shape_type<2> ref_shape_n0crop_n1pad = {n0_crop, n1_pad}; shape_type<2> ref_shape_n0crop_n1crop = {n0_crop, n1_crop}; - EXPECT_TRUE( shape_n0_n1 == ref_shape_n0_n1 ); - EXPECT_TRUE( shape_n0_n1pad == ref_shape_n0_n1pad ); - EXPECT_TRUE( shape_n0_n1crop == ref_shape_n0_n1crop ); - EXPECT_TRUE( shape_n0pad_n1 == ref_shape_n0pad_n1 ); - EXPECT_TRUE( shape_n0pad_n1pad == ref_shape_n0pad_n1pad ); - EXPECT_TRUE( shape_n0pad_n1crop == ref_shape_n0pad_n1crop ); - EXPECT_TRUE( shape_n0crop_n1 == ref_shape_n0crop_n1 ); - EXPECT_TRUE( shape_n0crop_n1pad == ref_shape_n0crop_n1pad ); - EXPECT_TRUE( shape_n0crop_n1crop == ref_shape_n0crop_n1crop ); + EXPECT_TRUE(shape_n0_n1 == ref_shape_n0_n1); + EXPECT_TRUE(shape_n0_n1pad == ref_shape_n0_n1pad); + EXPECT_TRUE(shape_n0_n1crop == ref_shape_n0_n1crop); + EXPECT_TRUE(shape_n0pad_n1 == ref_shape_n0pad_n1); + EXPECT_TRUE(shape_n0pad_n1pad == ref_shape_n0pad_n1pad); + EXPECT_TRUE(shape_n0pad_n1crop == ref_shape_n0pad_n1crop); + EXPECT_TRUE(shape_n0crop_n1 == ref_shape_n0crop_n1); + EXPECT_TRUE(shape_n0crop_n1pad == ref_shape_n0crop_n1pad); + EXPECT_TRUE(shape_n0crop_n1crop == ref_shape_n0crop_n1crop); } TEST(ModifyShape2D, View3D) { @@ -108,15 +123,24 @@ TEST(ModifyShape2D, View3D) { View3D x("x", n0, n1, n2); - auto shape_n0_n1 = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0, n1}); - auto shape_n0_n1pad = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0, n1_pad}); - auto shape_n0_n1crop = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0, n1_crop}); - auto shape_n0pad_n1 = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_pad, n1}); - auto shape_n0pad_n1pad = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_pad, n1_pad}); - auto shape_n0pad_n1crop = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_pad, n1_crop}); - auto shape_n0crop_n1 = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_crop, n1}); - auto shape_n0crop_n1pad = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_crop, n1_pad}); - auto shape_n0crop_n1crop = KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_crop, n1_crop}); + auto shape_n0_n1 = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0, n1}); + auto shape_n0_n1pad = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0, n1_pad}); + auto shape_n0_n1crop = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0, n1_crop}); + auto shape_n0pad_n1 = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_pad, n1}); + auto shape_n0pad_n1pad = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_pad, n1_pad}); + auto shape_n0pad_n1crop = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_pad, n1_crop}); + auto shape_n0crop_n1 = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_crop, n1}); + auto shape_n0crop_n1pad = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_crop, n1_pad}); + auto shape_n0crop_n1crop = + KokkosFFT::Impl::get_modified_shape(x, shape_type<2>{n0_crop, n1_crop}); shape_type<3> ref_shape_n0_n1 = {n0, n1, n2}; shape_type<3> ref_shape_n0_n1pad = {n0, n1_pad, n2}; @@ -128,32 +152,32 @@ TEST(ModifyShape2D, View3D) { shape_type<3> ref_shape_n0crop_n1pad = {n0_crop, n1_pad, n2}; shape_type<3> ref_shape_n0crop_n1crop = {n0_crop, n1_crop, n2}; - EXPECT_TRUE( shape_n0_n1 == ref_shape_n0_n1 ); - EXPECT_TRUE( shape_n0_n1pad == ref_shape_n0_n1pad ); - EXPECT_TRUE( shape_n0_n1crop == ref_shape_n0_n1crop ); - EXPECT_TRUE( shape_n0pad_n1 == ref_shape_n0pad_n1 ); - EXPECT_TRUE( shape_n0pad_n1pad == ref_shape_n0pad_n1pad ); - EXPECT_TRUE( shape_n0pad_n1crop == ref_shape_n0pad_n1crop ); - EXPECT_TRUE( shape_n0crop_n1 == ref_shape_n0crop_n1 ); - EXPECT_TRUE( shape_n0crop_n1pad == ref_shape_n0crop_n1pad ); - EXPECT_TRUE( shape_n0crop_n1crop == ref_shape_n0crop_n1crop ); + EXPECT_TRUE(shape_n0_n1 == ref_shape_n0_n1); + EXPECT_TRUE(shape_n0_n1pad == ref_shape_n0_n1pad); + EXPECT_TRUE(shape_n0_n1crop == ref_shape_n0_n1crop); + EXPECT_TRUE(shape_n0pad_n1 == ref_shape_n0pad_n1); + EXPECT_TRUE(shape_n0pad_n1pad == ref_shape_n0pad_n1pad); + EXPECT_TRUE(shape_n0pad_n1crop == ref_shape_n0pad_n1crop); + EXPECT_TRUE(shape_n0crop_n1 == ref_shape_n0crop_n1); + EXPECT_TRUE(shape_n0crop_n1pad == ref_shape_n0crop_n1pad); + EXPECT_TRUE(shape_n0crop_n1crop == ref_shape_n0crop_n1crop); } TEST(ModifyShape3D, View3D) { const int n0 = 30, n1 = 15, n2 = 8; View3D x("x", n0, n1, n2); - for(int i0=-1; i0<=1; i0++) { - for(int i1=-1; i1<=1; i1++) { - for(int i2=-1; i2<=1; i2++) { - std::size_t n0_new = static_cast(n0+i0); - std::size_t n1_new = static_cast(n1+i1); - std::size_t n2_new = static_cast(n2+i2); + for (int i0 = -1; i0 <= 1; i0++) { + for (int i1 = -1; i1 <= 1; i1++) { + for (int i2 = -1; i2 <= 1; i2++) { + std::size_t n0_new = static_cast(n0 + i0); + std::size_t n1_new = static_cast(n1 + i1); + std::size_t n2_new = static_cast(n2 + i2); shape_type<3> ref_shape = {n0_new, n1_new, n2_new}; auto shape = KokkosFFT::Impl::get_modified_shape(x, ref_shape); - EXPECT_TRUE( shape == ref_shape ); + EXPECT_TRUE(shape == ref_shape); } } } @@ -163,25 +187,27 @@ TEST(IsCropOrPadNeeded, View1D) { const int len = 30, len_pad = 32, len_crop = 28; View1D x("x", len); - EXPECT_FALSE( KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_type<1>{len}) ); - EXPECT_TRUE( KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_type<1>{len_pad}) ); - EXPECT_TRUE( KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_type<1>{len_crop}) ); + EXPECT_FALSE(KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_type<1>{len})); + EXPECT_TRUE( + KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_type<1>{len_pad})); + EXPECT_TRUE( + KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_type<1>{len_crop})); } TEST(IsCropOrPadNeeded, View2D) { const int n0 = 30, n1 = 15; View2D x("x", n0, n1); - for(int i0=-1; i0<=1; i0++) { - for(int i1=-1; i1<=1; i1++) { - std::size_t n0_new = static_cast(n0+i0); - std::size_t n1_new = static_cast(n1+i1); + for (int i0 = -1; i0 <= 1; i0++) { + for (int i1 = -1; i1 <= 1; i1++) { + std::size_t n0_new = static_cast(n0 + i0); + std::size_t n1_new = static_cast(n1 + i1); shape_type<2> shape_new = {n0_new, n1_new}; - if(i0 == 0 && i1 == 0) { - EXPECT_FALSE( KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_new) ); + if (i0 == 0 && i1 == 0) { + EXPECT_FALSE(KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_new)); } else { - EXPECT_TRUE( KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_new) ); + EXPECT_TRUE(KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_new)); } } } @@ -191,18 +217,18 @@ TEST(IsCropOrPadNeeded, View3D) { const int n0 = 30, n1 = 15, n2 = 8; View3D x("x", n0, n1, n2); - for(int i0=-1; i0<=1; i0++) { - for(int i1=-1; i1<=1; i1++) { - for(int i2=-1; i2<=1; i2++) { - std::size_t n0_new = static_cast(n0+i0); - std::size_t n1_new = static_cast(n1+i1); - std::size_t n2_new = static_cast(n2+i2); + for (int i0 = -1; i0 <= 1; i0++) { + for (int i1 = -1; i1 <= 1; i1++) { + for (int i2 = -1; i2 <= 1; i2++) { + std::size_t n0_new = static_cast(n0 + i0); + std::size_t n1_new = static_cast(n1 + i1); + std::size_t n2_new = static_cast(n2 + i2); shape_type<3> shape_new = {n0_new, n1_new, n2_new}; - if(i0 == 0 && i1 == 0 && i2 == 0) { - EXPECT_FALSE( KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_new) ); + if (i0 == 0 && i1 == 0 && i2 == 0) { + EXPECT_FALSE(KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_new)); } else { - EXPECT_TRUE( KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_new) ); + EXPECT_TRUE(KokkosFFT::Impl::is_crop_or_pad_needed(x, shape_new)); } } } @@ -214,7 +240,8 @@ TEST(CropOrPad1D, View1D) { View1D x("x", len); View1D _x, _x_pad, _x_crop; - View1D ref_x("ref_x", len), ref_x_pad("ref_x_pad", len_pad), ref_x_crop("ref_x_crop", len_crop); + View1D ref_x("ref_x", len), ref_x_pad("ref_x_pad", len_pad), + ref_x_crop("ref_x_crop", len_crop); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, 1.0); @@ -222,22 +249,24 @@ TEST(CropOrPad1D, View1D) { Kokkos::deep_copy(ref_x, x); // Copying the first len elements, others are initialized with zeros - auto range0 = std::pair(0, len); + auto range0 = std::pair(0, len); auto sub_ref_x_pad = Kokkos::subview(ref_x_pad, range0); Kokkos::deep_copy(sub_ref_x_pad, x); // Copying the cropped part - auto range1 = std::pair(0, len_crop); - auto sub_x = Kokkos::subview(x, range1); + auto range1 = std::pair(0, len_crop); + auto sub_x = Kokkos::subview(x, range1); Kokkos::deep_copy(ref_x_crop, sub_x); KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x, shape_type<1>{len}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_pad, shape_type<1>{len_pad}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_crop, shape_type<1>{len_crop}); - - EXPECT_TRUE( allclose(_x, ref_x, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_pad, ref_x_pad, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_crop, ref_x_crop, 1.e-5, 1.e-12) ); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_pad, + shape_type<1>{len_pad}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_crop, + shape_type<1>{len_crop}); + + EXPECT_TRUE(allclose(_x, ref_x, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_pad, ref_x_pad, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_crop, ref_x_crop, 1.e-5, 1.e-12)); } TEST(CropOrPad1D, View2D) { @@ -246,15 +275,18 @@ TEST(CropOrPad1D, View2D) { View2D x("x", n0, n1); View2D _x, _x_pad_axis0, _x_pad_axis1, _x_crop_axis0, _x_crop_axis1; - View2D ref_x("ref_x", n0, n1), ref_x_pad_axis0("ref_x_pad_axis0", n0_pad, n1), ref_x_crop_axis0("ref_x_crop_axis0", n0_crop, n1); - View2D ref_x_pad_axis1("ref_x_pad_axis1", n0, n1_pad), ref_x_crop_axis1("ref_x_cro_axis1", n0, n1_crop); + View2D ref_x("ref_x", n0, n1), + ref_x_pad_axis0("ref_x_pad_axis0", n0_pad, n1), + ref_x_crop_axis0("ref_x_crop_axis0", n0_crop, n1); + View2D ref_x_pad_axis1("ref_x_pad_axis1", n0, n1_pad), + ref_x_crop_axis1("ref_x_cro_axis1", n0, n1_crop); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, 1.0); Kokkos::deep_copy(ref_x, x); - auto h_x = Kokkos::create_mirror_view(x); + auto h_x = Kokkos::create_mirror_view(x); auto h_ref_x_pad_axis0 = Kokkos::create_mirror_view(ref_x_pad_axis0); auto h_ref_x_crop_axis0 = Kokkos::create_mirror_view(ref_x_crop_axis0); auto h_ref_x_pad_axis1 = Kokkos::create_mirror_view(ref_x_pad_axis1); @@ -266,27 +298,27 @@ TEST(CropOrPad1D, View2D) { Kokkos::deep_copy(h_ref_x_crop_axis1, ref_x_crop_axis1); // Pad or crop along axis 0 - for(int i1=0; i1{n0, n1}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_pad_axis0, shape_type<2>{n0_pad, n1}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_crop_axis0, shape_type<2>{n0_crop, n1}); - - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_pad_axis1, shape_type<2>{n0, n1_pad}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_crop_axis1, shape_type<2>{n0, n1_crop}); - - EXPECT_TRUE( allclose(_x, ref_x, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_pad_axis0, ref_x_pad_axis0, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_crop_axis0, ref_x_crop_axis0, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_pad_axis1, ref_x_pad_axis1, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_crop_axis1, ref_x_crop_axis1, 1.e-5, 1.e-12) ); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_pad_axis0, + shape_type<2>{n0_pad, n1}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_crop_axis0, + shape_type<2>{n0_crop, n1}); + + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_pad_axis1, + shape_type<2>{n0, n1_pad}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_crop_axis1, + shape_type<2>{n0, n1_crop}); + + EXPECT_TRUE(allclose(_x, ref_x, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_pad_axis0, ref_x_pad_axis0, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_crop_axis0, ref_x_crop_axis0, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_pad_axis1, ref_x_pad_axis1, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_crop_axis1, ref_x_crop_axis1, 1.e-5, 1.e-12)); } TEST(CropOrPad1D, View3D) { @@ -315,17 +351,22 @@ TEST(CropOrPad1D, View3D) { const int n2 = 8, n2_pad = 10, n2_crop = 6; View3D x("x", n0, n1, n2); - View3D _x, _x_pad_axis0, _x_crop_axis0, _x_pad_axis1, _x_crop_axis1, _x_pad_axis2, _x_crop_axis2; - View3D ref_x("ref_x", n0, n1, n2), ref_x_pad_axis0("ref_x_pad_axis0", n0_pad, n1, n2), ref_x_crop_axis0("ref_x_crop_axis0", n0_crop, n1, n2); - View3D ref_x_pad_axis1("ref_x_pad_axis1", n0, n1_pad, n2), ref_x_crop_axis1("ref_x_cro_axis1", n0, n1_crop, n2); - View3D ref_x_pad_axis2("ref_x_pad_axis2", n0, n1, n2_pad), ref_x_crop_axis2("ref_x_cro_axis2", n0, n1, n2_crop); + View3D _x, _x_pad_axis0, _x_crop_axis0, _x_pad_axis1, _x_crop_axis1, + _x_pad_axis2, _x_crop_axis2; + View3D ref_x("ref_x", n0, n1, n2), + ref_x_pad_axis0("ref_x_pad_axis0", n0_pad, n1, n2), + ref_x_crop_axis0("ref_x_crop_axis0", n0_crop, n1, n2); + View3D ref_x_pad_axis1("ref_x_pad_axis1", n0, n1_pad, n2), + ref_x_crop_axis1("ref_x_cro_axis1", n0, n1_crop, n2); + View3D ref_x_pad_axis2("ref_x_pad_axis2", n0, n1, n2_pad), + ref_x_crop_axis2("ref_x_cro_axis2", n0, n1, n2_crop); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, 1.0); Kokkos::deep_copy(ref_x, x); - auto h_x = Kokkos::create_mirror_view(x); + auto h_x = Kokkos::create_mirror_view(x); auto h_ref_x_pad_axis0 = Kokkos::create_mirror_view(ref_x_pad_axis0); auto h_ref_x_crop_axis0 = Kokkos::create_mirror_view(ref_x_crop_axis0); auto h_ref_x_pad_axis1 = Kokkos::create_mirror_view(ref_x_pad_axis1); @@ -341,45 +382,45 @@ TEST(CropOrPad1D, View3D) { Kokkos::deep_copy(h_ref_x_crop_axis2, ref_x_crop_axis2); // Pad or crop along axis 0 - for(int i2=0; i2{n0, n1, n2}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_pad_axis0, shape_type<3>{n0_pad, n1, n2}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_crop_axis0, shape_type<3>{n0_crop, n1, n2}); - - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_pad_axis1, shape_type<3>{n0, n1_pad, n2}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_crop_axis1, shape_type<3>{n0, n1_crop, n2}); - - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_pad_axis2, shape_type<3>{n0, n1, n2_pad}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_crop_axis2, shape_type<3>{n0, n1, n2_crop}); - - EXPECT_TRUE( allclose(_x, ref_x, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_pad_axis0, ref_x_pad_axis0, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_crop_axis0, ref_x_crop_axis0, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_pad_axis1, ref_x_pad_axis1, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_crop_axis1, ref_x_crop_axis1, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_pad_axis2, ref_x_pad_axis2, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_crop_axis2, ref_x_crop_axis2, 1.e-5, 1.e-12) ); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x, + shape_type<3>{n0, n1, n2}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_pad_axis0, + shape_type<3>{n0_pad, n1, n2}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_crop_axis0, + shape_type<3>{n0_crop, n1, n2}); + + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_pad_axis1, + shape_type<3>{n0, n1_pad, n2}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_crop_axis1, + shape_type<3>{n0, n1_crop, n2}); + + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_pad_axis2, + shape_type<3>{n0, n1, n2_pad}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_crop_axis2, + shape_type<3>{n0, n1, n2_crop}); + + EXPECT_TRUE(allclose(_x, ref_x, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_pad_axis0, ref_x_pad_axis0, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_crop_axis0, ref_x_crop_axis0, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_pad_axis1, ref_x_pad_axis1, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_crop_axis1, ref_x_crop_axis1, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_pad_axis2, ref_x_pad_axis2, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_crop_axis2, ref_x_crop_axis2, 1.e-5, 1.e-12)); } TEST(CropOrPad2D, View2D) { @@ -416,19 +464,25 @@ TEST(CropOrPad2D, View2D) { const int n1 = 5, n1_pad = 7, n1_crop = 3; View2D x("x", n0, n1); - View2D _x, _x_0_1p, _x_0_1c, _x_0p_1, _x_0p_1p, _x_0p_1c, _x_0c_1, _x_0c_1p, _x_0c_1c; - View2D ref_x("ref_x", n0, n1), ref_x_0_1p("ref_x_0_1p", n0, n1_pad), ref_x_0_1c("ref_x_0_1c", n0, n1_crop); - View2D ref_x_0p_1("ref_x_0p_1", n0_pad, n1), ref_x_0p_1p("ref_x_0p_1p", n0_pad, n1_pad), ref_x_0p_1c("ref_x_0p_1c", n0_pad, n1_crop); - View2D ref_x_0c_1("ref_x_0c_1", n0_crop, n1), ref_x_0c_1p("ref_x_0c_1p", n0_crop, n1_pad), ref_x_0c_1c("ref_x_0c_1c", n0_crop, n1_crop); + View2D _x, _x_0_1p, _x_0_1c, _x_0p_1, _x_0p_1p, _x_0p_1c, _x_0c_1, + _x_0c_1p, _x_0c_1c; + View2D ref_x("ref_x", n0, n1), ref_x_0_1p("ref_x_0_1p", n0, n1_pad), + ref_x_0_1c("ref_x_0_1c", n0, n1_crop); + View2D ref_x_0p_1("ref_x_0p_1", n0_pad, n1), + ref_x_0p_1p("ref_x_0p_1p", n0_pad, n1_pad), + ref_x_0p_1c("ref_x_0p_1c", n0_pad, n1_crop); + View2D ref_x_0c_1("ref_x_0c_1", n0_crop, n1), + ref_x_0c_1p("ref_x_0c_1p", n0_crop, n1_pad), + ref_x_0c_1c("ref_x_0c_1c", n0_crop, n1_crop); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, 1.0); Kokkos::deep_copy(ref_x, x); - auto h_x = Kokkos::create_mirror_view(x); - auto h_ref_x_0_1p = Kokkos::create_mirror_view(ref_x_0_1p); - auto h_ref_x_0_1c = Kokkos::create_mirror_view(ref_x_0_1c); + auto h_x = Kokkos::create_mirror_view(x); + auto h_ref_x_0_1p = Kokkos::create_mirror_view(ref_x_0_1p); + auto h_ref_x_0_1c = Kokkos::create_mirror_view(ref_x_0_1c); auto h_ref_x_0p_1 = Kokkos::create_mirror_view(ref_x_0p_1); auto h_ref_x_0p_1p = Kokkos::create_mirror_view(ref_x_0p_1p); @@ -440,70 +494,78 @@ TEST(CropOrPad2D, View2D) { Kokkos::deep_copy(h_x, x); Kokkos::deep_copy(h_ref_x_0_1p, ref_x_0_1p); Kokkos::deep_copy(h_ref_x_0_1c, ref_x_0_1c); - Kokkos::deep_copy(h_ref_x_0p_1, ref_x_0p_1); + Kokkos::deep_copy(h_ref_x_0p_1, ref_x_0p_1); Kokkos::deep_copy(h_ref_x_0p_1p, ref_x_0p_1p); Kokkos::deep_copy(h_ref_x_0p_1c, ref_x_0p_1c); - Kokkos::deep_copy(h_ref_x_0c_1, ref_x_0c_1); + Kokkos::deep_copy(h_ref_x_0c_1, ref_x_0c_1); Kokkos::deep_copy(h_ref_x_0c_1p, ref_x_0c_1p); Kokkos::deep_copy(h_ref_x_0c_1c, ref_x_0c_1c); // Along axis 0 - for(int i1=0; i1{n0, n1}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0_1p, shape_type<2>{n0, n1_pad}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0_1c, shape_type<2>{n0, n1_crop}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0p_1, shape_type<2>{n0_pad, n1}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0p_1p, shape_type<2>{n0_pad, n1_pad}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0p_1c, shape_type<2>{n0_pad, n1_crop}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0c_1, shape_type<2>{n0_crop, n1}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0c_1p, shape_type<2>{n0_crop, n1_pad}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0c_1c, shape_type<2>{n0_crop, n1_crop}); - - EXPECT_TRUE( allclose(_x, ref_x, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0_1p, ref_x_0_1p, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0_1c, ref_x_0_1c, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0p_1, ref_x_0p_1, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0p_1p, ref_x_0p_1p, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0p_1c, ref_x_0p_1c, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0c_1, ref_x_0c_1, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0c_1p, ref_x_0c_1p, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0c_1c, ref_x_0c_1c, 1.e-5, 1.e-12) ); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0_1p, + shape_type<2>{n0, n1_pad}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0_1c, + shape_type<2>{n0, n1_crop}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0p_1, + shape_type<2>{n0_pad, n1}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0p_1p, + shape_type<2>{n0_pad, n1_pad}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0p_1c, + shape_type<2>{n0_pad, n1_crop}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0c_1, + shape_type<2>{n0_crop, n1}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0c_1p, + shape_type<2>{n0_crop, n1_pad}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0c_1c, + shape_type<2>{n0_crop, n1_crop}); + + EXPECT_TRUE(allclose(_x, ref_x, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0_1p, ref_x_0_1p, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0_1c, ref_x_0_1c, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0p_1, ref_x_0p_1, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0p_1p, ref_x_0p_1p, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0p_1c, ref_x_0p_1c, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0c_1, ref_x_0c_1, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0c_1p, ref_x_0c_1p, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0c_1c, ref_x_0c_1c, 1.e-5, 1.e-12)); } TEST(CropOrPad2D, View3D) { @@ -512,19 +574,26 @@ TEST(CropOrPad2D, View3D) { const int n2 = 8; View3D x("x", n0, n1, n2); - View3D _x, _x_0_1p, _x_0_1c, _x_0p_1, _x_0p_1p, _x_0p_1c, _x_0c_1, _x_0c_1p, _x_0c_1c; - View3D ref_x("ref_x", n0, n1, n2), ref_x_0_1p("ref_x_0_1p", n0, n1_pad, n2), ref_x_0_1c("ref_x_0_1c", n0, n1_crop, n2); - View3D ref_x_0p_1("ref_x_0p_1", n0_pad, n1, n2), ref_x_0p_1p("ref_x_0p_1p", n0_pad, n1_pad, n2), ref_x_0p_1c("ref_x_0p_1c", n0_pad, n1_crop, n2); - View3D ref_x_0c_1("ref_x_0c_1", n0_crop, n1, n2), ref_x_0c_1p("ref_x_0c_1p", n0_crop, n1_pad, n2), ref_x_0c_1c("ref_x_0c_1c", n0_crop, n1_crop, n2); + View3D _x, _x_0_1p, _x_0_1c, _x_0p_1, _x_0p_1p, _x_0p_1c, _x_0c_1, + _x_0c_1p, _x_0c_1c; + View3D ref_x("ref_x", n0, n1, n2), + ref_x_0_1p("ref_x_0_1p", n0, n1_pad, n2), + ref_x_0_1c("ref_x_0_1c", n0, n1_crop, n2); + View3D ref_x_0p_1("ref_x_0p_1", n0_pad, n1, n2), + ref_x_0p_1p("ref_x_0p_1p", n0_pad, n1_pad, n2), + ref_x_0p_1c("ref_x_0p_1c", n0_pad, n1_crop, n2); + View3D ref_x_0c_1("ref_x_0c_1", n0_crop, n1, n2), + ref_x_0c_1p("ref_x_0c_1p", n0_crop, n1_pad, n2), + ref_x_0c_1c("ref_x_0c_1c", n0_crop, n1_crop, n2); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, 1.0); Kokkos::deep_copy(ref_x, x); - auto h_x = Kokkos::create_mirror_view(x); - auto h_ref_x_0_1p = Kokkos::create_mirror_view(ref_x_0_1p); - auto h_ref_x_0_1c = Kokkos::create_mirror_view(ref_x_0_1c); + auto h_x = Kokkos::create_mirror_view(x); + auto h_ref_x_0_1p = Kokkos::create_mirror_view(ref_x_0_1p); + auto h_ref_x_0_1c = Kokkos::create_mirror_view(ref_x_0_1c); auto h_ref_x_0p_1 = Kokkos::create_mirror_view(ref_x_0p_1); auto h_ref_x_0p_1p = Kokkos::create_mirror_view(ref_x_0p_1p); @@ -536,72 +605,81 @@ TEST(CropOrPad2D, View3D) { Kokkos::deep_copy(h_x, x); Kokkos::deep_copy(h_ref_x_0_1p, ref_x_0_1p); Kokkos::deep_copy(h_ref_x_0_1c, ref_x_0_1c); - Kokkos::deep_copy(h_ref_x_0p_1, ref_x_0p_1); + Kokkos::deep_copy(h_ref_x_0p_1, ref_x_0p_1); Kokkos::deep_copy(h_ref_x_0p_1p, ref_x_0p_1p); Kokkos::deep_copy(h_ref_x_0p_1c, ref_x_0p_1c); - Kokkos::deep_copy(h_ref_x_0c_1, ref_x_0c_1); + Kokkos::deep_copy(h_ref_x_0c_1, ref_x_0c_1); Kokkos::deep_copy(h_ref_x_0c_1p, ref_x_0c_1p); Kokkos::deep_copy(h_ref_x_0c_1c, ref_x_0c_1c); // Along axis 0 - for(int i2=0; i2{n0, n1, n2}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0_1p, shape_type<3>{n0, n1_pad, n2}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0_1c, shape_type<3>{n0, n1_crop, n2}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0p_1, shape_type<3>{n0_pad, n1, n2}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0p_1p, shape_type<3>{n0_pad, n1_pad, n2}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0p_1c, shape_type<3>{n0_pad, n1_crop, n2}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0c_1, shape_type<3>{n0_crop, n1, n2}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0c_1p, shape_type<3>{n0_crop, n1_pad, n2}); - KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0c_1c, shape_type<3>{n0_crop, n1_crop, n2}); - - EXPECT_TRUE( allclose(_x, ref_x, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0_1p, ref_x_0_1p, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0_1c, ref_x_0_1c, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0p_1, ref_x_0p_1, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0p_1p, ref_x_0p_1p, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0p_1c, ref_x_0p_1c, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0c_1, ref_x_0c_1, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0c_1p, ref_x_0c_1p, 1.e-5, 1.e-12) ); - EXPECT_TRUE( allclose(_x_0c_1c, ref_x_0c_1c, 1.e-5, 1.e-12) ); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x, + shape_type<3>{n0, n1, n2}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0_1p, + shape_type<3>{n0, n1_pad, n2}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0_1c, + shape_type<3>{n0, n1_crop, n2}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0p_1, + shape_type<3>{n0_pad, n1, n2}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0p_1p, + shape_type<3>{n0_pad, n1_pad, n2}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0p_1c, + shape_type<3>{n0_pad, n1_crop, n2}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0c_1, + shape_type<3>{n0_crop, n1, n2}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0c_1p, + shape_type<3>{n0_crop, n1_pad, n2}); + KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x_0c_1c, + shape_type<3>{n0_crop, n1_crop, n2}); + + EXPECT_TRUE(allclose(_x, ref_x, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0_1p, ref_x_0_1p, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0_1c, ref_x_0_1c, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0p_1, ref_x_0p_1, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0p_1p, ref_x_0p_1p, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0p_1c, ref_x_0p_1c, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0c_1, ref_x_0c_1, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0c_1p, ref_x_0c_1p, 1.e-5, 1.e-12)); + EXPECT_TRUE(allclose(_x_0c_1c, ref_x_0c_1c, 1.e-5, 1.e-12)); } TEST(CropOrPad3D, View3D) { @@ -613,22 +691,24 @@ TEST(CropOrPad3D, View3D) { auto h_x = Kokkos::create_mirror_view(x); Kokkos::deep_copy(h_x, x); - for(int i0=-1; i0<=1; i0++) { - for(int i1=-1; i1<=1; i1++) { - for(int i2=-1; i2<=1; i2++) { - std::size_t n0_new = static_cast(n0+i0); - std::size_t n1_new = static_cast(n1+i1); - std::size_t n2_new = static_cast(n2+i2); + for (int i0 = -1; i0 <= 1; i0++) { + for (int i1 = -1; i1 <= 1; i1++) { + for (int i2 = -1; i2 <= 1; i2++) { + std::size_t n0_new = static_cast(n0 + i0); + std::size_t n1_new = static_cast(n1 + i1); + std::size_t n2_new = static_cast(n2 + i2); shape_type<3> shape_new = {n0_new, n1_new, n2_new}; View3D _x; View3D ref_x("ref_x", n0_new, n1_new, n2_new); auto h_ref_x = Kokkos::create_mirror_view(ref_x); - for(int i2=0; i2= h_ref_x.extent(0) || i1 >= h_ref_x.extent(1) || i2 >= h_ref_x.extent(2)) continue; + for (int i2 = 0; i2 < n2; i2++) { + for (int i1 = 0; i1 < n1; i1++) { + for (int i0 = 0; i0 < n0; i0++) { + if (i0 >= h_ref_x.extent(0) || i1 >= h_ref_x.extent(1) || + i2 >= h_ref_x.extent(2)) + continue; h_ref_x(i0, i1, i2) = h_x(i0, i1, i2); } } @@ -636,7 +716,7 @@ TEST(CropOrPad3D, View3D) { Kokkos::deep_copy(ref_x, h_ref_x); KokkosFFT::Impl::crop_or_pad(execution_space(), x, _x, shape_new); - EXPECT_TRUE( allclose(_x, ref_x, 1.e-5, 1.e-12) ); + EXPECT_TRUE(allclose(_x, ref_x, 1.e-5, 1.e-12)); } } } diff --git a/common/unit_test/Test_Transpose.cpp b/common/unit_test/Test_Transpose.cpp index ceb2b392..25c91eab 100644 --- a/common/unit_test/Test_Transpose.cpp +++ b/common/unit_test/Test_Transpose.cpp @@ -7,10 +7,7 @@ template using axes_type = std::array; -using test_types = ::testing::Types< - Kokkos::LayoutLeft, - Kokkos::LayoutRight ->; +using test_types = ::testing::Types; // Basically the same fixtures, used for labeling tests template @@ -29,20 +26,21 @@ TYPED_TEST_SUITE(Transpose, test_types); // Tests for map axes over ND views template void test_map_axes1d() { - const int len = 30; + const int len = 30; using RealView1Dtype = Kokkos::View; RealView1Dtype x("x", len); auto [map_axis, map_inv_axis] = KokkosFFT::Impl::get_map_axes(x, /*axis=*/0); - auto [map_axes, map_inv_axes] = KokkosFFT::Impl::get_map_axes(x, /*axes=*/ axes_type<1>({0})); + auto [map_axes, map_inv_axes] = + KokkosFFT::Impl::get_map_axes(x, /*axes=*/axes_type<1>({0})); axes_type<1> ref_map_axis = {0}; axes_type<1> ref_map_axes = {0}; - EXPECT_TRUE( map_axis == ref_map_axis ); - EXPECT_TRUE( map_axes == ref_map_axes ); - EXPECT_TRUE( map_inv_axis == ref_map_axis ); - EXPECT_TRUE( map_inv_axes == ref_map_axes ); + EXPECT_TRUE(map_axis == ref_map_axis); + EXPECT_TRUE(map_axes == ref_map_axes); + EXPECT_TRUE(map_inv_axis == ref_map_axis); + EXPECT_TRUE(map_inv_axes == ref_map_axes); } template @@ -51,16 +49,26 @@ void test_map_axes2d() { using RealView2Dtype = Kokkos::View; RealView2Dtype x("x", n0, n1); - auto [map_axis_0, map_inv_axis_0] = KokkosFFT::Impl::get_map_axes(x, /*axis=*/0); - auto [map_axis_1, map_inv_axis_1] = KokkosFFT::Impl::get_map_axes(x, /*axis=*/1); - auto [map_axis_minus1, map_inv_axis_minus1] = KokkosFFT::Impl::get_map_axes(x, /*axis=*/-1); - auto [map_axes_0, map_inv_axes_0] = KokkosFFT::Impl::get_map_axes(x, /*axes=*/ axes_type<1>({0})); - auto [map_axes_1, map_inv_axes_1] = KokkosFFT::Impl::get_map_axes(x, /*axes=*/ axes_type<1>({1})); - auto [map_axes_minus1, map_inv_axes_minus1] = KokkosFFT::Impl::get_map_axes(x, /*axes=*/ axes_type<1>({-1})); - auto [map_axes_0_minus1, map_inv_axes_0_minus1] = KokkosFFT::Impl::get_map_axes(x, /*axes=*/ axes_type<2>({0, -1})); - auto [map_axes_minus1_0, map_inv_axes_minus1_0] = KokkosFFT::Impl::get_map_axes(x, /*axes=*/ axes_type<2>({-1, 0})); - auto [map_axes_0_1, map_inv_axes_0_1] = KokkosFFT::Impl::get_map_axes(x, /*axes=*/ axes_type<2>({0, 1})); - auto [map_axes_1_0, map_inv_axes_1_0] = KokkosFFT::Impl::get_map_axes(x, /*axes=*/ axes_type<2>({1, 0})); + auto [map_axis_0, map_inv_axis_0] = + KokkosFFT::Impl::get_map_axes(x, /*axis=*/0); + auto [map_axis_1, map_inv_axis_1] = + KokkosFFT::Impl::get_map_axes(x, /*axis=*/1); + auto [map_axis_minus1, map_inv_axis_minus1] = + KokkosFFT::Impl::get_map_axes(x, /*axis=*/-1); + auto [map_axes_0, map_inv_axes_0] = + KokkosFFT::Impl::get_map_axes(x, /*axes=*/axes_type<1>({0})); + auto [map_axes_1, map_inv_axes_1] = + KokkosFFT::Impl::get_map_axes(x, /*axes=*/axes_type<1>({1})); + auto [map_axes_minus1, map_inv_axes_minus1] = + KokkosFFT::Impl::get_map_axes(x, /*axes=*/axes_type<1>({-1})); + auto [map_axes_0_minus1, map_inv_axes_0_minus1] = + KokkosFFT::Impl::get_map_axes(x, /*axes=*/axes_type<2>({0, -1})); + auto [map_axes_minus1_0, map_inv_axes_minus1_0] = + KokkosFFT::Impl::get_map_axes(x, /*axes=*/axes_type<2>({-1, 0})); + auto [map_axes_0_1, map_inv_axes_0_1] = + KokkosFFT::Impl::get_map_axes(x, /*axes=*/axes_type<2>({0, 1})); + auto [map_axes_1_0, map_inv_axes_1_0] = + KokkosFFT::Impl::get_map_axes(x, /*axes=*/axes_type<2>({1, 0})); axes_type<2> ref_map_axis_0, ref_map_inv_axis_0; axes_type<2> ref_map_axis_1, ref_map_inv_axis_1; @@ -74,7 +82,7 @@ void test_map_axes2d() { axes_type<2> ref_map_axes_0_1, ref_map_inv_axes_0_1; axes_type<2> ref_map_axes_1_0, ref_map_inv_axes_1_0; - if(std::is_same_v) { + if (std::is_same_v) { // Layout Left ref_map_axis_0 = {0, 1}, ref_map_inv_axis_0 = {0, 1}; ref_map_axis_1 = {1, 0}, ref_map_inv_axis_1 = {1, 0}; @@ -103,28 +111,28 @@ void test_map_axes2d() { } // Forward mapping - EXPECT_TRUE( map_axis_0 == ref_map_axis_0 ); - EXPECT_TRUE( map_axis_1 == ref_map_axis_1 ); - EXPECT_TRUE( map_axis_minus1 == ref_map_axis_minus1 ); - EXPECT_TRUE( map_axes_0 == ref_map_axes_0 ); - EXPECT_TRUE( map_axes_1 == ref_map_axes_1 ); - EXPECT_TRUE( map_axes_minus1 == ref_map_axes_minus1 ); - EXPECT_TRUE( map_axes_0_minus1 == ref_map_axes_0_minus1 ); - EXPECT_TRUE( map_axes_minus1_0 == ref_map_axes_minus1_0 ); - EXPECT_TRUE( map_axes_0_1 == ref_map_axes_0_1 ); - EXPECT_TRUE( map_axes_1_0 == ref_map_axes_1_0 ); + EXPECT_TRUE(map_axis_0 == ref_map_axis_0); + EXPECT_TRUE(map_axis_1 == ref_map_axis_1); + EXPECT_TRUE(map_axis_minus1 == ref_map_axis_minus1); + EXPECT_TRUE(map_axes_0 == ref_map_axes_0); + EXPECT_TRUE(map_axes_1 == ref_map_axes_1); + EXPECT_TRUE(map_axes_minus1 == ref_map_axes_minus1); + EXPECT_TRUE(map_axes_0_minus1 == ref_map_axes_0_minus1); + EXPECT_TRUE(map_axes_minus1_0 == ref_map_axes_minus1_0); + EXPECT_TRUE(map_axes_0_1 == ref_map_axes_0_1); + EXPECT_TRUE(map_axes_1_0 == ref_map_axes_1_0); // Inverse mapping - EXPECT_TRUE( map_inv_axis_0 == ref_map_inv_axis_0 ); - EXPECT_TRUE( map_inv_axis_1 == ref_map_inv_axis_1 ); - EXPECT_TRUE( map_inv_axis_minus1 == ref_map_inv_axis_minus1 ); - EXPECT_TRUE( map_inv_axes_0 == ref_map_inv_axes_0 ); - EXPECT_TRUE( map_inv_axes_1 == ref_map_inv_axes_1 ); - EXPECT_TRUE( map_inv_axes_minus1 == ref_map_inv_axes_minus1 ); - EXPECT_TRUE( map_inv_axes_0_minus1 == ref_map_inv_axes_0_minus1 ); - EXPECT_TRUE( map_inv_axes_minus1_0 == ref_map_inv_axes_minus1_0 ); - EXPECT_TRUE( map_inv_axes_0_1 == ref_map_inv_axes_0_1 ); - EXPECT_TRUE( map_inv_axes_1_0 == ref_map_inv_axes_1_0 ); + EXPECT_TRUE(map_inv_axis_0 == ref_map_inv_axis_0); + EXPECT_TRUE(map_inv_axis_1 == ref_map_inv_axis_1); + EXPECT_TRUE(map_inv_axis_minus1 == ref_map_inv_axis_minus1); + EXPECT_TRUE(map_inv_axes_0 == ref_map_inv_axes_0); + EXPECT_TRUE(map_inv_axes_1 == ref_map_inv_axes_1); + EXPECT_TRUE(map_inv_axes_minus1 == ref_map_inv_axes_minus1); + EXPECT_TRUE(map_inv_axes_0_minus1 == ref_map_inv_axes_0_minus1); + EXPECT_TRUE(map_inv_axes_minus1_0 == ref_map_inv_axes_minus1_0); + EXPECT_TRUE(map_inv_axes_0_1 == ref_map_inv_axes_0_1); + EXPECT_TRUE(map_inv_axes_1_0 == ref_map_inv_axes_1_0); } template @@ -133,27 +141,42 @@ void test_map_axes3d() { using RealView3Dtype = Kokkos::View; RealView3Dtype x("x", n0, n1, n2); - auto [map_axis_0, map_inv_axis_0] = KokkosFFT::Impl::get_map_axes(x, 0); - auto [map_axis_1, map_inv_axis_1] = KokkosFFT::Impl::get_map_axes(x, 1); - auto [map_axis_2, map_inv_axis_2] = KokkosFFT::Impl::get_map_axes(x, 2); - auto [map_axes_0, map_inv_axes_0] = KokkosFFT::Impl::get_map_axes(x, axes_type<1>({0})); - auto [map_axes_1, map_inv_axes_1] = KokkosFFT::Impl::get_map_axes(x, axes_type<1>({1})); - auto [map_axes_2, map_inv_axes_2] = KokkosFFT::Impl::get_map_axes(x, axes_type<1>({2})); - - auto [map_axes_0_1, map_inv_axes_0_1] = KokkosFFT::Impl::get_map_axes(x, axes_type<2>({0, 1})); - auto [map_axes_0_2, map_inv_axes_0_2] = KokkosFFT::Impl::get_map_axes(x, axes_type<2>({0, 2})); - auto [map_axes_1_0, map_inv_axes_1_0] = KokkosFFT::Impl::get_map_axes(x, axes_type<2>({1, 0})); - auto [map_axes_1_2, map_inv_axes_1_2] = KokkosFFT::Impl::get_map_axes(x, axes_type<2>({1, 2})); - auto [map_axes_2_0, map_inv_axes_2_0] = KokkosFFT::Impl::get_map_axes(x, axes_type<2>({2, 0})); - auto [map_axes_2_1, map_inv_axes_2_1] = KokkosFFT::Impl::get_map_axes(x, axes_type<2>({2, 1})); - - auto [map_axes_0_1_2, map_inv_axes_0_1_2] = KokkosFFT::Impl::get_map_axes(x, axes_type<3>({0, 1, 2})); - auto [map_axes_0_2_1, map_inv_axes_0_2_1] = KokkosFFT::Impl::get_map_axes(x, axes_type<3>({0, 2, 1})); - - auto [map_axes_1_0_2, map_inv_axes_1_0_2] = KokkosFFT::Impl::get_map_axes(x, axes_type<3>({1, 0, 2})); - auto [map_axes_1_2_0, map_inv_axes_1_2_0] = KokkosFFT::Impl::get_map_axes(x, axes_type<3>({1, 2, 0})); - auto [map_axes_2_0_1, map_inv_axes_2_0_1] = KokkosFFT::Impl::get_map_axes(x, axes_type<3>({2, 0, 1})); - auto [map_axes_2_1_0, map_inv_axes_2_1_0] = KokkosFFT::Impl::get_map_axes(x, axes_type<3>({2, 1, 0})); + auto [map_axis_0, map_inv_axis_0] = KokkosFFT::Impl::get_map_axes(x, 0); + auto [map_axis_1, map_inv_axis_1] = KokkosFFT::Impl::get_map_axes(x, 1); + auto [map_axis_2, map_inv_axis_2] = KokkosFFT::Impl::get_map_axes(x, 2); + auto [map_axes_0, map_inv_axes_0] = + KokkosFFT::Impl::get_map_axes(x, axes_type<1>({0})); + auto [map_axes_1, map_inv_axes_1] = + KokkosFFT::Impl::get_map_axes(x, axes_type<1>({1})); + auto [map_axes_2, map_inv_axes_2] = + KokkosFFT::Impl::get_map_axes(x, axes_type<1>({2})); + + auto [map_axes_0_1, map_inv_axes_0_1] = + KokkosFFT::Impl::get_map_axes(x, axes_type<2>({0, 1})); + auto [map_axes_0_2, map_inv_axes_0_2] = + KokkosFFT::Impl::get_map_axes(x, axes_type<2>({0, 2})); + auto [map_axes_1_0, map_inv_axes_1_0] = + KokkosFFT::Impl::get_map_axes(x, axes_type<2>({1, 0})); + auto [map_axes_1_2, map_inv_axes_1_2] = + KokkosFFT::Impl::get_map_axes(x, axes_type<2>({1, 2})); + auto [map_axes_2_0, map_inv_axes_2_0] = + KokkosFFT::Impl::get_map_axes(x, axes_type<2>({2, 0})); + auto [map_axes_2_1, map_inv_axes_2_1] = + KokkosFFT::Impl::get_map_axes(x, axes_type<2>({2, 1})); + + auto [map_axes_0_1_2, map_inv_axes_0_1_2] = + KokkosFFT::Impl::get_map_axes(x, axes_type<3>({0, 1, 2})); + auto [map_axes_0_2_1, map_inv_axes_0_2_1] = + KokkosFFT::Impl::get_map_axes(x, axes_type<3>({0, 2, 1})); + + auto [map_axes_1_0_2, map_inv_axes_1_0_2] = + KokkosFFT::Impl::get_map_axes(x, axes_type<3>({1, 0, 2})); + auto [map_axes_1_2_0, map_inv_axes_1_2_0] = + KokkosFFT::Impl::get_map_axes(x, axes_type<3>({1, 2, 0})); + auto [map_axes_2_0_1, map_inv_axes_2_0_1] = + KokkosFFT::Impl::get_map_axes(x, axes_type<3>({2, 0, 1})); + auto [map_axes_2_1_0, map_inv_axes_2_1_0] = + KokkosFFT::Impl::get_map_axes(x, axes_type<3>({2, 1, 0})); axes_type<3> ref_map_axis_0, ref_map_inv_axis_0; axes_type<3> ref_map_axis_1, ref_map_inv_axis_1; @@ -177,7 +200,7 @@ void test_map_axes3d() { axes_type<3> ref_map_axes_2_0_1, ref_map_inv_axes_2_0_1; axes_type<3> ref_map_axes_2_1_0, ref_map_inv_axes_2_1_0; - if(std::is_same_v) { + if (std::is_same_v) { // Layout Left ref_map_axis_0 = {0, 1, 2}, ref_map_inv_axis_0 = {0, 1, 2}; ref_map_axis_1 = {1, 0, 2}, ref_map_inv_axis_1 = {1, 0, 2}; @@ -226,48 +249,48 @@ void test_map_axes3d() { } // Forward mapping - EXPECT_TRUE( map_axis_0 == ref_map_axis_0 ); - EXPECT_TRUE( map_axis_1 == ref_map_axis_1 ); - EXPECT_TRUE( map_axis_2 == ref_map_axis_2 ); - EXPECT_TRUE( map_axes_0 == ref_map_axes_0 ); - EXPECT_TRUE( map_axes_1 == ref_map_axes_1 ); - EXPECT_TRUE( map_axes_2 == ref_map_axes_2 ); - - EXPECT_TRUE( map_axes_0_1 == ref_map_axes_0_1 ); - EXPECT_TRUE( map_axes_0_2 == ref_map_axes_0_2 ); - EXPECT_TRUE( map_axes_1_0 == ref_map_axes_1_0 ); - EXPECT_TRUE( map_axes_1_2 == ref_map_axes_1_2 ); - EXPECT_TRUE( map_axes_2_0 == ref_map_axes_2_0 ); - EXPECT_TRUE( map_axes_2_1 == ref_map_axes_2_1 ); - - EXPECT_TRUE( map_axes_0_1_2 == ref_map_axes_0_1_2 ); - EXPECT_TRUE( map_axes_0_2_1 == ref_map_axes_0_2_1 ); - EXPECT_TRUE( map_axes_1_0_2 == ref_map_axes_1_0_2 ); - EXPECT_TRUE( map_axes_1_2_0 == ref_map_axes_1_2_0 ); - EXPECT_TRUE( map_axes_2_0_1 == ref_map_axes_2_0_1 ); - EXPECT_TRUE( map_axes_2_1_0 == ref_map_axes_2_1_0 ); + EXPECT_TRUE(map_axis_0 == ref_map_axis_0); + EXPECT_TRUE(map_axis_1 == ref_map_axis_1); + EXPECT_TRUE(map_axis_2 == ref_map_axis_2); + EXPECT_TRUE(map_axes_0 == ref_map_axes_0); + EXPECT_TRUE(map_axes_1 == ref_map_axes_1); + EXPECT_TRUE(map_axes_2 == ref_map_axes_2); + + EXPECT_TRUE(map_axes_0_1 == ref_map_axes_0_1); + EXPECT_TRUE(map_axes_0_2 == ref_map_axes_0_2); + EXPECT_TRUE(map_axes_1_0 == ref_map_axes_1_0); + EXPECT_TRUE(map_axes_1_2 == ref_map_axes_1_2); + EXPECT_TRUE(map_axes_2_0 == ref_map_axes_2_0); + EXPECT_TRUE(map_axes_2_1 == ref_map_axes_2_1); + + EXPECT_TRUE(map_axes_0_1_2 == ref_map_axes_0_1_2); + EXPECT_TRUE(map_axes_0_2_1 == ref_map_axes_0_2_1); + EXPECT_TRUE(map_axes_1_0_2 == ref_map_axes_1_0_2); + EXPECT_TRUE(map_axes_1_2_0 == ref_map_axes_1_2_0); + EXPECT_TRUE(map_axes_2_0_1 == ref_map_axes_2_0_1); + EXPECT_TRUE(map_axes_2_1_0 == ref_map_axes_2_1_0); // Inverse mapping - EXPECT_TRUE( map_inv_axis_0 == ref_map_inv_axis_0 ); - EXPECT_TRUE( map_inv_axis_1 == ref_map_inv_axis_1 ); - EXPECT_TRUE( map_inv_axis_2 == ref_map_inv_axis_2 ); - EXPECT_TRUE( map_inv_axes_0 == ref_map_inv_axes_0 ); - EXPECT_TRUE( map_inv_axes_1 == ref_map_inv_axes_1 ); - EXPECT_TRUE( map_inv_axes_2 == ref_map_inv_axes_2 ); - - EXPECT_TRUE( map_inv_axes_0_1 == ref_map_inv_axes_0_1 ); - EXPECT_TRUE( map_inv_axes_0_2 == ref_map_inv_axes_0_2 ); - EXPECT_TRUE( map_inv_axes_1_0 == ref_map_inv_axes_1_0 ); - EXPECT_TRUE( map_inv_axes_1_2 == ref_map_inv_axes_1_2 ); - EXPECT_TRUE( map_inv_axes_2_0 == ref_map_inv_axes_2_0 ); - EXPECT_TRUE( map_inv_axes_2_1 == ref_map_inv_axes_2_1 ); - - EXPECT_TRUE( map_inv_axes_0_1_2 == ref_map_inv_axes_0_1_2 ); - EXPECT_TRUE( map_inv_axes_0_2_1 == ref_map_inv_axes_0_2_1 ); - EXPECT_TRUE( map_inv_axes_1_0_2 == ref_map_inv_axes_1_0_2 ); - EXPECT_TRUE( map_inv_axes_1_2_0 == ref_map_inv_axes_1_2_0 ); - EXPECT_TRUE( map_inv_axes_2_0_1 == ref_map_inv_axes_2_0_1 ); - EXPECT_TRUE( map_inv_axes_2_1_0 == ref_map_inv_axes_2_1_0 ); + EXPECT_TRUE(map_inv_axis_0 == ref_map_inv_axis_0); + EXPECT_TRUE(map_inv_axis_1 == ref_map_inv_axis_1); + EXPECT_TRUE(map_inv_axis_2 == ref_map_inv_axis_2); + EXPECT_TRUE(map_inv_axes_0 == ref_map_inv_axes_0); + EXPECT_TRUE(map_inv_axes_1 == ref_map_inv_axes_1); + EXPECT_TRUE(map_inv_axes_2 == ref_map_inv_axes_2); + + EXPECT_TRUE(map_inv_axes_0_1 == ref_map_inv_axes_0_1); + EXPECT_TRUE(map_inv_axes_0_2 == ref_map_inv_axes_0_2); + EXPECT_TRUE(map_inv_axes_1_0 == ref_map_inv_axes_1_0); + EXPECT_TRUE(map_inv_axes_1_2 == ref_map_inv_axes_1_2); + EXPECT_TRUE(map_inv_axes_2_0 == ref_map_inv_axes_2_0); + EXPECT_TRUE(map_inv_axes_2_1 == ref_map_inv_axes_2_1); + + EXPECT_TRUE(map_inv_axes_0_1_2 == ref_map_inv_axes_0_1_2); + EXPECT_TRUE(map_inv_axes_0_2_1 == ref_map_inv_axes_0_2_1); + EXPECT_TRUE(map_inv_axes_1_0_2 == ref_map_inv_axes_1_0_2); + EXPECT_TRUE(map_inv_axes_1_2_0 == ref_map_inv_axes_1_2_0); + EXPECT_TRUE(map_inv_axes_2_0_1 == ref_map_inv_axes_2_0_1); + EXPECT_TRUE(map_inv_axes_2_1_0 == ref_map_inv_axes_2_1_0); } // Tests for 1D View @@ -307,9 +330,8 @@ void test_transpose_1d() { Kokkos::fence(); EXPECT_THROW( - KokkosFFT::Impl::transpose(execution_space(), x, xt, axes_type<1>({0})), - std::runtime_error - ); + KokkosFFT::Impl::transpose(execution_space(), x, xt, axes_type<1>({0})), + std::runtime_error); } TYPED_TEST(Transpose, 1DView) { @@ -331,19 +353,19 @@ TEST(Transpose, 1DRightView) { TEST(Transpose, 2DLeftView) { const int n0 = 3, n1 = 5; LeftView2D x("x", n0, n1); - LeftView2D xt_axis0, xt_axis1; // views are allocated internally + LeftView2D xt_axis0, xt_axis1; // views are allocated internally LeftView2D ref_axis1("ref_axis1", n1, n0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, 1.0); // Transposed views - auto h_x = Kokkos::create_mirror_view(x); + auto h_x = Kokkos::create_mirror_view(x); auto h_ref_axis1 = Kokkos::create_mirror_view(ref_axis1); Kokkos::deep_copy(h_x, x); - for(int i0=0; i0({0, 1})), // xt is identical to x - std::runtime_error - ); + KokkosFFT::Impl::transpose(execution_space(), x, xt_axis0, + axes_type<2>({0, 1})), // xt is identical to x + std::runtime_error); - KokkosFFT::Impl::transpose(execution_space(), x, xt_axis1, axes_type<2>({1, 0})); // xt is the transpose of x - EXPECT_TRUE( allclose(xt_axis1, ref_axis1, 1.e-5, 1.e-12) ); + KokkosFFT::Impl::transpose(execution_space(), x, xt_axis1, + axes_type<2>({1, 0})); // xt is the transpose of x + EXPECT_TRUE(allclose(xt_axis1, ref_axis1, 1.e-5, 1.e-12)); } TEST(Transpose, 2DRightView) { const int n0 = 3, n1 = 5; RightView2D x("x", n0, n1), ref_axis0("ref_axis0", n1, n0); - RightView2D xt_axis0, xt_axis1; // views are allocated internally + RightView2D xt_axis0, xt_axis1; // views are allocated internally Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, 1.0); // Transposed views - auto h_x = Kokkos::create_mirror_view(x); + auto h_x = Kokkos::create_mirror_view(x); auto h_ref_axis0 = Kokkos::create_mirror_view(ref_axis0); Kokkos::deep_copy(h_x, x); - for(int i0=0; i0({1, 0})); // xt is the transpose of x - EXPECT_TRUE( allclose(xt_axis0, ref_axis0, 1.e-5, 1.e-12) ); + KokkosFFT::Impl::transpose(execution_space(), x, xt_axis0, + axes_type<2>({1, 0})); // xt is the transpose of x + EXPECT_TRUE(allclose(xt_axis0, ref_axis0, 1.e-5, 1.e-12)); EXPECT_THROW( - KokkosFFT::Impl::transpose(execution_space(), x, xt_axis1, axes_type<2>({0, 1})), // xt is identical to x - std::runtime_error - ); + KokkosFFT::Impl::transpose(execution_space(), x, xt_axis1, + axes_type<2>({0, 1})), // xt is identical to x + std::runtime_error); } TEST(Transpose, 3DLeftView) { const int n0 = 3, n1 = 5, n2 = 8; LeftView3D x("x", n0, n1, n2); - LeftView3D xt_axis012, xt_axis021, xt_axis102, xt_axis120, xt_axis201, xt_axis210; // views are allocated internally - LeftView3D ref_axis021("ref_axis021", n0, n2, n1), ref_axis102("ref_axis102", n1, n0, n2); - LeftView3D ref_axis120("ref_axis120", n1, n2, n0), ref_axis201("ref_axis201", n2, n0, n1), ref_axis210("ref_axis210", n2, n1, n0); + LeftView3D xt_axis012, xt_axis021, xt_axis102, xt_axis120, xt_axis201, + xt_axis210; // views are allocated internally + LeftView3D ref_axis021("ref_axis021", n0, n2, n1), + ref_axis102("ref_axis102", n1, n0, n2); + LeftView3D ref_axis120("ref_axis120", n1, n2, n0), + ref_axis201("ref_axis201", n2, n0, n1), + ref_axis210("ref_axis210", n2, n1, n0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, 1.0); // Transposed views - auto h_x = Kokkos::create_mirror_view(x); + auto h_x = Kokkos::create_mirror_view(x); auto h_ref_axis021 = Kokkos::create_mirror_view(ref_axis021); auto h_ref_axis102 = Kokkos::create_mirror_view(ref_axis102); auto h_ref_axis120 = Kokkos::create_mirror_view(ref_axis120); @@ -409,9 +437,9 @@ TEST(Transpose, 3DLeftView) { Kokkos::deep_copy(h_x, x); - for(int i0=0; i0({0, 1, 2})), // xt is identical to x - std::runtime_error - ); - - KokkosFFT::Impl::transpose(execution_space(), x, xt_axis021, axes_type<3>({0, 2, 1})); // xt is the transpose of x - EXPECT_TRUE( allclose(xt_axis021, ref_axis021, 1.e-5, 1.e-12) ); - - KokkosFFT::Impl::transpose(execution_space(), x, xt_axis102, axes_type<3>({1, 0, 2})); // xt is the transpose of x - EXPECT_TRUE( allclose(xt_axis102, ref_axis102, 1.e-5, 1.e-12) ); - - KokkosFFT::Impl::transpose(execution_space(), x, xt_axis120, axes_type<3>({1, 2, 0})); // xt is the transpose of x - EXPECT_TRUE( allclose(xt_axis120, ref_axis120, 1.e-5, 1.e-12) ); - - KokkosFFT::Impl::transpose(execution_space(), x, xt_axis201, axes_type<3>({2, 0, 1})); // xt is the transpose of x - EXPECT_TRUE( allclose(xt_axis201, ref_axis201, 1.e-5, 1.e-12) ); - - KokkosFFT::Impl::transpose(execution_space(), x, xt_axis210, axes_type<3>({2, 1, 0})); // xt is the transpose of x - EXPECT_TRUE( allclose(xt_axis210, ref_axis210, 1.e-5, 1.e-12) ); + EXPECT_THROW(KokkosFFT::Impl::transpose( + execution_space(), x, xt_axis012, + axes_type<3>({0, 1, 2})), // xt is identical to x + std::runtime_error); + + KokkosFFT::Impl::transpose( + execution_space(), x, xt_axis021, + axes_type<3>({0, 2, 1})); // xt is the transpose of x + EXPECT_TRUE(allclose(xt_axis021, ref_axis021, 1.e-5, 1.e-12)); + + KokkosFFT::Impl::transpose( + execution_space(), x, xt_axis102, + axes_type<3>({1, 0, 2})); // xt is the transpose of x + EXPECT_TRUE(allclose(xt_axis102, ref_axis102, 1.e-5, 1.e-12)); + + KokkosFFT::Impl::transpose( + execution_space(), x, xt_axis120, + axes_type<3>({1, 2, 0})); // xt is the transpose of x + EXPECT_TRUE(allclose(xt_axis120, ref_axis120, 1.e-5, 1.e-12)); + + KokkosFFT::Impl::transpose( + execution_space(), x, xt_axis201, + axes_type<3>({2, 0, 1})); // xt is the transpose of x + EXPECT_TRUE(allclose(xt_axis201, ref_axis201, 1.e-5, 1.e-12)); + + KokkosFFT::Impl::transpose( + execution_space(), x, xt_axis210, + axes_type<3>({2, 1, 0})); // xt is the transpose of x + EXPECT_TRUE(allclose(xt_axis210, ref_axis210, 1.e-5, 1.e-12)); } TEST(Transpose, 3DRightView) { const int n0 = 3, n1 = 5, n2 = 8; RightView3D x("x", n0, n1, n2); - RightView3D xt_axis012, xt_axis021, xt_axis102, xt_axis120, xt_axis201, xt_axis210; // views are allocated internally - RightView3D ref_axis021("ref_axis021", n0, n2, n1), ref_axis102("ref_axis102", n1, n0, n2); - RightView3D ref_axis120("ref_axis120", n1, n2, n0), ref_axis201("ref_axis201", n2, n0, n1), ref_axis210("ref_axis210", n2, n1, n0); + RightView3D xt_axis012, xt_axis021, xt_axis102, xt_axis120, + xt_axis201, xt_axis210; // views are allocated internally + RightView3D ref_axis021("ref_axis021", n0, n2, n1), + ref_axis102("ref_axis102", n1, n0, n2); + RightView3D ref_axis120("ref_axis120", n1, n2, n0), + ref_axis201("ref_axis201", n2, n0, n1), + ref_axis210("ref_axis210", n2, n1, n0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, 1.0); // Transposed views - auto h_x = Kokkos::create_mirror_view(x); + auto h_x = Kokkos::create_mirror_view(x); auto h_ref_axis021 = Kokkos::create_mirror_view(ref_axis021); auto h_ref_axis102 = Kokkos::create_mirror_view(ref_axis102); auto h_ref_axis120 = Kokkos::create_mirror_view(ref_axis120); @@ -470,9 +512,9 @@ TEST(Transpose, 3DRightView) { Kokkos::deep_copy(h_x, x); - for(int i0=0; i0({0, 1, 2})), // xt is identical to x - std::runtime_error - ); - - KokkosFFT::Impl::transpose(execution_space(), x, xt_axis021, axes_type<3>({0, 2, 1})); // xt is the transpose of x - EXPECT_TRUE( allclose(xt_axis021, ref_axis021, 1.e-5, 1.e-12) ); - - KokkosFFT::Impl::transpose(execution_space(), x, xt_axis102, axes_type<3>({1, 0, 2})); // xt is the transpose of x - EXPECT_TRUE( allclose(xt_axis102, ref_axis102, 1.e-5, 1.e-12) ); - - KokkosFFT::Impl::transpose(execution_space(), x, xt_axis120, axes_type<3>({1, 2, 0})); // xt is the transpose of x - EXPECT_TRUE( allclose(xt_axis120, ref_axis120, 1.e-5, 1.e-12) ); - - KokkosFFT::Impl::transpose(execution_space(), x, xt_axis201, axes_type<3>({2, 0, 1})); // xt is the transpose of x - EXPECT_TRUE( allclose(xt_axis201, ref_axis201, 1.e-5, 1.e-12) ); - - KokkosFFT::Impl::transpose(execution_space(), x, xt_axis210, axes_type<3>({2, 1, 0})); // xt is the transpose of x - EXPECT_TRUE( allclose(xt_axis210, ref_axis210, 1.e-5, 1.e-12) ); + EXPECT_THROW(KokkosFFT::Impl::transpose( + execution_space(), x, xt_axis012, + axes_type<3>({0, 1, 2})), // xt is identical to x + std::runtime_error); + + KokkosFFT::Impl::transpose( + execution_space(), x, xt_axis021, + axes_type<3>({0, 2, 1})); // xt is the transpose of x + EXPECT_TRUE(allclose(xt_axis021, ref_axis021, 1.e-5, 1.e-12)); + + KokkosFFT::Impl::transpose( + execution_space(), x, xt_axis102, + axes_type<3>({1, 0, 2})); // xt is the transpose of x + EXPECT_TRUE(allclose(xt_axis102, ref_axis102, 1.e-5, 1.e-12)); + + KokkosFFT::Impl::transpose( + execution_space(), x, xt_axis120, + axes_type<3>({1, 2, 0})); // xt is the transpose of x + EXPECT_TRUE(allclose(xt_axis120, ref_axis120, 1.e-5, 1.e-12)); + + KokkosFFT::Impl::transpose( + execution_space(), x, xt_axis201, + axes_type<3>({2, 0, 1})); // xt is the transpose of x + EXPECT_TRUE(allclose(xt_axis201, ref_axis201, 1.e-5, 1.e-12)); + + KokkosFFT::Impl::transpose( + execution_space(), x, xt_axis210, + axes_type<3>({2, 1, 0})); // xt is the transpose of x + EXPECT_TRUE(allclose(xt_axis210, ref_axis210, 1.e-5, 1.e-12)); } \ No newline at end of file diff --git a/common/unit_test/Test_Types.hpp b/common/unit_test/Test_Types.hpp index 9a3894b8..b973593d 100644 --- a/common/unit_test/Test_Types.hpp +++ b/common/unit_test/Test_Types.hpp @@ -3,16 +3,23 @@ #include using execution_space = Kokkos::DefaultExecutionSpace; -template using View1D = Kokkos::View; -template using View2D = Kokkos::View; -template using View3D = Kokkos::View; +template +using View1D = Kokkos::View; +template +using View2D = Kokkos::View; +template +using View3D = Kokkos::View; // Layout Left -template using LeftView2D = Kokkos::View; -template using LeftView3D = Kokkos::View; +template +using LeftView2D = Kokkos::View; +template +using LeftView3D = Kokkos::View; // Layout Right -template using RightView2D = Kokkos::View; -template using RightView3D = Kokkos::View; +template +using RightView2D = Kokkos::View; +template +using RightView3D = Kokkos::View; #endif \ No newline at end of file diff --git a/common/unit_test/Test_Utils.cpp b/common/unit_test/Test_Utils.cpp index 82cc1e83..c70222e0 100644 --- a/common/unit_test/Test_Utils.cpp +++ b/common/unit_test/Test_Utils.cpp @@ -2,10 +2,7 @@ #include "KokkosFFT_utils.hpp" #include "Test_Types.hpp" -using test_types = ::testing::Types< - Kokkos::LayoutLeft, - Kokkos::LayoutRight ->; +using test_types = ::testing::Types; // Basically the same fixtures, used for labeling tests template @@ -24,18 +21,19 @@ TYPED_TEST_SUITE(ConvertNegativeShift, test_types); // Tests for convert_negative_axes over ND views template void test_convert_negative_axes_1d() { - const int len = 30; + const int len = 30; using RealView1Dtype = Kokkos::View; RealView1Dtype x("x", len); int converted_axis_0 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/0); - int converted_axis_minus1 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-1); + int converted_axis_minus1 = + KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-1); - int ref_converted_axis_0 = 0; + int ref_converted_axis_0 = 0; int ref_converted_axis_minus1 = 0; - EXPECT_EQ( converted_axis_0, ref_converted_axis_0 ); - EXPECT_EQ( converted_axis_minus1, ref_converted_axis_minus1 ); + EXPECT_EQ(converted_axis_0, ref_converted_axis_0); + EXPECT_EQ(converted_axis_minus1, ref_converted_axis_minus1); } template @@ -44,17 +42,18 @@ void test_convert_negative_axes_2d() { using RealView2Dtype = Kokkos::View; RealView2Dtype x("x", n0, n1); - int converted_axis_0 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/0); - int converted_axis_1 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/1); - int converted_axis_minus1 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-1); + int converted_axis_0 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/0); + int converted_axis_1 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/1); + int converted_axis_minus1 = + KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-1); int ref_converted_axis_0 = 0; int ref_converted_axis_1 = 1; int ref_converted_axis_minus1 = 1; - EXPECT_EQ( converted_axis_0, ref_converted_axis_0 ); - EXPECT_EQ( converted_axis_1, ref_converted_axis_1 ); - EXPECT_EQ( converted_axis_minus1, ref_converted_axis_minus1 ); + EXPECT_EQ(converted_axis_0, ref_converted_axis_0); + EXPECT_EQ(converted_axis_1, ref_converted_axis_1); + EXPECT_EQ(converted_axis_minus1, ref_converted_axis_minus1); } template @@ -63,11 +62,13 @@ void test_convert_negative_axes_3d() { using RealView3Dtype = Kokkos::View; RealView3Dtype x("x", n0, n1, n2); - int converted_axis_0 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/0); - int converted_axis_1 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/1); - int converted_axis_2 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/2); - int converted_axis_minus1 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-1); - int converted_axis_minus2 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-2); + int converted_axis_0 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/0); + int converted_axis_1 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/1); + int converted_axis_2 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/2); + int converted_axis_minus1 = + KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-1); + int converted_axis_minus2 = + KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-2); int ref_converted_axis_0 = 0; int ref_converted_axis_1 = 1; @@ -75,11 +76,11 @@ void test_convert_negative_axes_3d() { int ref_converted_axis_minus1 = 2; int ref_converted_axis_minus2 = 1; - EXPECT_EQ( converted_axis_0, ref_converted_axis_0 ); - EXPECT_EQ( converted_axis_1, ref_converted_axis_1 ); - EXPECT_EQ( converted_axis_2, ref_converted_axis_2 ); - EXPECT_EQ( converted_axis_minus1, ref_converted_axis_minus1 ); - EXPECT_EQ( converted_axis_minus2, ref_converted_axis_minus2 ); + EXPECT_EQ(converted_axis_0, ref_converted_axis_0); + EXPECT_EQ(converted_axis_1, ref_converted_axis_1); + EXPECT_EQ(converted_axis_2, ref_converted_axis_2); + EXPECT_EQ(converted_axis_minus1, ref_converted_axis_minus1); + EXPECT_EQ(converted_axis_minus2, ref_converted_axis_minus2); } template @@ -88,13 +89,16 @@ void test_convert_negative_axes_4d() { using RealView4Dtype = Kokkos::View; RealView4Dtype x("x", n0, n1, n2, n3); - int converted_axis_0 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/0); - int converted_axis_1 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/1); - int converted_axis_2 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/2); - int converted_axis_3 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/3); - int converted_axis_minus1 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-1); - int converted_axis_minus2 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-2); - int converted_axis_minus3 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-3); + int converted_axis_0 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/0); + int converted_axis_1 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/1); + int converted_axis_2 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/2); + int converted_axis_3 = KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/3); + int converted_axis_minus1 = + KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-1); + int converted_axis_minus2 = + KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-2); + int converted_axis_minus3 = + KokkosFFT::Impl::convert_negative_axis(x, /*axis=*/-3); int ref_converted_axis_0 = 0; int ref_converted_axis_1 = 1; @@ -104,13 +108,13 @@ void test_convert_negative_axes_4d() { int ref_converted_axis_minus2 = 2; int ref_converted_axis_minus3 = 1; - EXPECT_EQ( converted_axis_0, ref_converted_axis_0 ); - EXPECT_EQ( converted_axis_1, ref_converted_axis_1 ); - EXPECT_EQ( converted_axis_2, ref_converted_axis_2 ); - EXPECT_EQ( converted_axis_3, ref_converted_axis_3 ); - EXPECT_EQ( converted_axis_minus1, ref_converted_axis_minus1 ); - EXPECT_EQ( converted_axis_minus2, ref_converted_axis_minus2 ); - EXPECT_EQ( converted_axis_minus3, ref_converted_axis_minus3 ); + EXPECT_EQ(converted_axis_0, ref_converted_axis_0); + EXPECT_EQ(converted_axis_1, ref_converted_axis_1); + EXPECT_EQ(converted_axis_2, ref_converted_axis_2); + EXPECT_EQ(converted_axis_3, ref_converted_axis_3); + EXPECT_EQ(converted_axis_minus1, ref_converted_axis_minus1); + EXPECT_EQ(converted_axis_minus2, ref_converted_axis_minus2); + EXPECT_EQ(converted_axis_minus3, ref_converted_axis_minus3); } // Tests for 1D View @@ -145,44 +149,56 @@ TYPED_TEST(ConvertNegativeAxis, 4DView) { template void test_convert_negative_shift_1d() { const int n0_odd = 29, n0_even = 30; - const int shift = 5; + const int shift = 5; using RealView1Dtype = Kokkos::View; RealView1Dtype x_odd("x_odd", n0_odd), x_even("x_even", n0_even); - auto [shift_5_0_odd, shift_5_1_odd, shift_5_2_odd] = KokkosFFT::Impl::convert_negative_shift(x_odd, shift, 0); - auto [shift_5_0_even, shift_5_1_even, shift_5_2_even] = KokkosFFT::Impl::convert_negative_shift(x_even, shift, 0); - auto [shift_0_0_odd, shift_0_1_odd, shift_0_2_odd] = KokkosFFT::Impl::convert_negative_shift(x_odd, 0, 0); - auto [shift_0_0_even, shift_0_1_even, shift_0_2_even] = KokkosFFT::Impl::convert_negative_shift(x_even, 0, 0); - auto [shift_m5_0_odd, shift_m5_1_odd, shift_m5_2_odd] = KokkosFFT::Impl::convert_negative_shift(x_odd, -shift, 0); - auto [shift_m5_0_even, shift_m5_1_even, shift_m5_2_even] = KokkosFFT::Impl::convert_negative_shift(x_even, -shift, 0); - - int ref_shift_5_0_odd = shift, ref_shift_5_1_odd = shift+1, ref_shift_5_2_odd = 0; - int ref_shift_5_0_even = shift, ref_shift_5_1_even = shift, ref_shift_5_2_even = 0; - int ref_shift_0_0_odd = 0, ref_shift_0_1_odd = 0, ref_shift_0_2_odd = n0_odd / 2; - int ref_shift_0_0_even = 0, ref_shift_0_1_even = 0, ref_shift_0_2_even = n0_even / 2; - int ref_shift_m5_0_odd = shift+1, ref_shift_m5_1_odd = shift, ref_shift_m5_2_odd = 0; - int ref_shift_m5_0_even = shift, ref_shift_m5_1_even = shift, ref_shift_m5_2_even = 0; - - EXPECT_EQ( shift_5_0_odd, ref_shift_5_0_odd ); - EXPECT_EQ( shift_5_0_even, ref_shift_5_0_even ); - EXPECT_EQ( shift_0_0_odd, ref_shift_0_0_odd ); - EXPECT_EQ( shift_0_0_even, ref_shift_0_0_even ); - EXPECT_EQ( shift_m5_0_odd, ref_shift_m5_0_odd ); - EXPECT_EQ( shift_m5_0_even, ref_shift_m5_0_even ); - - EXPECT_EQ( shift_5_1_odd, ref_shift_5_1_odd ); - EXPECT_EQ( shift_5_1_even, ref_shift_5_1_even ); - EXPECT_EQ( shift_0_1_odd, ref_shift_0_1_odd ); - EXPECT_EQ( shift_0_1_even, ref_shift_0_1_even ); - EXPECT_EQ( shift_m5_1_odd, ref_shift_m5_1_odd ); - EXPECT_EQ( shift_m5_1_even, ref_shift_m5_1_even ); - - EXPECT_EQ( shift_5_2_odd, ref_shift_5_2_odd ); - EXPECT_EQ( shift_5_2_even, ref_shift_5_2_even ); - EXPECT_EQ( shift_0_2_odd, ref_shift_0_2_odd ); - EXPECT_EQ( shift_0_2_even, ref_shift_0_2_even ); - EXPECT_EQ( shift_m5_2_odd, ref_shift_m5_2_odd ); - EXPECT_EQ( shift_m5_2_even, ref_shift_m5_2_even ); + auto [shift_5_0_odd, shift_5_1_odd, shift_5_2_odd] = + KokkosFFT::Impl::convert_negative_shift(x_odd, shift, 0); + auto [shift_5_0_even, shift_5_1_even, shift_5_2_even] = + KokkosFFT::Impl::convert_negative_shift(x_even, shift, 0); + auto [shift_0_0_odd, shift_0_1_odd, shift_0_2_odd] = + KokkosFFT::Impl::convert_negative_shift(x_odd, 0, 0); + auto [shift_0_0_even, shift_0_1_even, shift_0_2_even] = + KokkosFFT::Impl::convert_negative_shift(x_even, 0, 0); + auto [shift_m5_0_odd, shift_m5_1_odd, shift_m5_2_odd] = + KokkosFFT::Impl::convert_negative_shift(x_odd, -shift, 0); + auto [shift_m5_0_even, shift_m5_1_even, shift_m5_2_even] = + KokkosFFT::Impl::convert_negative_shift(x_even, -shift, 0); + + int ref_shift_5_0_odd = shift, ref_shift_5_1_odd = shift + 1, + ref_shift_5_2_odd = 0; + int ref_shift_5_0_even = shift, ref_shift_5_1_even = shift, + ref_shift_5_2_even = 0; + int ref_shift_0_0_odd = 0, ref_shift_0_1_odd = 0, + ref_shift_0_2_odd = n0_odd / 2; + int ref_shift_0_0_even = 0, ref_shift_0_1_even = 0, + ref_shift_0_2_even = n0_even / 2; + int ref_shift_m5_0_odd = shift + 1, ref_shift_m5_1_odd = shift, + ref_shift_m5_2_odd = 0; + int ref_shift_m5_0_even = shift, ref_shift_m5_1_even = shift, + ref_shift_m5_2_even = 0; + + EXPECT_EQ(shift_5_0_odd, ref_shift_5_0_odd); + EXPECT_EQ(shift_5_0_even, ref_shift_5_0_even); + EXPECT_EQ(shift_0_0_odd, ref_shift_0_0_odd); + EXPECT_EQ(shift_0_0_even, ref_shift_0_0_even); + EXPECT_EQ(shift_m5_0_odd, ref_shift_m5_0_odd); + EXPECT_EQ(shift_m5_0_even, ref_shift_m5_0_even); + + EXPECT_EQ(shift_5_1_odd, ref_shift_5_1_odd); + EXPECT_EQ(shift_5_1_even, ref_shift_5_1_even); + EXPECT_EQ(shift_0_1_odd, ref_shift_0_1_odd); + EXPECT_EQ(shift_0_1_even, ref_shift_0_1_even); + EXPECT_EQ(shift_m5_1_odd, ref_shift_m5_1_odd); + EXPECT_EQ(shift_m5_1_even, ref_shift_m5_1_even); + + EXPECT_EQ(shift_5_2_odd, ref_shift_5_2_odd); + EXPECT_EQ(shift_5_2_even, ref_shift_5_2_even); + EXPECT_EQ(shift_0_2_odd, ref_shift_0_2_odd); + EXPECT_EQ(shift_0_2_even, ref_shift_0_2_even); + EXPECT_EQ(shift_m5_2_odd, ref_shift_m5_2_odd); + EXPECT_EQ(shift_m5_2_even, ref_shift_m5_2_even); } // Tests for 1D View @@ -193,53 +209,47 @@ TYPED_TEST(ConvertNegativeShift, 1DView) { } TEST(IsTransposeNeeded, 1Dto3D) { - std::array map1D ={0}; - EXPECT_FALSE( KokkosFFT::Impl::is_transpose_needed(map1D) ); + std::array map1D = {0}; + EXPECT_FALSE(KokkosFFT::Impl::is_transpose_needed(map1D)); - std::array map2D ={0, 1}, map2D_axis0 = {1, 0}; - EXPECT_FALSE( KokkosFFT::Impl::is_transpose_needed(map2D) ); - EXPECT_TRUE( KokkosFFT::Impl::is_transpose_needed(map2D_axis0) ); + std::array map2D = {0, 1}, map2D_axis0 = {1, 0}; + EXPECT_FALSE(KokkosFFT::Impl::is_transpose_needed(map2D)); + EXPECT_TRUE(KokkosFFT::Impl::is_transpose_needed(map2D_axis0)); - std::array map3D = {0, 1, 2}; + std::array map3D = {0, 1, 2}; std::array map3D_021 = {0, 2, 1}; std::array map3D_102 = {1, 0, 2}; std::array map3D_120 = {1, 2, 0}; std::array map3D_201 = {2, 0, 1}; std::array map3D_210 = {2, 1, 0}; - EXPECT_FALSE( KokkosFFT::Impl::is_transpose_needed(map3D) ); - EXPECT_TRUE( KokkosFFT::Impl::is_transpose_needed(map3D_021) ); - EXPECT_TRUE( KokkosFFT::Impl::is_transpose_needed(map3D_102) ); - EXPECT_TRUE( KokkosFFT::Impl::is_transpose_needed(map3D_120) ); - EXPECT_TRUE( KokkosFFT::Impl::is_transpose_needed(map3D_201) ); - EXPECT_TRUE( KokkosFFT::Impl::is_transpose_needed(map3D_210) ); + EXPECT_FALSE(KokkosFFT::Impl::is_transpose_needed(map3D)); + EXPECT_TRUE(KokkosFFT::Impl::is_transpose_needed(map3D_021)); + EXPECT_TRUE(KokkosFFT::Impl::is_transpose_needed(map3D_102)); + EXPECT_TRUE(KokkosFFT::Impl::is_transpose_needed(map3D_120)); + EXPECT_TRUE(KokkosFFT::Impl::is_transpose_needed(map3D_201)); + EXPECT_TRUE(KokkosFFT::Impl::is_transpose_needed(map3D_210)); } TEST(GetIndex, Vectors) { std::vector v = {0, 1, 4, 2, 3}; - EXPECT_EQ( KokkosFFT::Impl::get_index(v, 0), 0 ); - EXPECT_EQ( KokkosFFT::Impl::get_index(v, 1), 1 ); - EXPECT_EQ( KokkosFFT::Impl::get_index(v, 2), 3 ); - EXPECT_EQ( KokkosFFT::Impl::get_index(v, 3), 4 ); - EXPECT_EQ( KokkosFFT::Impl::get_index(v, 4), 2 ); - - EXPECT_THROW( - KokkosFFT::Impl::get_index(v, -1), - std::runtime_error - ); - - EXPECT_THROW( - KokkosFFT::Impl::get_index(v, 5), - std::runtime_error - ); + EXPECT_EQ(KokkosFFT::Impl::get_index(v, 0), 0); + EXPECT_EQ(KokkosFFT::Impl::get_index(v, 1), 1); + EXPECT_EQ(KokkosFFT::Impl::get_index(v, 2), 3); + EXPECT_EQ(KokkosFFT::Impl::get_index(v, 3), 4); + EXPECT_EQ(KokkosFFT::Impl::get_index(v, 4), 2); + + EXPECT_THROW(KokkosFFT::Impl::get_index(v, -1), std::runtime_error); + + EXPECT_THROW(KokkosFFT::Impl::get_index(v, 5), std::runtime_error); } TEST(IsOutOfRangeValueIncluded, Array) { std::array v = {0, 1, 2, 3}; - EXPECT_TRUE( KokkosFFT::Impl::is_out_of_range_value_included(v, 2) ); - EXPECT_TRUE( KokkosFFT::Impl::is_out_of_range_value_included(v, 3) ); - EXPECT_FALSE( KokkosFFT::Impl::is_out_of_range_value_included(v, 4) ); - EXPECT_FALSE( KokkosFFT::Impl::is_out_of_range_value_included(v, 5) ); + EXPECT_TRUE(KokkosFFT::Impl::is_out_of_range_value_included(v, 2)); + EXPECT_TRUE(KokkosFFT::Impl::is_out_of_range_value_included(v, 3)); + EXPECT_FALSE(KokkosFFT::Impl::is_out_of_range_value_included(v, 4)); + EXPECT_FALSE(KokkosFFT::Impl::is_out_of_range_value_included(v, 5)); } \ No newline at end of file diff --git a/common/unit_test/Test_Utils.hpp b/common/unit_test/Test_Utils.hpp index b7ee99bb..e3ddeba4 100644 --- a/common/unit_test/Test_Utils.hpp +++ b/common/unit_test/Test_Utils.hpp @@ -4,14 +4,11 @@ #include "Test_Types.hpp" template -bool allclose(const ViewType& a, - const ViewType& b, - double rtol=1.e-5, - double atol=1.e-8 - ) { +bool allclose(const ViewType& a, const ViewType& b, double rtol = 1.e-5, + double atol = 1.e-8) { constexpr std::size_t rank = ViewType::rank; - for(std::size_t i=0; i>{0, n}, - KOKKOS_LAMBDA (const int& i, int& err) { - auto tmp_a = ptr_a[i]; - auto tmp_b = ptr_b[i]; - bool not_close = Kokkos::abs(tmp_a - tmp_b) > (atol + rtol * Kokkos::abs(tmp_b)); - err += static_cast(not_close); - }, error); + Kokkos::RangePolicy>{0, + n}, + KOKKOS_LAMBDA(const int& i, int& err) { + auto tmp_a = ptr_a[i]; + auto tmp_b = ptr_b[i]; + bool not_close = + Kokkos::abs(tmp_a - tmp_b) > (atol + rtol * Kokkos::abs(tmp_b)); + err += static_cast(not_close); + }, + error); return error == 0; } @@ -34,20 +34,18 @@ bool allclose(const ViewType& a, template void multiply(ViewType& x, T a) { const auto n = x.size(); - auto* ptr_x = x.data(); + auto* ptr_x = x.data(); Kokkos::parallel_for( - Kokkos::RangePolicy>{0, n}, - KOKKOS_LAMBDA (const int& i) { - ptr_x[i] = ptr_x[i] * a; - } - ); + Kokkos::RangePolicy>{0, + n}, + KOKKOS_LAMBDA(const int& i) { ptr_x[i] = ptr_x[i] * a; }); } template void display(std::string name, std::vector& values) { std::cout << name << std::endl; - for(auto value: values) { + for (auto value : values) { std::cout << value << std::endl; } } From 5efac33532f395a1419624543d7c29eca0f63f5e Mon Sep 17 00:00:00 2001 From: Yuuichi Asahi Date: Mon, 8 Jan 2024 19:18:09 +0900 Subject: [PATCH 3/5] Apply format to fft/ --- fft/src/KokkosFFT_Cuda_plans.hpp | 357 ++--- fft/src/KokkosFFT_Cuda_transform.hpp | 80 +- fft/src/KokkosFFT_HIP_plans.hpp | 369 ++--- fft/src/KokkosFFT_HIP_transform.hpp | 80 +- fft/src/KokkosFFT_Helpers.hpp | 380 ++--- fft/src/KokkosFFT_OpenMP_plans.hpp | 422 +++--- fft/src/KokkosFFT_OpenMP_transform.hpp | 68 +- fft/src/KokkosFFT_Plans.hpp | 211 +-- fft/src/KokkosFFT_Transform.hpp | 1765 ++++++++++++------------ fft/unit_test/Test_Helpers.cpp | 314 +++-- fft/unit_test/Test_Main.cpp | 6 +- fft/unit_test/Test_Plans.cpp | 346 +++-- fft/unit_test/Test_Transform.cpp | 1548 ++++++++++++--------- fft/unit_test/Test_Types.hpp | 21 +- fft/unit_test/Test_Utils.hpp | 42 +- 15 files changed, 3218 insertions(+), 2791 deletions(-) diff --git a/fft/src/KokkosFFT_Cuda_plans.hpp b/fft/src/KokkosFFT_Cuda_plans.hpp index 2977ca2c..e78dd43c 100644 --- a/fft/src/KokkosFFT_Cuda_plans.hpp +++ b/fft/src/KokkosFFT_Cuda_plans.hpp @@ -7,183 +7,204 @@ namespace KokkosFFT { namespace Impl { - // 1D transform - template , std::nullptr_t> = nullptr> - auto _create(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, const OutViewType& out, [[maybe_unused]] Direction direction) { - static_assert(Kokkos::is_view::value, +// 1D transform +template , + std::nullptr_t> = nullptr> +auto _create(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, const OutViewType& out, + [[maybe_unused]] Direction direction) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - cufftResult cufft_rt = cufftCreate(&plan); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftCreate failed"); - - const int batch = 1; - const int axis = 0; - - auto type = KokkosFFT::Impl::transform_type::type(); - auto [in_extents, out_extents, fft_extents] = KokkosFFT::Impl::get_extents(in, out, axis); - const int nx = fft_extents.at(0); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - - cufft_rt = cufftPlan1d(&plan, nx, type, batch); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftPlan1d failed"); - return fft_size; - } - - // 2D transform - template , std::nullptr_t> = nullptr> - auto _create(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, const OutViewType& out, [[maybe_unused]] Direction direction) { - static_assert(Kokkos::is_view::value, + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + cufftResult cufft_rt = cufftCreate(&plan); + if (cufft_rt != CUFFT_SUCCESS) throw std::runtime_error("cufftCreate failed"); + + const int batch = 1; + const int axis = 0; + + auto type = KokkosFFT::Impl::transform_type::type(); + auto [in_extents, out_extents, fft_extents] = + KokkosFFT::Impl::get_extents(in, out, axis); + const int nx = fft_extents.at(0); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + + cufft_rt = cufftPlan1d(&plan, nx, type, batch); + if (cufft_rt != CUFFT_SUCCESS) throw std::runtime_error("cufftPlan1d failed"); + return fft_size; +} + +// 2D transform +template , + std::nullptr_t> = nullptr> +auto _create(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, const OutViewType& out, + [[maybe_unused]] Direction direction) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - cufftResult cufft_rt = cufftCreate(&plan); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftCreate failed"); - - const int axis = 0; - auto type = KokkosFFT::Impl::transform_type::type(); - auto [in_extents, out_extents, fft_extents] = KokkosFFT::Impl::get_extents(in, out, axis); - const int nx = fft_extents.at(0), ny = fft_extents.at(1); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - - cufft_rt = cufftPlan2d(&plan, nx, ny, type); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftPlan2d failed"); - return fft_size; - } - - // 3D transform - template , std::nullptr_t> = nullptr> - auto _create(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, const OutViewType& out, [[maybe_unused]] Direction direction) { - static_assert(Kokkos::is_view::value, + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + cufftResult cufft_rt = cufftCreate(&plan); + if (cufft_rt != CUFFT_SUCCESS) throw std::runtime_error("cufftCreate failed"); + + const int axis = 0; + auto type = KokkosFFT::Impl::transform_type::type(); + auto [in_extents, out_extents, fft_extents] = + KokkosFFT::Impl::get_extents(in, out, axis); + const int nx = fft_extents.at(0), ny = fft_extents.at(1); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + + cufft_rt = cufftPlan2d(&plan, nx, ny, type); + if (cufft_rt != CUFFT_SUCCESS) throw std::runtime_error("cufftPlan2d failed"); + return fft_size; +} + +// 3D transform +template , + std::nullptr_t> = nullptr> +auto _create(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, const OutViewType& out, + [[maybe_unused]] Direction direction) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - cufftResult cufft_rt = cufftCreate(&plan); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftCreate failed"); - - const int axis = 0; - - auto type = KokkosFFT::Impl::transform_type::type(); - auto [in_extents, out_extents, fft_extents] = KokkosFFT::Impl::get_extents(in, out, axis); - - const int nx = fft_extents.at(0), ny = fft_extents.at(1), nz = fft_extents.at(2); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - - cufft_rt = cufftPlan3d(&plan, nx, ny, nz, type); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftPlan3d failed"); - return fft_size; - } - - // ND transform - template , std::nullptr_t> = nullptr> - auto _create(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, const OutViewType& out, [[maybe_unused]] Direction direction) { - static_assert(Kokkos::is_view::value, + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + cufftResult cufft_rt = cufftCreate(&plan); + if (cufft_rt != CUFFT_SUCCESS) throw std::runtime_error("cufftCreate failed"); + + const int axis = 0; + + auto type = KokkosFFT::Impl::transform_type::type(); + auto [in_extents, out_extents, fft_extents] = + KokkosFFT::Impl::get_extents(in, out, axis); + + const int nx = fft_extents.at(0), ny = fft_extents.at(1), + nz = fft_extents.at(2); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + + cufft_rt = cufftPlan3d(&plan, nx, ny, nz, type); + if (cufft_rt != CUFFT_SUCCESS) throw std::runtime_error("cufftPlan3d failed"); + return fft_size; +} + +// ND transform +template , + std::nullptr_t> = nullptr> +auto _create(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, const OutViewType& out, + [[maybe_unused]] Direction direction) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - cufftResult cufft_rt = cufftCreate(&plan); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftCreate failed"); - - const int rank = InViewType::rank(); - const int batch = 1; - const int axis = 0; - auto type = KokkosFFT::Impl::transform_type::type(); - auto [in_extents, out_extents, fft_extents] = KokkosFFT::Impl::get_extents(in, out, axis); - int idist = std::accumulate(in_extents.begin(), in_extents.end(), 1, std::multiplies<>()); - int odist = std::accumulate(out_extents.begin(), out_extents.end(), 1, std::multiplies<>()); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - - cufft_rt = cufftPlanMany( - &plan, - rank, - fft_extents.data(), - nullptr, - 1, - idist, - nullptr, - 1, - odist, - type, - batch); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftPlanMany failed"); - return fft_size; - } - - // batched transform, over ND Views - template , std::nullptr_t> = nullptr> - auto _create(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, const OutViewType& out, [[maybe_unused]] Direction direction, axis_type axes) { - static_assert(Kokkos::is_view::value, + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + cufftResult cufft_rt = cufftCreate(&plan); + if (cufft_rt != CUFFT_SUCCESS) throw std::runtime_error("cufftCreate failed"); + + const int rank = InViewType::rank(); + const int batch = 1; + const int axis = 0; + auto type = KokkosFFT::Impl::transform_type::type(); + auto [in_extents, out_extents, fft_extents] = + KokkosFFT::Impl::get_extents(in, out, axis); + int idist = std::accumulate(in_extents.begin(), in_extents.end(), 1, + std::multiplies<>()); + int odist = std::accumulate(out_extents.begin(), out_extents.end(), 1, + std::multiplies<>()); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + + cufft_rt = cufftPlanMany(&plan, rank, fft_extents.data(), nullptr, 1, idist, + nullptr, 1, odist, type, batch); + if (cufft_rt != CUFFT_SUCCESS) + throw std::runtime_error("cufftPlanMany failed"); + return fft_size; +} + +// batched transform, over ND Views +template , + std::nullptr_t> = nullptr> +auto _create(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, const OutViewType& out, + [[maybe_unused]] Direction direction, axis_type axes) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - static_assert(InViewType::rank() >= fft_rank, - "KokkosFFT::_create: Rank of View must be larger than Rank of FFT."); - const int rank = fft_rank; - constexpr auto type = KokkosFFT::Impl::transform_type::type(); - auto [in_extents, out_extents, fft_extents, howmany] = KokkosFFT::Impl::get_extents_batched(in, out, axes); - int idist = std::accumulate(in_extents.begin(), in_extents.end(), 1, std::multiplies<>()); - int odist = std::accumulate(out_extents.begin(), out_extents.end(), 1, std::multiplies<>()); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - - // For the moment, considering the contiguous layout only - int istride = 1, ostride = 1; - - cufftResult cufft_rt = cufftCreate(&plan); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftCreate failed"); - - cufft_rt = cufftPlanMany( - &plan, - rank, - fft_extents.data(), - in_extents.data(), - istride, - idist, - out_extents.data(), - ostride, - odist, - type, - howmany); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftPlanMany failed"); - - return fft_size; - } - - template , std::nullptr_t> = nullptr> - void _destroy(typename KokkosFFT::Impl::FFTPlanType::type& plan) { - cufftDestroy(plan); - } -} // namespace Impl -} // namespace KokkosFFT + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + static_assert( + InViewType::rank() >= fft_rank, + "KokkosFFT::_create: Rank of View must be larger than Rank of FFT."); + const int rank = fft_rank; + constexpr auto type = + KokkosFFT::Impl::transform_type::type(); + auto [in_extents, out_extents, fft_extents, howmany] = + KokkosFFT::Impl::get_extents_batched(in, out, axes); + int idist = std::accumulate(in_extents.begin(), in_extents.end(), 1, + std::multiplies<>()); + int odist = std::accumulate(out_extents.begin(), out_extents.end(), 1, + std::multiplies<>()); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + + // For the moment, considering the contiguous layout only + int istride = 1, ostride = 1; + + cufftResult cufft_rt = cufftCreate(&plan); + if (cufft_rt != CUFFT_SUCCESS) throw std::runtime_error("cufftCreate failed"); + + cufft_rt = + cufftPlanMany(&plan, rank, fft_extents.data(), in_extents.data(), istride, + idist, out_extents.data(), ostride, odist, type, howmany); + if (cufft_rt != CUFFT_SUCCESS) + throw std::runtime_error("cufftPlanMany failed"); + + return fft_size; +} + +template , + std::nullptr_t> = nullptr> +void _destroy( + typename KokkosFFT::Impl::FFTPlanType::type& plan) { + cufftDestroy(plan); +} +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/fft/src/KokkosFFT_Cuda_transform.hpp b/fft/src/KokkosFFT_Cuda_transform.hpp index 8250963e..478e16a0 100644 --- a/fft/src/KokkosFFT_Cuda_transform.hpp +++ b/fft/src/KokkosFFT_Cuda_transform.hpp @@ -5,42 +5,48 @@ namespace KokkosFFT { namespace Impl { - void _exec(cufftHandle plan, cufftReal* idata, cufftComplex* odata, [[maybe_unused]] int direction) { - cufftResult cufft_rt = cufftExecR2C(plan, idata, odata); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftExecR2C failed"); - } - - void _exec(cufftHandle plan, cufftDoubleReal* idata, cufftDoubleComplex* odata, [[maybe_unused]] int direction) { - cufftResult cufft_rt = cufftExecD2Z(plan, idata, odata); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftExecD2Z failed"); - } - - void _exec(cufftHandle plan, cufftComplex* idata, cufftReal* odata, [[maybe_unused]] int direction) { - cufftResult cufft_rt = cufftExecC2R(plan, idata, odata); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftExecC2R failed"); - } - - void _exec(cufftHandle plan, cufftDoubleComplex* idata, cufftDoubleReal* odata, [[maybe_unused]] int direction) { - cufftResult cufft_rt = cufftExecZ2D(plan, idata, odata); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftExecZ2D failed"); - } - - void _exec(cufftHandle plan, cufftComplex* idata, cufftComplex* odata, int direction) { - cufftResult cufft_rt = cufftExecC2C(plan, idata, odata, direction); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftExecC2C failed"); - } - - void _exec(cufftHandle plan, cufftDoubleComplex* idata, cufftDoubleComplex* odata, int direction) { - cufftResult cufft_rt = cufftExecZ2Z(plan, idata, odata, direction); - if(cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftExecZ2Z failed"); - } -} // namespace Impl -} // namespace KokkosFFT +void _exec(cufftHandle plan, cufftReal* idata, cufftComplex* odata, + [[maybe_unused]] int direction) { + cufftResult cufft_rt = cufftExecR2C(plan, idata, odata); + if (cufft_rt != CUFFT_SUCCESS) + throw std::runtime_error("cufftExecR2C failed"); +} + +void _exec(cufftHandle plan, cufftDoubleReal* idata, cufftDoubleComplex* odata, + [[maybe_unused]] int direction) { + cufftResult cufft_rt = cufftExecD2Z(plan, idata, odata); + if (cufft_rt != CUFFT_SUCCESS) + throw std::runtime_error("cufftExecD2Z failed"); +} + +void _exec(cufftHandle plan, cufftComplex* idata, cufftReal* odata, + [[maybe_unused]] int direction) { + cufftResult cufft_rt = cufftExecC2R(plan, idata, odata); + if (cufft_rt != CUFFT_SUCCESS) + throw std::runtime_error("cufftExecC2R failed"); +} + +void _exec(cufftHandle plan, cufftDoubleComplex* idata, cufftDoubleReal* odata, + [[maybe_unused]] int direction) { + cufftResult cufft_rt = cufftExecZ2D(plan, idata, odata); + if (cufft_rt != CUFFT_SUCCESS) + throw std::runtime_error("cufftExecZ2D failed"); +} + +void _exec(cufftHandle plan, cufftComplex* idata, cufftComplex* odata, + int direction) { + cufftResult cufft_rt = cufftExecC2C(plan, idata, odata, direction); + if (cufft_rt != CUFFT_SUCCESS) + throw std::runtime_error("cufftExecC2C failed"); +} + +void _exec(cufftHandle plan, cufftDoubleComplex* idata, + cufftDoubleComplex* odata, int direction) { + cufftResult cufft_rt = cufftExecZ2Z(plan, idata, odata, direction); + if (cufft_rt != CUFFT_SUCCESS) + throw std::runtime_error("cufftExecZ2Z failed"); +} +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/fft/src/KokkosFFT_HIP_plans.hpp b/fft/src/KokkosFFT_HIP_plans.hpp index 0b2a43d6..7d494d32 100644 --- a/fft/src/KokkosFFT_HIP_plans.hpp +++ b/fft/src/KokkosFFT_HIP_plans.hpp @@ -7,184 +7,215 @@ namespace KokkosFFT { namespace Impl { - // 1D transform - template , std::nullptr_t> = nullptr> - auto _create(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, const OutViewType& out, [[maybe_unused]] FFTDirectionType direction) { - static_assert(Kokkos::is_view::value, +// 1D transform +template , + std::nullptr_t> = nullptr> +auto _create(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, const OutViewType& out, + [[maybe_unused]] FFTDirectionType direction) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - hipfftResult hipfft_rt = hipfftCreate(&plan); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftCreate failed"); - - const int batch = 1; - const int axis = 0; - - auto type = KokkosFFT::Impl::transform_type::type(); - auto [in_extents, out_extents, fft_extents] = KokkosFFT::Impl::get_extents(in, out, axis); - const int nx = fft_extents.at(0); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - - hipfft_rt = hipfftPlan1d(&plan, nx, type, batch); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftPlan1d failed"); - return fft_size; - } - - // 2D transform - template , std::nullptr_t> = nullptr> - auto _create(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, const OutViewType& out, [[maybe_unused]] FFTDirectionType direction) { - static_assert(Kokkos::is_view::value, + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + hipfftResult hipfft_rt = hipfftCreate(&plan); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftCreate failed"); + + const int batch = 1; + const int axis = 0; + + auto type = KokkosFFT::Impl::transform_type::type(); + auto [in_extents, out_extents, fft_extents] = + KokkosFFT::Impl::get_extents(in, out, axis); + const int nx = fft_extents.at(0); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + + hipfft_rt = hipfftPlan1d(&plan, nx, type, batch); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftPlan1d failed"); + return fft_size; +} + +// 2D transform +template , + std::nullptr_t> = nullptr> +auto _create(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, const OutViewType& out, + [[maybe_unused]] FFTDirectionType direction) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - hipfftResult hipfft_rt = hipfftCreate(&plan); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftCreate failed"); - - const int axis = 0; - auto type = KokkosFFT::Impl::transform_type::type(); - auto [in_extents, out_extents, fft_extents] = KokkosFFT::Impl::get_extents(in, out, axis); - const int nx = fft_extents.at(0), ny = fft_extents.at(1); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - - hipfft_rt = hipfftPlan2d(&plan, nx, ny, type); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftPlan2d failed"); - return fft_size; - } - - // 3D transform - template , std::nullptr_t> = nullptr> - auto _create(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, const OutViewType& out, [[maybe_unused]] FFTDirectionType direction) { - static_assert(Kokkos::is_view::value, + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + hipfftResult hipfft_rt = hipfftCreate(&plan); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftCreate failed"); + + const int axis = 0; + auto type = KokkosFFT::Impl::transform_type::type(); + auto [in_extents, out_extents, fft_extents] = + KokkosFFT::Impl::get_extents(in, out, axis); + const int nx = fft_extents.at(0), ny = fft_extents.at(1); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + + hipfft_rt = hipfftPlan2d(&plan, nx, ny, type); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftPlan2d failed"); + return fft_size; +} + +// 3D transform +template , + std::nullptr_t> = nullptr> +auto _create(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, const OutViewType& out, + [[maybe_unused]] FFTDirectionType direction) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - hipfftResult hipfft_rt = hipfftCreate(&plan); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftCreate failed"); - - const int batch = 1; - const int axis = 0; - - auto type = KokkosFFT::Impl::transform_type::type(); - auto [in_extents, out_extents, fft_extents] = KokkosFFT::Impl::get_extents(in, out, axis); - - const int nx = fft_extents.at(0), ny = fft_extents.at(1), nz = fft_extents.at(2); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - - hipfft_rt = hipfftPlan3d(&plan, nx, ny, nz, type); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftPlan3d failed"); - return fft_size; - } - - // ND transform - template , std::nullptr_t> = nullptr> - auto _create(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, const OutViewType& out, [[maybe_unused]] FFTDirectionType direction) { - static_assert(Kokkos::is_view::value, + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + hipfftResult hipfft_rt = hipfftCreate(&plan); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftCreate failed"); + + const int batch = 1; + const int axis = 0; + + auto type = KokkosFFT::Impl::transform_type::type(); + auto [in_extents, out_extents, fft_extents] = + KokkosFFT::Impl::get_extents(in, out, axis); + + const int nx = fft_extents.at(0), ny = fft_extents.at(1), + nz = fft_extents.at(2); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + + hipfft_rt = hipfftPlan3d(&plan, nx, ny, nz, type); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftPlan3d failed"); + return fft_size; +} + +// ND transform +template , + std::nullptr_t> = nullptr> +auto _create(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, const OutViewType& out, + [[maybe_unused]] FFTDirectionType direction) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - hipfftResult hipfft_rt = hipfftCreate(&plan); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftCreate failed"); - - const int rank = InViewType::rank(); - const int batch = 1; - const int axis = 0; - auto type = KokkosFFT::Impl::transform_type::type(); - auto [in_extents, out_extents, fft_extents] = KokkosFFT::Impl::get_extents(in, out, axis); - int idist = std::accumulate(in_extents.begin(), in_extents.end(), 1, std::multiplies<>()); - int odist = std::accumulate(out_extents.begin(), out_extents.end(), 1, std::multiplies<>()); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - - hipfft_rt = hipfftPlanMany( - &plan, - rank, - fft_extents.data(), - nullptr, - 1, - idist, - nullptr, - 1, - odist, - type, - batch); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftPlanMany failed"); - return fft_size; - } - - // batched transform, over ND Views - template , std::nullptr_t> = nullptr> - auto _create(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, const OutViewType& out, [[maybe_unused]] FFTDirectionType direction, axis_type axes) { - static_assert(Kokkos::is_view::value, + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + hipfftResult hipfft_rt = hipfftCreate(&plan); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftCreate failed"); + + const int rank = InViewType::rank(); + const int batch = 1; + const int axis = 0; + auto type = KokkosFFT::Impl::transform_type::type(); + auto [in_extents, out_extents, fft_extents] = + KokkosFFT::Impl::get_extents(in, out, axis); + int idist = std::accumulate(in_extents.begin(), in_extents.end(), 1, + std::multiplies<>()); + int odist = std::accumulate(out_extents.begin(), out_extents.end(), 1, + std::multiplies<>()); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + + hipfft_rt = hipfftPlanMany(&plan, rank, fft_extents.data(), nullptr, 1, idist, + nullptr, 1, odist, type, batch); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftPlanMany failed"); + return fft_size; +} + +// batched transform, over ND Views +template , + std::nullptr_t> = nullptr> +auto _create(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, const OutViewType& out, + [[maybe_unused]] FFTDirectionType direction, + axis_type axes) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - static_assert(InViewType::rank() >= fft_rank, - "KokkosFFT::_create: Rank of View must be larger than Rank of FFT."); - const int rank = fft_rank; - constexpr auto type = KokkosFFT::Impl::transform_type::type(); - auto [in_extents, out_extents, fft_extents, howmany] = KokkosFFT::Impl::get_extents_batched(in, out, axes); - int idist = std::accumulate(in_extents.begin(), in_extents.end(), 1, std::multiplies<>()); - int odist = std::accumulate(out_extents.begin(), out_extents.end(), 1, std::multiplies<>()); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - - // For the moment, considering the contiguous layout only - int istride = 1, ostride = 1; - - hipfftResult hipfft_rt = hipfftCreate(&plan); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftCreate failed"); - - hipfft_rt = hipfftPlanMany( - &plan, - rank, - fft_extents.data(), - in_extents.data(), - istride, - idist, - out_extents.data(), - ostride, - odist, - type, - howmany); - - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftPlan failed"); - return fft_size; - } - - template , std::nullptr_t> = nullptr> - void _destroy(typename KokkosFFT::Impl::FFTPlanType::type& plan) { - hipfftDestroy(plan); - } -} // namespace Impl -} // namespace KokkosFFT + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + static_assert( + InViewType::rank() >= fft_rank, + "KokkosFFT::_create: Rank of View must be larger than Rank of FFT."); + const int rank = fft_rank; + constexpr auto type = + KokkosFFT::Impl::transform_type::type(); + auto [in_extents, out_extents, fft_extents, howmany] = + KokkosFFT::Impl::get_extents_batched(in, out, axes); + int idist = std::accumulate(in_extents.begin(), in_extents.end(), 1, + std::multiplies<>()); + int odist = std::accumulate(out_extents.begin(), out_extents.end(), 1, + std::multiplies<>()); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + + // For the moment, considering the contiguous layout only + int istride = 1, ostride = 1; + + hipfftResult hipfft_rt = hipfftCreate(&plan); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftCreate failed"); + + hipfft_rt = hipfftPlanMany(&plan, rank, fft_extents.data(), in_extents.data(), + istride, idist, out_extents.data(), ostride, odist, + type, howmany); + + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftPlan failed"); + return fft_size; +} + +template , + std::nullptr_t> = nullptr> +void _destroy( + typename KokkosFFT::Impl::FFTPlanType::type& plan) { + hipfftDestroy(plan); +} +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/fft/src/KokkosFFT_HIP_transform.hpp b/fft/src/KokkosFFT_HIP_transform.hpp index 220ab500..dd63fbbb 100644 --- a/fft/src/KokkosFFT_HIP_transform.hpp +++ b/fft/src/KokkosFFT_HIP_transform.hpp @@ -5,42 +5,48 @@ namespace KokkosFFT { namespace Impl { - void _exec(hipfftHandle plan, hipfftReal* idata, hipfftComplex* odata, [[maybe_unused]] int direction) { - hipfftResult hipfft_rt = hipfftExecR2C(plan, idata, odata); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftExecR2C failed"); - } - - void _exec(hipfftHandle plan, hipfftDoubleReal* idata, hipfftDoubleComplex* odata, [[maybe_unused]] int direction) { - hipfftResult hipfft_rt = hipfftExecD2Z(plan, idata, odata); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftExecD2Z failed"); - } - - void _exec(hipfftHandle plan, hipfftComplex* idata, hipfftReal* odata, [[maybe_unused]] int direction) { - hipfftResult hipfft_rt = hipfftExecC2R(plan, idata, odata); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftExecC2R failed"); - } - - void _exec(hipfftHandle plan, hipfftDoubleComplex* idata, hipfftDoubleReal* odata, [[maybe_unused]] int direction) { - hipfftResult hipfft_rt = hipfftExecZ2D(plan, idata, odata); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftExecZ2D failed"); - } - - void _exec(hipfftHandle plan, hipfftComplex* idata, hipfftComplex* odata, int direction) { - hipfftResult hipfft_rt = hipfftExecC2C(plan, idata, odata, direction); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftExecC2C failed"); - } - - void _exec(hipfftHandle plan, hipfftDoubleComplex* idata, hipfftDoubleComplex* odata, int direction) { - hipfftResult hipfft_rt = hipfftExecZ2Z(plan, idata, odata, direction); - if(hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftExecZ2Z failed"); - } -} // namespace Impl -} // namespace KokkosFFT +void _exec(hipfftHandle plan, hipfftReal* idata, hipfftComplex* odata, + [[maybe_unused]] int direction) { + hipfftResult hipfft_rt = hipfftExecR2C(plan, idata, odata); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftExecR2C failed"); +} + +void _exec(hipfftHandle plan, hipfftDoubleReal* idata, + hipfftDoubleComplex* odata, [[maybe_unused]] int direction) { + hipfftResult hipfft_rt = hipfftExecD2Z(plan, idata, odata); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftExecD2Z failed"); +} + +void _exec(hipfftHandle plan, hipfftComplex* idata, hipfftReal* odata, + [[maybe_unused]] int direction) { + hipfftResult hipfft_rt = hipfftExecC2R(plan, idata, odata); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftExecC2R failed"); +} + +void _exec(hipfftHandle plan, hipfftDoubleComplex* idata, + hipfftDoubleReal* odata, [[maybe_unused]] int direction) { + hipfftResult hipfft_rt = hipfftExecZ2D(plan, idata, odata); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftExecZ2D failed"); +} + +void _exec(hipfftHandle plan, hipfftComplex* idata, hipfftComplex* odata, + int direction) { + hipfftResult hipfft_rt = hipfftExecC2C(plan, idata, odata, direction); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftExecC2C failed"); +} + +void _exec(hipfftHandle plan, hipfftDoubleComplex* idata, + hipfftDoubleComplex* odata, int direction) { + hipfftResult hipfft_rt = hipfftExecZ2Z(plan, idata, odata, direction); + if (hipfft_rt != HIPFFT_SUCCESS) + throw std::runtime_error("hipfftExecZ2Z failed"); +} +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/fft/src/KokkosFFT_Helpers.hpp b/fft/src/KokkosFFT_Helpers.hpp index 9cc6cdbb..df0ac35f 100644 --- a/fft/src/KokkosFFT_Helpers.hpp +++ b/fft/src/KokkosFFT_Helpers.hpp @@ -7,213 +7,235 @@ namespace KokkosFFT { namespace Impl { - template - auto _get_shift(const ViewType& inout, axis_type _axes, int direction=1) { - static_assert(DIM > 0, - "KokkosFFT::Impl::_get_shift: Rank of shift axes must be larger than or equal to 1."); - - // Convert the input axes to be in the range of [0, rank-1] - std::vector axes; - for(std::size_t i=0; i shift = {0}; - for(int i=0; i +auto _get_shift(const ViewType& inout, axis_type _axes, + int direction = 1) { + static_assert(DIM > 0, + "KokkosFFT::Impl::_get_shift: Rank of shift axes must be " + "larger than or equal to 1."); + + // Convert the input axes to be in the range of [0, rank-1] + std::vector axes; + for (std::size_t i = 0; i < DIM; i++) { + int axis = KokkosFFT::Impl::convert_negative_axis(inout, _axes.at(i)); + axes.push_back(axis); } - template - void _roll(const ExecutionSpace& exec_space, ViewType& inout, axis_type<1> shift, axis_type<1> axes) { - static_assert(ViewType::rank() == 1, - "KokkosFFT::Impl::_roll: Rank of View must be 1."); - std::size_t n0 = inout.extent(0); + // Assert if the elements are overlapped + constexpr int rank = ViewType::rank(); + assert(!KokkosFFT::Impl::has_duplicate_values(axes)); + assert(!KokkosFFT::Impl::is_out_of_range_value_included(axes, rank)); - ViewType tmp("tmp", n0); - std::size_t len = (n0-1) / 2 + 1; - - auto [_shift0, _shift1, _shift2] = KokkosFFT::Impl::convert_negative_shift(inout, shift.at(0), 0); - int shift0 = _shift0, shift1 = _shift1, shift2 = _shift2; - - // shift2 == 0 means shift - if(shift2 == 0) { - Kokkos::parallel_for( - Kokkos::RangePolicy>(exec_space, 0, len), + axis_type shift = {0}; + for (int i = 0; i < DIM; i++) { + int axis = axes.at(i); + shift.at(axis) = inout.extent(axis) / 2 * direction; + } + return shift; +} + +template +void _roll(const ExecutionSpace& exec_space, ViewType& inout, + axis_type<1> shift, axis_type<1> axes) { + static_assert(ViewType::rank() == 1, + "KokkosFFT::Impl::_roll: Rank of View must be 1."); + std::size_t n0 = inout.extent(0); + + ViewType tmp("tmp", n0); + std::size_t len = (n0 - 1) / 2 + 1; + + auto [_shift0, _shift1, _shift2] = + KokkosFFT::Impl::convert_negative_shift(inout, shift.at(0), 0); + int shift0 = _shift0, shift1 = _shift1, shift2 = _shift2; + + // shift2 == 0 means shift + if (shift2 == 0) { + Kokkos::parallel_for( + Kokkos::RangePolicy>( + exec_space, 0, len), KOKKOS_LAMBDA(const int& i) { - tmp(i+shift0) = inout(i); - if(i+shift1 +void _roll(const ExecutionSpace& exec_space, ViewType& inout, + axis_type<2> shift, axis_type axes) { + constexpr std::size_t DIM0 = 2; + static_assert(ViewType::rank() == DIM0, + "KokkosFFT::Impl::_roll: Rank of View must be 2."); + int n0 = inout.extent(0), n1 = inout.extent(1); + + ViewType tmp("tmp", n0, n1); + int len0 = (n0 - 1) / 2 + 1; + int len1 = (n1 - 1) / 2 + 1; + + using range_type = Kokkos::MDRangePolicy< + ExecutionSpace, + Kokkos::Rank<2, Kokkos::Iterate::Default, Kokkos::Iterate::Default>>; + using tile_type = typename range_type::tile_type; + using point_type = typename range_type::point_type; + + range_type range( + point_type{{0, 0}}, point_type{{len0, len1}}, tile_type{{4, 4}} + // [TO DO] Choose optimal tile sizes for each device + ); + + axis_type<2> shift0 = {0}, shift1 = {0}, shift2 = {n0 / 2, n1 / 2}; + for (int i = 0; i < DIM1; i++) { + int axis = axes.at(i); + + auto [_shift0, _shift1, _shift2] = + KokkosFFT::Impl::convert_negative_shift(inout, shift.at(axis), axis); + shift0.at(axis) = _shift0; + shift1.at(axis) = _shift1; + shift2.at(axis) = _shift2; } - template - void _roll(const ExecutionSpace& exec_space, ViewType& inout, axis_type<2> shift, axis_type axes) { - constexpr std::size_t DIM0 = 2; - static_assert(ViewType::rank() == DIM0, - "KokkosFFT::Impl::_roll: Rank of View must be 2."); - int n0 = inout.extent(0), n1 = inout.extent(1); - - ViewType tmp("tmp", n0, n1); - int len0 = (n0-1) / 2 + 1; - int len1 = (n1-1) / 2 + 1; - - using range_type = Kokkos::MDRangePolicy >; - using tile_type = typename range_type::tile_type; - using point_type = typename range_type::point_type; - - range_type range( - point_type{{0, 0}}, - point_type{{len0, len1}}, - tile_type{{4, 4}} // [TO DO] Choose optimal tile sizes for each device - ); - - axis_type<2> shift0 = {0}, shift1 = {0}, shift2 = {n0/2, n1/2}; - for(int i=0; i - void _fftshift(const ExecutionSpace& exec_space, ViewType& inout, axis_type axes) { - static_assert(ViewType::rank() >= DIM, - "KokkosFFT::Impl::_fftshift: Rank of View must be larger thane or equal to the Rank of shift axes."); - auto shift = _get_shift(inout, axes); - _roll(exec_space, inout, shift, axes); - } - - template - void _ifftshift(const ExecutionSpace& exec_space, ViewType& inout, axis_type axes) { - static_assert(ViewType::rank() >= DIM, - "KokkosFFT::Impl::_ifftshift: Rank of View must be larger thane or equal to the Rank of shift axes."); - auto shift = _get_shift(inout, axes, -1); - _roll(exec_space, inout, shift, axes); - } -} // namespace Impl -} // namespace KokkosFFT + if (i0 + shift_00 < n0 && i1 + shift_11 < n1) { + tmp(i0 + shift_00 + shift_02, i1 + shift_12) = + inout(i0 + shift_02, i1 + shift_11 + shift_12); + } + }); + + inout = tmp; +} + +template +void _fftshift(const ExecutionSpace& exec_space, ViewType& inout, + axis_type axes) { + static_assert(ViewType::rank() >= DIM, + "KokkosFFT::Impl::_fftshift: Rank of View must be larger thane " + "or equal to the Rank of shift axes."); + auto shift = _get_shift(inout, axes); + _roll(exec_space, inout, shift, axes); +} + +template +void _ifftshift(const ExecutionSpace& exec_space, ViewType& inout, + axis_type axes) { + static_assert(ViewType::rank() >= DIM, + "KokkosFFT::Impl::_ifftshift: Rank of View must be larger " + "thane or equal to the Rank of shift axes."); + auto shift = _get_shift(inout, axes, -1); + _roll(exec_space, inout, shift, axes); +} +} // namespace Impl +} // namespace KokkosFFT namespace KokkosFFT { - template - auto fftfreq(const ExecutionSpace& exec_space,const std::size_t n, const RealType d = 1.0) { - static_assert(std::is_floating_point::value, - "KokkosFFT::fftfreq: d must be real"); - using ViewType = Kokkos::View; - ViewType freq("freq", n); - - RealType val = 1.0 / ( static_cast(n) * d ); - int N1 = (n - 1) / 2 + 1; - int N2 = n/2; +template +auto fftfreq(const ExecutionSpace& exec_space, const std::size_t n, + const RealType d = 1.0) { + static_assert(std::is_floating_point::value, + "KokkosFFT::fftfreq: d must be real"); + using ViewType = Kokkos::View; + ViewType freq("freq", n); - auto h_freq = Kokkos::create_mirror_view(freq); + RealType val = 1.0 / (static_cast(n) * d); + int N1 = (n - 1) / 2 + 1; + int N2 = n / 2; - auto p1 = KokkosFFT::Impl::arange(0, N1); - auto p2 = KokkosFFT::Impl::arange(-N2, 0); + auto h_freq = Kokkos::create_mirror_view(freq); - for(int i=0; i( p1.at(i) ) * val; } - for(int i=0; i( p2.at(i) ) * val; } - Kokkos::deep_copy(freq, h_freq); + auto p1 = KokkosFFT::Impl::arange(0, N1); + auto p2 = KokkosFFT::Impl::arange(-N2, 0); - return freq; + for (int i = 0; i < N1; i++) { + h_freq(i) = static_cast(p1.at(i)) * val; } - - template - auto rfftfreq(const ExecutionSpace& exec_space,const std::size_t n, const RealType d = 1.0) { - static_assert(std::is_floating_point::value, - "KokkosFFT::fftfreq: d must be real"); - using ViewType = Kokkos::View; - - RealType val = 1.0 / ( static_cast(n) * d ); - int N = n/2 + 1; - ViewType freq("freq", N); - - auto h_freq = Kokkos::create_mirror_view(freq); - auto p = KokkosFFT::Impl::arange(0, N); - - for(int i=0; i( p.at(i) ) * val; } - Kokkos::deep_copy(freq, h_freq); - - return freq; + for (int i = 0; i < N2; i++) { + h_freq(i + N1) = static_cast(p2.at(i)) * val; } + Kokkos::deep_copy(freq, h_freq); - template - void fftshift(const ExecutionSpace& exec_space, ViewType& inout) { - constexpr std::size_t rank = ViewType::rank(); - constexpr int start = -static_cast(rank); - axis_type axes = KokkosFFT::Impl::index_sequence(start); - KokkosFFT::Impl::_fftshift(exec_space, inout, axes); - } - - template - void fftshift(const ExecutionSpace& exec_space, ViewType& inout, int axes) { - KokkosFFT::Impl::_fftshift(exec_space, inout, axis_type<1>{axes}); - } + return freq; +} - template - void fftshift(const ExecutionSpace& exec_space, ViewType& inout, axis_type axes) { - KokkosFFT::Impl::_fftshift(exec_space, inout, axes); - } +template +auto rfftfreq(const ExecutionSpace& exec_space, const std::size_t n, + const RealType d = 1.0) { + static_assert(std::is_floating_point::value, + "KokkosFFT::fftfreq: d must be real"); + using ViewType = Kokkos::View; - template - void ifftshift(const ExecutionSpace& exec_space, ViewType& inout) { - constexpr std::size_t rank = ViewType::rank(); - constexpr int start = -static_cast(rank); - axis_type axes = KokkosFFT::Impl::index_sequence(start); - KokkosFFT::Impl::_ifftshift(exec_space, inout, axes); - } + RealType val = 1.0 / (static_cast(n) * d); + int N = n / 2 + 1; + ViewType freq("freq", N); - template - void ifftshift(const ExecutionSpace& exec_space, ViewType& inout, int axes) { - KokkosFFT::Impl::_ifftshift(exec_space, inout, axis_type<1>{axes}); - } + auto h_freq = Kokkos::create_mirror_view(freq); + auto p = KokkosFFT::Impl::arange(0, N); - template - void ifftshift(const ExecutionSpace& exec_space, ViewType& inout, axis_type axes) { - KokkosFFT::Impl::_ifftshift(exec_space, inout, axes); + for (int i = 0; i < N; i++) { + h_freq(i) = static_cast(p.at(i)) * val; } -} // namespace KokkosFFT + Kokkos::deep_copy(freq, h_freq); + + return freq; +} + +template +void fftshift(const ExecutionSpace& exec_space, ViewType& inout) { + constexpr std::size_t rank = ViewType::rank(); + constexpr int start = -static_cast(rank); + axis_type axes = KokkosFFT::Impl::index_sequence(start); + KokkosFFT::Impl::_fftshift(exec_space, inout, axes); +} + +template +void fftshift(const ExecutionSpace& exec_space, ViewType& inout, int axes) { + KokkosFFT::Impl::_fftshift(exec_space, inout, axis_type<1>{axes}); +} + +template +void fftshift(const ExecutionSpace& exec_space, ViewType& inout, + axis_type axes) { + KokkosFFT::Impl::_fftshift(exec_space, inout, axes); +} + +template +void ifftshift(const ExecutionSpace& exec_space, ViewType& inout) { + constexpr std::size_t rank = ViewType::rank(); + constexpr int start = -static_cast(rank); + axis_type axes = KokkosFFT::Impl::index_sequence(start); + KokkosFFT::Impl::_ifftshift(exec_space, inout, axes); +} + +template +void ifftshift(const ExecutionSpace& exec_space, ViewType& inout, int axes) { + KokkosFFT::Impl::_ifftshift(exec_space, inout, axis_type<1>{axes}); +} + +template +void ifftshift(const ExecutionSpace& exec_space, ViewType& inout, + axis_type axes) { + KokkosFFT::Impl::_ifftshift(exec_space, inout, axes); +} +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/fft/src/KokkosFFT_OpenMP_plans.hpp b/fft/src/KokkosFFT_OpenMP_plans.hpp index 18519cb0..4f62aa80 100644 --- a/fft/src/KokkosFFT_OpenMP_plans.hpp +++ b/fft/src/KokkosFFT_OpenMP_plans.hpp @@ -7,270 +7,178 @@ namespace KokkosFFT { namespace Impl { - template - void _init_threads(const ExecutionSpace& exec_space) { - #if defined(KOKKOS_ENABLE_OPENMP) || defined(KOKKOS_ENABLE_THREADS) - int nthreads = exec_space.concurrency(); - - if constexpr (std::is_same_v) { - fftwf_init_threads(); - fftwf_plan_with_nthreads(nthreads); - } else { - fftw_init_threads(); - fftw_plan_with_nthreads(nthreads); - } - #endif +template +void _init_threads(const ExecutionSpace& exec_space) { +#if defined(KOKKOS_ENABLE_OPENMP) || defined(KOKKOS_ENABLE_THREADS) + int nthreads = exec_space.concurrency(); + + if constexpr (std::is_same_v) { + fftwf_init_threads(); + fftwf_plan_with_nthreads(nthreads); + } else { + fftw_init_threads(); + fftw_plan_with_nthreads(nthreads); } - - // ND transform - template , std::nullptr_t> = nullptr> - auto _create(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, const OutViewType& out, [[maybe_unused]] Direction direction) { - static_assert(Kokkos::is_view::value, +#endif +} + +// ND transform +template , + std::nullptr_t> = nullptr> +auto _create(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, const OutViewType& out, + [[maybe_unused]] Direction direction) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_create: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - _init_threads>(exec_space); - - const int rank = InViewType::rank(); - const int axis = -1; - const int howmany = 1; - constexpr auto type = KokkosFFT::Impl::transform_type::type(); - auto [in_extents, out_extents, fft_extents] = KokkosFFT::Impl::get_extents(in, out, axis); - int idist = std::accumulate(in_extents.begin(), in_extents.end(), 1, std::multiplies<>()); - int odist = std::accumulate(out_extents.begin(), out_extents.end(), 1, std::multiplies<>()); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - - auto* idata = reinterpret_cast::type*>(in.data()); - auto* odata = reinterpret_cast::type*>(out.data()); - int istride = 1, ostride = 1; - auto sign = KokkosFFT::Impl::direction_type(direction); - - if constexpr(type == KokkosFFT::Impl::FFTWTransformType::R2C) { - plan = fftwf_plan_many_dft_r2c(rank, - fft_extents.data(), - howmany, - idata, - in_extents.data(), - istride, - idist, - odata, - out_extents.data(), - ostride, - odist, - FFTW_ESTIMATE - ); - } else if constexpr(type == KokkosFFT::Impl::FFTWTransformType::D2Z) { - plan = fftw_plan_many_dft_r2c(rank, - fft_extents.data(), - howmany, - idata, - in_extents.data(), - istride, - idist, - odata, - out_extents.data(), - ostride, - odist, - FFTW_ESTIMATE - ); - } else if constexpr(type == KokkosFFT::Impl::FFTWTransformType::C2R) { - plan = fftwf_plan_many_dft_c2r(rank, - fft_extents.data(), - howmany, - idata, - in_extents.data(), - istride, - idist, - odata, - out_extents.data(), - ostride, - odist, - FFTW_ESTIMATE - ); - } else if constexpr(type == KokkosFFT::Impl::FFTWTransformType::Z2D) { - plan = fftw_plan_many_dft_c2r(rank, - fft_extents.data(), - howmany, - idata, - in_extents.data(), - istride, - idist, - odata, - out_extents.data(), - ostride, - odist, - FFTW_ESTIMATE - ); - } else if constexpr(type == KokkosFFT::Impl::FFTWTransformType::C2C) { - plan = fftwf_plan_many_dft(rank, - fft_extents.data(), - howmany, - idata, - in_extents.data(), - istride, - idist, - odata, - out_extents.data(), - ostride, - odist, - sign, - FFTW_ESTIMATE - ); - } else if constexpr(type == KokkosFFT::Impl::FFTWTransformType::Z2Z) { - plan = fftw_plan_many_dft(rank, - fft_extents.data(), - howmany, - idata, - in_extents.data(), - istride, - idist, - odata, - out_extents.data(), - ostride, - odist, - sign, - FFTW_ESTIMATE - ); - } - - return fft_size; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + _init_threads>( + exec_space); + + const int rank = InViewType::rank(); + const int axis = -1; + const int howmany = 1; + constexpr auto type = + KokkosFFT::Impl::transform_type::type(); + auto [in_extents, out_extents, fft_extents] = + KokkosFFT::Impl::get_extents(in, out, axis); + int idist = std::accumulate(in_extents.begin(), in_extents.end(), 1, + std::multiplies<>()); + int odist = std::accumulate(out_extents.begin(), out_extents.end(), 1, + std::multiplies<>()); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + + auto* idata = reinterpret_cast::type*>(in.data()); + auto* odata = reinterpret_cast::type*>(out.data()); + int istride = 1, ostride = 1; + auto sign = KokkosFFT::Impl::direction_type(direction); + + if constexpr (type == KokkosFFT::Impl::FFTWTransformType::R2C) { + plan = fftwf_plan_many_dft_r2c( + rank, fft_extents.data(), howmany, idata, in_extents.data(), istride, + idist, odata, out_extents.data(), ostride, odist, FFTW_ESTIMATE); + } else if constexpr (type == KokkosFFT::Impl::FFTWTransformType::D2Z) { + plan = fftw_plan_many_dft_r2c( + rank, fft_extents.data(), howmany, idata, in_extents.data(), istride, + idist, odata, out_extents.data(), ostride, odist, FFTW_ESTIMATE); + } else if constexpr (type == KokkosFFT::Impl::FFTWTransformType::C2R) { + plan = fftwf_plan_many_dft_c2r( + rank, fft_extents.data(), howmany, idata, in_extents.data(), istride, + idist, odata, out_extents.data(), ostride, odist, FFTW_ESTIMATE); + } else if constexpr (type == KokkosFFT::Impl::FFTWTransformType::Z2D) { + plan = fftw_plan_many_dft_c2r( + rank, fft_extents.data(), howmany, idata, in_extents.data(), istride, + idist, odata, out_extents.data(), ostride, odist, FFTW_ESTIMATE); + } else if constexpr (type == KokkosFFT::Impl::FFTWTransformType::C2C) { + plan = fftwf_plan_many_dft( + rank, fft_extents.data(), howmany, idata, in_extents.data(), istride, + idist, odata, out_extents.data(), ostride, odist, sign, FFTW_ESTIMATE); + } else if constexpr (type == KokkosFFT::Impl::FFTWTransformType::Z2Z) { + plan = fftw_plan_many_dft( + rank, fft_extents.data(), howmany, idata, in_extents.data(), istride, + idist, odata, out_extents.data(), ostride, odist, sign, FFTW_ESTIMATE); } - // batched transform, over ND Views - template , std::nullptr_t> = nullptr> - auto _create(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, const OutViewType& out, [[maybe_unused]] Direction direction, axis_type axes) { - static_assert(Kokkos::is_view::value, - "KokkosFFT::_create: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, - "KokkosFFT::_create: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - static_assert(InViewType::rank() >= fft_rank, - "KokkosFFT::_create: Rank of View must be larger than Rank of FFT."); - const int rank = fft_rank; - - _init_threads>(exec_space); - - constexpr auto type = KokkosFFT::Impl::transform_type::type(); - auto [in_extents, out_extents, fft_extents, howmany] = KokkosFFT::Impl::get_extents_batched(in, out, axes); - int idist = std::accumulate(in_extents.begin(), in_extents.end(), 1, std::multiplies<>()); - int odist = std::accumulate(out_extents.begin(), out_extents.end(), 1, std::multiplies<>()); - int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, std::multiplies<>()); - - auto* idata = reinterpret_cast::type*>(in.data()); - auto* odata = reinterpret_cast::type*>(out.data()); - - // For the moment, considering the contiguous layout only - int istride = 1, ostride = 1; - auto sign = KokkosFFT::Impl::direction_type(direction); - - if constexpr(type == KokkosFFT::Impl::FFTWTransformType::R2C) { - plan = fftwf_plan_many_dft_r2c(rank, - fft_extents.data(), - howmany, - idata, - in_extents.data(), - istride, - idist, - odata, - out_extents.data(), - ostride, - odist, - FFTW_ESTIMATE - ); - } else if constexpr(type == KokkosFFT::Impl::FFTWTransformType::D2Z) { - plan = fftw_plan_many_dft_r2c(rank, - fft_extents.data(), - howmany, - idata, - in_extents.data(), - istride, - idist, - odata, - out_extents.data(), - ostride, - odist, - FFTW_ESTIMATE - ); - } else if constexpr(type == KokkosFFT::Impl::FFTWTransformType::C2R) { - plan = fftwf_plan_many_dft_c2r(rank, - fft_extents.data(), - howmany, - idata, - in_extents.data(), - istride, - idist, - odata, - out_extents.data(), - ostride, - odist, - FFTW_ESTIMATE - ); - } else if constexpr(type == KokkosFFT::Impl::FFTWTransformType::Z2D) { - plan = fftw_plan_many_dft_c2r(rank, - fft_extents.data(), - howmany, - idata, - in_extents.data(), - istride, - idist, - odata, - out_extents.data(), - ostride, - odist, - FFTW_ESTIMATE - ); - } else if constexpr(type == KokkosFFT::Impl::FFTWTransformType::C2C) { - plan = fftwf_plan_many_dft(rank, - fft_extents.data(), - howmany, - idata, - in_extents.data(), - istride, - idist, - odata, - out_extents.data(), - ostride, - odist, - sign, - FFTW_ESTIMATE - ); - } else if constexpr(type == KokkosFFT::Impl::FFTWTransformType::Z2Z) { - plan = fftw_plan_many_dft(rank, - fft_extents.data(), - howmany, - idata, - in_extents.data(), - istride, - idist, - odata, - out_extents.data(), - ostride, - odist, - sign, - FFTW_ESTIMATE - ); - } - - return fft_size; + return fft_size; +} + +// batched transform, over ND Views +template , + std::nullptr_t> = nullptr> +auto _create(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, const OutViewType& out, + [[maybe_unused]] Direction direction, axis_type axes) { + static_assert(Kokkos::is_view::value, + "KokkosFFT::_create: InViewType is not a Kokkos::View."); + static_assert(Kokkos::is_view::value, + "KokkosFFT::_create: OutViewType is not a Kokkos::View."); + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + static_assert( + InViewType::rank() >= fft_rank, + "KokkosFFT::_create: Rank of View must be larger than Rank of FFT."); + const int rank = fft_rank; + + _init_threads>( + exec_space); + + constexpr auto type = + KokkosFFT::Impl::transform_type::type(); + auto [in_extents, out_extents, fft_extents, howmany] = + KokkosFFT::Impl::get_extents_batched(in, out, axes); + int idist = std::accumulate(in_extents.begin(), in_extents.end(), 1, + std::multiplies<>()); + int odist = std::accumulate(out_extents.begin(), out_extents.end(), 1, + std::multiplies<>()); + int fft_size = std::accumulate(fft_extents.begin(), fft_extents.end(), 1, + std::multiplies<>()); + + auto* idata = reinterpret_cast::type*>(in.data()); + auto* odata = reinterpret_cast::type*>(out.data()); + + // For the moment, considering the contiguous layout only + int istride = 1, ostride = 1; + auto sign = KokkosFFT::Impl::direction_type(direction); + + if constexpr (type == KokkosFFT::Impl::FFTWTransformType::R2C) { + plan = fftwf_plan_many_dft_r2c( + rank, fft_extents.data(), howmany, idata, in_extents.data(), istride, + idist, odata, out_extents.data(), ostride, odist, FFTW_ESTIMATE); + } else if constexpr (type == KokkosFFT::Impl::FFTWTransformType::D2Z) { + plan = fftw_plan_many_dft_r2c( + rank, fft_extents.data(), howmany, idata, in_extents.data(), istride, + idist, odata, out_extents.data(), ostride, odist, FFTW_ESTIMATE); + } else if constexpr (type == KokkosFFT::Impl::FFTWTransformType::C2R) { + plan = fftwf_plan_many_dft_c2r( + rank, fft_extents.data(), howmany, idata, in_extents.data(), istride, + idist, odata, out_extents.data(), ostride, odist, FFTW_ESTIMATE); + } else if constexpr (type == KokkosFFT::Impl::FFTWTransformType::Z2D) { + plan = fftw_plan_many_dft_c2r( + rank, fft_extents.data(), howmany, idata, in_extents.data(), istride, + idist, odata, out_extents.data(), ostride, odist, FFTW_ESTIMATE); + } else if constexpr (type == KokkosFFT::Impl::FFTWTransformType::C2C) { + plan = fftwf_plan_many_dft( + rank, fft_extents.data(), howmany, idata, in_extents.data(), istride, + idist, odata, out_extents.data(), ostride, odist, sign, FFTW_ESTIMATE); + } else if constexpr (type == KokkosFFT::Impl::FFTWTransformType::Z2Z) { + plan = fftw_plan_many_dft( + rank, fft_extents.data(), howmany, idata, in_extents.data(), istride, + idist, odata, out_extents.data(), ostride, odist, sign, FFTW_ESTIMATE); } - template , std::nullptr_t> = nullptr> - void _destroy(typename KokkosFFT::Impl::FFTPlanType::type& plan) { - if constexpr (std::is_same_v) { - fftwf_destroy_plan(plan); - } else { - fftw_destroy_plan(plan); - } + return fft_size; +} + +template , + std::nullptr_t> = nullptr> +void _destroy( + typename KokkosFFT::Impl::FFTPlanType::type& plan) { + if constexpr (std::is_same_v) { + fftwf_destroy_plan(plan); + } else { + fftw_destroy_plan(plan); } -} // namespace Impl -} // namespace KokkosFFT +} +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/fft/src/KokkosFFT_OpenMP_transform.hpp b/fft/src/KokkosFFT_OpenMP_transform.hpp index 9c646061..c16d1b7f 100644 --- a/fft/src/KokkosFFT_OpenMP_transform.hpp +++ b/fft/src/KokkosFFT_OpenMP_transform.hpp @@ -5,36 +5,42 @@ namespace KokkosFFT { namespace Impl { - template - void _exec(PlanType plan, float* idata, fftwf_complex* odata, [[maybe_unused]] int direction) { - fftwf_execute_dft_r2c(plan, idata, odata); - } - - template - void _exec(PlanType plan, double* idata, fftw_complex* odata, [[maybe_unused]] int direction) { - fftw_execute_dft_r2c(plan, idata, odata); - } - - template - void _exec(PlanType plan, fftwf_complex* idata, float* odata, [[maybe_unused]] int direction) { - fftwf_execute_dft_c2r(plan, idata, odata); - } - - template - void _exec(PlanType plan, fftw_complex* idata, double* odata, [[maybe_unused]] int direction) { - fftw_execute_dft_c2r(plan, idata, odata); - } - - template - void _exec(PlanType plan, fftwf_complex* idata, fftwf_complex* odata, [[maybe_unused]] int direction) { - fftwf_execute_dft(plan, idata, odata); - } - - template - void _exec(PlanType plan, fftw_complex* idata, fftw_complex* odata, [[maybe_unused]] int direction) { - fftw_execute_dft(plan, idata, odata); - } -} // namespace Impl -} // namespace KokkosFFT +template +void _exec(PlanType plan, float* idata, fftwf_complex* odata, + [[maybe_unused]] int direction) { + fftwf_execute_dft_r2c(plan, idata, odata); +} + +template +void _exec(PlanType plan, double* idata, fftw_complex* odata, + [[maybe_unused]] int direction) { + fftw_execute_dft_r2c(plan, idata, odata); +} + +template +void _exec(PlanType plan, fftwf_complex* idata, float* odata, + [[maybe_unused]] int direction) { + fftwf_execute_dft_c2r(plan, idata, odata); +} + +template +void _exec(PlanType plan, fftw_complex* idata, double* odata, + [[maybe_unused]] int direction) { + fftw_execute_dft_c2r(plan, idata, odata); +} + +template +void _exec(PlanType plan, fftwf_complex* idata, fftwf_complex* odata, + [[maybe_unused]] int direction) { + fftwf_execute_dft(plan, idata, odata); +} + +template +void _exec(PlanType plan, fftw_complex* idata, fftw_complex* odata, + [[maybe_unused]] int direction) { + fftw_execute_dft(plan, idata, odata); +} +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/fft/src/KokkosFFT_Plans.hpp b/fft/src/KokkosFFT_Plans.hpp index d303cd08..e1c03fd4 100644 --- a/fft/src/KokkosFFT_Plans.hpp +++ b/fft/src/KokkosFFT_Plans.hpp @@ -7,113 +7,132 @@ #include "KokkosFFT_utils.hpp" #if defined(KOKKOS_ENABLE_CUDA) - using default_device = Kokkos::Cuda; - #include "KokkosFFT_Cuda_plans.hpp" - #ifdef ENABLE_HOST_AND_DEVICE - #include "KokkosFFT_OpenMP_plans.hpp" - #endif +using default_device = Kokkos::Cuda; +#include "KokkosFFT_Cuda_plans.hpp" +#ifdef ENABLE_HOST_AND_DEVICE +#include "KokkosFFT_OpenMP_plans.hpp" +#endif #elif defined(KOKKOS_ENABLE_HIP) - using default_device = Kokkos::HIP; - #include "KokkosFFT_HIP_plans.hpp" - #ifdef ENABLE_HOST_AND_DEVICE - #include "KokkosFFT_OpenMP_plans.hpp" - #endif +using default_device = Kokkos::HIP; +#include "KokkosFFT_HIP_plans.hpp" +#ifdef ENABLE_HOST_AND_DEVICE +#include "KokkosFFT_OpenMP_plans.hpp" +#endif #elif defined(KOKKOS_ENABLE_OPENMP) - using default_device = Kokkos::OpenMP; - #include "KokkosFFT_OpenMP_plans.hpp" +using default_device = Kokkos::OpenMP; +#include "KokkosFFT_OpenMP_plans.hpp" #elif defined(KOKKOS_ENABLE_THREADS) - using default_device = Kokkos::Threads; - #include "KokkosFFT_OpenMP_plans.hpp" +using default_device = Kokkos::Threads; +#include "KokkosFFT_OpenMP_plans.hpp" #else - using default_device = Kokkos::Serial; - #include "KokkosFFT_OpenMP_plans.hpp" +using default_device = Kokkos::Serial; +#include "KokkosFFT_OpenMP_plans.hpp" #endif namespace KokkosFFT { namespace Impl { - template - class Plan { - using execSpace = ExecutionSpace; - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - using float_type = KokkosFFT::Impl::real_type_t; - using fft_plan_type = typename KokkosFFT::Impl::FFTPlanType::type; - using fft_size_type = std::size_t; - using map_type = axis_type; - using nonConstInViewType = std::remove_cv_t; - using nonConstOutViewType = std::remove_cv_t; - - fft_plan_type m_plan; - fft_size_type m_fft_size; - map_type m_map, m_map_inv; - bool m_is_transpose_needed; - axis_type m_axes; - KokkosFFT::Impl::Direction m_direction; - - // Only used when transpose needed - nonConstInViewType m_in_T; - nonConstOutViewType m_out_T; - - public: - explicit Plan(const ExecutionSpace& exec_space, InViewType& in, OutViewType& out, KokkosFFT::Impl::Direction direction, int axis) : m_fft_size(1), m_is_transpose_needed(false), m_direction(direction) { - static_assert(Kokkos::is_view::value, - "KokkosFFT::Plan: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, - "KokkosFFT::Plan: OutViewType is not a Kokkos::View."); - - m_axes = {axis}; - std::tie(m_map, m_map_inv) = KokkosFFT::Impl::get_map_axes(in, axis); - m_is_transpose_needed = KokkosFFT::Impl::is_transpose_needed(m_map); - m_fft_size = KokkosFFT::Impl::_create(exec_space, m_plan, in, out, direction, m_axes); - } - - explicit Plan(const ExecutionSpace& exec_space, InViewType& in, OutViewType& out, KokkosFFT::Impl::Direction direction, axis_type axes) : m_fft_size(1), m_is_transpose_needed(false), m_direction(direction), m_axes(axes) { - static_assert(Kokkos::is_view::value, - "KokkosFFT::Plan: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, - "KokkosFFT::Plan: OutViewType is not a Kokkos::View."); - - std::tie(m_map, m_map_inv) = KokkosFFT::Impl::get_map_axes(in, axes); - m_is_transpose_needed = KokkosFFT::Impl::is_transpose_needed(m_map); - m_fft_size = KokkosFFT::Impl::_create(exec_space, m_plan, in, out, direction, axes); +template +class Plan { + using execSpace = ExecutionSpace; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + using float_type = KokkosFFT::Impl::real_type_t; + using fft_plan_type = + typename KokkosFFT::Impl::FFTPlanType::type; + using fft_size_type = std::size_t; + using map_type = axis_type; + using nonConstInViewType = std::remove_cv_t; + using nonConstOutViewType = std::remove_cv_t; + + fft_plan_type m_plan; + fft_size_type m_fft_size; + map_type m_map, m_map_inv; + bool m_is_transpose_needed; + axis_type m_axes; + KokkosFFT::Impl::Direction m_direction; + + // Only used when transpose needed + nonConstInViewType m_in_T; + nonConstOutViewType m_out_T; + + public: + explicit Plan(const ExecutionSpace& exec_space, InViewType& in, + OutViewType& out, KokkosFFT::Impl::Direction direction, + int axis) + : m_fft_size(1), m_is_transpose_needed(false), m_direction(direction) { + static_assert(Kokkos::is_view::value, + "KokkosFFT::Plan: InViewType is not a Kokkos::View."); + static_assert(Kokkos::is_view::value, + "KokkosFFT::Plan: OutViewType is not a Kokkos::View."); + + m_axes = {axis}; + std::tie(m_map, m_map_inv) = KokkosFFT::Impl::get_map_axes(in, axis); + m_is_transpose_needed = KokkosFFT::Impl::is_transpose_needed(m_map); + m_fft_size = KokkosFFT::Impl::_create(exec_space, m_plan, in, out, + direction, m_axes); + } + + explicit Plan(const ExecutionSpace& exec_space, InViewType& in, + OutViewType& out, KokkosFFT::Impl::Direction direction, + axis_type axes) + : m_fft_size(1), + m_is_transpose_needed(false), + m_direction(direction), + m_axes(axes) { + static_assert(Kokkos::is_view::value, + "KokkosFFT::Plan: InViewType is not a Kokkos::View."); + static_assert(Kokkos::is_view::value, + "KokkosFFT::Plan: OutViewType is not a Kokkos::View."); + + std::tie(m_map, m_map_inv) = KokkosFFT::Impl::get_map_axes(in, axes); + m_is_transpose_needed = KokkosFFT::Impl::is_transpose_needed(m_map); + m_fft_size = + KokkosFFT::Impl::_create(exec_space, m_plan, in, out, direction, axes); + } + + ~Plan() { _destroy(m_plan); } + + template + void good(KokkosFFT::Impl::Direction direction, axis_type axes) const { + static_assert(std::is_same_v, + "KokkosFFT::Plan: is_good: Execution spaces for plan and " + "execution are inconsistent."); + + using nonConstInViewType2 = std::remove_cv_t; + using nonConstOutViewType2 = std::remove_cv_t; + static_assert(std::is_same_v, + "KokkosFFT::Plan: is_good: InViewType for plan and execution " + "are inconsistent."); + static_assert(std::is_same_v, + "KokkosFFT::Plan: is_good: OutViewType for plan and " + "execution are inconsistent."); + + if (direction != m_direction) { + throw std::runtime_error( + "KokkosFFT::Impl::Plan::good: direction for plan and execution are " + "inconsistent."); } - ~Plan() { - _destroy(m_plan); - } - - template - void good(KokkosFFT::Impl::Direction direction, axis_type axes) const { - static_assert(std::is_same_v, - "KokkosFFT::Plan: is_good: Execution spaces for plan and execution are inconsistent."); - - using nonConstInViewType2 = std::remove_cv_t; - using nonConstOutViewType2 = std::remove_cv_t; - static_assert(std::is_same_v, - "KokkosFFT::Plan: is_good: InViewType for plan and execution are inconsistent."); - static_assert(std::is_same_v, - "KokkosFFT::Plan: is_good: OutViewType for plan and execution are inconsistent."); - - if(direction != m_direction) { - throw std::runtime_error("KokkosFFT::Impl::Plan::good: direction for plan and execution are inconsistent."); - } - - if(axes != m_axes) { - throw std::runtime_error("KokkosFFT::Impl::Plan::good: axes for plan and execution are inconsistent."); - } - - // [TO DO] Check view extents + if (axes != m_axes) { + throw std::runtime_error( + "KokkosFFT::Impl::Plan::good: axes for plan and execution are " + "inconsistent."); } - fft_plan_type plan() const { return m_plan; } - fft_size_type fft_size() const { return m_fft_size; } - bool is_transpose_needed() const { return m_is_transpose_needed; } - map_type map() const { return m_map; } - map_type map_inv() const { return m_map_inv; } - nonConstInViewType& in_T() { return m_in_T; } - nonConstOutViewType& out_T() { return m_out_T; } - }; -} // namespace Impl -} // namespace KokkosFFT + // [TO DO] Check view extents + } + + fft_plan_type plan() const { return m_plan; } + fft_size_type fft_size() const { return m_fft_size; } + bool is_transpose_needed() const { return m_is_transpose_needed; } + map_type map() const { return m_map; } + map_type map_inv() const { return m_map_inv; } + nonConstInViewType& in_T() { return m_in_T; } + nonConstOutViewType& out_T() { return m_out_T; } +}; +} // namespace Impl +} // namespace KokkosFFT #endif \ No newline at end of file diff --git a/fft/src/KokkosFFT_Transform.hpp b/fft/src/KokkosFFT_Transform.hpp index d57ff1ac..3c9500e6 100644 --- a/fft/src/KokkosFFT_Transform.hpp +++ b/fft/src/KokkosFFT_Transform.hpp @@ -10,1179 +10,1168 @@ #include "KokkosFFT_Plans.hpp" #if defined(KOKKOS_ENABLE_CUDA) - using default_device = Kokkos::Cuda; - #include "KokkosFFT_Cuda_transform.hpp" - #ifdef ENABLE_HOST_AND_DEVICE - #include "KokkosFFT_OpenMP_transform.hpp" - #endif +using default_device = Kokkos::Cuda; +#include "KokkosFFT_Cuda_transform.hpp" +#ifdef ENABLE_HOST_AND_DEVICE +#include "KokkosFFT_OpenMP_transform.hpp" +#endif #elif defined(KOKKOS_ENABLE_HIP) - using default_device = Kokkos::HIP; - #include "KokkosFFT_HIP_transform.hpp" - #ifdef ENABLE_HOST_AND_DEVICE - #include "KokkosFFT_OpenMP_transform.hpp" - #endif +using default_device = Kokkos::HIP; +#include "KokkosFFT_HIP_transform.hpp" +#ifdef ENABLE_HOST_AND_DEVICE +#include "KokkosFFT_OpenMP_transform.hpp" +#endif #elif defined(KOKKOS_ENABLE_OPENMP) - using default_device = Kokkos::OpenMP; - #include "KokkosFFT_OpenMP_transform.hpp" +using default_device = Kokkos::OpenMP; +#include "KokkosFFT_OpenMP_transform.hpp" #elif defined(KOKKOS_ENABLE_THREADS) - using default_device = Kokkos::Threads; - #include "KokkosFFT_OpenMP_transform.hpp" +using default_device = Kokkos::Threads; +#include "KokkosFFT_OpenMP_transform.hpp" #else - using default_device = Kokkos::Serial; - #include "KokkosFFT_OpenMP_transform.hpp" +using default_device = Kokkos::Serial; +#include "KokkosFFT_OpenMP_transform.hpp" #endif // General Transform Interface namespace KokkosFFT { namespace Impl { - template - void _fft(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, OutViewType& out, KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD) { - static_assert(Kokkos::is_view::value, +template +void _fft(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_fft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_fft: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - auto* idata = reinterpret_cast::type*>(in.data()); - auto* odata = reinterpret_cast::type*>(out.data()); - - auto forward = direction_type(KokkosFFT::Impl::Direction::Forward); - KokkosFFT::Impl::_exec(plan.plan(), idata, odata, forward); - KokkosFFT::Impl::normalize(exec_space, out, KokkosFFT::Impl::Direction::Forward, norm, plan.fft_size()); - } - - template - void _ifft(const ExecutionSpace& exec_space, PlanType& plan, const InViewType& in, OutViewType& out, KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD) { - static_assert(Kokkos::is_view::value, + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + auto* idata = reinterpret_cast::type*>(in.data()); + auto* odata = reinterpret_cast::type*>(out.data()); + + auto forward = + direction_type(KokkosFFT::Impl::Direction::Forward); + KokkosFFT::Impl::_exec(plan.plan(), idata, odata, forward); + KokkosFFT::Impl::normalize(exec_space, out, + KokkosFFT::Impl::Direction::Forward, norm, + plan.fft_size()); +} + +template +void _ifft(const ExecutionSpace& exec_space, PlanType& plan, + const InViewType& in, OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD) { + static_assert(Kokkos::is_view::value, "KokkosFFT::_ifft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::_ifft: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - auto* idata = reinterpret_cast::type*>(in.data()); - auto* odata = reinterpret_cast::type*>(out.data()); + auto* idata = reinterpret_cast::type*>(in.data()); + auto* odata = reinterpret_cast::type*>(out.data()); - auto backward = direction_type(KokkosFFT::Impl::Direction::Backward); - KokkosFFT::Impl::_exec(plan.plan(), idata, odata, backward); - KokkosFFT::Impl::normalize(exec_space, out, KokkosFFT::Impl::Direction::Backward, norm, plan.fft_size()); - } -} // namespace Impl -} // namespace KokkosFFT + auto backward = + direction_type(KokkosFFT::Impl::Direction::Backward); + KokkosFFT::Impl::_exec(plan.plan(), idata, odata, backward); + KokkosFFT::Impl::normalize(exec_space, out, + KokkosFFT::Impl::Direction::Backward, norm, + plan.fft_size()); +} +} // namespace Impl +} // namespace KokkosFFT namespace KokkosFFT { - // 1D FFT - template - void fft(const ExecutionSpace& exec_space, - const InViewType& in, OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - int axis=-1, - std::optional n = std::nullopt) { - static_assert(Kokkos::is_view::value, - "KokkosFFT::fft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, - "KokkosFFT::fft: OutViewType is not a Kokkos::View."); - - static_assert( +// 1D FFT +template +void fft(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + int axis = -1, std::optional n = std::nullopt) { + static_assert(Kokkos::is_view::value, + "KokkosFFT::fft: InViewType is not a Kokkos::View."); + static_assert(Kokkos::is_view::value, + "KokkosFFT::fft: OutViewType is not a Kokkos::View."); + + static_assert( Kokkos::SpaceAccessibility::accessible, - "KokkosFFT::fft: execution_space cannot access data in InViewType" - ); - - static_assert( - Kokkos::SpaceAccessibility::accessible, - "KokkosFFT::fft: execution_space cannot access data in OutViewType" - ); - - InViewType _in; - if(n) { - std::size_t _n = n.value(); - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, shape_type<1>({_n})); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + "KokkosFFT::fft: execution_space cannot access data in InViewType"); + + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "KokkosFFT::fft: execution_space cannot access data in OutViewType"); + + InViewType _in; + if (n) { + std::size_t _n = n.value(); + auto modified_shape = + KokkosFFT::Impl::get_modified_shape(in, shape_type<1>({_n})); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - KokkosFFT::Impl::Plan plan(exec_space, _in, out, KokkosFFT::Impl::Direction::Forward, axis); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + KokkosFFT::Impl::Plan plan(exec_space, _in, out, + KokkosFFT::Impl::Direction::Forward, axis); + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); - } + } else { + KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); } - - template - void fft(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - int axis=-1, - std::optional n = std::nullopt) { - static_assert(Kokkos::is_view::value, - "KokkosFFT::fft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, - "KokkosFFT::fft: OutViewType is not a Kokkos::View."); - - static_assert( +} + +template +void fft(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + int axis = -1, std::optional n = std::nullopt) { + static_assert(Kokkos::is_view::value, + "KokkosFFT::fft: InViewType is not a Kokkos::View."); + static_assert(Kokkos::is_view::value, + "KokkosFFT::fft: OutViewType is not a Kokkos::View."); + + static_assert( Kokkos::SpaceAccessibility::accessible, - "KokkosFFT::fft: execution_space cannot access data in InViewType" - ); - - static_assert( - Kokkos::SpaceAccessibility::accessible, - "KokkosFFT::fft: execution_space cannot access data in OutViewType" - ); - - InViewType _in; - if(n) { - std::size_t _n = n.value(); - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, shape_type<1>({_n})); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + "KokkosFFT::fft: execution_space cannot access data in InViewType"); + + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename OutViewType::memory_space>::accessible, + "KokkosFFT::fft: execution_space cannot access data in OutViewType"); + + InViewType _in; + if (n) { + std::size_t _n = n.value(); + auto modified_shape = + KokkosFFT::Impl::get_modified_shape(in, shape_type<1>({_n})); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - plan.template good(KokkosFFT::Impl::Direction::Forward, axis_type<1>{axis}); + plan.template good( + KokkosFFT::Impl::Direction::Forward, axis_type<1>{axis}); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); - } + } else { + KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); } - - template - void ifft(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - int axis=-1, - std::optional n = std::nullopt) { - static_assert(Kokkos::is_view::value, +} + +template +void ifft(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + int axis = -1, std::optional n = std::nullopt) { + static_assert(Kokkos::is_view::value, "KokkosFFT::ifft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::ifft: OutViewType is not a Kokkos::View."); - InViewType _in; - // [TO DO] Modify crop_or_pad to perform the following lines - // KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, n); - if(n) { - std::size_t _n = n.value(); - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, shape_type<1>({_n})); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + InViewType _in; + // [TO DO] Modify crop_or_pad to perform the following lines + // KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, n); + if (n) { + std::size_t _n = n.value(); + auto modified_shape = + KokkosFFT::Impl::get_modified_shape(in, shape_type<1>({_n})); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - KokkosFFT::Impl::Plan plan(exec_space, _in, out, KokkosFFT::Impl::Direction::Backward, axis); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + KokkosFFT::Impl::Plan plan(exec_space, _in, out, + KokkosFFT::Impl::Direction::Backward, axis); + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); - } + } else { + KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); } - - template - void ifft(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - int axis=-1, - std::optional n = std::nullopt) { - static_assert(Kokkos::is_view::value, +} + +template +void ifft(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + int axis = -1, std::optional n = std::nullopt) { + static_assert(Kokkos::is_view::value, "KokkosFFT::ifft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::ifft: OutViewType is not a Kokkos::View."); - InViewType _in; - // [TO DO] Modify crop_or_pad to perform the following lines - // KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, n); - if(n) { - std::size_t _n = n.value(); - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, shape_type<1>({_n})); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + InViewType _in; + // [TO DO] Modify crop_or_pad to perform the following lines + // KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, n); + if (n) { + std::size_t _n = n.value(); + auto modified_shape = + KokkosFFT::Impl::get_modified_shape(in, shape_type<1>({_n})); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - plan.template good(KokkosFFT::Impl::Direction::Backward, axis_type<1>{axis}); + plan.template good( + KokkosFFT::Impl::Direction::Backward, axis_type<1>{axis}); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); - } + } else { + KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); } - - template - void rfft(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - int axis=-1, - std::optional n = std::nullopt) { - static_assert(Kokkos::is_view::value, +} + +template +void rfft(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + int axis = -1, std::optional n = std::nullopt) { + static_assert(Kokkos::is_view::value, "KokkosFFT::rfft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::rfft: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - static_assert(std::is_floating_point::value, - "KokkosFFT::rfft: InViewType must be real"); - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfft: OutViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::rfft: InViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::rfft: OutViewType must be complex"); - fft(exec_space, in, out, norm, axis, n); - } + fft(exec_space, in, out, norm, axis, n); +} - template - void rfft(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - int axis=-1, - std::optional n = std::nullopt) { - static_assert(Kokkos::is_view::value, +template +void rfft(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + int axis = -1, std::optional n = std::nullopt) { + static_assert(Kokkos::is_view::value, "KokkosFFT::rfft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::rfft: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - static_assert(std::is_floating_point::value, - "KokkosFFT::rfft: InViewType must be real"); - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfft: OutViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::rfft: InViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::rfft: OutViewType must be complex"); - fft(exec_space, in, out, plan, norm, axis, n); - } + fft(exec_space, in, out, plan, norm, axis, n); +} - template - void irfft(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - int axis=-1, - std::optional n = std::nullopt) { - static_assert(Kokkos::is_view::value, +template +void irfft(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + int axis = -1, std::optional n = std::nullopt) { + static_assert(Kokkos::is_view::value, "KokkosFFT::irfft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::irfft: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfft: InViewType must be complex"); - static_assert(std::is_floating_point::value, - "KokkosFFT::irfft: OutViewType must be real"); - if(n) { - std::size_t _n = n.value() / 2 + 1; - ifft(exec_space, in, out, norm, axis, _n); - } else { - ifft(exec_space, in, out, norm, axis); - } + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::irfft: InViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::irfft: OutViewType must be real"); + if (n) { + std::size_t _n = n.value() / 2 + 1; + ifft(exec_space, in, out, norm, axis, _n); + } else { + ifft(exec_space, in, out, norm, axis); } - - template - void irfft(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - int axis=-1, - std::optional n = std::nullopt) { - static_assert(Kokkos::is_view::value, +} + +template +void irfft(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + int axis = -1, std::optional n = std::nullopt) { + static_assert(Kokkos::is_view::value, "KokkosFFT::irfft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::irfft: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfft: InViewType must be complex"); - static_assert(std::is_floating_point::value, - "KokkosFFT::irfft: OutViewType must be real"); - if(n) { - std::size_t _n = n.value() / 2 + 1; - ifft(exec_space, in, out, plan, norm, axis, _n); - } else { - ifft(exec_space, in, out, plan, norm, axis); - } + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::irfft: InViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::irfft: OutViewType must be real"); + if (n) { + std::size_t _n = n.value() / 2 + 1; + ifft(exec_space, in, out, plan, norm, axis, _n); + } else { + ifft(exec_space, in, out, plan, norm, axis); } - - template - void hfft(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - int axis=-1, - std::optional n = std::nullopt) { - static_assert(Kokkos::is_view::value, +} + +template +void hfft(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + int axis = -1, std::optional n = std::nullopt) { + static_assert(Kokkos::is_view::value, "KokkosFFT::hfft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::hfft: OutViewType is not a Kokkos::View."); - // [TO DO] - // allow real type as input, need to obtain complex view type from in view type - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::hfft: InViewType must be complex"); - static_assert(std::is_floating_point::value, - "KokkosFFT::hfft: OutViewType must be real"); - auto new_norm = KokkosFFT::Impl::swap_direction(norm); - //using ComplexViewType = typename KokkosFFT::Impl::complex_view_type::type; - //ComplexViewType in_conj; - InViewType in_conj; - KokkosFFT::Impl::conjugate(exec_space, in, in_conj); - irfft(exec_space, in_conj, out, new_norm, axis, n); - } - - template - void hfft(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - int axis=-1, - std::optional n = std::nullopt) { - static_assert(Kokkos::is_view::value, + // [TO DO] + // allow real type as input, need to obtain complex view type from in view + // type + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::hfft: InViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::hfft: OutViewType must be real"); + auto new_norm = KokkosFFT::Impl::swap_direction(norm); + // using ComplexViewType = typename + // KokkosFFT::Impl::complex_view_type::type; + // ComplexViewType in_conj; + InViewType in_conj; + KokkosFFT::Impl::conjugate(exec_space, in, in_conj); + irfft(exec_space, in_conj, out, new_norm, axis, n); +} + +template +void hfft(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + int axis = -1, std::optional n = std::nullopt) { + static_assert(Kokkos::is_view::value, "KokkosFFT::hfft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::hfft: OutViewType is not a Kokkos::View."); - // [TO DO] - // allow real type as input, need to obtain complex view type from in view type - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::hfft: InViewType must be complex"); - static_assert(std::is_floating_point::value, - "KokkosFFT::hfft: OutViewType must be real"); - auto new_norm = KokkosFFT::Impl::swap_direction(norm); - //using ComplexViewType = typename KokkosFFT::Impl::complex_view_type::type; - //ComplexViewType in_conj; - InViewType in_conj; - KokkosFFT::Impl::conjugate(exec_space, in, in_conj); - irfft(exec_space, in_conj, out, plan, new_norm, axis, n); - } - - template - void ihfft(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - int axis=-1, - std::optional n = std::nullopt) { - static_assert(Kokkos::is_view::value, + // [TO DO] + // allow real type as input, need to obtain complex view type from in view + // type + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::hfft: InViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::hfft: OutViewType must be real"); + auto new_norm = KokkosFFT::Impl::swap_direction(norm); + // using ComplexViewType = typename + // KokkosFFT::Impl::complex_view_type::type; + // ComplexViewType in_conj; + InViewType in_conj; + KokkosFFT::Impl::conjugate(exec_space, in, in_conj); + irfft(exec_space, in_conj, out, plan, new_norm, axis, n); +} + +template +void ihfft(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + int axis = -1, std::optional n = std::nullopt) { + static_assert(Kokkos::is_view::value, "KokkosFFT::ihfft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::ihfft: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - static_assert(std::is_floating_point::value, - "KokkosFFT::rfft: InViewType must be real"); - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::ihfft: OutViewType must be complex"); - - auto new_norm = KokkosFFT::Impl::swap_direction(norm); - OutViewType out_conj; - rfft(exec_space, in, out, new_norm, axis, n); - KokkosFFT::Impl::conjugate(exec_space, out, out_conj); - out = out_conj; - } - - template - void ihfft(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - int axis=-1, - std::optional n = std::nullopt) { - static_assert(Kokkos::is_view::value, + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + static_assert(std::is_floating_point::value, + "KokkosFFT::rfft: InViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::ihfft: OutViewType must be complex"); + + auto new_norm = KokkosFFT::Impl::swap_direction(norm); + OutViewType out_conj; + rfft(exec_space, in, out, new_norm, axis, n); + KokkosFFT::Impl::conjugate(exec_space, out, out_conj); + out = out_conj; +} + +template +void ihfft(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + int axis = -1, std::optional n = std::nullopt) { + static_assert(Kokkos::is_view::value, "KokkosFFT::ihfft: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::ihfft: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; - static_assert(std::is_floating_point::value, - "KokkosFFT::rfft: InViewType must be real"); - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::ihfft: OutViewType must be complex"); - - auto new_norm = KokkosFFT::Impl::swap_direction(norm); - OutViewType out_conj; - rfft(exec_space, in, out, plan, new_norm, axis, n); - KokkosFFT::Impl::conjugate(exec_space, out, out_conj); - out = out_conj; - } - - template - void fft2(const ExecutionSpace& exec_space, - const InViewType& in, OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - axis_type<2> axes={-2, -1}, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; + static_assert(std::is_floating_point::value, + "KokkosFFT::rfft: InViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::ihfft: OutViewType must be complex"); + + auto new_norm = KokkosFFT::Impl::swap_direction(norm); + OutViewType out_conj; + rfft(exec_space, in, out, plan, new_norm, axis, n); + KokkosFFT::Impl::conjugate(exec_space, out, out_conj); + out = out_conj; +} + +template +void fft2(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + axis_type<2> axes = {-2, -1}, shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::fft2: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::fft2: OutViewType is not a Kokkos::View."); - InViewType _in; - shape_type zeros = {0}; // default shape means no crop or pad - if( s != zeros ) { - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + InViewType _in; + shape_type zeros = {0}; // default shape means no crop or pad + if (s != zeros) { + auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - KokkosFFT::Impl::Plan plan(exec_space, _in, out, KokkosFFT::Impl::Direction::Forward, axes); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + KokkosFFT::Impl::Plan plan(exec_space, _in, out, + KokkosFFT::Impl::Direction::Forward, axes); + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); - } + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + } else { + KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); } - - // 2D FFT - template - void fft2(const ExecutionSpace& exec_space, - const InViewType& in, OutViewType& out, - const PlanType& plan, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - axis_type<2> axes={-2, -1}, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +} + +// 2D FFT +template +void fft2(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + axis_type<2> axes = {-2, -1}, shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::fft2: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::fft2: OutViewType is not a Kokkos::View."); - InViewType _in; - shape_type zeros = {0}; // default shape means no crop or pad - if( s != zeros ) { - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + InViewType _in; + shape_type zeros = {0}; // default shape means no crop or pad + if (s != zeros) { + auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - plan.template good(KokkosFFT::Impl::Direction::Forward, axes); + plan.template good( + KokkosFFT::Impl::Direction::Forward, axes); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); - } + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + } else { + KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); } +} - template - void ifft2(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - axis_type<2> axes={-2, -1}, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +template +void ifft2(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + axis_type<2> axes = {-2, -1}, shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::ifft2: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::ifft2: OutViewType is not a Kokkos::View."); - InViewType _in; - shape_type zeros = {0}; // default shape means no crop or pad - if( s != zeros ) { - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + InViewType _in; + shape_type zeros = {0}; // default shape means no crop or pad + if (s != zeros) { + auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - KokkosFFT::Impl::Plan plan(exec_space, _in, out, KokkosFFT::Impl::Direction::Backward, axes); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + KokkosFFT::Impl::Plan plan(exec_space, _in, out, + KokkosFFT::Impl::Direction::Backward, axes); + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); - } + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + } else { + KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); } - - template - void ifft2(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - axis_type<2> axes={-2, -1}, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +} + +template +void ifft2(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + axis_type<2> axes = {-2, -1}, shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::ifft2: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::ifft2: OutViewType is not a Kokkos::View."); - InViewType _in; - shape_type zeros = {0}; // default shape means no crop or pad - if( s != zeros ) { - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + InViewType _in; + shape_type zeros = {0}; // default shape means no crop or pad + if (s != zeros) { + auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - plan.template good(KokkosFFT::Impl::Direction::Backward, axes); + plan.template good( + KokkosFFT::Impl::Direction::Backward, axes); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); - } + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + } else { + KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); } +} - template - void rfft2(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - axis_type<2> axes={-2, -1}, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +template +void rfft2(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + axis_type<2> axes = {-2, -1}, shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::rfft2: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::rfft2: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - static_assert(std::is_floating_point::value, - "KokkosFFT::rfft2: InViewType must be real"); - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfft2: OutViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::rfft2: InViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::rfft2: OutViewType must be complex"); - fft2(exec_space, in, out, norm, axes, s); - } + fft2(exec_space, in, out, norm, axes, s); +} - template - void rfft2(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - axis_type<2> axes={-2, -1}, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +template +void rfft2(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + axis_type<2> axes = {-2, -1}, shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::rfft2: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::rfft2: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - static_assert(std::is_floating_point::value, - "KokkosFFT::rfft2: InViewType must be real"); - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfft2: OutViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::rfft2: InViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::rfft2: OutViewType must be complex"); - fft2(exec_space, in, out, plan, norm, axes, s); - } + fft2(exec_space, in, out, plan, norm, axes, s); +} - template - void irfft2(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - axis_type<2> axes={-2, -1}, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +template +void irfft2(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + axis_type<2> axes = {-2, -1}, shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::irfft2: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::irfft2: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfft2: InViewType must be complex"); - static_assert(std::is_floating_point::value, - "KokkosFFT::irfft2: OutViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::irfft2: InViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::irfft2: OutViewType must be real"); - shape_type zeros = {0}; // default shape means no crop or pad - shape_type _s = {0}; - if( s != zeros ) { - for(int i=0; i zeros = {0}; // default shape means no crop or pad + shape_type _s = {0}; + if (s != zeros) { + for (int i = 0; i < DIM; i++) { + _s.at(i) = s.at(i) / 2 + 1; } - - ifft2(exec_space, in, out, norm, axes, _s); } - template - void irfft2(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - axis_type<2> axes={-2, -1}, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, + ifft2(exec_space, in, out, norm, axes, _s); +} + +template +void irfft2(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + axis_type<2> axes = {-2, -1}, shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::irfft2: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::irfft2: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfft2: InViewType must be complex"); - static_assert(std::is_floating_point::value, - "KokkosFFT::irfft2: OutViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::irfft2: InViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::irfft2: OutViewType must be real"); - shape_type zeros = {0}; // default shape means no crop or pad - shape_type _s = {0}; - if( s != zeros ) { - for(int i=0; i zeros = {0}; // default shape means no crop or pad + shape_type _s = {0}; + if (s != zeros) { + for (int i = 0; i < DIM; i++) { + _s.at(i) = s.at(i) / 2 + 1; } - - ifft2(exec_space, in, out, plan, norm, axes, _s); } - // ND FFT - template - void fftn(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, + ifft2(exec_space, in, out, plan, norm, axes, _s); +} + +// ND FFT +template +void fftn(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::fftn: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::fftn: OutViewType is not a Kokkos::View."); - // Create a default sequence of axes {-rank, -(rank-1), ..., -1} - constexpr std::size_t rank = InViewType::rank(); - constexpr int start = -static_cast(rank); - axis_type axes = KokkosFFT::Impl::index_sequence(start); - - InViewType _in; - shape_type zeros = {0}; // default shape means no crop or pad - if( s != zeros ) { - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + // Create a default sequence of axes {-rank, -(rank-1), ..., -1} + constexpr std::size_t rank = InViewType::rank(); + constexpr int start = -static_cast(rank); + axis_type axes = KokkosFFT::Impl::index_sequence(start); + + InViewType _in; + shape_type zeros = {0}; // default shape means no crop or pad + if (s != zeros) { + auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - KokkosFFT::Impl::Plan plan(exec_space, _in, out, KokkosFFT::Impl::Direction::Forward, axes); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + KokkosFFT::Impl::Plan plan(exec_space, _in, out, + KokkosFFT::Impl::Direction::Forward, axes); + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); - } + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + } else { + KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); } - - template - void fftn(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - axis_type axes, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +} + +template +void fftn(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, axis_type axes, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::fftn: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::fftn: OutViewType is not a Kokkos::View."); - InViewType _in; - shape_type zeros = {0}; // default shape means no crop or pad - if( s != zeros ) { - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + InViewType _in; + shape_type zeros = {0}; // default shape means no crop or pad + if (s != zeros) { + auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - KokkosFFT::Impl::Plan plan(exec_space, _in, out, KokkosFFT::Impl::Direction::Forward, axes); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + KokkosFFT::Impl::Plan plan(exec_space, _in, out, + KokkosFFT::Impl::Direction::Forward, axes); + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); - } + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + } else { + KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); } - - template - void fftn(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - axis_type axes, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +} + +template +void fftn(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, axis_type axes, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::fftn: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::fftn: OutViewType is not a Kokkos::View."); - InViewType _in; - shape_type zeros = {0}; // default shape means no crop or pad - if( s != zeros ) { - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + InViewType _in; + shape_type zeros = {0}; // default shape means no crop or pad + if (s != zeros) { + auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - plan.template good(KokkosFFT::Impl::Direction::Forward, axes); + plan.template good( + KokkosFFT::Impl::Direction::Forward, axes); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_fft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); - } + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + } else { + KokkosFFT::Impl::_fft(exec_space, plan, _in, out, norm); } +} - template - void ifftn(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +template +void ifftn(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::ifftn: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::ifftn: OutViewType is not a Kokkos::View."); - // Create a default sequence of axes {-rank, -(rank-1), ..., -1} - constexpr std::size_t rank = InViewType::rank(); - constexpr int start = -static_cast(rank); - axis_type axes = KokkosFFT::Impl::index_sequence(start); - - InViewType _in; - shape_type zeros = {0}; // default shape means no crop or pad - if( s != zeros ) { - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + // Create a default sequence of axes {-rank, -(rank-1), ..., -1} + constexpr std::size_t rank = InViewType::rank(); + constexpr int start = -static_cast(rank); + axis_type axes = KokkosFFT::Impl::index_sequence(start); + + InViewType _in; + shape_type zeros = {0}; // default shape means no crop or pad + if (s != zeros) { + auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - KokkosFFT::Impl::Plan plan(exec_space, _in, out, KokkosFFT::Impl::Direction::Backward, axes); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + KokkosFFT::Impl::Plan plan(exec_space, _in, out, + KokkosFFT::Impl::Direction::Backward, axes); + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); - } + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + } else { + KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); } - - template - void ifftn(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - axis_type axes, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +} + +template +void ifftn(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, axis_type axes, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::ifftn: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::ifftn: OutViewType is not a Kokkos::View."); - InViewType _in; - shape_type zeros = {0}; // default shape means no crop or pad - if( s != zeros ) { - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + InViewType _in; + shape_type zeros = {0}; // default shape means no crop or pad + if (s != zeros) { + auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - KokkosFFT::Impl::Plan plan(exec_space, _in, out, KokkosFFT::Impl::Direction::Backward, axes); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + KokkosFFT::Impl::Plan plan(exec_space, _in, out, + KokkosFFT::Impl::Direction::Backward, axes); + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); - } + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + } else { + KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); } - - template - void ifftn(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - axis_type axes, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +} + +template +void ifftn(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, axis_type axes, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::ifftn: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::ifftn: OutViewType is not a Kokkos::View."); - InViewType _in; - shape_type zeros = {0}; // default shape means no crop or pad - if( s != zeros ) { - auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); - if( KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape) ) { - KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); - } else { - _in = in; - } + InViewType _in; + shape_type zeros = {0}; // default shape means no crop or pad + if (s != zeros) { + auto modified_shape = KokkosFFT::Impl::get_modified_shape(in, s); + if (KokkosFFT::Impl::is_crop_or_pad_needed(in, modified_shape)) { + KokkosFFT::Impl::crop_or_pad(exec_space, in, _in, modified_shape); } else { _in = in; } + } else { + _in = in; + } - plan.template good(KokkosFFT::Impl::Direction::Backward, axes); + plan.template good( + KokkosFFT::Impl::Direction::Backward, axes); - if(plan.is_transpose_needed()) { - InViewType in_T; - OutViewType out_T; + if (plan.is_transpose_needed()) { + InViewType in_T; + OutViewType out_T; - KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); - KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, _in, in_T, plan.map()); + KokkosFFT::Impl::transpose(exec_space, out, out_T, plan.map()); - KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); + KokkosFFT::Impl::_ifft(exec_space, plan, in_T, out_T, norm); - KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); - } else { - KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); - } + KokkosFFT::Impl::transpose(exec_space, out_T, out, plan.map_inv()); + } else { + KokkosFFT::Impl::_ifft(exec_space, plan, _in, out, norm); } +} - template - void rfftn(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +template +void rfftn(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::rfftn: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::rfftn: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - static_assert(std::is_floating_point::value, - "KokkosFFT::rfftn: InViewType must be real"); - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfftn: OutViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::rfftn: InViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::rfftn: OutViewType must be complex"); - fftn(exec_space, in, out, norm, s); - } + fftn(exec_space, in, out, norm, s); +} - template - void rfftn(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - axis_type axes, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +template +void rfftn(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, axis_type axes, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::rfftn: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::rfftn: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - static_assert(std::is_floating_point::value, - "KokkosFFT::rfftn: InViewType must be real"); - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfftn: OutViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::rfftn: InViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::rfftn: OutViewType must be complex"); - fftn(exec_space, in, out, plan, axes, norm, s); - } + fftn(exec_space, in, out, plan, axes, norm, s); +} - template - void rfftn(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - axis_type axes, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +template +void rfftn(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, axis_type axes, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::rfftn: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::rfftn: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - static_assert(std::is_floating_point::value, - "KokkosFFT::rfftn: InViewType must be real"); - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::rfftn: OutViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::rfftn: InViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::rfftn: OutViewType must be complex"); - fftn(exec_space, in, out, axes, norm, s); - } + fftn(exec_space, in, out, axes, norm, s); +} - template - void irfftn(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, +template +void irfftn(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::irfftn: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::irfftn: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfftn: InViewType must be complex"); - static_assert(std::is_floating_point::value, - "KokkosFFT::irfftn: OutViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::irfftn: InViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::irfftn: OutViewType must be real"); - shape_type zeros = {0}; // default shape means no crop or pad - shape_type _s = {0}; - if( s != zeros ) { - for(int i=0; i zeros = {0}; // default shape means no crop or pad + shape_type _s = {0}; + if (s != zeros) { + for (int i = 0; i < DIM; i++) { + _s.at(i) = s.at(i) / 2 + 1; } - - ifftn(exec_space, in, out, norm, _s); } - template - void irfftn(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - axis_type axes, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, + ifftn(exec_space, in, out, norm, _s); +} + +template +void irfftn(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, axis_type axes, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::irfftn: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::irfftn: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfftn: InViewType must be complex"); - static_assert(std::is_floating_point::value, - "KokkosFFT::irfftn: OutViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::irfftn: InViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::irfftn: OutViewType must be real"); - shape_type zeros = {0}; // default shape means no crop or pad - shape_type _s = {0}; - if( s != zeros ) { - for(int i=0; i zeros = {0}; // default shape means no crop or pad + shape_type _s = {0}; + if (s != zeros) { + for (int i = 0; i < DIM2; i++) { + _s.at(i) = s.at(i) / 2 + 1; } - - ifftn(exec_space, in, out, axes, norm, _s); } - template - void irfftn(const ExecutionSpace& exec_space, - const InViewType& in, - OutViewType& out, - const PlanType& plan, - axis_type axes, - KokkosFFT::Normalization norm=KokkosFFT::Normalization::BACKWARD, - shape_type s={0}) { - static_assert(Kokkos::is_view::value, + ifftn(exec_space, in, out, axes, norm, _s); +} + +template +void irfftn(const ExecutionSpace& exec_space, const InViewType& in, + OutViewType& out, const PlanType& plan, axis_type axes, + KokkosFFT::Normalization norm = KokkosFFT::Normalization::BACKWARD, + shape_type s = {0}) { + static_assert(Kokkos::is_view::value, "KokkosFFT::irfftn: InViewType is not a Kokkos::View."); - static_assert(Kokkos::is_view::value, + static_assert(Kokkos::is_view::value, "KokkosFFT::irfftn: OutViewType is not a Kokkos::View."); - using in_value_type = typename InViewType::non_const_value_type; - using out_value_type = typename OutViewType::non_const_value_type; + using in_value_type = typename InViewType::non_const_value_type; + using out_value_type = typename OutViewType::non_const_value_type; - static_assert(KokkosFFT::Impl::is_complex::value, - "KokkosFFT::irfftn: InViewType must be complex"); - static_assert(std::is_floating_point::value, - "KokkosFFT::irfftn: OutViewType must be real"); + static_assert(KokkosFFT::Impl::is_complex::value, + "KokkosFFT::irfftn: InViewType must be complex"); + static_assert(std::is_floating_point::value, + "KokkosFFT::irfftn: OutViewType must be real"); - shape_type zeros = {0}; // default shape means no crop or pad - shape_type _s = {0}; - if( s != zeros ) { - for(int i=0; i zeros = {0}; // default shape means no crop or pad + shape_type _s = {0}; + if (s != zeros) { + for (int i = 0; i < DIM2; i++) { + _s.at(i) = s.at(i) / 2 + 1; } - - ifftn(exec_space, in, out, plan, axes, norm, _s); } -}; + + ifftn(exec_space, in, out, plan, axes, norm, _s); +} +}; // namespace KokkosFFT #endif \ No newline at end of file diff --git a/fft/unit_test/Test_Helpers.cpp b/fft/unit_test/Test_Helpers.cpp index 5fb5b8c9..fcbe46af 100644 --- a/fft/unit_test/Test_Helpers.cpp +++ b/fft/unit_test/Test_Helpers.cpp @@ -8,17 +8,15 @@ template using axes_type = std::array; -using test_types = ::testing::Types< - std::pair, - std::pair, - std::pair, - std::pair ->; +using test_types = ::testing::Types, + std::pair, + std::pair, + std::pair >; // Basically the same fixtures, used for labeling tests template struct FFTHelper : public ::testing::Test { - using float_type = typename T::first_type; + using float_type = typename T::first_type; using layout_type = typename T::second_type; }; @@ -26,89 +24,98 @@ TYPED_TEST_SUITE(FFTHelper, test_types); // Tests for FFT Freq template -void test_fft_freq(T atol=1.0e-12) { +void test_fft_freq(T atol = 1.0e-12) { constexpr std::size_t n_odd = 9, n_even = 10; using RealView1DType = Kokkos::View; - RealView1DType x_odd_ref("x_odd_ref", n_odd), x_even_ref("x_even_ref", n_even); + RealView1DType x_odd_ref("x_odd_ref", n_odd), + x_even_ref("x_even_ref", n_even); auto h_x_odd_ref = Kokkos::create_mirror_view(x_odd_ref); auto h_x_even_ref = Kokkos::create_mirror_view(x_even_ref); - std::vector _x_odd_ref = {0, 1, 2, 3, 4, -4, -3, -2, -1}; + std::vector _x_odd_ref = {0, 1, 2, 3, 4, -4, -3, -2, -1}; std::vector _x_even_ref = {0, 1, 2, 3, 4, -5, -4, -3, -2, -1}; - for(std::size_t i=0; i<_x_odd_ref.size(); i++) { - h_x_odd_ref(i) = static_cast( _x_odd_ref.at(i) ); + for (std::size_t i = 0; i < _x_odd_ref.size(); i++) { + h_x_odd_ref(i) = static_cast(_x_odd_ref.at(i)); } - for(std::size_t i=0; i<_x_even_ref.size(); i++) { - h_x_even_ref(i) = static_cast( _x_even_ref.at(i) ); + for (std::size_t i = 0; i < _x_even_ref.size(); i++) { + h_x_even_ref(i) = static_cast(_x_even_ref.at(i)); } Kokkos::deep_copy(x_odd_ref, h_x_odd_ref); Kokkos::deep_copy(x_even_ref, h_x_even_ref); - T pi = static_cast(M_PI); + T pi = static_cast(M_PI); auto x_odd = KokkosFFT::fftfreq(execution_space(), n_odd); - auto x_odd_pi = KokkosFFT::fftfreq(execution_space(), n_odd, pi); + auto x_odd_pi = + KokkosFFT::fftfreq(execution_space(), n_odd, pi); multiply(x_odd, static_cast(n_odd)); - multiply(x_odd_pi, static_cast(n_odd)*pi); + multiply(x_odd_pi, static_cast(n_odd) * pi); - EXPECT_TRUE( allclose(x_odd, x_odd_ref, 1.e-5, atol) ); - EXPECT_TRUE( allclose(x_odd_pi, x_odd_ref, 1.e-5, atol) ); + EXPECT_TRUE(allclose(x_odd, x_odd_ref, 1.e-5, atol)); + EXPECT_TRUE(allclose(x_odd_pi, x_odd_ref, 1.e-5, atol)); - auto x_even = KokkosFFT::fftfreq(execution_space(), n_even); - auto x_even_pi = KokkosFFT::fftfreq(execution_space(), n_even, pi); + auto x_even = + KokkosFFT::fftfreq(execution_space(), n_even); + auto x_even_pi = + KokkosFFT::fftfreq(execution_space(), n_even, pi); multiply(x_even, static_cast(n_even)); multiply(x_even_pi, static_cast(n_even) * pi); - EXPECT_TRUE( allclose(x_even, x_even_ref, 1.e-5, atol) ); - EXPECT_TRUE( allclose(x_even_pi, x_even_ref, 1.e-5, atol) ); + EXPECT_TRUE(allclose(x_even, x_even_ref, 1.e-5, atol)); + EXPECT_TRUE(allclose(x_even_pi, x_even_ref, 1.e-5, atol)); } // Tests for RFFT Freq template -void test_rfft_freq(T atol=1.0e-12) { +void test_rfft_freq(T atol = 1.0e-12) { constexpr std::size_t n_odd = 9, n_even = 10; using RealView1DType = Kokkos::View; - RealView1DType x_odd_ref("x_odd_ref", n_odd/2+1), x_even_ref("x_even_ref", n_even/2+1); + RealView1DType x_odd_ref("x_odd_ref", n_odd / 2 + 1), + x_even_ref("x_even_ref", n_even / 2 + 1); auto h_x_odd_ref = Kokkos::create_mirror_view(x_odd_ref); auto h_x_even_ref = Kokkos::create_mirror_view(x_even_ref); - std::vector _x_odd_ref = {0, 1, 2, 3, 4}; + std::vector _x_odd_ref = {0, 1, 2, 3, 4}; std::vector _x_even_ref = {0, 1, 2, 3, 4, 5}; - for(std::size_t i=0; i<_x_odd_ref.size(); i++) { - h_x_odd_ref(i) = static_cast( _x_odd_ref.at(i) ); + for (std::size_t i = 0; i < _x_odd_ref.size(); i++) { + h_x_odd_ref(i) = static_cast(_x_odd_ref.at(i)); } - for(std::size_t i=0; i<_x_even_ref.size(); i++) { - h_x_even_ref(i) = static_cast( _x_even_ref.at(i) ); + for (std::size_t i = 0; i < _x_even_ref.size(); i++) { + h_x_even_ref(i) = static_cast(_x_even_ref.at(i)); } Kokkos::deep_copy(x_odd_ref, h_x_odd_ref); Kokkos::deep_copy(x_even_ref, h_x_even_ref); T pi = static_cast(M_PI); - auto x_odd = KokkosFFT::rfftfreq(execution_space(), n_odd); - auto x_odd_pi = KokkosFFT::rfftfreq(execution_space(), n_odd, pi); + auto x_odd = + KokkosFFT::rfftfreq(execution_space(), n_odd); + auto x_odd_pi = + KokkosFFT::rfftfreq(execution_space(), n_odd, pi); multiply(x_odd, static_cast(n_odd)); - multiply(x_odd_pi, static_cast(n_odd)*pi); + multiply(x_odd_pi, static_cast(n_odd) * pi); - EXPECT_TRUE( allclose(x_odd, x_odd_ref, 1.e-5, atol) ); - EXPECT_TRUE( allclose(x_odd_pi, x_odd_ref, 1.e-5, atol) ); + EXPECT_TRUE(allclose(x_odd, x_odd_ref, 1.e-5, atol)); + EXPECT_TRUE(allclose(x_odd_pi, x_odd_ref, 1.e-5, atol)); - auto x_even = KokkosFFT::rfftfreq(execution_space(), n_even); - auto x_even_pi = KokkosFFT::rfftfreq(execution_space(), n_even, pi); + auto x_even = + KokkosFFT::rfftfreq(execution_space(), n_even); + auto x_even_pi = + KokkosFFT::rfftfreq(execution_space(), n_even, pi); multiply(x_even, static_cast(n_even)); multiply(x_even_pi, static_cast(n_even) * pi); - EXPECT_TRUE( allclose(x_even, x_even_ref, 1.e-5, atol) ); - EXPECT_TRUE( allclose(x_even_pi, x_even_ref, 1.e-5, atol) ); + EXPECT_TRUE(allclose(x_even, x_even_ref, 1.e-5, atol)); + EXPECT_TRUE(allclose(x_even_pi, x_even_ref, 1.e-5, atol)); } // Tests for fftfreq TYPED_TEST(FFTHelper, fftfreq) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; float_type atol = std::is_same_v ? 1.0e-6 : 1.0e-12; @@ -117,7 +124,7 @@ TYPED_TEST(FFTHelper, fftfreq) { // Tests for rfftfreq TYPED_TEST(FFTHelper, rfftfreq) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; float_type atol = std::is_same_v ? 1.0e-6 : 1.0e-12; @@ -132,48 +139,53 @@ void test_get_shift(int direction) { RealView1DType x1_odd("x1_odd", n_odd), x1_even("x1_even", n_even); RealView2DType x2_odd("x2_odd", n_odd, n2), x2_even("x2_even", n_even, n2); - KokkosFFT::axis_type<1> shift1_odd_ref = {direction * n_odd/2}; - KokkosFFT::axis_type<1> shift1_even_ref = {direction * n_even/2}; - KokkosFFT::axis_type<2> shift1_axis0_odd_ref = {direction * n_odd/2, 0}; - KokkosFFT::axis_type<2> shift1_axis0_even_ref = {direction * n_even/2, 0}; - KokkosFFT::axis_type<2> shift1_axis1_odd_ref = {0, direction * n2/2}; - KokkosFFT::axis_type<2> shift1_axis1_even_ref = {0, direction * n2/2}; - KokkosFFT::axis_type<2> shift2_odd_ref = {direction * n_odd/2, direction * n2/2}; - KokkosFFT::axis_type<2> shift2_even_ref = {direction * n_even/2, direction * n2/2}; - - auto shift1_odd = KokkosFFT::Impl::_get_shift(x1_odd, KokkosFFT::axis_type<1>({0}), direction); - auto shift1_even = KokkosFFT::Impl::_get_shift(x1_even, KokkosFFT::axis_type<1>({0}), direction); - auto shift1_axis0_odd = KokkosFFT::Impl::_get_shift(x2_odd, KokkosFFT::axis_type<1>({0}), direction); - auto shift1_axis0_even = KokkosFFT::Impl::_get_shift(x2_even, KokkosFFT::axis_type<1>({0}), direction); - auto shift1_axis1_odd = KokkosFFT::Impl::_get_shift(x2_odd, KokkosFFT::axis_type<1>({1}), direction); - auto shift1_axis1_even = KokkosFFT::Impl::_get_shift(x2_even, KokkosFFT::axis_type<1>({1}), direction); - auto shift2_odd = KokkosFFT::Impl::_get_shift(x2_odd, KokkosFFT::axis_type<2>({0, 1}), direction); - auto shift2_even = KokkosFFT::Impl::_get_shift(x2_even, KokkosFFT::axis_type<2>({0, 1}), direction); - - EXPECT_TRUE( shift1_odd == shift1_odd_ref ); - EXPECT_TRUE( shift1_even == shift1_even_ref ); - EXPECT_TRUE( shift1_axis0_odd == shift1_axis0_odd_ref ); - EXPECT_TRUE( shift1_axis0_even == shift1_axis0_even_ref ); - EXPECT_TRUE( shift1_axis1_odd == shift1_axis1_odd_ref ); - EXPECT_TRUE( shift1_axis1_even == shift1_axis1_even_ref ); - EXPECT_TRUE( shift2_odd == shift2_odd_ref ); - EXPECT_TRUE( shift2_even == shift2_even_ref ); + KokkosFFT::axis_type<1> shift1_odd_ref = {direction * n_odd / 2}; + KokkosFFT::axis_type<1> shift1_even_ref = {direction * n_even / 2}; + KokkosFFT::axis_type<2> shift1_axis0_odd_ref = {direction * n_odd / 2, 0}; + KokkosFFT::axis_type<2> shift1_axis0_even_ref = {direction * n_even / 2, 0}; + KokkosFFT::axis_type<2> shift1_axis1_odd_ref = {0, direction * n2 / 2}; + KokkosFFT::axis_type<2> shift1_axis1_even_ref = {0, direction * n2 / 2}; + KokkosFFT::axis_type<2> shift2_odd_ref = {direction * n_odd / 2, + direction * n2 / 2}; + KokkosFFT::axis_type<2> shift2_even_ref = {direction * n_even / 2, + direction * n2 / 2}; + + auto shift1_odd = KokkosFFT::Impl::_get_shift( + x1_odd, KokkosFFT::axis_type<1>({0}), direction); + auto shift1_even = KokkosFFT::Impl::_get_shift( + x1_even, KokkosFFT::axis_type<1>({0}), direction); + auto shift1_axis0_odd = KokkosFFT::Impl::_get_shift( + x2_odd, KokkosFFT::axis_type<1>({0}), direction); + auto shift1_axis0_even = KokkosFFT::Impl::_get_shift( + x2_even, KokkosFFT::axis_type<1>({0}), direction); + auto shift1_axis1_odd = KokkosFFT::Impl::_get_shift( + x2_odd, KokkosFFT::axis_type<1>({1}), direction); + auto shift1_axis1_even = KokkosFFT::Impl::_get_shift( + x2_even, KokkosFFT::axis_type<1>({1}), direction); + auto shift2_odd = KokkosFFT::Impl::_get_shift( + x2_odd, KokkosFFT::axis_type<2>({0, 1}), direction); + auto shift2_even = KokkosFFT::Impl::_get_shift( + x2_even, KokkosFFT::axis_type<2>({0, 1}), direction); + + EXPECT_TRUE(shift1_odd == shift1_odd_ref); + EXPECT_TRUE(shift1_even == shift1_even_ref); + EXPECT_TRUE(shift1_axis0_odd == shift1_axis0_odd_ref); + EXPECT_TRUE(shift1_axis0_even == shift1_axis0_even_ref); + EXPECT_TRUE(shift1_axis1_odd == shift1_axis1_odd_ref); + EXPECT_TRUE(shift1_axis1_even == shift1_axis1_even_ref); + EXPECT_TRUE(shift2_odd == shift2_odd_ref); + EXPECT_TRUE(shift2_even == shift2_even_ref); } -class GetShiftParamTests: public ::testing::TestWithParam {}; +class GetShiftParamTests : public ::testing::TestWithParam {}; TEST_P(GetShiftParamTests, ForwardAndInverse) { int direction = GetParam(); test_get_shift(direction); } -INSTANTIATE_TEST_SUITE_P( - GetShift, - GetShiftParamTests, - ::testing::Values( - 1, -1 - ) -); +INSTANTIATE_TEST_SUITE_P(GetShift, GetShiftParamTests, + ::testing::Values(1, -1)); // Identity Tests for fftshift1D on 1D View void test_fftshift1D_1DView_identity(int n0) { @@ -190,7 +202,7 @@ void test_fftshift1D_1DView_identity(int n0) { KokkosFFT::fftshift(execution_space(), x); KokkosFFT::ifftshift(execution_space(), x); - EXPECT_TRUE( allclose(x, x_ref, 1.e-5, 1.e-12) ); + EXPECT_TRUE(allclose(x, x_ref, 1.e-5, 1.e-12)); } // Tests for fftshift1D on 1D View @@ -199,13 +211,13 @@ void test_fftshift1D_1DView(int n0) { RealView1DType x("x", n0), y("y", n0); RealView1DType x_ref("x_ref", n0), y_ref("y_ref", n0); - auto h_x_ref = Kokkos::create_mirror_view(x_ref); - auto h_y_ref = Kokkos::create_mirror_view(y_ref); + auto h_x_ref = Kokkos::create_mirror_view(x_ref); + auto h_y_ref = Kokkos::create_mirror_view(y_ref); std::vector _x_ref; std::vector _y_ref; - if(n0 % 2 == 0) { + if (n0 % 2 == 0) { _x_ref = {0, 1, 2, 3, 4, -5, -4, -3, -2, -1}; _y_ref = {-5, -4, -3, -2, -1, 0, 1, 2, 3, 4}; } else { @@ -213,9 +225,9 @@ void test_fftshift1D_1DView(int n0) { _y_ref = {-4, -3, -2, -1, 0, 1, 2, 3, 4}; } - for(std::size_t i=0; i( _x_ref.at(i) ); - h_y_ref(i) = static_cast( _y_ref.at(i) ); + for (std::size_t i = 0; i < n0; i++) { + h_x_ref(i) = static_cast(_x_ref.at(i)); + h_y_ref(i) = static_cast(_y_ref.at(i)); } Kokkos::deep_copy(x_ref, h_x_ref); @@ -226,59 +238,53 @@ void test_fftshift1D_1DView(int n0) { KokkosFFT::fftshift(execution_space(), x); KokkosFFT::ifftshift(execution_space(), y); - EXPECT_TRUE( allclose(x, y_ref) ); - EXPECT_TRUE( allclose(y, x_ref) ); + EXPECT_TRUE(allclose(x, y_ref)); + EXPECT_TRUE(allclose(y, x_ref)); } // Tests for fftshift1D on 2D View void test_fftshift1D_2DView(int n0) { - using RealView2DType = Kokkos::View; + using RealView2DType = + Kokkos::View; constexpr int n1 = 3; - RealView2DType x("x", n0, n1), y_axis0("y_axis0", n0, n1), y_axis1("y_axis1", n0, n1); + RealView2DType x("x", n0, n1), y_axis0("y_axis0", n0, n1), + y_axis1("y_axis1", n0, n1); RealView2DType x_ref("x_ref", n0, n1); - RealView2DType y_axis0_ref("y_axis0_ref", n0, n1), y_axis1_ref("y_axis1_ref", n0, n1); + RealView2DType y_axis0_ref("y_axis0_ref", n0, n1), + y_axis1_ref("y_axis1_ref", n0, n1); - auto h_x_ref = Kokkos::create_mirror_view(x_ref); - auto h_y_axis0_ref = Kokkos::create_mirror_view(y_axis0_ref); - auto h_y_axis1_ref = Kokkos::create_mirror_view(y_axis1_ref); + auto h_x_ref = Kokkos::create_mirror_view(x_ref); + auto h_y_axis0_ref = Kokkos::create_mirror_view(y_axis0_ref); + auto h_y_axis1_ref = Kokkos::create_mirror_view(y_axis1_ref); std::vector _x_ref; std::vector _y0_ref, _y1_ref; - if(n0 % 2 == 0) { - _x_ref = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, -15, -14, -13, -12, -11, - -10, -9, -8, -7, -6, -5, -4, -3, -2, -1 - }; - _y0_ref = {5, 6, 7, 8, 9, 0, 1, 2, 3, 4, - -15, -14, -13, -12, -11, 10, 11, 12, 13, 14, - -5, -4, -3, -2, -1, -10, -9, -8, -7, -6, - }; - _y1_ref = {-10, -9, -8, -7, -6, -5, -4, -3, -2, -1, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, -15, -14, -13, -12, -11 - }; + if (n0 % 2 == 0) { + _x_ref = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + -15, -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1}; + _y0_ref = { + 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, -15, -14, -13, -12, -11, + 10, 11, 12, 13, 14, -5, -4, -3, -2, -1, -10, -9, -8, -7, -6, + }; + _y1_ref = {-10, -9, -8, -7, -6, -5, -4, -3, -2, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, -15, -14, -13, -12, -11}; } else { - _x_ref = {0, 1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 13, -13, -12, -11, -10, - -9, -8, -7, -6, -5, -4, -3, -2, -1 - }; - _y0_ref = {5, 6, 7, 8, 0, 1, 2, 3, 4, - -13, -12, -11, -10, 9, 10, 11, 12, 13, - -4, -3, -2, -1, -9, -8, -7, -6, -5 - }; - _y1_ref = {-9, -8, -7, -6, -5, -4, -3, -2, -1, - 0, 1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 13, -13, -12, -11, -10 - }; + _x_ref = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1}; + _y0_ref = {5, 6, 7, 8, 0, 1, 2, 3, 4, -13, -12, -11, -10, 9, + 10, 11, 12, 13, -4, -3, -2, -1, -9, -8, -7, -6, -5}; + _y1_ref = {-9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, + 5, 6, 7, 8, 9, 10, 11, 12, 13, -13, -12, -11, -10}; } - for(std::size_t i1=0; i1( _x_ref.at(i) ); - h_y_axis0_ref(i0, i1) = static_cast( _y0_ref.at(i) ); - h_y_axis1_ref(i0, i1) = static_cast( _y1_ref.at(i) ); + for (std::size_t i1 = 0; i1 < n1; i1++) { + for (std::size_t i0 = 0; i0 < n0; i0++) { + std::size_t i = i0 + i1 * n0; + h_x_ref(i0, i1) = static_cast(_x_ref.at(i)); + h_y_axis0_ref(i0, i1) = static_cast(_y0_ref.at(i)); + h_y_axis1_ref(i0, i1) = static_cast(_y1_ref.at(i)); } } @@ -292,56 +298,49 @@ void test_fftshift1D_2DView(int n0) { KokkosFFT::fftshift(execution_space(), x, axes_type<1>({0})); KokkosFFT::ifftshift(execution_space(), y_axis0, axes_type<1>({0})); - EXPECT_TRUE( allclose(x, y_axis0_ref) ); - EXPECT_TRUE( allclose(y_axis0, x_ref) ); + EXPECT_TRUE(allclose(x, y_axis0_ref)); + EXPECT_TRUE(allclose(y_axis0, x_ref)); Kokkos::deep_copy(x, h_x_ref); KokkosFFT::fftshift(execution_space(), x, axes_type<1>({1})); KokkosFFT::ifftshift(execution_space(), y_axis1, axes_type<1>({1})); - EXPECT_TRUE( allclose(x, y_axis1_ref) ); - EXPECT_TRUE( allclose(y_axis1, x_ref) ); + EXPECT_TRUE(allclose(x, y_axis1_ref)); + EXPECT_TRUE(allclose(y_axis1, x_ref)); } // Tests for fftshift2D on 2D View void test_fftshift2D_2DView(int n0) { - using RealView2DType = Kokkos::View; + using RealView2DType = + Kokkos::View; constexpr int n1 = 3; RealView2DType x("x", n0, n1), y("y", n0, n1); RealView2DType x_ref("x_ref", n0, n1), y_ref("y_ref", n0, n1); - auto h_x_ref = Kokkos::create_mirror_view(x_ref); - auto h_y_ref = Kokkos::create_mirror_view(y_ref); + auto h_x_ref = Kokkos::create_mirror_view(x_ref); + auto h_y_ref = Kokkos::create_mirror_view(y_ref); std::vector _x_ref; std::vector _y_ref; - if(n0 % 2 == 0) { - _x_ref = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, -15, -14, -13, -12, -11, - -10, -9, -8, -7, -6, -5, -4, -3, -2, -1 - }; - _y_ref = {-5, -4, -3, -2, -1, -10, -9, -8, -7, -6, - 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, - -15, -14, -13, -12, -11, 10, 11, 12, 13, 14 - }; + if (n0 % 2 == 0) { + _x_ref = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + -15, -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1}; + _y_ref = {-5, -4, -3, -2, -1, -10, -9, -8, -7, -6, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, -15, -14, -13, -12, -11, 10, 11, 12, 13, 14}; } else { - _x_ref = {0, 1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 13, -13, -12, -11, -10, - -9, -8, -7, -6, -5, -4, -3, -2, -1 - }; - _y_ref = {-4, -3, -2, -1, -9, -8, -7, -6, -5, - 5, 6, 7, 8, 0, 1, 2, 3, 4, - -13, -12, -11, -10, 9, 10, 11, 12, 13 - }; + _x_ref = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1}; + _y_ref = {-4, -3, -2, -1, -9, -8, -7, -6, -5, 5, 6, 7, 8, 0, + 1, 2, 3, 4, -13, -12, -11, -10, 9, 10, 11, 12, 13}; } - for(std::size_t i1=0; i1( _x_ref.at(i) ); - h_y_ref(i0, i1) = static_cast( _y_ref.at(i) ); + for (std::size_t i1 = 0; i1 < n1; i1++) { + for (std::size_t i0 = 0; i0 < n0; i0++) { + std::size_t i = i0 + i1 * n0; + h_x_ref(i0, i1) = static_cast(_x_ref.at(i)); + h_y_ref(i0, i1) = static_cast(_y_ref.at(i)); } } @@ -353,11 +352,11 @@ void test_fftshift2D_2DView(int n0) { KokkosFFT::fftshift(execution_space(), x, axes_type<2>({0, 1})); KokkosFFT::ifftshift(execution_space(), y, axes_type<2>({0, 1})); - EXPECT_TRUE( allclose(x, y_ref) ); - EXPECT_TRUE( allclose(y, x_ref) ); + EXPECT_TRUE(allclose(x, y_ref)); + EXPECT_TRUE(allclose(y, x_ref)); } -class FFTShiftParamTests: public ::testing::TestWithParam {}; +class FFTShiftParamTests : public ::testing::TestWithParam {}; // Identity Tests for fftshift1D on 1D View TEST_P(FFTShiftParamTests, Identity) { @@ -383,10 +382,5 @@ TEST_P(FFTShiftParamTests, 2DShift2DView) { test_fftshift2D_2DView(n0); } -INSTANTIATE_TEST_SUITE_P( - FFTShift, - FFTShiftParamTests, - ::testing::Values( - 9, 10 - ) -); \ No newline at end of file +INSTANTIATE_TEST_SUITE_P(FFTShift, FFTShiftParamTests, + ::testing::Values(9, 10)); \ No newline at end of file diff --git a/fft/unit_test/Test_Main.cpp b/fft/unit_test/Test_Main.cpp index c02cf018..b4cfc19a 100644 --- a/fft/unit_test/Test_Main.cpp +++ b/fft/unit_test/Test_Main.cpp @@ -17,12 +17,12 @@ #include namespace testing::internal { -// accessing gtest internals is not very clean, but gtest provides no public access... +// accessing gtest internals is not very clean, but gtest provides no public +// access... extern bool g_help_flag; -} // namespace testing::internal +} // namespace testing::internal int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); int result = 0; if (::testing::GTEST_FLAG(list_tests) || ::testing::internal::g_help_flag) { diff --git a/fft/unit_test/Test_Plans.cpp b/fft/unit_test/Test_Plans.cpp index d796898c..197ff67d 100644 --- a/fft/unit_test/Test_Plans.cpp +++ b/fft/unit_test/Test_Plans.cpp @@ -5,29 +5,27 @@ template using axes_type = std::array; -using test_types = ::testing::Types< - std::pair, - std::pair, - std::pair, - std::pair ->; +using test_types = ::testing::Types, + std::pair, + std::pair, + std::pair >; // Basically the same fixtures, used for labeling tests template struct Plans1D : public ::testing::Test { - using float_type = typename T::first_type; + using float_type = typename T::first_type; using layout_type = typename T::second_type; }; template struct Plans2D : public ::testing::Test { - using float_type = typename T::first_type; + using float_type = typename T::first_type; using layout_type = typename T::second_type; }; template struct Plans3D : public ::testing::Test { - using float_type = typename T::first_type; + using float_type = typename T::first_type; using layout_type = typename T::second_type; }; @@ -38,83 +36,141 @@ TYPED_TEST_SUITE(Plans3D, test_types); // Tests for 1D FFT Plans template void test_plan_1dfft_1dview() { - const int n = 30; + const int n = 30; using RealView1DType = Kokkos::View; - using ComplexView1DType = Kokkos::View*, LayoutType, execution_space>; + using ComplexView1DType = + Kokkos::View*, LayoutType, execution_space>; RealView1DType x("x", n); - ComplexView1DType x_c("x_c", n/2+1); + ComplexView1DType x_c("x_c", n / 2 + 1); ComplexView1DType x_cin("x_cin", n), x_cout("x_cout", n); // R2C plan - KokkosFFT::Impl::Plan plan_r2c_axis_0(execution_space(), x, x_c, KokkosFFT::Impl::Direction::Forward, /*axis=*/0); - KokkosFFT::Impl::Plan plan_r2c_axes_0(execution_space(), x, x_c, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<1>({0})); + KokkosFFT::Impl::Plan plan_r2c_axis_0(execution_space(), x, x_c, + KokkosFFT::Impl::Direction::Forward, + /*axis=*/0); + KokkosFFT::Impl::Plan plan_r2c_axes_0(execution_space(), x, x_c, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<1>({0})); // C2R plan - KokkosFFT::Impl::Plan plan_c2r_axis0(execution_space(), x_c, x, KokkosFFT::Impl::Direction::Backward, /*axis=*/0); - KokkosFFT::Impl::Plan plan_c2r_axes0(execution_space(), x_c, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<1>({0})); + KokkosFFT::Impl::Plan plan_c2r_axis0(execution_space(), x_c, x, + KokkosFFT::Impl::Direction::Backward, + /*axis=*/0); + KokkosFFT::Impl::Plan plan_c2r_axes0(execution_space(), x_c, x, + KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<1>({0})); // C2C plan - KokkosFFT::Impl::Plan plan_c2c_f_axis0(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axis=*/0); - KokkosFFT::Impl::Plan plan_c2c_f_axes0(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<1>({0})); + KokkosFFT::Impl::Plan plan_c2c_f_axis0(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axis=*/0); + KokkosFFT::Impl::Plan plan_c2c_f_axes0(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<1>({0})); } template void test_plan_1dfft_2dview() { const int n0 = 10, n1 = 6; using RealView2DType = Kokkos::View; - using ComplexView2DType = Kokkos::View**, LayoutType, execution_space>; + using ComplexView2DType = + Kokkos::View**, LayoutType, execution_space>; RealView2DType x("x", n0, n1); - ComplexView2DType x_c_axis_0("x_c_axis_0", n0/2+1, n1), x_c_axis_1("x_c_axis_1", n0, n1/2+1); + ComplexView2DType x_c_axis_0("x_c_axis_0", n0 / 2 + 1, n1), + x_c_axis_1("x_c_axis_1", n0, n1 / 2 + 1); ComplexView2DType x_cin("x_cin", n0, n1), x_cout("x_cout", n0, n1); // R2C plan - KokkosFFT::Impl::Plan plan_r2c_axis_0(execution_space(), x, x_c_axis_0, KokkosFFT::Impl::Direction::Forward, /*axis=*/0); - KokkosFFT::Impl::Plan plan_r2c_axis_1(execution_space(), x, x_c_axis_1, KokkosFFT::Impl::Direction::Forward, /*axis=*/1); - KokkosFFT::Impl::Plan plan_r2c_axis_minus1(execution_space(), x, x_c_axis_1, KokkosFFT::Impl::Direction::Forward, /*axis=*/-1); + KokkosFFT::Impl::Plan plan_r2c_axis_0(execution_space(), x, x_c_axis_0, + KokkosFFT::Impl::Direction::Forward, + /*axis=*/0); + KokkosFFT::Impl::Plan plan_r2c_axis_1(execution_space(), x, x_c_axis_1, + KokkosFFT::Impl::Direction::Forward, + /*axis=*/1); + KokkosFFT::Impl::Plan plan_r2c_axis_minus1( + execution_space(), x, x_c_axis_1, KokkosFFT::Impl::Direction::Forward, + /*axis=*/-1); // C2R plan - KokkosFFT::Impl::Plan plan_c2r_axis_0(execution_space(), x_c_axis_0, x, KokkosFFT::Impl::Direction::Backward, /*axis=*/0); - KokkosFFT::Impl::Plan plan_c2r_axis_1(execution_space(), x_c_axis_1, x, KokkosFFT::Impl::Direction::Backward, /*axis=*/1); - KokkosFFT::Impl::Plan plan_c2r_axis_minus1(execution_space(), x_c_axis_1, x, KokkosFFT::Impl::Direction::Backward, /*axis=*/-1); + KokkosFFT::Impl::Plan plan_c2r_axis_0(execution_space(), x_c_axis_0, x, + KokkosFFT::Impl::Direction::Backward, + /*axis=*/0); + KokkosFFT::Impl::Plan plan_c2r_axis_1(execution_space(), x_c_axis_1, x, + KokkosFFT::Impl::Direction::Backward, + /*axis=*/1); + KokkosFFT::Impl::Plan plan_c2r_axis_minus1( + execution_space(), x_c_axis_1, x, KokkosFFT::Impl::Direction::Backward, + /*axis=*/-1); // C2C plan - KokkosFFT::Impl::Plan plan_c2c_f_axis_0(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axis=*/0); - KokkosFFT::Impl::Plan plan_c2c_f_axis_1(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axis=*/1); + KokkosFFT::Impl::Plan plan_c2c_f_axis_0(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axis=*/0); + KokkosFFT::Impl::Plan plan_c2c_f_axis_1(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axis=*/1); } template void test_plan_1dfft_3dview() { const int n0 = 10, n1 = 6, n2 = 8; - using RealView3DType = Kokkos::View; - using ComplexView3DType = Kokkos::View***, LayoutType, execution_space>; + using RealView3DType = Kokkos::View; + using ComplexView3DType = + Kokkos::View***, LayoutType, execution_space>; RealView3DType x("x", n0, n1, n2); - ComplexView3DType x_c_axis_0("x_c_axis_0", n0/2+1, n1, n2), x_c_axis_1("x_c_axis_1", n0, n1/2+1, n2), x_c_axis_2("x_c_axis_2", n0, n1, n2/2+1); + ComplexView3DType x_c_axis_0("x_c_axis_0", n0 / 2 + 1, n1, n2), + x_c_axis_1("x_c_axis_1", n0, n1 / 2 + 1, n2), + x_c_axis_2("x_c_axis_2", n0, n1, n2 / 2 + 1); ComplexView3DType x_cin("x_cin", n0, n1, n2), x_cout("x_cout", n0, n1, n2); // R2C plan - KokkosFFT::Impl::Plan plan_r2c_axis_0(execution_space(), x, x_c_axis_0, KokkosFFT::Impl::Direction::Forward, /*axis=*/0); - KokkosFFT::Impl::Plan plan_r2c_axis_1(execution_space(), x, x_c_axis_1, KokkosFFT::Impl::Direction::Forward, /*axis=*/1); - KokkosFFT::Impl::Plan plan_r2c_axis_2(execution_space(), x, x_c_axis_2, KokkosFFT::Impl::Direction::Forward, /*axis=*/2); + KokkosFFT::Impl::Plan plan_r2c_axis_0(execution_space(), x, x_c_axis_0, + KokkosFFT::Impl::Direction::Forward, + /*axis=*/0); + KokkosFFT::Impl::Plan plan_r2c_axis_1(execution_space(), x, x_c_axis_1, + KokkosFFT::Impl::Direction::Forward, + /*axis=*/1); + KokkosFFT::Impl::Plan plan_r2c_axis_2(execution_space(), x, x_c_axis_2, + KokkosFFT::Impl::Direction::Forward, + /*axis=*/2); // C2R plan - KokkosFFT::Impl::Plan plan_c2r_axis_0(execution_space(), x_c_axis_0, x, KokkosFFT::Impl::Direction::Backward, /*axis=*/0); - KokkosFFT::Impl::Plan plan_c2r_axis_1(execution_space(), x_c_axis_1, x, KokkosFFT::Impl::Direction::Backward, /*axis=*/1); - KokkosFFT::Impl::Plan plan_c2r_axis_2(execution_space(), x_c_axis_2, x, KokkosFFT::Impl::Direction::Backward, /*axis=*/2); + KokkosFFT::Impl::Plan plan_c2r_axis_0(execution_space(), x_c_axis_0, x, + KokkosFFT::Impl::Direction::Backward, + /*axis=*/0); + KokkosFFT::Impl::Plan plan_c2r_axis_1(execution_space(), x_c_axis_1, x, + KokkosFFT::Impl::Direction::Backward, + /*axis=*/1); + KokkosFFT::Impl::Plan plan_c2r_axis_2(execution_space(), x_c_axis_2, x, + KokkosFFT::Impl::Direction::Backward, + /*axis=*/2); // C2C plan - KokkosFFT::Impl::Plan plan_c2c_f_axis_0(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axis=*/0); - KokkosFFT::Impl::Plan plan_c2c_f_axis_1(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axis=*/1); - KokkosFFT::Impl::Plan plan_c2c_f_axis_2(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axis=*/2); - KokkosFFT::Impl::Plan plan_c2c_b_axis_0(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Backward, /*axis=*/0); - KokkosFFT::Impl::Plan plan_c2c_b_axis_1(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Backward, /*axis=*/1); - KokkosFFT::Impl::Plan plan_c2c_b_axis_2(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Backward, /*axis=*/2); + KokkosFFT::Impl::Plan plan_c2c_f_axis_0(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axis=*/0); + KokkosFFT::Impl::Plan plan_c2c_f_axis_1(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axis=*/1); + KokkosFFT::Impl::Plan plan_c2c_f_axis_2(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axis=*/2); + KokkosFFT::Impl::Plan plan_c2c_b_axis_0(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Backward, + /*axis=*/0); + KokkosFFT::Impl::Plan plan_c2c_b_axis_1(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Backward, + /*axis=*/1); + KokkosFFT::Impl::Plan plan_c2c_b_axis_2(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Backward, + /*axis=*/2); } // Tests for 1D FFT plan on 1D View TYPED_TEST(Plans1D, 1DFFT_1DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_plan_1dfft_1dview(); @@ -122,7 +178,7 @@ TYPED_TEST(Plans1D, 1DFFT_1DView) { // Tests for 1D batched FFT plan on 2D View TYPED_TEST(Plans1D, 1DFFT_batched_2DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_plan_1dfft_2dview(); @@ -130,7 +186,7 @@ TYPED_TEST(Plans1D, 1DFFT_batched_2DView) { // Tests for 1D batched FFT plan on 3D View TYPED_TEST(Plans1D, 1DFFT_batched_3DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_plan_1dfft_3dview(); @@ -141,62 +197,115 @@ template void test_plan_2dfft_2dview() { const int n0 = 10, n1 = 6; using RealView2DType = Kokkos::View; - using ComplexView2DType = Kokkos::View**, LayoutType, execution_space>; + using ComplexView2DType = + Kokkos::View**, LayoutType, execution_space>; RealView2DType x("x", n0, n1); - ComplexView2DType x_c_axis_0("x_c_axis_0", n0/2+1, n1), x_c_axis_1("x_c_axis_1", n0, n1/2+1); + ComplexView2DType x_c_axis_0("x_c_axis_0", n0 / 2 + 1, n1), + x_c_axis_1("x_c_axis_1", n0, n1 / 2 + 1); ComplexView2DType x_cin("x_cin", n0, n1), x_cout("x_cout", n0, n1); // R2C plan - KokkosFFT::Impl::Plan plan_r2c_axes_0_1(execution_space(), x, x_c_axis_1, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({0, 1})); - KokkosFFT::Impl::Plan plan_r2c_axes_1_0(execution_space(), x, x_c_axis_0, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({1, 0})); + KokkosFFT::Impl::Plan plan_r2c_axes_0_1(execution_space(), x, x_c_axis_1, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({0, 1})); + KokkosFFT::Impl::Plan plan_r2c_axes_1_0(execution_space(), x, x_c_axis_0, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({1, 0})); // C2R plan - KokkosFFT::Impl::Plan plan_c2r_axes_0_1(execution_space(), x_c_axis_1, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<2>({0, 1})); - KokkosFFT::Impl::Plan plan_c2r_axes_1_0(execution_space(), x_c_axis_0, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<2>({1, 0})); + KokkosFFT::Impl::Plan plan_c2r_axes_0_1(execution_space(), x_c_axis_1, x, + KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<2>({0, 1})); + KokkosFFT::Impl::Plan plan_c2r_axes_1_0(execution_space(), x_c_axis_0, x, + KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<2>({1, 0})); // C2C plan - KokkosFFT::Impl::Plan plan_c2c_f_axes_0_1(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({0, 1})); - KokkosFFT::Impl::Plan plan_c2c_f_axes_1_0(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({1, 0})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_0_1(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({0, 1})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_1_0(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({1, 0})); } template void test_plan_2dfft_3dview() { const int n0 = 10, n1 = 6, n2 = 8; - using RealView3DType = Kokkos::View; - using ComplexView3DType = Kokkos::View***, LayoutType, execution_space>; + using RealView3DType = Kokkos::View; + using ComplexView3DType = + Kokkos::View***, LayoutType, execution_space>; RealView3DType x("x", n0, n1, n2); - ComplexView3DType x_c_axis_0("x_c_axis_0", n0/2+1, n1, n2), x_c_axis_1("x_c_axis_1", n0, n1/2+1, n2), x_c_axis_2("x_c_axis_2", n0, n1, n2/2+1); + ComplexView3DType x_c_axis_0("x_c_axis_0", n0 / 2 + 1, n1, n2), + x_c_axis_1("x_c_axis_1", n0, n1 / 2 + 1, n2), + x_c_axis_2("x_c_axis_2", n0, n1, n2 / 2 + 1); ComplexView3DType x_cin("x_cin", n0, n1, n2), x_cout("x_cout", n0, n1, n2); // R2C plan - KokkosFFT::Impl::Plan plan_r2c_axes_0_1(execution_space(), x, x_c_axis_1, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({0, 1})); - KokkosFFT::Impl::Plan plan_r2c_axes_0_2(execution_space(), x, x_c_axis_2, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({0, 2})); - KokkosFFT::Impl::Plan plan_r2c_axes_1_0(execution_space(), x, x_c_axis_0, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({1, 0})); - KokkosFFT::Impl::Plan plan_r2c_axes_1_2(execution_space(), x, x_c_axis_2, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({1, 2})); - KokkosFFT::Impl::Plan plan_r2c_axes_2_0(execution_space(), x, x_c_axis_0, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({2, 0})); - KokkosFFT::Impl::Plan plan_r2c_axes_2_1(execution_space(), x, x_c_axis_1, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({2, 1})); + KokkosFFT::Impl::Plan plan_r2c_axes_0_1(execution_space(), x, x_c_axis_1, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({0, 1})); + KokkosFFT::Impl::Plan plan_r2c_axes_0_2(execution_space(), x, x_c_axis_2, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({0, 2})); + KokkosFFT::Impl::Plan plan_r2c_axes_1_0(execution_space(), x, x_c_axis_0, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({1, 0})); + KokkosFFT::Impl::Plan plan_r2c_axes_1_2(execution_space(), x, x_c_axis_2, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({1, 2})); + KokkosFFT::Impl::Plan plan_r2c_axes_2_0(execution_space(), x, x_c_axis_0, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({2, 0})); + KokkosFFT::Impl::Plan plan_r2c_axes_2_1(execution_space(), x, x_c_axis_1, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({2, 1})); // C2R plan - KokkosFFT::Impl::Plan plan_c2r_axes_0_1(execution_space(), x_c_axis_1, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<2>({0, 1})); - KokkosFFT::Impl::Plan plan_c2r_axes_0_2(execution_space(), x_c_axis_2, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<2>({0, 2})); - KokkosFFT::Impl::Plan plan_c2r_axes_1_0(execution_space(), x_c_axis_0, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<2>({1, 0})); - KokkosFFT::Impl::Plan plan_c2r_axes_1_2(execution_space(), x_c_axis_2, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<2>({1, 2})); - KokkosFFT::Impl::Plan plan_c2r_axes_2_0(execution_space(), x_c_axis_0, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<2>({2, 0})); - KokkosFFT::Impl::Plan plan_c2r_axes_2_1(execution_space(), x_c_axis_1, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<2>({2, 1})); + KokkosFFT::Impl::Plan plan_c2r_axes_0_1(execution_space(), x_c_axis_1, x, + KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<2>({0, 1})); + KokkosFFT::Impl::Plan plan_c2r_axes_0_2(execution_space(), x_c_axis_2, x, + KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<2>({0, 2})); + KokkosFFT::Impl::Plan plan_c2r_axes_1_0(execution_space(), x_c_axis_0, x, + KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<2>({1, 0})); + KokkosFFT::Impl::Plan plan_c2r_axes_1_2(execution_space(), x_c_axis_2, x, + KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<2>({1, 2})); + KokkosFFT::Impl::Plan plan_c2r_axes_2_0(execution_space(), x_c_axis_0, x, + KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<2>({2, 0})); + KokkosFFT::Impl::Plan plan_c2r_axes_2_1(execution_space(), x_c_axis_1, x, + KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<2>({2, 1})); // C2C plan - KokkosFFT::Impl::Plan plan_c2c_f_axes_0_1(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({0, 1})); - KokkosFFT::Impl::Plan plan_c2c_f_axes_0_2(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({0, 2})); - KokkosFFT::Impl::Plan plan_c2c_f_axes_1_0(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({1, 0})); - KokkosFFT::Impl::Plan plan_c2c_f_axes_1_2(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({1, 2})); - KokkosFFT::Impl::Plan plan_c2c_f_axes_2_0(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({2, 0})); - KokkosFFT::Impl::Plan plan_c2c_f_axes_2_1(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<2>({2, 1})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_0_1(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({0, 1})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_0_2(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({0, 2})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_1_0(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({1, 0})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_1_2(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({1, 2})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_2_0(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({2, 0})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_2_1(execution_space(), x_cin, x_cout, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<2>({2, 1})); } // Tests for 2D FFT plan on 2D View TYPED_TEST(Plans2D, 2DFFT_2DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_plan_2dfft_2dview(); @@ -204,7 +313,7 @@ TYPED_TEST(Plans2D, 2DFFT_2DView) { // Tests for 2D batched FFT plan on 3D View TYPED_TEST(Plans2D, 2DFFT_3DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_plan_2dfft_3dview(); @@ -214,41 +323,80 @@ TYPED_TEST(Plans2D, 2DFFT_3DView) { template void test_plan_3dfft_3dview() { const int n0 = 10, n1 = 6, n2 = 8; - using RealView3DType = Kokkos::View; - using ComplexView3DType = Kokkos::View***, LayoutType, execution_space>; + using RealView3DType = Kokkos::View; + using ComplexView3DType = + Kokkos::View***, LayoutType, execution_space>; RealView3DType x("x", n0, n1, n2); - ComplexView3DType x_c_axis_0("x_c_axis_0", n0/2+1, n1, n2), x_c_axis_1("x_c_axis_1", n0, n1/2+1, n2), x_c_axis_2("x_c_axis_2", n0, n1, n2/2+1); + ComplexView3DType x_c_axis_0("x_c_axis_0", n0 / 2 + 1, n1, n2), + x_c_axis_1("x_c_axis_1", n0, n1 / 2 + 1, n2), + x_c_axis_2("x_c_axis_2", n0, n1, n2 / 2 + 1); ComplexView3DType x_cin("x_cin", n0, n1, n2), x_cout("x_cout", n0, n1, n2); // R2C plan - KokkosFFT::Impl::Plan plan_r2c_axes_0_1_2(execution_space(), x, x_c_axis_2, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<3>({0, 1, 2})); - KokkosFFT::Impl::Plan plan_r2c_axes_0_2_1(execution_space(), x, x_c_axis_1, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<3>({0, 2, 1})); - KokkosFFT::Impl::Plan plan_r2c_axes_1_0_2(execution_space(), x, x_c_axis_2, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<3>({1, 0, 2})); - KokkosFFT::Impl::Plan plan_r2c_axes_1_2_0(execution_space(), x, x_c_axis_0, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<3>({1, 2, 0})); - KokkosFFT::Impl::Plan plan_r2c_axes_2_0_1(execution_space(), x, x_c_axis_1, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<3>({2, 0, 1})); - KokkosFFT::Impl::Plan plan_r2c_axes_2_1_0(execution_space(), x, x_c_axis_0, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<3>({2, 1, 0})); + KokkosFFT::Impl::Plan plan_r2c_axes_0_1_2(execution_space(), x, x_c_axis_2, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<3>({0, 1, 2})); + KokkosFFT::Impl::Plan plan_r2c_axes_0_2_1(execution_space(), x, x_c_axis_1, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<3>({0, 2, 1})); + KokkosFFT::Impl::Plan plan_r2c_axes_1_0_2(execution_space(), x, x_c_axis_2, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<3>({1, 0, 2})); + KokkosFFT::Impl::Plan plan_r2c_axes_1_2_0(execution_space(), x, x_c_axis_0, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<3>({1, 2, 0})); + KokkosFFT::Impl::Plan plan_r2c_axes_2_0_1(execution_space(), x, x_c_axis_1, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<3>({2, 0, 1})); + KokkosFFT::Impl::Plan plan_r2c_axes_2_1_0(execution_space(), x, x_c_axis_0, + KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<3>({2, 1, 0})); // C2R plan - KokkosFFT::Impl::Plan plan_c2r_axes_0_1_2(execution_space(), x_c_axis_2, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<3>({0, 1, 2})); - KokkosFFT::Impl::Plan plan_c2r_axes_0_2_1(execution_space(), x_c_axis_1, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<3>({0, 2, 1})); - KokkosFFT::Impl::Plan plan_c2r_axes_1_0_2(execution_space(), x_c_axis_2, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<3>({1, 0, 2})); - KokkosFFT::Impl::Plan plan_c2r_axes_1_2_0(execution_space(), x_c_axis_0, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<3>({1, 2, 0})); - KokkosFFT::Impl::Plan plan_c2r_axes_2_0_1(execution_space(), x_c_axis_1, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<3>({2, 0, 1})); - KokkosFFT::Impl::Plan plan_c2r_axes_2_1_0(execution_space(), x_c_axis_0, x, KokkosFFT::Impl::Direction::Backward, /*axes=*/axes_type<3>({2, 1, 0})); + KokkosFFT::Impl::Plan plan_c2r_axes_0_1_2( + execution_space(), x_c_axis_2, x, KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<3>({0, 1, 2})); + KokkosFFT::Impl::Plan plan_c2r_axes_0_2_1( + execution_space(), x_c_axis_1, x, KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<3>({0, 2, 1})); + KokkosFFT::Impl::Plan plan_c2r_axes_1_0_2( + execution_space(), x_c_axis_2, x, KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<3>({1, 0, 2})); + KokkosFFT::Impl::Plan plan_c2r_axes_1_2_0( + execution_space(), x_c_axis_0, x, KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<3>({1, 2, 0})); + KokkosFFT::Impl::Plan plan_c2r_axes_2_0_1( + execution_space(), x_c_axis_1, x, KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<3>({2, 0, 1})); + KokkosFFT::Impl::Plan plan_c2r_axes_2_1_0( + execution_space(), x_c_axis_0, x, KokkosFFT::Impl::Direction::Backward, + /*axes=*/axes_type<3>({2, 1, 0})); // C2C plan - KokkosFFT::Impl::Plan plan_c2c_f_axes_0_1_2(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<3>({0, 1, 2})); - KokkosFFT::Impl::Plan plan_c2c_f_axes_0_2_1(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<3>({0, 2, 1})); - KokkosFFT::Impl::Plan plan_c2c_f_axes_1_0_2(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<3>({1, 0, 2})); - KokkosFFT::Impl::Plan plan_c2c_f_axes_1_2_0(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<3>({1, 2, 0})); - KokkosFFT::Impl::Plan plan_c2c_f_axes_2_0_1(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<3>({2, 0, 1})); - KokkosFFT::Impl::Plan plan_c2c_f_axes_2_1_0(execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, /*axes=*/axes_type<3>({2, 1, 0})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_0_1_2( + execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<3>({0, 1, 2})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_0_2_1( + execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<3>({0, 2, 1})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_1_0_2( + execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<3>({1, 0, 2})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_1_2_0( + execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<3>({1, 2, 0})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_2_0_1( + execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<3>({2, 0, 1})); + KokkosFFT::Impl::Plan plan_c2c_f_axes_2_1_0( + execution_space(), x_cin, x_cout, KokkosFFT::Impl::Direction::Forward, + /*axes=*/axes_type<3>({2, 1, 0})); } // Tests for 3D FFT plan on 3D View TYPED_TEST(Plans3D, 3DFFT_3DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_plan_3dfft_3dview(); diff --git a/fft/unit_test/Test_Transform.cpp b/fft/unit_test/Test_Transform.cpp index 90f3bacf..36fe65ef 100644 --- a/fft/unit_test/Test_Transform.cpp +++ b/fft/unit_test/Test_Transform.cpp @@ -12,7 +12,7 @@ /// return np.sum(x*np.exp(phase), axis=1) template void fft1(ViewType& in, ViewType& out) { - using value_type = typename ViewType::non_const_value_type; + using value_type = typename ViewType::non_const_value_type; using real_value_type = KokkosFFT::Impl::real_type_t; static_assert(KokkosFFT::Impl::is_complex::value, @@ -22,26 +22,26 @@ void fft1(ViewType& in, ViewType& out) { std::size_t L = in.size(); Kokkos::parallel_for( - Kokkos::TeamPolicy(L, Kokkos::AUTO), - KOKKOS_LAMBDA (const Kokkos::TeamPolicy::member_type& team_member) { - const int j = team_member.league_rank(); - - value_type sum = 0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team_member, L), - [&](const int i, value_type& lsum) { - auto phase = -2 * I * M_PI * - static_cast(i) / static_cast(L); - - auto tmp_in = in(i); - lsum += tmp_in * Kokkos::exp( static_cast(j) * phase ); - }, - sum - ); - - out(j) = sum; - } - ); + Kokkos::TeamPolicy(L, Kokkos::AUTO), + KOKKOS_LAMBDA( + const Kokkos::TeamPolicy::member_type& team_member) { + const int j = team_member.league_rank(); + + value_type sum = 0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team_member, L), + [&](const int i, value_type& lsum) { + auto phase = -2 * I * M_PI * static_cast(i) / + static_cast(L); + + auto tmp_in = in(i); + lsum += + tmp_in * Kokkos::exp(static_cast(j) * phase); + }, + sum); + + out(j) = sum; + }); } /// Kokkos equivalent of ifft1 with numpy @@ -52,7 +52,7 @@ void fft1(ViewType& in, ViewType& out) { /// return np.sum(x*np.exp(phase), axis=1) template void ifft1(ViewType& in, ViewType& out) { - using value_type = typename ViewType::non_const_value_type; + using value_type = typename ViewType::non_const_value_type; using real_value_type = KokkosFFT::Impl::real_type_t; static_assert(KokkosFFT::Impl::is_complex::value, @@ -62,51 +62,49 @@ void ifft1(ViewType& in, ViewType& out) { std::size_t L = in.size(); Kokkos::parallel_for( - Kokkos::TeamPolicy(L, Kokkos::AUTO), - KOKKOS_LAMBDA (const Kokkos::TeamPolicy::member_type& team_member) { - const int j = team_member.league_rank(); - - value_type sum = 0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team_member, L), - [&](const int i, value_type& lsum) { - auto phase = 2 * I * M_PI * - static_cast(i) / static_cast(L); - - auto tmp_in = in(i); - lsum += tmp_in * Kokkos::exp( static_cast(j) * phase ); - }, - sum - ); - - out(j) = sum; - } - ); + Kokkos::TeamPolicy(L, Kokkos::AUTO), + KOKKOS_LAMBDA( + const Kokkos::TeamPolicy::member_type& team_member) { + const int j = team_member.league_rank(); + + value_type sum = 0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team_member, L), + [&](const int i, value_type& lsum) { + auto phase = 2 * I * M_PI * static_cast(i) / + static_cast(L); + + auto tmp_in = in(i); + lsum += + tmp_in * Kokkos::exp(static_cast(j) * phase); + }, + sum); + + out(j) = sum; + }); } -using test_types = ::testing::Types< - std::pair, - std::pair, - std::pair, - std::pair ->; +using test_types = ::testing::Types, + std::pair, + std::pair, + std::pair >; // Basically the same fixtures, used for labeling tests template struct FFT1D : public ::testing::Test { - using float_type = typename T::first_type; + using float_type = typename T::first_type; using layout_type = typename T::second_type; }; template struct FFT2D : public ::testing::Test { - using float_type = typename T::first_type; + using float_type = typename T::first_type; using layout_type = typename T::second_type; }; template struct FFTND : public ::testing::Test { - using float_type = typename T::first_type; + using float_type = typename T::first_type; using layout_type = typename T::second_type; }; @@ -116,18 +114,19 @@ TYPED_TEST_SUITE(FFTND, test_types); // Tests for 1D FFT template -void test_fft1_identity(T atol=1.0e-12) { - const int maxlen = 30; +void test_fft1_identity(T atol = 1.0e-12) { + const int maxlen = 30; using RealView1DType = Kokkos::View; - using ComplexView1DType = Kokkos::View*, LayoutType, execution_space>; + using ComplexView1DType = + Kokkos::View*, LayoutType, execution_space>; ComplexView1DType a("a", maxlen), _a("_a", maxlen), a_ref("a_ref", maxlen); - ComplexView1DType out("out", maxlen), outr("outr", maxlen/2+1); + ComplexView1DType out("out", maxlen), outr("outr", maxlen / 2 + 1); RealView1DType ar("ar", maxlen), _ar("_ar", maxlen), ar_ref("ar_ref", maxlen); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(/*seed=*/12345); - Kokkos::fill_random(a, random_pool, I); + Kokkos::fill_random(a, random_pool, I); Kokkos::fill_random(ar, random_pool, 1.0); Kokkos::deep_copy(a_ref, a); Kokkos::deep_copy(ar_ref, ar); @@ -140,23 +139,24 @@ void test_fft1_identity(T atol=1.0e-12) { KokkosFFT::rfft(execution_space(), ar, outr); KokkosFFT::irfft(execution_space(), outr, _ar); - EXPECT_TRUE( allclose(_a, a_ref, 1.e-5, atol) ); - EXPECT_TRUE( allclose(_ar, ar_ref, 1.e-5, atol) ); + EXPECT_TRUE(allclose(_a, a_ref, 1.e-5, atol)); + EXPECT_TRUE(allclose(_ar, ar_ref, 1.e-5, atol)); } template -void test_fft1_identity_reuse_plan(T atol=1.0e-12) { - const int maxlen = 30; +void test_fft1_identity_reuse_plan(T atol = 1.0e-12) { + const int maxlen = 30; using RealView1DType = Kokkos::View; - using ComplexView1DType = Kokkos::View*, LayoutType, execution_space>; + using ComplexView1DType = + Kokkos::View*, LayoutType, execution_space>; ComplexView1DType a("a", maxlen), _a("_a", maxlen), a_ref("a_ref", maxlen); - ComplexView1DType out("out", maxlen), outr("outr", maxlen/2+1); + ComplexView1DType out("out", maxlen), outr("outr", maxlen / 2 + 1); RealView1DType ar("ar", maxlen), _ar("_ar", maxlen), ar_ref("ar_ref", maxlen); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(/*seed=*/12345); - Kokkos::fill_random(a, random_pool, I); + Kokkos::fill_random(a, random_pool, I); Kokkos::fill_random(ar, random_pool, 1.0); Kokkos::deep_copy(a_ref, a); Kokkos::deep_copy(ar_ref, ar); @@ -164,81 +164,85 @@ void test_fft1_identity_reuse_plan(T atol=1.0e-12) { Kokkos::fence(); int axis = -1; - KokkosFFT::Impl::Plan fft_plan(execution_space(), a, out, KokkosFFT::Impl::Direction::Forward, axis); + KokkosFFT::Impl::Plan fft_plan(execution_space(), a, out, + KokkosFFT::Impl::Direction::Forward, axis); KokkosFFT::fft(execution_space(), a, out, fft_plan); - KokkosFFT::Impl::Plan ifft_plan(execution_space(), out, _a, KokkosFFT::Impl::Direction::Backward, axis); + KokkosFFT::Impl::Plan ifft_plan(execution_space(), out, _a, + KokkosFFT::Impl::Direction::Backward, axis); KokkosFFT::ifft(execution_space(), out, _a, ifft_plan); - KokkosFFT::Impl::Plan rfft_plan(execution_space(), ar, outr, KokkosFFT::Impl::Direction::Forward, axis); + KokkosFFT::Impl::Plan rfft_plan(execution_space(), ar, outr, + KokkosFFT::Impl::Direction::Forward, axis); KokkosFFT::rfft(execution_space(), ar, outr, rfft_plan); - KokkosFFT::Impl::Plan irfft_plan(execution_space(), outr, _ar, KokkosFFT::Impl::Direction::Backward, axis); + KokkosFFT::Impl::Plan irfft_plan(execution_space(), outr, _ar, + KokkosFFT::Impl::Direction::Backward, axis); KokkosFFT::irfft(execution_space(), outr, _ar, irfft_plan); - EXPECT_TRUE( allclose(_a, a_ref, 1.e-5, atol) ); - EXPECT_TRUE( allclose(_ar, ar_ref, 1.e-5, atol) ); + EXPECT_TRUE(allclose(_a, a_ref, 1.e-5, atol)); + EXPECT_TRUE(allclose(_ar, ar_ref, 1.e-5, atol)); // Check if errors are correctly raised aginst wrong axis int wrong_axis = 0; - EXPECT_THROW( - KokkosFFT::fft(execution_space(), a, out, fft_plan, KokkosFFT::Normalization::BACKWARD, wrong_axis), - std::runtime_error - ); - - EXPECT_THROW( - KokkosFFT::ifft(execution_space(), out, _a, ifft_plan, KokkosFFT::Normalization::BACKWARD, wrong_axis), - std::runtime_error - ); - - EXPECT_THROW( - KokkosFFT::rfft(execution_space(), ar, outr, rfft_plan, KokkosFFT::Normalization::BACKWARD, wrong_axis), - std::runtime_error - ); - - EXPECT_THROW( - KokkosFFT::irfft(execution_space(), outr, _ar, irfft_plan, KokkosFFT::Normalization::BACKWARD, wrong_axis), - std::runtime_error - ); + EXPECT_THROW(KokkosFFT::fft(execution_space(), a, out, fft_plan, + KokkosFFT::Normalization::BACKWARD, wrong_axis), + std::runtime_error); + + EXPECT_THROW(KokkosFFT::ifft(execution_space(), out, _a, ifft_plan, + KokkosFFT::Normalization::BACKWARD, wrong_axis), + std::runtime_error); + + EXPECT_THROW(KokkosFFT::rfft(execution_space(), ar, outr, rfft_plan, + KokkosFFT::Normalization::BACKWARD, wrong_axis), + std::runtime_error); + + EXPECT_THROW(KokkosFFT::irfft(execution_space(), outr, _ar, irfft_plan, + KokkosFFT::Normalization::BACKWARD, wrong_axis), + std::runtime_error); // Check if errors are correctly raised aginst wrong dirction - KokkosFFT::Impl::Direction wrong_fft_direction = KokkosFFT::Impl::Direction::Backward; - KokkosFFT::Impl::Plan wrong_fft_plan(execution_space(), a, out, wrong_fft_direction, axis); - - KokkosFFT::Impl::Direction wrong_ifft_direction = KokkosFFT::Impl::Direction::Forward; - KokkosFFT::Impl::Plan wrong_ifft_plan(execution_space(), out, _a, wrong_ifft_direction, axis); - - KokkosFFT::Impl::Plan wrong_rfft_plan(execution_space(), ar, outr, wrong_fft_direction, axis); - KokkosFFT::Impl::Plan wrong_irfft_plan(execution_space(), outr, _ar, wrong_ifft_direction, axis); - - EXPECT_THROW( - KokkosFFT::fft(execution_space(), a, out, wrong_fft_plan, KokkosFFT::Normalization::BACKWARD, axis), - std::runtime_error - ); - - EXPECT_THROW( - KokkosFFT::ifft(execution_space(), out, _a, wrong_ifft_plan, KokkosFFT::Normalization::BACKWARD, axis), - std::runtime_error - ); - - EXPECT_THROW( - KokkosFFT::rfft(execution_space(), ar, outr, wrong_rfft_plan, KokkosFFT::Normalization::BACKWARD, axis), - std::runtime_error - ); - - EXPECT_THROW( - KokkosFFT::irfft(execution_space(), outr, _ar, wrong_irfft_plan, KokkosFFT::Normalization::BACKWARD, axis), - std::runtime_error - ); + KokkosFFT::Impl::Direction wrong_fft_direction = + KokkosFFT::Impl::Direction::Backward; + KokkosFFT::Impl::Plan wrong_fft_plan(execution_space(), a, out, + wrong_fft_direction, axis); + + KokkosFFT::Impl::Direction wrong_ifft_direction = + KokkosFFT::Impl::Direction::Forward; + KokkosFFT::Impl::Plan wrong_ifft_plan(execution_space(), out, _a, + wrong_ifft_direction, axis); + + KokkosFFT::Impl::Plan wrong_rfft_plan(execution_space(), ar, outr, + wrong_fft_direction, axis); + KokkosFFT::Impl::Plan wrong_irfft_plan(execution_space(), outr, _ar, + wrong_ifft_direction, axis); + + EXPECT_THROW(KokkosFFT::fft(execution_space(), a, out, wrong_fft_plan, + KokkosFFT::Normalization::BACKWARD, axis), + std::runtime_error); + + EXPECT_THROW(KokkosFFT::ifft(execution_space(), out, _a, wrong_ifft_plan, + KokkosFFT::Normalization::BACKWARD, axis), + std::runtime_error); + + EXPECT_THROW(KokkosFFT::rfft(execution_space(), ar, outr, wrong_rfft_plan, + KokkosFFT::Normalization::BACKWARD, axis), + std::runtime_error); + + EXPECT_THROW(KokkosFFT::irfft(execution_space(), outr, _ar, wrong_irfft_plan, + KokkosFFT::Normalization::BACKWARD, axis), + std::runtime_error); } template void test_fft1_1dfft_1dview() { const int len = 30; - using ComplexView1DType = Kokkos::View*, LayoutType, execution_space>; + using ComplexView1DType = + Kokkos::View*, LayoutType, execution_space>; ComplexView1DType x("x", len), out("out", len), ref("ref", len); - ComplexView1DType out_b("out_b", len), out_o("out_o", len), out_f("out_f", len); + ComplexView1DType out_b("out_b", len), out_o("out_o", len), + out_f("out_f", len); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); @@ -246,28 +250,33 @@ void test_fft1_1dfft_1dview() { Kokkos::fence(); - KokkosFFT::fft(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::fft(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::fft(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::fft(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); KokkosFFT::fft(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); - KokkosFFT::fft(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::fft(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); fft1(x, ref); multiply(out_o, sqrt(static_cast(len))); multiply(out_f, static_cast(len)); - EXPECT_TRUE( allclose(out, ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, ref, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, ref, 1.e-5, 1.e-6)); } template void test_fft1_1difft_1dview() { const int len = 30; - using ComplexView1DType = Kokkos::View*, LayoutType, execution_space>; + using ComplexView1DType = + Kokkos::View*, LayoutType, execution_space>; ComplexView1DType x("x", len), out("out", len), ref("ref", len); - ComplexView1DType out_b("out_b", len), out_o("out_o", len), out_f("out_f", len); + ComplexView1DType out_b("out_b", len), out_o("out_o", len), + out_f("out_f", len); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); @@ -275,29 +284,34 @@ void test_fft1_1difft_1dview() { Kokkos::fence(); - KokkosFFT::ifft(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::ifft(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::ifft(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::ifft(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); KokkosFFT::ifft(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); - KokkosFFT::ifft(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::ifft(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); ifft1(x, ref); multiply(out_o, sqrt(static_cast(len))); multiply(out_b, static_cast(len)); multiply(out, static_cast(len)); - EXPECT_TRUE( allclose(out, ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, ref, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, ref, 1.e-5, 1.e-6)); } template void test_fft1_1dhfft_1dview() { const int len_herm = 16, len = len_herm * 2 - 2; - using RealView1DType = Kokkos::View; - using ComplexView1DType = Kokkos::View*, LayoutType, execution_space>; + using RealView1DType = Kokkos::View; + using ComplexView1DType = + Kokkos::View*, LayoutType, execution_space>; - ComplexView1DType x_herm("x_herm", len_herm), x_herm_ref("x_herm_ref", len_herm); + ComplexView1DType x_herm("x_herm", len_herm), + x_herm_ref("x_herm_ref", len_herm); ComplexView1DType x("x", len), ref("ref", len); RealView1DType out("out", len); RealView1DType out_b("out_b", len), out_o("out_o", len), out_f("out_f", len); @@ -310,17 +324,17 @@ void test_fft1_1dhfft_1dview() { auto h_x_herm = Kokkos::create_mirror_view(x_herm); Kokkos::deep_copy(h_x_herm, x_herm); - auto last = h_x_herm.extent(0) - 1; + auto last = h_x_herm.extent(0) - 1; h_x_herm(0) = h_x_herm(0).real(); h_x_herm(last) = h_x_herm(last).real(); - for(int i=0; i0; i--) { - h_x(len-i) = Kokkos::conj(h_x_herm(i)); + for (int i = last - 1; i > 0; i--) { + h_x(len - i) = Kokkos::conj(h_x_herm(i)); } Kokkos::deep_copy(x_herm, h_x_herm); @@ -332,60 +346,72 @@ void test_fft1_1dhfft_1dview() { KokkosFFT::fft(execution_space(), x, ref); Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::hfft(execution_space(), x_herm, + out); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out_b, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::hfft(execution_space(), x_herm, out_b, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out_o, KokkosFFT::Normalization::ORTHO); + KokkosFFT::hfft(execution_space(), x_herm, out_o, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::hfft(execution_space(), x_herm, out_f, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(len))); multiply(out_f, static_cast(len)); - EXPECT_TRUE( allclose(out, ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out, 1.e-5, 1.e-6)); // Reuse plans int axis = -1; Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::Impl::Plan hfft_plan(execution_space(), x_herm, out, KokkosFFT::Impl::Direction::Backward, axis); + KokkosFFT::Impl::Plan hfft_plan(execution_space(), x_herm, out, + KokkosFFT::Impl::Direction::Backward, axis); KokkosFFT::hfft(execution_space(), x_herm, out, hfft_plan); Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out_b, hfft_plan, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::hfft(execution_space(), x_herm, out_b, hfft_plan, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out_o, hfft_plan, KokkosFFT::Normalization::ORTHO); + KokkosFFT::hfft(execution_space(), x_herm, out_o, hfft_plan, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out_f, hfft_plan, KokkosFFT::Normalization::FORWARD); + KokkosFFT::hfft(execution_space(), x_herm, out_f, hfft_plan, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(len))); multiply(out_f, static_cast(len)); - EXPECT_TRUE( allclose(out, ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out, 1.e-5, 1.e-6)); } template void test_fft1_1dihfft_1dview() { const int len_herm = 16, len = len_herm * 2 - 2; - using RealView1DType = Kokkos::View; - using ComplexView1DType = Kokkos::View*, LayoutType, execution_space>; + using RealView1DType = Kokkos::View; + using ComplexView1DType = + Kokkos::View*, LayoutType, execution_space>; - ComplexView1DType x_herm("x_herm", len_herm), x_herm_ref("x_herm_ref", len_herm); + ComplexView1DType x_herm("x_herm", len_herm), + x_herm_ref("x_herm_ref", len_herm); RealView1DType out1("out1", len); - RealView1DType out1_b("out1_b", len), out1_o("out1_o", len), out1_f("out1_f", len); - ComplexView1DType out2("out2", len_herm), out2_b("out2_b", len_herm), out2_o("out2_o", len_herm), out2_f("out2_f", len_herm); + RealView1DType out1_b("out1_b", len), out1_o("out1_o", len), + out1_f("out1_f", len); + ComplexView1DType out2("out2", len_herm), out2_b("out2_b", len_herm), + out2_o("out2_o", len_herm), out2_f("out2_f", len_herm); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); @@ -394,7 +420,7 @@ void test_fft1_1dihfft_1dview() { auto h_x_herm = Kokkos::create_mirror_view(x_herm); Kokkos::deep_copy(h_x_herm, x_herm); - auto last = h_x_herm.extent(0) - 1; + auto last = h_x_herm.extent(0) - 1; h_x_herm(0) = h_x_herm(0).real(); h_x_herm(last) = h_x_herm(last).real(); @@ -403,74 +429,97 @@ void test_fft1_1dihfft_1dview() { Kokkos::fence(); Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out1); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::ihfft(execution_space(), out1, out2); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::hfft(execution_space(), x_herm, + out1); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::ihfft(execution_space(), out1, + out2); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out1_b, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::ihfft(execution_space(), out1_b, out2_b, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::hfft(execution_space(), x_herm, out1_b, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::ihfft(execution_space(), out1_b, out2_b, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out1_o, KokkosFFT::Normalization::ORTHO); - KokkosFFT::ihfft(execution_space(), out1_o, out2_o, KokkosFFT::Normalization::ORTHO); + KokkosFFT::hfft(execution_space(), x_herm, out1_o, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::ihfft(execution_space(), out1_o, out2_o, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out1_f, KokkosFFT::Normalization::FORWARD); - KokkosFFT::ihfft(execution_space(), out1_f, out2_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::hfft(execution_space(), x_herm, out1_f, + KokkosFFT::Normalization::FORWARD); + KokkosFFT::ihfft(execution_space(), out1_f, out2_f, + KokkosFFT::Normalization::FORWARD); - EXPECT_TRUE( allclose(out2, x_herm_ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out2_b, x_herm_ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out2_o, x_herm_ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out2_f, x_herm_ref, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out2, x_herm_ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out2_b, x_herm_ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out2_o, x_herm_ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out2_f, x_herm_ref, 1.e-5, 1.e-6)); // Reuse plans int axis = -1; - KokkosFFT::Impl::Plan hfft_plan(execution_space(), x_herm, out1, KokkosFFT::Impl::Direction::Backward, axis); - KokkosFFT::Impl::Plan ihfft_plan(execution_space(), out1, out2, KokkosFFT::Impl::Direction::Forward, axis); + KokkosFFT::Impl::Plan hfft_plan(execution_space(), x_herm, out1, + KokkosFFT::Impl::Direction::Backward, axis); + KokkosFFT::Impl::Plan ihfft_plan(execution_space(), out1, out2, + KokkosFFT::Impl::Direction::Forward, axis); Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out1, hfft_plan); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::ihfft(execution_space(), out1, out2, ihfft_plan); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::hfft(execution_space(), x_herm, out1, + hfft_plan); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::ihfft(execution_space(), out1, out2, + ihfft_plan); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out1_b, hfft_plan, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::ihfft(execution_space(), out1_b, out2_b, ihfft_plan, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::hfft(execution_space(), x_herm, out1_b, hfft_plan, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::ihfft(execution_space(), out1_b, out2_b, ihfft_plan, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out1_o, hfft_plan, KokkosFFT::Normalization::ORTHO); - KokkosFFT::ihfft(execution_space(), out1_o, out2_o, ihfft_plan, KokkosFFT::Normalization::ORTHO); + KokkosFFT::hfft(execution_space(), x_herm, out1_o, hfft_plan, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::ihfft(execution_space(), out1_o, out2_o, ihfft_plan, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x_herm, x_herm_ref); - KokkosFFT::hfft(execution_space(), x_herm, out1_f, hfft_plan, KokkosFFT::Normalization::FORWARD); - KokkosFFT::ihfft(execution_space(), out1_f, out2_f, ihfft_plan, KokkosFFT::Normalization::FORWARD); - - EXPECT_TRUE( allclose(out2, x_herm_ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out2_b, x_herm_ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out2_o, x_herm_ref, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out2_f, x_herm_ref, 1.e-5, 1.e-6) ); + KokkosFFT::hfft(execution_space(), x_herm, out1_f, hfft_plan, + KokkosFFT::Normalization::FORWARD); + KokkosFFT::ihfft(execution_space(), out1_f, out2_f, ihfft_plan, + KokkosFFT::Normalization::FORWARD); + + EXPECT_TRUE(allclose(out2, x_herm_ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out2_b, x_herm_ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out2_o, x_herm_ref, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out2_f, x_herm_ref, 1.e-5, 1.e-6)); } template -void test_fft1_1dfft_2dview(T atol=1.e-12) { +void test_fft1_1dfft_2dview(T atol = 1.e-12) { const int n0 = 10, n1 = 12; - using RealView2DType = Kokkos::View; - using ComplexView2DType = Kokkos::View**, LayoutType, execution_space>; + using RealView2DType = Kokkos::View; + using ComplexView2DType = + Kokkos::View**, LayoutType, execution_space>; ComplexView2DType x("x", n0, n1), ref_x("ref_x", n0, n1); ComplexView2DType x_axis0("x_axis0", n0, n1), x_axis1("x_axis1", n0, n1); - ComplexView2DType out_axis0("out_axis0", n0, n1), ref_out_axis0("ref_out_axis0", n0, n1); - ComplexView2DType out_axis1("out_axis1", n0, n1), ref_out_axis1("ref_out_axis1", n0, n1); + ComplexView2DType out_axis0("out_axis0", n0, n1), + ref_out_axis0("ref_out_axis0", n0, n1); + ComplexView2DType out_axis1("out_axis1", n0, n1), + ref_out_axis1("ref_out_axis1", n0, n1); RealView2DType xr("xr", n0, n1), ref_xr("ref_xr", n0, n1); RealView2DType xr_axis0("xr_axis0", n0, n1), xr_axis1("xr_axis1", n0, n1); - ComplexView2DType outr_axis0("outr_axis0", n0/2+1, n1), outr_axis1("outr_axis1", n0, n1/2+1); + ComplexView2DType outr_axis0("outr_axis0", n0 / 2 + 1, n1), + outr_axis1("outr_axis1", n0, n1 / 2 + 1); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, I); Kokkos::fill_random(xr, random_pool, 1); - // Since HIP FFT destructs the input data, we need to keep the input data in different place + // Since HIP FFT destructs the input data, we need to keep the input data in + // different place Kokkos::deep_copy(ref_x, x); Kokkos::deep_copy(ref_xr, xr); @@ -478,23 +527,27 @@ void test_fft1_1dfft_2dview(T atol=1.e-12) { // Along axis 0 (transpose neeed) // Perform batched 1D (along 0th axis) FFT sequentially - for(int i1=0; i1 -void test_fft1_1dfft_3dview(T atol=1.e-12) { +void test_fft1_1dfft_3dview(T atol = 1.e-12) { const int n0 = 10, n1 = 12, n2 = 8; - using RealView3DType = Kokkos::View; - using ComplexView3DType = Kokkos::View***, LayoutType, execution_space>; + using RealView3DType = Kokkos::View; + using ComplexView3DType = + Kokkos::View***, LayoutType, execution_space>; ComplexView3DType x("x", n0, n1, n2), ref_x("ref_x", n0, n1, n2); - ComplexView3DType x_axis0("x_axis0", n0, n1, n2), x_axis1("x_axis1", n0, n1, n2), x_axis2("x_axis2", n0, n1, n2); - ComplexView3DType out_axis0("out_axis0", n0, n1, n2), ref_out_axis0("ref_out_axis0", n0, n1, n2); - ComplexView3DType out_axis1("out_axis1", n0, n1, n2), ref_out_axis1("ref_out_axis1", n0, n1, n2); - ComplexView3DType out_axis2("out_axis2", n0, n1, n2), ref_out_axis2("ref_out_axis2", n0, n1, n2); + ComplexView3DType x_axis0("x_axis0", n0, n1, n2), + x_axis1("x_axis1", n0, n1, n2), x_axis2("x_axis2", n0, n1, n2); + ComplexView3DType out_axis0("out_axis0", n0, n1, n2), + ref_out_axis0("ref_out_axis0", n0, n1, n2); + ComplexView3DType out_axis1("out_axis1", n0, n1, n2), + ref_out_axis1("ref_out_axis1", n0, n1, n2); + ComplexView3DType out_axis2("out_axis2", n0, n1, n2), + ref_out_axis2("ref_out_axis2", n0, n1, n2); RealView3DType xr("xr", n0, n1, n2), ref_xr("ref_xr", n0, n1, n2); - RealView3DType xr_axis0("xr_axis0", n0, n1, n2), xr_axis1("xr_axis1", n0, n1, n2), xr_axis2("xr_axis2", n0, n1, n2); - ComplexView3DType outr_axis0("outr_axis0", n0/2+1, n1, n2), outr_axis1("outr_axis1", n0, n1/2+1, n2), outr_axis2("outr_axis2", n0, n1, n2/2+1); + RealView3DType xr_axis0("xr_axis0", n0, n1, n2), + xr_axis1("xr_axis1", n0, n1, n2), xr_axis2("xr_axis2", n0, n1, n2); + ComplexView3DType outr_axis0("outr_axis0", n0 / 2 + 1, n1, n2), + outr_axis1("outr_axis1", n0, n1 / 2 + 1, n2), + outr_axis2("outr_axis2", n0, n1, n2 / 2 + 1); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, I); Kokkos::fill_random(xr, random_pool, 1); - // Since HIP FFT destructs the input data, we need to keep the input data in different place + // Since HIP FFT destructs the input data, we need to keep the input data in + // different place Kokkos::deep_copy(ref_x, x); Kokkos::deep_copy(ref_xr, xr); @@ -550,25 +616,29 @@ void test_fft1_1dfft_3dview(T atol=1.e-12) { // Along axis 0 (transpose neeed) // Perform batched 1D (along 0th axis) FFT sequentially - for(int i2=0; i2 ? 1.0e-6 : 1.0e-12; @@ -634,7 +712,7 @@ TYPED_TEST(FFT1D, Identity_1DView) { // Identity tests on 1D Views with plan reuse TYPED_TEST(FFT1D, Identity_1DView_reuse_plans) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; float_type atol = std::is_same_v ? 1.0e-6 : 1.0e-12; @@ -643,7 +721,7 @@ TYPED_TEST(FFT1D, Identity_1DView_reuse_plans) { // fft on 1D Views TYPED_TEST(FFT1D, FFT_1DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_fft1_1dfft_1dview(); @@ -651,7 +729,7 @@ TYPED_TEST(FFT1D, FFT_1DView) { // ifft on 1D Views TYPED_TEST(FFT1D, IFFT_1DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_fft1_1difft_1dview(); @@ -659,7 +737,7 @@ TYPED_TEST(FFT1D, IFFT_1DView) { // hfft on 1D Views TYPED_TEST(FFT1D, HFFT_1DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_fft1_1dhfft_1dview(); @@ -667,7 +745,7 @@ TYPED_TEST(FFT1D, HFFT_1DView) { // ihfft on 1D Views TYPED_TEST(FFT1D, IHFFT_1DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_fft1_1dihfft_1dview(); @@ -675,7 +753,7 @@ TYPED_TEST(FFT1D, IHFFT_1DView) { // batced fft1 on 2D Views TYPED_TEST(FFT1D, FFT_batched_2DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; float_type atol = std::is_same_v ? 1.0e-6 : 1.0e-12; @@ -684,7 +762,7 @@ TYPED_TEST(FFT1D, FFT_batched_2DView) { // batced fft1 on 3D Views TYPED_TEST(FFT1D, FFT_batched_3DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; float_type atol = std::is_same_v ? 1.0e-6 : 1.0e-12; @@ -695,11 +773,14 @@ TYPED_TEST(FFT1D, FFT_batched_3DView) { template void test_fft2_2dfft_2dview() { const int n0 = 4, n1 = 6; - using ComplexView2DType = Kokkos::View**, LayoutType, execution_space>; + using ComplexView2DType = + Kokkos::View**, LayoutType, execution_space>; ComplexView2DType x("x", n0, n1); - ComplexView2DType out("out", n0, n1), out1("out1", n0, n1), out2("out2", n0, n1); - ComplexView2DType out_b("out_b", n0, n1), out_o("out_o", n0, n1), out_f("out_f", n0, n1); + ComplexView2DType out("out", n0, n1), out1("out1", n0, n1), + out2("out2", n0, n1); + ComplexView2DType out_b("out_b", n0, n1), out_o("out_o", n0, n1), + out_f("out_f", n0, n1); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); @@ -708,48 +789,61 @@ void test_fft2_2dfft_2dview() { Kokkos::fence(); // np.fft2 is identical to np.fft(np.fft(x, axis=1), axis=0) - KokkosFFT::fft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, /*axis=*/1); - KokkosFFT::fft(execution_space(), out1, out2, KokkosFFT::Normalization::BACKWARD, /*axis=*/0); - - KokkosFFT::fft2(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::fft2(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::fft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, + /*axis=*/1); + KokkosFFT::fft(execution_space(), out1, out2, + KokkosFFT::Normalization::BACKWARD, /*axis=*/0); + + KokkosFFT::fft2(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::fft2(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); KokkosFFT::fft2(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); - KokkosFFT::fft2(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::fft2(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1))); multiply(out_f, static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); // Reuse plans using axes_type = KokkosFFT::axis_type<2>; - axes_type axes={-2, -1}; - KokkosFFT::Impl::Plan fft2_plan(execution_space(), x, out, KokkosFFT::Impl::Direction::Forward, axes); - KokkosFFT::fft2(execution_space(), x, out, fft2_plan); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::fft2(execution_space(), x, out_b, fft2_plan, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::fft2(execution_space(), x, out_o, fft2_plan, KokkosFFT::Normalization::ORTHO); - KokkosFFT::fft2(execution_space(), x, out_f, fft2_plan, KokkosFFT::Normalization::FORWARD); + axes_type axes = {-2, -1}; + KokkosFFT::Impl::Plan fft2_plan(execution_space(), x, out, + KokkosFFT::Impl::Direction::Forward, axes); + KokkosFFT::fft2(execution_space(), x, out, + fft2_plan); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::fft2(execution_space(), x, out_b, fft2_plan, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::fft2(execution_space(), x, out_o, fft2_plan, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::fft2(execution_space(), x, out_f, fft2_plan, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1))); multiply(out_f, static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); } template void test_fft2_2difft_2dview() { const int n0 = 4, n1 = 6; - using ComplexView2DType = Kokkos::View**, LayoutType, execution_space>; + using ComplexView2DType = + Kokkos::View**, LayoutType, execution_space>; ComplexView2DType x("x", n0, n1); - ComplexView2DType out("out", n0, n1), out1("out1", n0, n1), out2("out2", n0, n1); - ComplexView2DType out_b("out_b", n0, n1), out_o("out_o", n0, n1), out_f("out_f", n0, n1); + ComplexView2DType out("out", n0, n1), out1("out1", n0, n1), + out2("out2", n0, n1); + ComplexView2DType out_b("out_b", n0, n1), out_o("out_o", n0, n1), + out_f("out_f", n0, n1); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); @@ -758,50 +852,64 @@ void test_fft2_2difft_2dview() { Kokkos::fence(); // np.ifft2 is identical to np.ifft(np.ifft(x, axis=1), axis=0) - KokkosFFT::ifft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, /*axis=*/1); - KokkosFFT::ifft(execution_space(), out1, out2, KokkosFFT::Normalization::BACKWARD, /*axis=*/0); - - KokkosFFT::ifft2(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::ifft2(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::ifft2(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); - KokkosFFT::ifft2(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); - - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1))); - multiply(out_f, 1.0/static_cast(n0 * n1)); - - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + KokkosFFT::ifft(execution_space(), x, out1, + KokkosFFT::Normalization::BACKWARD, /*axis=*/1); + KokkosFFT::ifft(execution_space(), out1, out2, + KokkosFFT::Normalization::BACKWARD, /*axis=*/0); + + KokkosFFT::ifft2(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::ifft2(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::ifft2(execution_space(), x, out_o, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::ifft2(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); + + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1))); + multiply(out_f, 1.0 / static_cast(n0 * n1)); + + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); // Reuse plans using axes_type = KokkosFFT::axis_type<2>; - axes_type axes={-2, -1}; - KokkosFFT::Impl::Plan ifft2_plan(execution_space(), x, out, KokkosFFT::Impl::Direction::Backward, axes); - - KokkosFFT::ifft2(execution_space(), x, out, ifft2_plan); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::ifft2(execution_space(), x, out_b, ifft2_plan, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::ifft2(execution_space(), x, out_o, ifft2_plan, KokkosFFT::Normalization::ORTHO); - KokkosFFT::ifft2(execution_space(), x, out_f, ifft2_plan, KokkosFFT::Normalization::FORWARD); - - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1))); - multiply(out_f, 1.0/static_cast(n0 * n1)); - - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + axes_type axes = {-2, -1}; + KokkosFFT::Impl::Plan ifft2_plan(execution_space(), x, out, + KokkosFFT::Impl::Direction::Backward, axes); + + KokkosFFT::ifft2(execution_space(), x, out, + ifft2_plan); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::ifft2(execution_space(), x, out_b, ifft2_plan, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::ifft2(execution_space(), x, out_o, ifft2_plan, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::ifft2(execution_space(), x, out_f, ifft2_plan, + KokkosFFT::Normalization::FORWARD); + + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1))); + multiply(out_f, 1.0 / static_cast(n0 * n1)); + + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); } template void test_fft2_2drfft_2dview() { const int n0 = 4, n1 = 6; - using RealView2DType = Kokkos::View; - using ComplexView2DType = Kokkos::View**, LayoutType, execution_space>; + using RealView2DType = Kokkos::View; + using ComplexView2DType = + Kokkos::View**, LayoutType, execution_space>; RealView2DType x("x", n0, n1), x_ref("x_ref", n0, n1); - ComplexView2DType out("out", n0, n1/2+1), out1("out1", n0, n1/2+1), out2("out2", n0, n1/2+1); - ComplexView2DType out_b("out_b", n0, n1/2+1), out_o("out_o", n0, n1/2+1), out_f("out_f", n0, n1/2+1); + ComplexView2DType out("out", n0, n1 / 2 + 1), out1("out1", n0, n1 / 2 + 1), + out2("out2", n0, n1 / 2 + 1); + ComplexView2DType out_b("out_b", n0, n1 / 2 + 1), + out_o("out_o", n0, n1 / 2 + 1), out_f("out_f", n0, n1 / 2 + 1); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, 1); @@ -809,65 +917,78 @@ void test_fft2_2drfft_2dview() { Kokkos::fence(); // np.rfft2 is identical to np.fft(np.rfft(x, axis=1), axis=0) - KokkosFFT::rfft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, /*axis=*/1); - KokkosFFT::fft(execution_space(), out1, out2, KokkosFFT::Normalization::BACKWARD, /*axis=*/0); + KokkosFFT::rfft(execution_space(), x, out1, + KokkosFFT::Normalization::BACKWARD, /*axis=*/1); + KokkosFFT::fft(execution_space(), out1, out2, + KokkosFFT::Normalization::BACKWARD, /*axis=*/0); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfft2(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::rfft2(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfft2(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::rfft2(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfft2(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); + KokkosFFT::rfft2(execution_space(), x, out_o, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfft2(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::rfft2(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1))); multiply(out_f, static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); // Reuse plans using axes_type = KokkosFFT::axis_type<2>; - axes_type axes={-2, -1}; - KokkosFFT::Impl::Plan rfft2_plan(execution_space(), x, out, KokkosFFT::Impl::Direction::Forward, axes); + axes_type axes = {-2, -1}; + KokkosFFT::Impl::Plan rfft2_plan(execution_space(), x, out, + KokkosFFT::Impl::Direction::Forward, axes); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfft2(execution_space(), x, out, rfft2_plan); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::rfft2(execution_space(), x, out, + rfft2_plan); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfft2(execution_space(), x, out_b, rfft2_plan, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::rfft2(execution_space(), x, out_b, rfft2_plan, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfft2(execution_space(), x, out_o, rfft2_plan, KokkosFFT::Normalization::ORTHO); + KokkosFFT::rfft2(execution_space(), x, out_o, rfft2_plan, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfft2(execution_space(), x, out_f, rfft2_plan, KokkosFFT::Normalization::FORWARD); + KokkosFFT::rfft2(execution_space(), x, out_f, rfft2_plan, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1))); multiply(out_f, static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); } template void test_fft2_2dirfft_2dview() { const int n0 = 4, n1 = 6; - using RealView2DType = Kokkos::View; - using ComplexView2DType = Kokkos::View**, LayoutType, execution_space>; + using RealView2DType = Kokkos::View; + using ComplexView2DType = + Kokkos::View**, LayoutType, execution_space>; - ComplexView2DType x("x", n0, n1/2+1), x_ref("x_ref", n0, n1/2+1); - ComplexView2DType out1("out1", n0, n1/2+1); + ComplexView2DType x("x", n0, n1 / 2 + 1), x_ref("x_ref", n0, n1 / 2 + 1); + ComplexView2DType out1("out1", n0, n1 / 2 + 1); RealView2DType out2("out2", n0, n1), out("out", n0, n1); - RealView2DType out_b("out_b", n0, n1), out_o("out_o", n0, n1), out_f("out_f", n0, n1); + RealView2DType out_b("out_b", n0, n1), out_o("out_o", n0, n1), + out_f("out_f", n0, n1); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); @@ -875,58 +996,70 @@ void test_fft2_2dirfft_2dview() { Kokkos::deep_copy(x_ref, x); // np.irfft2 is identical to np.irfft(np.ifft(x, axis=0), axis=1) - KokkosFFT::ifft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, 0); - KokkosFFT::irfft(execution_space(), out1, out2, KokkosFFT::Normalization::BACKWARD, 1); + KokkosFFT::ifft(execution_space(), x, out1, + KokkosFFT::Normalization::BACKWARD, 0); + KokkosFFT::irfft(execution_space(), out1, out2, + KokkosFFT::Normalization::BACKWARD, 1); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfft2(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::irfft2(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfft2(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::irfft2(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfft2(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); + KokkosFFT::irfft2(execution_space(), x, out_o, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfft2(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::irfft2(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1))); - multiply(out_f, 1.0/static_cast(n0 * n1)); + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1))); + multiply(out_f, 1.0 / static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); // Reuse plans using axes_type = KokkosFFT::axis_type<2>; - axes_type axes={-2, -1}; - KokkosFFT::Impl::Plan irfft2_plan(execution_space(), x, out, KokkosFFT::Impl::Direction::Backward, axes); + axes_type axes = {-2, -1}; + KokkosFFT::Impl::Plan irfft2_plan(execution_space(), x, out, + KokkosFFT::Impl::Direction::Backward, axes); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfft2(execution_space(), x, out, irfft2_plan); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::irfft2( + execution_space(), x, out, + irfft2_plan); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfft2(execution_space(), x, out_b, irfft2_plan, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::irfft2(execution_space(), x, out_b, irfft2_plan, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfft2(execution_space(), x, out_o, irfft2_plan, KokkosFFT::Normalization::ORTHO); + KokkosFFT::irfft2(execution_space(), x, out_o, irfft2_plan, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfft2(execution_space(), x, out_f, irfft2_plan, KokkosFFT::Normalization::FORWARD); + KokkosFFT::irfft2(execution_space(), x, out_f, irfft2_plan, + KokkosFFT::Normalization::FORWARD); - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1))); - multiply(out_f, 1.0/static_cast(n0 * n1)); + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1))); + multiply(out_f, 1.0 / static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); } // fft2 on 2D Views TYPED_TEST(FFT2D, FFT2_2DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_fft2_2dfft_2dview(); @@ -934,7 +1067,7 @@ TYPED_TEST(FFT2D, FFT2_2DView) { // ifft2 on 2D Views TYPED_TEST(FFT2D, IFFT2_2DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_fft2_2difft_2dview(); @@ -942,7 +1075,7 @@ TYPED_TEST(FFT2D, IFFT2_2DView) { // rfft2 on 2D Views TYPED_TEST(FFT2D, RFFT2_2DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_fft2_2drfft_2dview(); @@ -950,7 +1083,7 @@ TYPED_TEST(FFT2D, RFFT2_2DView) { // irfft2 on 2D Views TYPED_TEST(FFT2D, IRFFT2_2DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_fft2_2dirfft_2dview(); @@ -960,11 +1093,14 @@ TYPED_TEST(FFT2D, IRFFT2_2DView) { template void test_fftn_2dfft_2dview() { const int n0 = 4, n1 = 6; - using ComplexView2DType = Kokkos::View**, LayoutType, execution_space>; + using ComplexView2DType = + Kokkos::View**, LayoutType, execution_space>; ComplexView2DType x("x", n0, n1); - ComplexView2DType out("out", n0, n1), out1("out1", n0, n1), out2("out2", n0, n1); - ComplexView2DType out_b("out_b", n0, n1), out_o("out_o", n0, n1), out_f("out_f", n0, n1); + ComplexView2DType out("out", n0, n1), out1("out1", n0, n1), + out2("out2", n0, n1); + ComplexView2DType out_b("out_b", n0, n1), out_o("out_o", n0, n1), + out_f("out_f", n0, n1); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); @@ -973,65 +1109,82 @@ void test_fftn_2dfft_2dview() { Kokkos::fence(); // np.fftn for 2D array is identical to np.fft(np.fft(x, axis=1), axis=0) - KokkosFFT::fft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, /*axis=*/1); - KokkosFFT::fft(execution_space(), out1, out2, KokkosFFT::Normalization::BACKWARD, /*axis=*/0); - - KokkosFFT::fftn(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::fftn(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::fft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, + /*axis=*/1); + KokkosFFT::fft(execution_space(), out1, out2, + KokkosFFT::Normalization::BACKWARD, /*axis=*/0); + + KokkosFFT::fftn(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::fftn(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); KokkosFFT::fftn(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); - KokkosFFT::fftn(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::fftn(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1))); multiply(out_f, static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); // Same tests with specifying axes // np.fftn for 2D array is identical to np.fft(np.fft(x, axis=1), axis=0) using axes_type = KokkosFFT::axis_type<2>; - axes_type axes = {-2, -1}; + axes_type axes = {-2, -1}; - KokkosFFT::fftn(execution_space(), x, out, axes); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::fftn(execution_space(), x, out_b, axes, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::fftn(execution_space(), x, out_o, axes, KokkosFFT::Normalization::ORTHO); - KokkosFFT::fftn(execution_space(), x, out_f, axes, KokkosFFT::Normalization::FORWARD); + KokkosFFT::fftn(execution_space(), x, out, + axes); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::fftn(execution_space(), x, out_b, axes, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::fftn(execution_space(), x, out_o, axes, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::fftn(execution_space(), x, out_f, axes, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1))); multiply(out_f, static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); // Reuse plans - KokkosFFT::Impl::Plan fftn_plan(execution_space(), x, out, KokkosFFT::Impl::Direction::Forward, axes); - - KokkosFFT::fftn(execution_space(), x, out, fftn_plan, axes); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::fftn(execution_space(), x, out_b, fftn_plan, axes, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::fftn(execution_space(), x, out_o, fftn_plan, axes, KokkosFFT::Normalization::ORTHO); - KokkosFFT::fftn(execution_space(), x, out_f, fftn_plan, axes, KokkosFFT::Normalization::FORWARD); + KokkosFFT::Impl::Plan fftn_plan(execution_space(), x, out, + KokkosFFT::Impl::Direction::Forward, axes); + + KokkosFFT::fftn(execution_space(), x, out, fftn_plan, + axes); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::fftn(execution_space(), x, out_b, fftn_plan, axes, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::fftn(execution_space(), x, out_o, fftn_plan, axes, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::fftn(execution_space(), x, out_f, fftn_plan, axes, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1))); multiply(out_f, static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); } template -void test_fftn_3dfft_3dview(T atol=1.0e-6) { +void test_fftn_3dfft_3dview(T atol = 1.0e-6) { const int n0 = 4, n1 = 6, n2 = 8; - using ComplexView3DType = Kokkos::View***, LayoutType, execution_space>; + using ComplexView3DType = + Kokkos::View***, LayoutType, execution_space>; ComplexView3DType x("x", n0, n1, n2); - ComplexView3DType out("out", n0, n1, n2), out1("out1", n0, n1, n2), out2("out2", n0, n1, n2), out3("out3", n0, n1, n2); - ComplexView3DType out_b("out_b", n0, n1, n2), out_o("out_o", n0, n1, n2), out_f("out_f", n0, n1, n2); + ComplexView3DType out("out", n0, n1, n2), out1("out1", n0, n1, n2), + out2("out2", n0, n1, n2), out3("out3", n0, n1, n2); + ComplexView3DType out_b("out_b", n0, n1, n2), out_o("out_o", n0, n1, n2), + out_f("out_f", n0, n1, n2); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); @@ -1039,50 +1192,66 @@ void test_fftn_3dfft_3dview(T atol=1.0e-6) { Kokkos::fence(); - // np.fftn for 3D array is identical to np.fft(np.fft(np.fft(x, axis=2), axis=1), axis=0) - KokkosFFT::fft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, /*axis=*/2); - KokkosFFT::fft(execution_space(), out1, out2, KokkosFFT::Normalization::BACKWARD, /*axis=*/1); - KokkosFFT::fft(execution_space(), out2, out3, KokkosFFT::Normalization::BACKWARD, /*axis=*/0); - - KokkosFFT::fftn(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::fftn(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); + // np.fftn for 3D array is identical to np.fft(np.fft(np.fft(x, axis=2), + // axis=1), axis=0) + KokkosFFT::fft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, + /*axis=*/2); + KokkosFFT::fft(execution_space(), out1, out2, + KokkosFFT::Normalization::BACKWARD, /*axis=*/1); + KokkosFFT::fft(execution_space(), out2, out3, + KokkosFFT::Normalization::BACKWARD, /*axis=*/0); + + KokkosFFT::fftn(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::fftn(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); KokkosFFT::fftn(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); - KokkosFFT::fftn(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::fftn(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1 * n2))); multiply(out_f, static_cast(n0 * n1 * n2)); - EXPECT_TRUE( allclose(out, out3, 1.e-5, atol) ); - EXPECT_TRUE( allclose(out_b, out3, 1.e-5, atol) ); - EXPECT_TRUE( allclose(out_o, out3, 1.e-5, atol) ); - EXPECT_TRUE( allclose(out_f, out3, 1.e-5, atol) ); + EXPECT_TRUE(allclose(out, out3, 1.e-5, atol)); + EXPECT_TRUE(allclose(out_b, out3, 1.e-5, atol)); + EXPECT_TRUE(allclose(out_o, out3, 1.e-5, atol)); + EXPECT_TRUE(allclose(out_f, out3, 1.e-5, atol)); // Same tests with specifying axes - // np.fftn for 3D array is identical to np.fft(np.fft(np.fft(x, axis=2), axis=1), axis=0) + // np.fftn for 3D array is identical to np.fft(np.fft(np.fft(x, axis=2), + // axis=1), axis=0) using axes_type = KokkosFFT::axis_type<3>; - KokkosFFT::fftn(execution_space(), x, out, axes_type{-3, -2, -1}); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::fftn(execution_space(), x, out_b, axes_type{-3, -2, -1}, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::fftn(execution_space(), x, out_o, axes_type{-3, -2, -1}, KokkosFFT::Normalization::ORTHO); - KokkosFFT::fftn(execution_space(), x, out_f, axes_type{-3, -2, -1}, KokkosFFT::Normalization::FORWARD); + KokkosFFT::fftn( + execution_space(), x, out, + axes_type{-3, -2, -1}); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::fftn(execution_space(), x, out_b, axes_type{-3, -2, -1}, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::fftn(execution_space(), x, out_o, axes_type{-3, -2, -1}, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::fftn(execution_space(), x, out_f, axes_type{-3, -2, -1}, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1 * n2))); multiply(out_f, static_cast(n0 * n1 * n2)); - EXPECT_TRUE( allclose(out, out3, 1.e-5, atol) ); - EXPECT_TRUE( allclose(out_b, out3, 1.e-5, atol) ); - EXPECT_TRUE( allclose(out_o, out3, 1.e-5, atol) ); - EXPECT_TRUE( allclose(out_f, out3, 1.e-5, atol) ); + EXPECT_TRUE(allclose(out, out3, 1.e-5, atol)); + EXPECT_TRUE(allclose(out_b, out3, 1.e-5, atol)); + EXPECT_TRUE(allclose(out_o, out3, 1.e-5, atol)); + EXPECT_TRUE(allclose(out_f, out3, 1.e-5, atol)); } template void test_ifftn_2dfft_2dview() { const int n0 = 4, n1 = 6; - using ComplexView2DType = Kokkos::View**, LayoutType, execution_space>; + using ComplexView2DType = + Kokkos::View**, LayoutType, execution_space>; ComplexView2DType x("x", n0, n1); - ComplexView2DType out("out", n0, n1), out1("out1", n0, n1), out2("out2", n0, n1); - ComplexView2DType out_b("out_b", n0, n1), out_o("out_o", n0, n1), out_f("out_f", n0, n1); + ComplexView2DType out("out", n0, n1), out1("out1", n0, n1), + out2("out2", n0, n1); + ComplexView2DType out_b("out_b", n0, n1), out_o("out_o", n0, n1), + out_f("out_f", n0, n1); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); @@ -1091,64 +1260,82 @@ void test_ifftn_2dfft_2dview() { Kokkos::fence(); // np.ifftn for 2D array is identical to np.ifft(np.ifft(x, axis=1), axis=0) - KokkosFFT::ifft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, /*axis=*/1); - KokkosFFT::ifft(execution_space(), out1, out2, KokkosFFT::Normalization::BACKWARD, /*axis=*/0); - - KokkosFFT::ifftn(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::ifftn(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::ifftn(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); - KokkosFFT::ifftn(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); - - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1))); - multiply(out_f, 1.0/static_cast(n0 * n1)); - - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + KokkosFFT::ifft(execution_space(), x, out1, + KokkosFFT::Normalization::BACKWARD, /*axis=*/1); + KokkosFFT::ifft(execution_space(), out1, out2, + KokkosFFT::Normalization::BACKWARD, /*axis=*/0); + + KokkosFFT::ifftn(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::ifftn(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::ifftn(execution_space(), x, out_o, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::ifftn(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); + + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1))); + multiply(out_f, 1.0 / static_cast(n0 * n1)); + + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); // Same tests with specifying axes // np.fftn for 2D array is identical to np.fft(np.fft(x, axis=1), axis=0) using axes_type = KokkosFFT::axis_type<2>; - axes_type axes = {-2, -1}; + axes_type axes = {-2, -1}; - KokkosFFT::ifftn(execution_space(), x, out, axes); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::ifftn(execution_space(), x, out_b, axes, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::ifftn(execution_space(), x, out_o, axes, KokkosFFT::Normalization::ORTHO); - KokkosFFT::ifftn(execution_space(), x, out_f, axes, KokkosFFT::Normalization::FORWARD); + KokkosFFT::ifftn(execution_space(), x, out, + axes); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::ifftn(execution_space(), x, out_b, axes, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::ifftn(execution_space(), x, out_o, axes, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::ifftn(execution_space(), x, out_f, axes, + KokkosFFT::Normalization::FORWARD); - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1))); - multiply(out_f, 1.0/static_cast(n0 * n1)); + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1))); + multiply(out_f, 1.0 / static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); // Reuse plans - KokkosFFT::Impl::Plan ifftn_plan(execution_space(), x, out, KokkosFFT::Impl::Direction::Backward, axes); - KokkosFFT::ifftn(execution_space(), x, out, ifftn_plan, axes); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::ifftn(execution_space(), x, out_b, ifftn_plan, axes, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::ifftn(execution_space(), x, out_o, ifftn_plan, axes, KokkosFFT::Normalization::ORTHO); - KokkosFFT::ifftn(execution_space(), x, out_f, ifftn_plan, axes, KokkosFFT::Normalization::FORWARD); - - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1))); - multiply(out_f, 1.0/static_cast(n0 * n1)); - - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + KokkosFFT::Impl::Plan ifftn_plan(execution_space(), x, out, + KokkosFFT::Impl::Direction::Backward, axes); + KokkosFFT::ifftn(execution_space(), x, out, ifftn_plan, + axes); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::ifftn(execution_space(), x, out_b, ifftn_plan, axes, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::ifftn(execution_space(), x, out_o, ifftn_plan, axes, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::ifftn(execution_space(), x, out_f, ifftn_plan, axes, + KokkosFFT::Normalization::FORWARD); + + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1))); + multiply(out_f, 1.0 / static_cast(n0 * n1)); + + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); } template void test_ifftn_3dfft_3dview() { const int n0 = 4, n1 = 6, n2 = 8; - using ComplexView3DType = Kokkos::View***, LayoutType, execution_space>; + using ComplexView3DType = + Kokkos::View***, LayoutType, execution_space>; ComplexView3DType x("x", n0, n1, n2); - ComplexView3DType out("out", n0, n1, n2), out1("out1", n0, n1, n2), out2("out2", n0, n1, n2), out3("out3", n0, n1, n2); - ComplexView3DType out_b("out_b", n0, n1, n2), out_o("out_o", n0, n1, n2), out_f("out_f", n0, n1, n2); + ComplexView3DType out("out", n0, n1, n2), out1("out1", n0, n1, n2), + out2("out2", n0, n1, n2), out3("out3", n0, n1, n2); + ComplexView3DType out_b("out_b", n0, n1, n2), out_o("out_o", n0, n1, n2), + out_f("out_f", n0, n1, n2); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); @@ -1156,51 +1343,68 @@ void test_ifftn_3dfft_3dview() { Kokkos::fence(); - // np.ifftn for 3D array is identical to np.ifft(np.ifft(np.ifft(x, axis=2), axis=1), axis=0) - KokkosFFT::ifft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, /*axis=*/2); - KokkosFFT::ifft(execution_space(), out1, out2, KokkosFFT::Normalization::BACKWARD, /*axis=*/1); - KokkosFFT::ifft(execution_space(), out2, out3, KokkosFFT::Normalization::BACKWARD, /*axis=*/0); - - KokkosFFT::ifftn(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::ifftn(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::ifftn(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); - KokkosFFT::ifftn(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); - - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1 * n2))); - multiply(out_f, 1.0/static_cast(n0 * n1 * n2)); - - EXPECT_TRUE( allclose(out, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out3, 1.e-5, 1.e-6) ); + // np.ifftn for 3D array is identical to np.ifft(np.ifft(np.ifft(x, axis=2), + // axis=1), axis=0) + KokkosFFT::ifft(execution_space(), x, out1, + KokkosFFT::Normalization::BACKWARD, /*axis=*/2); + KokkosFFT::ifft(execution_space(), out1, out2, + KokkosFFT::Normalization::BACKWARD, /*axis=*/1); + KokkosFFT::ifft(execution_space(), out2, out3, + KokkosFFT::Normalization::BACKWARD, /*axis=*/0); + + KokkosFFT::ifftn(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::ifftn(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::ifftn(execution_space(), x, out_o, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::ifftn(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); + + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1 * n2))); + multiply(out_f, 1.0 / static_cast(n0 * n1 * n2)); + + EXPECT_TRUE(allclose(out, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out3, 1.e-5, 1.e-6)); // Same tests with specifying axes - // np.ifftn for 3D array is identical to np.ifft(np.ifft(np.ifft(x, axis=2), axis=1), axis=0) + // np.ifftn for 3D array is identical to np.ifft(np.ifft(np.ifft(x, axis=2), + // axis=1), axis=0) using axes_type = KokkosFFT::axis_type<3>; - KokkosFFT::ifftn(execution_space(), x, out, axes_type{-3, -2, -1}); // default: KokkosFFT::Normalization::BACKWARD - KokkosFFT::ifftn(execution_space(), x, out_b, axes_type{-3, -2, -1}, KokkosFFT::Normalization::BACKWARD); - KokkosFFT::ifftn(execution_space(), x, out_o, axes_type{-3, -2, -1}, KokkosFFT::Normalization::ORTHO); - KokkosFFT::ifftn(execution_space(), x, out_f, axes_type{-3, -2, -1}, KokkosFFT::Normalization::FORWARD); - - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1 * n2))); - multiply(out_f, 1.0/static_cast(n0 * n1 * n2)); - - EXPECT_TRUE( allclose(out, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out3, 1.e-5, 1.e-6) ); + KokkosFFT::ifftn( + execution_space(), x, out, + axes_type{-3, -2, -1}); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::ifftn(execution_space(), x, out_b, axes_type{-3, -2, -1}, + KokkosFFT::Normalization::BACKWARD); + KokkosFFT::ifftn(execution_space(), x, out_o, axes_type{-3, -2, -1}, + KokkosFFT::Normalization::ORTHO); + KokkosFFT::ifftn(execution_space(), x, out_f, axes_type{-3, -2, -1}, + KokkosFFT::Normalization::FORWARD); + + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1 * n2))); + multiply(out_f, 1.0 / static_cast(n0 * n1 * n2)); + + EXPECT_TRUE(allclose(out, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out3, 1.e-5, 1.e-6)); } template void test_rfftn_2dfft_2dview() { const int n0 = 4, n1 = 6; - using RealView2DType = Kokkos::View; - using ComplexView2DType = Kokkos::View**, LayoutType, execution_space>; + using RealView2DType = Kokkos::View; + using ComplexView2DType = + Kokkos::View**, LayoutType, execution_space>; RealView2DType x("x", n0, n1), x_ref("x_ref", n0, n1); - ComplexView2DType out("out", n0, n1/2+1), out1("out1", n0, n1/2+1), out2("out2", n0, n1/2+1); - ComplexView2DType out_b("out_b", n0, n1/2+1), out_o("out_o", n0, n1/2+1), out_f("out_f", n0, n1/2+1); + ComplexView2DType out("out", n0, n1 / 2 + 1), out1("out1", n0, n1 / 2 + 1), + out2("out2", n0, n1 / 2 + 1); + ComplexView2DType out_b("out_b", n0, n1 / 2 + 1), + out_o("out_o", n0, n1 / 2 + 1), out_f("out_f", n0, n1 / 2 + 1); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, 1); @@ -1208,153 +1412,188 @@ void test_rfftn_2dfft_2dview() { Kokkos::fence(); // np.rfftn for 2D array is identical to np.fft(np.rfft(x, axis=1), axis=0) - KokkosFFT::rfft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, /*axis=*/1); - KokkosFFT::fft(execution_space(), out1, out2, KokkosFFT::Normalization::BACKWARD, /*axis=*/0); + KokkosFFT::rfft(execution_space(), x, out1, + KokkosFFT::Normalization::BACKWARD, /*axis=*/1); + KokkosFFT::fft(execution_space(), out1, out2, + KokkosFFT::Normalization::BACKWARD, /*axis=*/0); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::rfftn(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::rfftn(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); + KokkosFFT::rfftn(execution_space(), x, out_o, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::rfftn(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1))); multiply(out_f, static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); // Same tests with specifying axes // np.rfftn for 2D array is identical to np.fft(np.rfft(x, axis=1), axis=0) using axes_type = KokkosFFT::axis_type<2>; - axes_type axes = {-2, -1}; + axes_type axes = {-2, -1}; Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out, axes); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::rfftn(execution_space(), x, out, + axes); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_b, axes, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::rfftn(execution_space(), x, out_b, axes, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_o, axes, KokkosFFT::Normalization::ORTHO); + KokkosFFT::rfftn(execution_space(), x, out_o, axes, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_f, axes, KokkosFFT::Normalization::FORWARD); + KokkosFFT::rfftn(execution_space(), x, out_f, axes, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1))); multiply(out_f, static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); // Reuse plans - KokkosFFT::Impl::Plan rfftn_plan(execution_space(), x, out, KokkosFFT::Impl::Direction::Forward, axes); + KokkosFFT::Impl::Plan rfftn_plan(execution_space(), x, out, + KokkosFFT::Impl::Direction::Forward, axes); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out, rfftn_plan, axes); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::rfftn(execution_space(), x, out, rfftn_plan, + axes); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_b, rfftn_plan, axes, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::rfftn(execution_space(), x, out_b, rfftn_plan, axes, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_o, rfftn_plan, axes, KokkosFFT::Normalization::ORTHO); + KokkosFFT::rfftn(execution_space(), x, out_o, rfftn_plan, axes, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_f, rfftn_plan, axes, KokkosFFT::Normalization::FORWARD); + KokkosFFT::rfftn(execution_space(), x, out_f, rfftn_plan, axes, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1))); multiply(out_f, static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); } template void test_rfftn_3dfft_3dview() { const int n0 = 4, n1 = 6, n2 = 8; - using RealView3DType = Kokkos::View; - using ComplexView3DType = Kokkos::View***, LayoutType, execution_space>; + using RealView3DType = Kokkos::View; + using ComplexView3DType = + Kokkos::View***, LayoutType, execution_space>; RealView3DType x("x", n0, n1, n2), x_ref("x_ref", n0, n1, n2); - ComplexView3DType out("out", n0, n1, n2/2+1), out1("out1", n0, n1, n2/2+1), out2("out2", n0, n1, n2/2+1), out3("out3", n0, n1, n2/2+1); - ComplexView3DType out_b("out_b", n0, n1, n2/2+1), out_o("out_o", n0, n1, n2/2+1), out_f("out_f", n0, n1, n2/2+1); + ComplexView3DType out("out", n0, n1, n2 / 2 + 1), + out1("out1", n0, n1, n2 / 2 + 1), out2("out2", n0, n1, n2 / 2 + 1), + out3("out3", n0, n1, n2 / 2 + 1); + ComplexView3DType out_b("out_b", n0, n1, n2 / 2 + 1), + out_o("out_o", n0, n1, n2 / 2 + 1), out_f("out_f", n0, n1, n2 / 2 + 1); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, 1); Kokkos::deep_copy(x_ref, x); Kokkos::fence(); - // np.rfftn for 3D array is identical to np.fft(np.fft(np.rfft(x, axis=2), axis=1), axis=0) - KokkosFFT::rfft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, /*axis=*/2); - KokkosFFT::fft(execution_space(), out1, out2, KokkosFFT::Normalization::BACKWARD, /*axis=*/1); - KokkosFFT::fft(execution_space(), out2, out3, KokkosFFT::Normalization::BACKWARD, /*axis=*/0); + // np.rfftn for 3D array is identical to np.fft(np.fft(np.rfft(x, axis=2), + // axis=1), axis=0) + KokkosFFT::rfft(execution_space(), x, out1, + KokkosFFT::Normalization::BACKWARD, /*axis=*/2); + KokkosFFT::fft(execution_space(), out1, out2, + KokkosFFT::Normalization::BACKWARD, /*axis=*/1); + KokkosFFT::fft(execution_space(), out2, out3, + KokkosFFT::Normalization::BACKWARD, /*axis=*/0); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::rfftn(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::rfftn(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); + KokkosFFT::rfftn(execution_space(), x, out_o, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::rfftn(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1 * n2))); multiply(out_f, static_cast(n0 * n1 * n2)); - EXPECT_TRUE( allclose(out, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out3, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out3, 1.e-5, 1.e-6)); // Same tests with specifying axes - // np.rfftn for 3D array is identical to np.fft(np.fft(np.rfft(x, axis=2), axis=1), axis=0) + // np.rfftn for 3D array is identical to np.fft(np.fft(np.rfft(x, axis=2), + // axis=1), axis=0) using axes_type = KokkosFFT::axis_type<3>; Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out, axes_type{-3, -2, -1}); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::rfftn( + execution_space(), x, out, + axes_type{-3, -2, -1}); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_b, axes_type{-3, -2, -1}, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::rfftn(execution_space(), x, out_b, axes_type{-3, -2, -1}, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_o, axes_type{-3, -2, -1}, KokkosFFT::Normalization::ORTHO); + KokkosFFT::rfftn(execution_space(), x, out_o, axes_type{-3, -2, -1}, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::rfftn(execution_space(), x, out_f, axes_type{-3, -2, -1}, KokkosFFT::Normalization::FORWARD); + KokkosFFT::rfftn(execution_space(), x, out_f, axes_type{-3, -2, -1}, + KokkosFFT::Normalization::FORWARD); multiply(out_o, sqrt(static_cast(n0 * n1 * n2))); multiply(out_f, static_cast(n0 * n1 * n2)); - EXPECT_TRUE( allclose(out, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out3, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out3, 1.e-5, 1.e-6)); } template void test_irfftn_2dfft_2dview() { const int n0 = 4, n1 = 6; - using RealView2DType = Kokkos::View; - using ComplexView2DType = Kokkos::View**, LayoutType, execution_space>; + using RealView2DType = Kokkos::View; + using ComplexView2DType = + Kokkos::View**, LayoutType, execution_space>; - ComplexView2DType x("x", n0, n1/2+1), x_ref("x_ref", n0, n1/2+1); - ComplexView2DType out1("out1", n0, n1/2+1); + ComplexView2DType x("x", n0, n1 / 2 + 1), x_ref("x_ref", n0, n1 / 2 + 1); + ComplexView2DType out1("out1", n0, n1 / 2 + 1); RealView2DType out2("out2", n0, n1), out("out", n0, n1); - RealView2DType out_b("out_b", n0, n1), out_o("out_o", n0, n1), out_f("out_f", n0, n1); + RealView2DType out_b("out_b", n0, n1), out_o("out_o", n0, n1), + out_f("out_f", n0, n1); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); @@ -1362,146 +1601,179 @@ void test_irfftn_2dfft_2dview() { Kokkos::deep_copy(x_ref, x); // np.irfftn for 2D array is identical to np.irfft(np.ifft(x, axis=0), axis=1) - KokkosFFT::ifft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, /*axis=*/0); - KokkosFFT::irfft(execution_space(), out1, out2, KokkosFFT::Normalization::BACKWARD, /*axis=*/1); + KokkosFFT::ifft(execution_space(), x, out1, + KokkosFFT::Normalization::BACKWARD, /*axis=*/0); + KokkosFFT::irfft(execution_space(), out1, out2, + KokkosFFT::Normalization::BACKWARD, /*axis=*/1); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::irfftn(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::irfftn(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); + KokkosFFT::irfftn(execution_space(), x, out_o, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::irfftn(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1))); - multiply(out_f, 1.0/static_cast(n0 * n1)); + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1))); + multiply(out_f, 1.0 / static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); // Same tests with specifying axes // np.irfftn for 2D array is identical to np.fft(np.rfft(x, axis=1), axis=0) using axes_type = KokkosFFT::axis_type<2>; - axes_type axes = {-2, -1}; + axes_type axes = {-2, -1}; Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out, axes); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::irfftn(execution_space(), x, out, + axes); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_b, axes, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::irfftn(execution_space(), x, out_b, axes, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_o, axes, KokkosFFT::Normalization::ORTHO); + KokkosFFT::irfftn(execution_space(), x, out_o, axes, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_f, axes, KokkosFFT::Normalization::FORWARD); + KokkosFFT::irfftn(execution_space(), x, out_f, axes, + KokkosFFT::Normalization::FORWARD); - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1))); - multiply(out_f, 1.0/static_cast(n0 * n1)); + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1))); + multiply(out_f, 1.0 / static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); // Reuse plans - KokkosFFT::Impl::Plan irfftn_plan(execution_space(), x, out, KokkosFFT::Impl::Direction::Backward, axes); + KokkosFFT::Impl::Plan irfftn_plan(execution_space(), x, out, + KokkosFFT::Impl::Direction::Backward, axes); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out, irfftn_plan, axes); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::irfftn(execution_space(), x, out, irfftn_plan, + axes); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_b, irfftn_plan, axes, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::irfftn(execution_space(), x, out_b, irfftn_plan, axes, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_o, irfftn_plan, axes, KokkosFFT::Normalization::ORTHO); + KokkosFFT::irfftn(execution_space(), x, out_o, irfftn_plan, axes, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_f, irfftn_plan, axes, KokkosFFT::Normalization::FORWARD); + KokkosFFT::irfftn(execution_space(), x, out_f, irfftn_plan, axes, + KokkosFFT::Normalization::FORWARD); - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1))); - multiply(out_f, 1.0/static_cast(n0 * n1)); + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1))); + multiply(out_f, 1.0 / static_cast(n0 * n1)); - EXPECT_TRUE( allclose(out, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out2, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out2, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out2, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out2, 1.e-5, 1.e-6)); } template void test_irfftn_3dfft_3dview() { const int n0 = 4, n1 = 6, n2 = 8; - using RealView3DType = Kokkos::View; - using ComplexView3DType = Kokkos::View***, LayoutType, execution_space>; - - ComplexView3DType x("x", n0, n1, n2/2+1), x_ref("x_ref", n0, n1, n2/2+1); - ComplexView3DType out1("out1", n0, n1, n2/2+1), out2("out2", n0, n1, n2/2+1); + using RealView3DType = Kokkos::View; + using ComplexView3DType = + Kokkos::View***, LayoutType, execution_space>; + + ComplexView3DType x("x", n0, n1, n2 / 2 + 1), + x_ref("x_ref", n0, n1, n2 / 2 + 1); + ComplexView3DType out1("out1", n0, n1, n2 / 2 + 1), + out2("out2", n0, n1, n2 / 2 + 1); RealView3DType out("out", n0, n1, n2), out3("out3", n0, n1, n2); - RealView3DType out_b("out_b", n0, n1, n2), out_o("out_o", n0, n1, n2), out_f("out_f", n0, n1, n2); + RealView3DType out_b("out_b", n0, n1, n2), out_o("out_o", n0, n1, n2), + out_f("out_f", n0, n1, n2); const Kokkos::complex I(1.0, 1.0); Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(x, random_pool, I); Kokkos::deep_copy(x_ref, x); - // np.irfftn for 3D array is identical to np.irfft(np.ifft(np.ifft(x, axis=0), axis=1), axis=2) - KokkosFFT::ifft(execution_space(), x, out1, KokkosFFT::Normalization::BACKWARD, /*axis=*/0); - KokkosFFT::ifft(execution_space(), out1, out2, KokkosFFT::Normalization::BACKWARD, /*axis=*/1); - KokkosFFT::irfft(execution_space(), out2, out3, KokkosFFT::Normalization::BACKWARD, /*axis=*/2); + // np.irfftn for 3D array is identical to np.irfft(np.ifft(np.ifft(x, axis=0), + // axis=1), axis=2) + KokkosFFT::ifft(execution_space(), x, out1, + KokkosFFT::Normalization::BACKWARD, /*axis=*/0); + KokkosFFT::ifft(execution_space(), out1, out2, + KokkosFFT::Normalization::BACKWARD, /*axis=*/1); + KokkosFFT::irfft(execution_space(), out2, out3, + KokkosFFT::Normalization::BACKWARD, /*axis=*/2); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::irfftn(execution_space(), x, + out); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_b, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::irfftn(execution_space(), x, out_b, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_o, KokkosFFT::Normalization::ORTHO); + KokkosFFT::irfftn(execution_space(), x, out_o, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_f, KokkosFFT::Normalization::FORWARD); + KokkosFFT::irfftn(execution_space(), x, out_f, + KokkosFFT::Normalization::FORWARD); - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1 * n2))); - multiply(out_f, 1.0/static_cast(n0 * n1 * n2)); + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1 * n2))); + multiply(out_f, 1.0 / static_cast(n0 * n1 * n2)); - EXPECT_TRUE( allclose(out, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out3, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out3, 1.e-5, 1.e-6)); // Same tests with specifying axes - // np.irfftn for 3D array is identical to np.irfft(np.ifft(np.ifft(x, axis=0), axis=1), axis=2) + // np.irfftn for 3D array is identical to np.irfft(np.ifft(np.ifft(x, axis=0), + // axis=1), axis=2) using axes_type = KokkosFFT::axis_type<3>; Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out, axes_type{-3, -2, -1}); // default: KokkosFFT::Normalization::BACKWARD + KokkosFFT::irfftn( + execution_space(), x, out, + axes_type{-3, -2, -1}); // default: KokkosFFT::Normalization::BACKWARD Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_b, axes_type{-3, -2, -1}, KokkosFFT::Normalization::BACKWARD); + KokkosFFT::irfftn(execution_space(), x, out_b, axes_type{-3, -2, -1}, + KokkosFFT::Normalization::BACKWARD); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_o, axes_type{-3, -2, -1}, KokkosFFT::Normalization::ORTHO); + KokkosFFT::irfftn(execution_space(), x, out_o, axes_type{-3, -2, -1}, + KokkosFFT::Normalization::ORTHO); Kokkos::deep_copy(x, x_ref); - KokkosFFT::irfftn(execution_space(), x, out_f, axes_type{-3, -2, -1}, KokkosFFT::Normalization::FORWARD); + KokkosFFT::irfftn(execution_space(), x, out_f, axes_type{-3, -2, -1}, + KokkosFFT::Normalization::FORWARD); - multiply(out_o, 1.0/sqrt(static_cast(n0 * n1 * n2))); - multiply(out_f, 1.0/static_cast(n0 * n1 * n2)); + multiply(out_o, 1.0 / sqrt(static_cast(n0 * n1 * n2))); + multiply(out_f, 1.0 / static_cast(n0 * n1 * n2)); - EXPECT_TRUE( allclose(out, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_b, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_o, out3, 1.e-5, 1.e-6) ); - EXPECT_TRUE( allclose(out_f, out3, 1.e-5, 1.e-6) ); + EXPECT_TRUE(allclose(out, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_b, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_o, out3, 1.e-5, 1.e-6)); + EXPECT_TRUE(allclose(out_f, out3, 1.e-5, 1.e-6)); } // fftn on 2D Views TYPED_TEST(FFTND, 2DFFT_2DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_fftn_2dfft_2dview(); @@ -1509,7 +1781,7 @@ TYPED_TEST(FFTND, 2DFFT_2DView) { // fftn on 3D Views TYPED_TEST(FFTND, 3DFFT_3DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; float_type atol = std::is_same_v ? 5.0e-5 : 1.0e-6; @@ -1518,7 +1790,7 @@ TYPED_TEST(FFTND, 3DFFT_3DView) { // ifftn on 2D Views TYPED_TEST(FFTND, 2DIFFT_2DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_ifftn_2dfft_2dview(); @@ -1526,7 +1798,7 @@ TYPED_TEST(FFTND, 2DIFFT_2DView) { // ifftn on 3D Views TYPED_TEST(FFTND, 3DIFFT_3DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_ifftn_3dfft_3dview(); @@ -1534,7 +1806,7 @@ TYPED_TEST(FFTND, 3DIFFT_3DView) { // rfftn on 2D Views TYPED_TEST(FFTND, 2DRFFT_2DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_rfftn_2dfft_2dview(); @@ -1542,7 +1814,7 @@ TYPED_TEST(FFTND, 2DRFFT_2DView) { // rfftn on 3D Views TYPED_TEST(FFTND, 3DRFFT_3DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_rfftn_3dfft_3dview(); @@ -1550,7 +1822,7 @@ TYPED_TEST(FFTND, 3DRFFT_3DView) { // irfftn on 2D Views TYPED_TEST(FFTND, 2DIRFFT_2DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_irfftn_2dfft_2dview(); @@ -1558,7 +1830,7 @@ TYPED_TEST(FFTND, 2DIRFFT_2DView) { // irfftn on 3D Views TYPED_TEST(FFTND, 3DIRFFT_3DView) { - using float_type = typename TestFixture::float_type; + using float_type = typename TestFixture::float_type; using layout_type = typename TestFixture::layout_type; test_irfftn_3dfft_3dview(); diff --git a/fft/unit_test/Test_Types.hpp b/fft/unit_test/Test_Types.hpp index 9a3894b8..b973593d 100644 --- a/fft/unit_test/Test_Types.hpp +++ b/fft/unit_test/Test_Types.hpp @@ -3,16 +3,23 @@ #include using execution_space = Kokkos::DefaultExecutionSpace; -template using View1D = Kokkos::View; -template using View2D = Kokkos::View; -template using View3D = Kokkos::View; +template +using View1D = Kokkos::View; +template +using View2D = Kokkos::View; +template +using View3D = Kokkos::View; // Layout Left -template using LeftView2D = Kokkos::View; -template using LeftView3D = Kokkos::View; +template +using LeftView2D = Kokkos::View; +template +using LeftView3D = Kokkos::View; // Layout Right -template using RightView2D = Kokkos::View; -template using RightView3D = Kokkos::View; +template +using RightView2D = Kokkos::View; +template +using RightView3D = Kokkos::View; #endif \ No newline at end of file diff --git a/fft/unit_test/Test_Utils.hpp b/fft/unit_test/Test_Utils.hpp index 997a4a8b..3faeed5a 100644 --- a/fft/unit_test/Test_Utils.hpp +++ b/fft/unit_test/Test_Utils.hpp @@ -6,14 +6,11 @@ #include "Test_Types.hpp" template -bool allclose(const AViewType& a, - const BViewType& b, - double rtol=1.e-5, - double atol=1.e-8 - ) { +bool allclose(const AViewType& a, const BViewType& b, double rtol = 1.e-5, + double atol = 1.e-8) { constexpr std::size_t rank = AViewType::rank; - for(std::size_t i=0; i>{0, n}, - KOKKOS_LAMBDA (const int& i, int& err) { - auto tmp_a = ptr_a[i]; - auto tmp_b = ptr_b[i]; - bool not_close = Kokkos::abs(tmp_a - tmp_b) > (atol + rtol * Kokkos::abs(tmp_b)); - err += static_cast(not_close); - }, error); + Kokkos::RangePolicy>{0, + n}, + KOKKOS_LAMBDA(const int& i, int& err) { + auto tmp_a = ptr_a[i]; + auto tmp_b = ptr_b[i]; + bool not_close = + Kokkos::abs(tmp_a - tmp_b) > (atol + rtol * Kokkos::abs(tmp_b)); + err += static_cast(not_close); + }, + error); return error == 0; } @@ -36,19 +36,17 @@ bool allclose(const AViewType& a, template void multiply(ViewType& x, T a) { const auto n = x.size(); - auto* ptr_x = x.data(); + auto* ptr_x = x.data(); Kokkos::parallel_for( - Kokkos::RangePolicy>{0, n}, - KOKKOS_LAMBDA (const int& i) { - ptr_x[i] = ptr_x[i] * a; - } - ); + Kokkos::RangePolicy>{0, + n}, + KOKKOS_LAMBDA(const int& i) { ptr_x[i] = ptr_x[i] * a; }); } template void display(ViewType& a) { - auto label = a.label(); + auto label = a.label(); const auto n = a.size(); auto h_a = Kokkos::create_mirror_view(a); @@ -56,7 +54,7 @@ void display(ViewType& a) { auto* data = h_a.data(); std::cout << std::scientific << std::setprecision(16) << std::flush; - for(int i=0; i Date: Mon, 8 Jan 2024 19:18:46 +0900 Subject: [PATCH 4/5] Apply format to examples --- examples/01_1DFFT/01_1DFFT.cpp | 11 ++-- examples/02_2DFFT/02_2DFFT.cpp | 11 ++-- examples/03_NDFFT/03_NDFFT.cpp | 11 ++-- examples/04_batchedFFT/04_batchedFFT.cpp | 23 +++++--- .../05_1DFFT_HOST_DEVICE.cpp | 56 ++++++++++--------- .../06_1DFFT_reuse_plans.cpp | 24 +++++--- 6 files changed, 76 insertions(+), 60 deletions(-) diff --git a/examples/01_1DFFT/01_1DFFT.cpp b/examples/01_1DFFT/01_1DFFT.cpp index f9a7872a..a7117df9 100644 --- a/examples/01_1DFFT/01_1DFFT.cpp +++ b/examples/01_1DFFT/01_1DFFT.cpp @@ -4,10 +4,11 @@ #include using execution_space = Kokkos::DefaultExecutionSpace; -template using View1D = Kokkos::View; +template +using View1D = Kokkos::View; -int main( int argc, char* argv[] ) { - Kokkos::initialize( argc, argv ); +int main(int argc, char* argv[]) { + Kokkos::initialize(argc, argv); { constexpr int n0 = 128; const Kokkos::complex I(1.0, 1.0); @@ -25,13 +26,13 @@ int main( int argc, char* argv[] ) { // 1D R2C FFT View1D xr2c("xr2c", n0); - View1D > xr2c_hat("xr2c_hat", n0/2+1); + View1D > xr2c_hat("xr2c_hat", n0 / 2 + 1); Kokkos::fill_random(xr2c, random_pool, 1); KokkosFFT::rfft(execution_space(), xr2c, xr2c_hat); // 1D C2R FFT - View1D > xc2r("xr2c_hat", n0/2+1); + View1D > xc2r("xr2c_hat", n0 / 2 + 1); View1D xc2r_hat("xc2r", n0); Kokkos::fill_random(xc2r, random_pool, I); diff --git a/examples/02_2DFFT/02_2DFFT.cpp b/examples/02_2DFFT/02_2DFFT.cpp index 5f837c85..992be796 100644 --- a/examples/02_2DFFT/02_2DFFT.cpp +++ b/examples/02_2DFFT/02_2DFFT.cpp @@ -4,10 +4,11 @@ #include using execution_space = Kokkos::DefaultExecutionSpace; -template using View2D = Kokkos::View; +template +using View2D = Kokkos::View; -int main( int argc, char* argv[] ) { - Kokkos::initialize( argc, argv ); +int main(int argc, char* argv[]) { + Kokkos::initialize(argc, argv); { constexpr int n0 = 128, n1 = 128; const Kokkos::complex I(1.0, 1.0); @@ -25,13 +26,13 @@ int main( int argc, char* argv[] ) { // 2D R2C FFT View2D xr2c("xr2c", n0, n1); - View2D > xr2c_hat("xr2c_hat", n0, n1/2+1); + View2D > xr2c_hat("xr2c_hat", n0, n1 / 2 + 1); Kokkos::fill_random(xr2c, random_pool, 1); KokkosFFT::rfft2(execution_space(), xr2c, xr2c_hat); // 2D C2R FFT - View2D > xc2r("xr2c_hat", n0, n1/2+1); + View2D > xc2r("xr2c_hat", n0, n1 / 2 + 1); View2D xc2r_hat("xc2r", n0, n1); Kokkos::fill_random(xc2r, random_pool, I); diff --git a/examples/03_NDFFT/03_NDFFT.cpp b/examples/03_NDFFT/03_NDFFT.cpp index 4bb7b8be..2c8a6c5b 100644 --- a/examples/03_NDFFT/03_NDFFT.cpp +++ b/examples/03_NDFFT/03_NDFFT.cpp @@ -4,10 +4,11 @@ #include using execution_space = Kokkos::DefaultExecutionSpace; -template using View3D = Kokkos::View; +template +using View3D = Kokkos::View; -int main( int argc, char* argv[] ) { - Kokkos::initialize( argc, argv ); +int main(int argc, char* argv[]) { + Kokkos::initialize(argc, argv); { constexpr int n0 = 128, n1 = 128, n2 = 16; const Kokkos::complex I(1.0, 1.0); @@ -25,13 +26,13 @@ int main( int argc, char* argv[] ) { // 3D R2C FFT View3D xr2c("xr2c", n0, n1, n2); - View3D > xr2c_hat("xr2c_hat", n0, n1, n2/2+1); + View3D > xr2c_hat("xr2c_hat", n0, n1, n2 / 2 + 1); Kokkos::fill_random(xr2c, random_pool, 1); KokkosFFT::rfftn(execution_space(), xr2c, xr2c_hat); // 3D C2R FFT - View3D > xc2r("xr2c_hat", n0, n1, n2/2+1); + View3D > xc2r("xr2c_hat", n0, n1, n2 / 2 + 1); View3D xc2r_hat("xc2r", n0, n1, n2); Kokkos::fill_random(xc2r, random_pool, I); diff --git a/examples/04_batchedFFT/04_batchedFFT.cpp b/examples/04_batchedFFT/04_batchedFFT.cpp index 36e32a97..afa23ada 100644 --- a/examples/04_batchedFFT/04_batchedFFT.cpp +++ b/examples/04_batchedFFT/04_batchedFFT.cpp @@ -4,10 +4,11 @@ #include using execution_space = Kokkos::DefaultExecutionSpace; -template using View3D = Kokkos::View; +template +using View3D = Kokkos::View; -int main( int argc, char* argv[] ) { - Kokkos::initialize( argc, argv ); +int main(int argc, char* argv[]) { + Kokkos::initialize(argc, argv); { constexpr int n0 = 128, n1 = 128, n2 = 16; const Kokkos::complex I(1.0, 1.0); @@ -20,22 +21,26 @@ int main( int argc, char* argv[] ) { Kokkos::Random_XorShift64_Pool<> random_pool(12345); Kokkos::fill_random(xc2c, random_pool, I); - KokkosFFT::fft(execution_space(), xc2c, xc2c_hat, KokkosFFT::Normalization::BACKWARD, /*axis=*/-1); - KokkosFFT::ifft(execution_space(), xc2c_hat, xc2c_inv, KokkosFFT::Normalization::BACKWARD, /*axis=*/-1); + KokkosFFT::fft(execution_space(), xc2c, xc2c_hat, + KokkosFFT::Normalization::BACKWARD, /*axis=*/-1); + KokkosFFT::ifft(execution_space(), xc2c_hat, xc2c_inv, + KokkosFFT::Normalization::BACKWARD, /*axis=*/-1); // 1D batched R2C FFT View3D xr2c("xr2c", n0, n1, n2); - View3D > xr2c_hat("xr2c_hat", n0, n1, n2/2+1); + View3D > xr2c_hat("xr2c_hat", n0, n1, n2 / 2 + 1); Kokkos::fill_random(xr2c, random_pool, 1); - KokkosFFT::rfft(execution_space(), xr2c, xr2c_hat, KokkosFFT::Normalization::BACKWARD, /*axis=*/-1); + KokkosFFT::rfft(execution_space(), xr2c, xr2c_hat, + KokkosFFT::Normalization::BACKWARD, /*axis=*/-1); // 1D batched C2R FFT - View3D > xc2r("xr2c_hat", n0, n1, n2/2+1); + View3D > xc2r("xr2c_hat", n0, n1, n2 / 2 + 1); View3D xc2r_hat("xc2r", n0, n1, n2); Kokkos::fill_random(xc2r, random_pool, I); - KokkosFFT::irfft(execution_space(), xc2r, xc2r_hat, KokkosFFT::Normalization::BACKWARD, /*axis=*/-1); + KokkosFFT::irfft(execution_space(), xc2r, xc2r_hat, + KokkosFFT::Normalization::BACKWARD, /*axis=*/-1); } Kokkos::finalize(); diff --git a/examples/05_1DFFT_HOST_DEVICE/05_1DFFT_HOST_DEVICE.cpp b/examples/05_1DFFT_HOST_DEVICE/05_1DFFT_HOST_DEVICE.cpp index 6c86162c..fa38101c 100644 --- a/examples/05_1DFFT_HOST_DEVICE/05_1DFFT_HOST_DEVICE.cpp +++ b/examples/05_1DFFT_HOST_DEVICE/05_1DFFT_HOST_DEVICE.cpp @@ -3,13 +3,15 @@ #include #include -using execution_space = Kokkos::DefaultExecutionSpace; +using execution_space = Kokkos::DefaultExecutionSpace; using host_execution_space = Kokkos::DefaultHostExecutionSpace; -template using View1D = Kokkos::View; -template using HostView1D = Kokkos::View; +template +using View1D = Kokkos::View; +template +using HostView1D = Kokkos::View; -int main( int argc, char* argv[] ) { - Kokkos::initialize( argc, argv ); +int main(int argc, char* argv[]) { + Kokkos::initialize(argc, argv); { constexpr int n0 = 128; const Kokkos::complex I(1.0, 1.0); @@ -27,44 +29,44 @@ int main( int argc, char* argv[] ) { // 1D R2C FFT View1D xr2c("xr2c", n0); - View1D > xr2c_hat("xr2c_hat", n0/2+1); + View1D > xr2c_hat("xr2c_hat", n0 / 2 + 1); Kokkos::fill_random(xr2c, random_pool, 1); KokkosFFT::rfft(execution_space(), xr2c, xr2c_hat); // 1D C2R FFT - View1D > xc2r("xr2c_hat", n0/2+1); + View1D > xc2r("xr2c_hat", n0 / 2 + 1); View1D xc2r_hat("xc2r", n0); Kokkos::fill_random(xc2r, random_pool, I); KokkosFFT::irfft(execution_space(), xc2r, xc2r_hat); - #ifdef ENABLE_HOST_AND_DEVICE - // FFTs on Host - // 1D C2C FFT (Forward and Backward) - HostView1D > h_xc2c("h_xc2c", n0); - HostView1D > h_xc2c_hat("h_xc2c_hat", n0); - HostView1D > h_xc2c_inv("h_xc2c_inv", n0); +#ifdef ENABLE_HOST_AND_DEVICE + // FFTs on Host + // 1D C2C FFT (Forward and Backward) + HostView1D > h_xc2c("h_xc2c", n0); + HostView1D > h_xc2c_hat("h_xc2c_hat", n0); + HostView1D > h_xc2c_inv("h_xc2c_inv", n0); - Kokkos::deep_copy(h_xc2c, xc2c); + Kokkos::deep_copy(h_xc2c, xc2c); - KokkosFFT::fft(host_execution_space(), h_xc2c, h_xc2c_hat); - KokkosFFT::ifft(host_execution_space(), h_xc2c_hat, h_xc2c_inv); + KokkosFFT::fft(host_execution_space(), h_xc2c, h_xc2c_hat); + KokkosFFT::ifft(host_execution_space(), h_xc2c_hat, h_xc2c_inv); - // 1D R2C FFT - HostView1D h_xr2c("h_xr2c", n0); - HostView1D > h_xr2c_hat("h_xr2c_hat", n0/2+1); + // 1D R2C FFT + HostView1D h_xr2c("h_xr2c", n0); + HostView1D > h_xr2c_hat("h_xr2c_hat", n0 / 2 + 1); - Kokkos::deep_copy(h_xr2c, xr2c); - KokkosFFT::rfft(host_execution_space(), h_xr2c, h_xr2c_hat); + Kokkos::deep_copy(h_xr2c, xr2c); + KokkosFFT::rfft(host_execution_space(), h_xr2c, h_xr2c_hat); - // 1D C2R FFT - HostView1D > h_xc2r("h_xr2c_hat", n0/2+1); - HostView1D h_xc2r_hat("h_xc2r", n0); + // 1D C2R FFT + HostView1D > h_xc2r("h_xr2c_hat", n0 / 2 + 1); + HostView1D h_xc2r_hat("h_xc2r", n0); - Kokkos::deep_copy(h_xc2r, xc2r); - KokkosFFT::irfft(host_execution_space(), h_xc2r, h_xc2r_hat); - #endif + Kokkos::deep_copy(h_xc2r, xc2r); + KokkosFFT::irfft(host_execution_space(), h_xc2r, h_xc2r_hat); +#endif } Kokkos::finalize(); diff --git a/examples/06_1DFFT_reuse_plans/06_1DFFT_reuse_plans.cpp b/examples/06_1DFFT_reuse_plans/06_1DFFT_reuse_plans.cpp index 801e61df..12d810e8 100644 --- a/examples/06_1DFFT_reuse_plans/06_1DFFT_reuse_plans.cpp +++ b/examples/06_1DFFT_reuse_plans/06_1DFFT_reuse_plans.cpp @@ -4,10 +4,11 @@ #include using execution_space = Kokkos::DefaultExecutionSpace; -template using View1D = Kokkos::View; +template +using View1D = Kokkos::View; -int main( int argc, char* argv[] ) { - Kokkos::initialize( argc, argv ); +int main(int argc, char* argv[]) { + Kokkos::initialize(argc, argv); { constexpr int n0 = 128; const Kokkos::complex I(1.0, 1.0); @@ -21,26 +22,31 @@ int main( int argc, char* argv[] ) { Kokkos::fill_random(xc2c, random_pool, I); int axis = -1; - KokkosFFT::Impl::Plan fft_plan(execution_space(), xc2c, xc2c_hat, KokkosFFT::Impl::Direction::Forward, axis); + KokkosFFT::Impl::Plan fft_plan(execution_space(), xc2c, xc2c_hat, + KokkosFFT::Impl::Direction::Forward, axis); KokkosFFT::fft(execution_space(), xc2c, xc2c_hat, fft_plan); - KokkosFFT::Impl::Plan ifft_plan(execution_space(), xc2c_hat, xc2c_inv, KokkosFFT::Impl::Direction::Backward, axis); + KokkosFFT::Impl::Plan ifft_plan(execution_space(), xc2c_hat, xc2c_inv, + KokkosFFT::Impl::Direction::Backward, axis); KokkosFFT::ifft(execution_space(), xc2c_hat, xc2c_inv, ifft_plan); // 1D R2C FFT View1D xr2c("xr2c", n0); - View1D > xr2c_hat("xr2c_hat", n0/2+1); + View1D > xr2c_hat("xr2c_hat", n0 / 2 + 1); Kokkos::fill_random(xr2c, random_pool, 1); - KokkosFFT::Impl::Plan rfft_plan(execution_space(), xr2c, xr2c_hat, KokkosFFT::Impl::Direction::Forward, axis); + KokkosFFT::Impl::Plan rfft_plan(execution_space(), xr2c, xr2c_hat, + KokkosFFT::Impl::Direction::Forward, axis); KokkosFFT::rfft(execution_space(), xr2c, xr2c_hat, rfft_plan); // 1D C2R FFT - View1D > xc2r("xc2r_hat", n0/2+1); + View1D > xc2r("xc2r_hat", n0 / 2 + 1); View1D xc2r_hat("xc2r", n0); Kokkos::fill_random(xc2r, random_pool, I); - KokkosFFT::Impl::Plan irfft_plan(execution_space(), xc2r, xc2r_hat, KokkosFFT::Impl::Direction::Backward, axis); + KokkosFFT::Impl::Plan irfft_plan(execution_space(), xc2r, xc2r_hat, + KokkosFFT::Impl::Direction::Backward, + axis); KokkosFFT::irfft(execution_space(), xc2r, xc2r_hat, irfft_plan); } Kokkos::finalize(); From 3dd1a5a4ea4d9fc1fbf28cf1a5a5ab524f83aa80 Mon Sep 17 00:00:00 2001 From: Yuuichi Asahi Date: Mon, 8 Jan 2024 19:19:24 +0900 Subject: [PATCH 5/5] Add format check in CI --- .github/workflows/cmake.yml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 708a87ca..4fc7f865 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -9,6 +9,16 @@ env: BUILD_TYPE: RelWithDebInfo jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: DoozyX/clang-format-lint-action@v0.13 + with: + source: 'common/ fft/ examples/' + exclude: '' + extensions: 'hpp,cpp' + clangFormatVersion: 12 build_nvidia: # The CMake configure and build commands are platform agnostic and should work equally well on Windows or Mac. @@ -115,21 +125,7 @@ jobs: -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_COMPILER=${{env.CMAKE_CXX_COMPILER}} \ -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_HIP=ON -DKokkos_ARCH_${{env.architecture}}=ON -DBUILD_TESTING=ON ${{matrix.backend.option}} - #- name: Configure CMake for HIP backend with HOST and DEVICE option - # # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. - # # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - # run: | - # docker run -v ${{github.workspace}}:/work ${{ env.container }} cmake -B build_HIP_HOST_DEVICE \ - # -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_CXX_COMPILER=${{env.CMAKE_CXX_COMPILER}} \ - # -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_HIP=ON -DKokkos_ARCH_${{env.architecture}}=ON -DBUILD_TESTING=ON \ - # -DKokkosFFT_ENABLE_HOST_AND_DEVICE=ON - - name: Build # Build your program with the given configuration run: | - docker run -v ${{github.workspace}}:/work ${{ env.container }} cmake --build build_${{matrix.backend.name}} --config ${{env.BUILD_TYPE}} -j 2 - - #run: | - # for backend in ${{ env.backends }}; do - # docker run -v ${{github.workspace}}:/work ${{ env.container }} cmake --build build_${backend} --config ${{env.BUILD_TYPE}} -j 2 - # done \ No newline at end of file + docker run -v ${{github.workspace}}:/work ${{ env.container }} cmake --build build_${{matrix.backend.name}} --config ${{env.BUILD_TYPE}} -j 2 \ No newline at end of file