diff --git a/include/sparrow/array.hpp b/include/sparrow/array.hpp index 10c9139b..a3b127ee 100644 --- a/include/sparrow/array.hpp +++ b/include/sparrow/array.hpp @@ -43,5 +43,4 @@ namespace sparrow { return sparrow::visit(std::forward(func), *p_array); } -} - +} \ No newline at end of file diff --git a/include/sparrow/array_api.hpp b/include/sparrow/array_api.hpp index e5c330c5..97595b73 100644 --- a/include/sparrow/array_api.hpp +++ b/include/sparrow/array_api.hpp @@ -21,6 +21,8 @@ #include "sparrow/layout/null_array.hpp" #include "sparrow/types/data_traits.hpp" +#include "sparrow/layout/array_access.hpp" + namespace sparrow { class array @@ -65,8 +67,11 @@ namespace sparrow visit_result_t visit(F&& func); private: + SPARROW_API cloning_ptr extract_array_wrapper() &&; cloning_ptr p_array = nullptr; + + friend class detail::array_access; }; } diff --git a/include/sparrow/buffer/buffer_adaptor.hpp b/include/sparrow/buffer/buffer_adaptor.hpp index 4ca29788..d4d46a84 100644 --- a/include/sparrow/buffer/buffer_adaptor.hpp +++ b/include/sparrow/buffer/buffer_adaptor.hpp @@ -184,6 +184,19 @@ namespace sparrow size_type m_max_size; }; + + template + class holder + { + public: + template + holder(Args&&... args) + : value(std::forward(args)...) + { + } + T value; + }; + template FromBufferRef> requires T_is_const_if_FromBufferRef_is_const buffer_adaptor::buffer_adaptor(FromBufferRef buf) diff --git a/include/sparrow/buffer/dynamic_bitset.hpp b/include/sparrow/buffer/dynamic_bitset.hpp index 4327d27e..78038abd 100644 --- a/include/sparrow/buffer/dynamic_bitset.hpp +++ b/include/sparrow/buffer/dynamic_bitset.hpp @@ -16,4 +16,4 @@ #include "sparrow/buffer/dynamic_bitset/dynamic_bitset_view.hpp" #include "sparrow/buffer/dynamic_bitset/non_owning_dynamic_bitset.hpp" -#include "sparrow/buffer/dynamic_bitset/dynamic_bitset.hpp" +#include "sparrow/buffer/dynamic_bitset/dynamic_bitset.hpp" \ No newline at end of file diff --git a/include/sparrow/buffer/dynamic_bitset/dynamic_bitset.hpp b/include/sparrow/buffer/dynamic_bitset/dynamic_bitset.hpp index c14560f3..29d7cafb 100644 --- a/include/sparrow/buffer/dynamic_bitset/dynamic_bitset.hpp +++ b/include/sparrow/buffer/dynamic_bitset/dynamic_bitset.hpp @@ -37,6 +37,21 @@ namespace sparrow using value_type = typename base_type::value_type; using size_type = typename base_type::size_type; + template + requires std::convertible_to, value_type> + explicit dynamic_bitset(const R& r) + : dynamic_bitset(std::ranges::size(r), true) + { + std::size_t i = 0; + for(auto value : r) + { + if(!value){ + this->set(i, false); + } + i++; + } + } + constexpr dynamic_bitset(); constexpr explicit dynamic_bitset(size_type n); constexpr dynamic_bitset(size_type n, value_type v); @@ -92,4 +107,79 @@ namespace sparrow : base_type(storage_type(p, this->compute_block_count(n)), n, null_count) { } -} + + + using validity_bitmap = dynamic_bitset; + + + namespace detail + { + using validity_bitmap = sparrow::validity_bitmap; + inline validity_bitmap ensure_validity_bitmap_impl(std::size_t size, const validity_bitmap & bitmap) + { + if(bitmap.size() == 0) + { + return validity_bitmap(size, true); + } + return bitmap; // copy + } + + inline validity_bitmap ensure_validity_bitmap_impl(std::size_t size, validity_bitmap && bitmap) + { + if(bitmap.size() == 0) + { + bitmap.resize(size, true); + } + return std::move(bitmap); + } + + // range of booleans + template + requires(std::same_as, bool>) + validity_bitmap ensure_validity_bitmap_impl(std::size_t size, R&& range) + { + SPARROW_ASSERT_TRUE(size == std::ranges::size(range) || std::ranges::size(range) == 0); + validity_bitmap bitmap(size, true); + std::size_t i = 0; + for(auto value : range) + { + if(!value){ + bitmap.set(i, false); + } + i++; + } + return bitmap; + } + + // range of indices / integers (but not booleans) + template + requires( + std::unsigned_integral> && + !std::same_as, bool> && + !std::same_as, validity_bitmap> + ) + validity_bitmap ensure_validity_bitmap_impl(std::size_t size, R&& range_of_indices) + { + validity_bitmap bitmap(size, true); + for(auto index : range_of_indices) + { + bitmap.set(index, false); + } + return bitmap; + } + } // namespace detail + + template + concept validity_bitmap_input = + std::same_as || + std::same_as || + (std::ranges::input_range && std::same_as, bool>) || + (std::ranges::input_range && std::unsigned_integral> ); + + template + validity_bitmap ensure_validity_bitmap(std::size_t size, R&& validity_input) + { + return detail::ensure_validity_bitmap_impl(size, std::forward(validity_input)); + } + +} // namespace sparrow \ No newline at end of file diff --git a/include/sparrow/buffer/dynamic_bitset/dynamic_bitset_base.hpp b/include/sparrow/buffer/dynamic_bitset/dynamic_bitset_base.hpp index ce17176a..43e6525a 100644 --- a/include/sparrow/buffer/dynamic_bitset/dynamic_bitset_base.hpp +++ b/include/sparrow/buffer/dynamic_bitset/dynamic_bitset_base.hpp @@ -110,7 +110,13 @@ namespace sparrow } } - static constexpr size_type compute_block_count(size_type bits_count) noexcept; + static constexpr size_type compute_block_count(size_type bits_count) noexcept; + + // storage_type is a value_type + storage_type extract_storage() noexcept requires std::same_as + { + return std::move(m_buffer); + } protected: diff --git a/include/sparrow/buffer/u8_buffer.hpp b/include/sparrow/buffer/u8_buffer.hpp new file mode 100644 index 00000000..7ef90753 --- /dev/null +++ b/include/sparrow/buffer/u8_buffer.hpp @@ -0,0 +1,73 @@ +// Copyright 2024 Man Group Operations Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "sparrow/buffer/buffer_adaptor.hpp" + +namespace sparrow +{ + + namespace detail + { + template + class holder + { + public: + template + holder(Args&&... args) + : value(std::forward(args)...) + { + } + T value; + }; + } + + // like buffer but for any type T, nut always use buffer as storage + // This internal storage can be extracted + template + class u8_buffer : private detail::holder>, + public buffer_adaptor&> + { + public: + using holder_type = detail::holder>; + using buffer_adaptor_type = buffer_adaptor&>; + + template + requires std::convertible_to, T> + u8_buffer(R&& range) + : holder_type{std::ranges::size(range) * sizeof(T)} + ,buffer_adaptor_type(holder_type::value) + { + std::ranges::copy(range, this->begin()); + } + + u8_buffer(std::size_t n, T val = T{}) + : holder_type{n * sizeof(T)} + , buffer_adaptor_type(holder_type::value) + { + std::fill(this->begin(), this->end(), val); + } + + buffer extract_storage() && + { + return std::move(holder_type::value); + } + }; + +} diff --git a/include/sparrow/layout/array_access.hpp b/include/sparrow/layout/array_access.hpp new file mode 100644 index 00000000..78f14594 --- /dev/null +++ b/include/sparrow/layout/array_access.hpp @@ -0,0 +1,51 @@ +// Copyright 2024 Man Group Operations Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "sparrow/arrow_array_schema_proxy.hpp" + +namespace sparrow::detail +{ + + class array_access + { + public: + template + static inline const sparrow::arrow_proxy& storage(const ARRAY& array) + { + return array.storage(); + } + + template + static inline sparrow::arrow_proxy& storage(ARRAY& array) + { + return array.storage(); + } + + template + requires(std::is_rvalue_reference_v) + static inline sparrow::arrow_proxy extract_arrow_proxy(ARRAY&& array) + { + return std::move(array).extract_arrow_proxy(); + } + + template + requires(std::is_rvalue_reference_v) + static inline auto extract_array_wrapper(ARRAY&& array) + { + return std::move(array).extract_array_wrapper(); + } + }; +} \ No newline at end of file diff --git a/include/sparrow/layout/array_base.hpp b/include/sparrow/layout/array_base.hpp index 1f44a853..057dc0a4 100644 --- a/include/sparrow/layout/array_base.hpp +++ b/include/sparrow/layout/array_base.hpp @@ -23,6 +23,7 @@ #include "sparrow/buffer/dynamic_bitset/dynamic_bitset_view.hpp" #include "sparrow/layout/layout_iterator.hpp" #include "sparrow/utils/crtp_base.hpp" +#include "sparrow/layout/array_access.hpp" #include "sparrow/utils/iterator.hpp" #include "sparrow/utils/nullable.hpp" @@ -108,7 +109,7 @@ namespace sparrow protected: - array_crtp_base(arrow_proxy); + explicit array_crtp_base(arrow_proxy); array_crtp_base(const array_crtp_base&) = default; array_crtp_base& operator=(const array_crtp_base&) = default; @@ -116,9 +117,11 @@ namespace sparrow array_crtp_base(array_crtp_base&&) = default; array_crtp_base& operator=(array_crtp_base&&) = default; + [[nodiscard]] arrow_proxy extract_arrow_proxy() &&; [[nodiscard]] arrow_proxy& get_arrow_proxy(); [[nodiscard]] const arrow_proxy& get_arrow_proxy() const; + bitmap_const_reference has_value(size_type i) const; const_bitmap_iterator bitmap_begin() const; @@ -134,6 +137,7 @@ namespace sparrow friend class layout_iterator; template friend class array_wrapper_impl; + friend class detail::array_access; }; template @@ -212,6 +216,12 @@ namespace sparrow { return m_proxy; } + + template + auto array_crtp_base::extract_arrow_proxy() && -> arrow_proxy + { + return std::move(m_proxy); + } template auto array_crtp_base::has_value(size_type i) const -> bitmap_const_reference diff --git a/include/sparrow/layout/array_wrapper.hpp b/include/sparrow/layout/array_wrapper.hpp index 249827fb..e5e1ff82 100644 --- a/include/sparrow/layout/array_wrapper.hpp +++ b/include/sparrow/layout/array_wrapper.hpp @@ -82,6 +82,8 @@ namespace sparrow enum data_type data_type() const; bool is_dictionary() const; + + [[nodiscard]] arrow_proxy extract_arrow_proxy() &&; [[nodiscard]] arrow_proxy& get_arrow_proxy(); [[nodiscard]] const arrow_proxy& get_arrow_proxy() const; @@ -96,6 +98,7 @@ namespace sparrow virtual bool is_dictionary_impl() const = 0; virtual arrow_proxy& get_arrow_proxy_impl() = 0; virtual const arrow_proxy& get_arrow_proxy_impl() const = 0; + virtual arrow_proxy extract_arrow_proxy_impl() = 0; virtual wrapper_ptr clone_impl() const = 0; }; @@ -123,6 +126,7 @@ namespace sparrow bool is_dictionary_impl() const override; arrow_proxy& get_arrow_proxy_impl() override; const arrow_proxy& get_arrow_proxy_impl() const override; + arrow_proxy extract_arrow_proxy_impl() override; wrapper_ptr clone_impl() const override; using storage_type = std::variant, std::shared_ptr, T*>; @@ -159,12 +163,18 @@ namespace sparrow { return get_arrow_proxy_impl(); } + + inline arrow_proxy array_wrapper::extract_arrow_proxy() && + { + return extract_arrow_proxy_impl(); + } inline const arrow_proxy& array_wrapper::get_arrow_proxy() const { return get_arrow_proxy_impl(); } + inline array_wrapper::array_wrapper(enum data_type dt) : m_data_type(dt) { @@ -256,6 +266,12 @@ namespace sparrow return wrapper_ptr{new array_wrapper_impl(*this)}; } + template + auto array_wrapper_impl::extract_arrow_proxy_impl() -> arrow_proxy + { + return std::move(*p_array).extract_arrow_proxy(); + } + template T& unwrap_array(array_wrapper& ar) { diff --git a/include/sparrow/layout/dictionary_encoded_array.hpp b/include/sparrow/layout/dictionary_encoded_array.hpp index 2e8a208c..ca43cb2b 100644 --- a/include/sparrow/layout/dictionary_encoded_array.hpp +++ b/include/sparrow/layout/dictionary_encoded_array.hpp @@ -25,6 +25,7 @@ #include "sparrow/utils/contracts.hpp" #include "sparrow/utils/functor_index_iterator.hpp" #include "sparrow/utils/memory.hpp" +#include "sparrow/layout/array_access.hpp" namespace sparrow { @@ -138,12 +139,16 @@ namespace sparrow [[nodiscard]] arrow_proxy& get_arrow_proxy(); [[nodiscard]] const arrow_proxy& get_arrow_proxy() const; + arrow_proxy extract_arrow_proxy() &&; + arrow_proxy m_proxy; keys_layout m_keys_layout; values_layout p_values_layout; template friend class array_wrapper_impl; + + friend class detail::array_access; }; template @@ -309,6 +314,12 @@ namespace sparrow return m_proxy; } + template + auto dictionary_encoded_array::extract_arrow_proxy() && -> arrow_proxy + { + return std::move(m_proxy); + } + template auto dictionary_encoded_array::get_arrow_proxy() const -> const arrow_proxy& { diff --git a/include/sparrow/layout/list_layout/list_array.hpp b/include/sparrow/layout/list_layout/list_array.hpp index f83441ad..1882b613 100644 --- a/include/sparrow/layout/list_layout/list_array.hpp +++ b/include/sparrow/layout/list_layout/list_array.hpp @@ -16,6 +16,9 @@ #include // for std::stoull + +#include "sparrow/arrow_interface/arrow_array.hpp" +#include "sparrow/arrow_interface/arrow_schema.hpp" #include "sparrow/array_factory.hpp" #include "sparrow/layout/array_bitmap_base.hpp" #include "sparrow/layout/array_wrapper.hpp" @@ -26,6 +29,8 @@ #include "sparrow/utils/iterator.hpp" #include "sparrow/utils/memory.hpp" #include "sparrow/utils/nullable.hpp" +#include "sparrow/array_api.hpp" +#include "sparrow/buffer/dynamic_bitset.hpp" namespace sparrow { @@ -255,7 +260,14 @@ namespace sparrow fixed_sized_list_array(self_type&&) = default; fixed_sized_list_array& operator=(self_type&&) = default; + template + requires(mpl::excludes_copy_and_move_ctor_v) + fixed_sized_list_array(ARGS&& ...args): self_type(create_proxy(std::forward(args)...)) + {} + private: + template + static arrow_proxy create_proxy(std::uint64_t list_size, array && flat_values, R && validity_input = validity_bitmap{}); static uint64_t list_size_from_format(const std::string_view format); std::pair offset_range(size_type i) const; @@ -491,4 +503,44 @@ namespace sparrow const auto offset = i * m_list_size; return std::make_pair(offset, offset + m_list_size); } + + template + inline arrow_proxy fixed_sized_list_array::create_proxy( + std::uint64_t list_size, array && flat_values, + R && validity_input + ) + { + const auto size = flat_values.size() / static_cast(list_size); + auto vbitmap = ensure_validity_bitmap(size, std::forward(validity_input)); + + auto wrapper = detail::array_access::extract_array_wrapper(std::move(flat_values)); + auto storage = std::move(*wrapper).extract_arrow_proxy(); + auto flat_schema = storage.extract_schema(); + auto flat_arr = storage.extract_array(); + const auto null_count = vbitmap.null_count(); + + std::string format = "+w:" + std::to_string(list_size); + ArrowSchema schema = make_arrow_schema( + format, + std::nullopt, // name + std::nullopt, // metadata + std::nullopt, // flags, + 1, // n_children + new ArrowSchema*[1]{new ArrowSchema(std::move(flat_schema))}, // children + nullptr // dictionary + + ); + std::vector> arr_buffs = {vbitmap.extract_storage()}; + + ArrowArray arr = make_arrow_array( + static_cast(size), // length + static_cast(null_count), + 0, // offset + std::move(arr_buffs), + 1, // n_children + new ArrowArray*[1]{new ArrowArray(std::move(flat_arr))}, // children + nullptr // dictionary + ); + return arrow_proxy{std::move(arr), std::move(schema)}; + } } diff --git a/include/sparrow/layout/null_array.hpp b/include/sparrow/layout/null_array.hpp index 6f42e47c..467be406 100644 --- a/include/sparrow/layout/null_array.hpp +++ b/include/sparrow/layout/null_array.hpp @@ -21,6 +21,7 @@ #include "sparrow/utils/contracts.hpp" #include "sparrow/utils/iterator.hpp" #include "sparrow/utils/nullable.hpp" +#include "sparrow/layout/array_access.hpp" namespace sparrow { @@ -101,13 +102,17 @@ namespace sparrow difference_type ssize() const; + [[nodiscard]] arrow_proxy extract_arrow_proxy() &&; [[nodiscard]] arrow_proxy& get_arrow_proxy(); [[nodiscard]] const arrow_proxy& get_arrow_proxy() const; + arrow_proxy m_proxy; template friend class array_wrapper_impl; + + friend class detail::array_access; }; bool operator==(const null_array& lhs, const null_array& rhs); @@ -241,6 +246,11 @@ namespace sparrow return m_proxy; } + inline arrow_proxy null_array::extract_arrow_proxy() && + { + return std::move(m_proxy); + } + inline const arrow_proxy& null_array::get_arrow_proxy() const { return m_proxy; diff --git a/include/sparrow/layout/primitive_array.hpp b/include/sparrow/layout/primitive_array.hpp index 1de03737..72201485 100644 --- a/include/sparrow/layout/primitive_array.hpp +++ b/include/sparrow/layout/primitive_array.hpp @@ -14,19 +14,27 @@ #pragma once -#include +#include +#include "sparrow/arrow_interface/arrow_array.hpp" +#include "sparrow/arrow_interface/arrow_schema.hpp" +#include #include "sparrow/arrow_array_schema_proxy.hpp" #include "sparrow/buffer/buffer_adaptor.hpp" #include "sparrow/layout/array_bitmap_base.hpp" #include "sparrow/utils/iterator.hpp" #include "sparrow/utils/nullable.hpp" +#include "sparrow/layout/primitive_array.hpp" +#include "sparrow/layout/array_access.hpp" +#include "sparrow/buffer/dynamic_bitset.hpp" +#include "sparrow/buffer/u8_buffer.hpp" namespace sparrow { template class primitive_array; + template struct array_inner_types> : array_inner_types_base { @@ -82,6 +90,11 @@ namespace sparrow explicit primitive_array(arrow_proxy); + template + requires(mpl::excludes_copy_and_move_ctor_v, Args...>) + primitive_array(Args&& ... args) : base_type(create_proxy(std::forward(args) ...)) + {} + using base_type::size; using base_type::get_arrow_proxy; @@ -98,7 +111,36 @@ namespace sparrow const_value_iterator value_cbegin() const; const_value_iterator value_cend() const; - private: + private: + + static arrow_proxy create_proxy(size_type n); + + template + static auto create_proxy( + u8_buffer&& data_buffer, + R && bitmaps = validity_bitmap{} + ) -> arrow_proxy; + + // range of values (no missing values) + template + requires std::convertible_to, T> + static auto create_proxy(R&& range) -> arrow_proxy; + + template + requires std::convertible_to + static arrow_proxy create_proxy(size_type n, const U& value = U{}); + + // range of values, validity_bitmap_input + template + requires(std::convertible_to, T>) + static arrow_proxy create_proxy(R&&, R2&&); + + // range of nullable values + template + requires std::is_same_v< + std::ranges::range_value_t, nullable> + static arrow_proxy create_proxy(R&&); + // Modifiers @@ -116,6 +158,7 @@ namespace sparrow static constexpr size_type DATA_BUFFER_INDEX = 1; friend class run_end_encoded_array; + friend class detail::array_access; friend base_type; friend base_type::base_type; }; @@ -154,6 +197,86 @@ namespace sparrow { SPARROW_ASSERT_TRUE(get_arrow_proxy().data_type() == arrow_traits::type_id); } + template + template + auto primitive_array::create_proxy( + u8_buffer&& data_buffer, + R && bitmap_input + ) -> arrow_proxy + { + const auto size = data_buffer.size(); + validity_bitmap bitmap = ensure_validity_bitmap(size, std::forward(bitmap_input)); + const auto null_count = bitmap.null_count(); + + // create arrow schema and array + ArrowSchema schema = make_arrow_schema( + sparrow::data_type_format_of(), + std::nullopt, // name + std::nullopt, // metadata + std::nullopt, // flags + 0, // n_children + nullptr, // children + nullptr // dictionary + ); + + std::vector> buffers(2); + buffers[0] = std::move(bitmap).extract_storage(); + buffers[1] = std::move(data_buffer).extract_storage(); + + // create arrow array + ArrowArray arr = make_arrow_array( + static_cast(size), // length + static_cast(null_count), + 0, // offset + std::move(buffers), + 0, // n_children + nullptr, // children + nullptr // dictionary + ); + return arrow_proxy(std::move(arr), std::move(schema)); + } + + template + template + requires(std::convertible_to, T>) + arrow_proxy primitive_array::create_proxy(VALUE_RANGE && values, R && validity_input) + { + u8_buffer data_buffer(std::forward(values)); + return create_proxy(std::move(data_buffer), std::forward(validity_input)); + } + + template + template + requires std::convertible_to + arrow_proxy primitive_array::create_proxy(size_type n, const U& value) + { + // create data_buffer + u8_buffer data_buffer(n, value); + return create_proxy(std::move(data_buffer)); + } + + template + template + requires std::convertible_to, T> + arrow_proxy primitive_array::create_proxy(R&& range) + { + const std::size_t n = std::ranges::size(range); + auto iota = std::ranges::iota_view{std::size_t(0), n}; + std::ranges::transform_view iota_to_is_non_missing(iota, [](std::size_t) { return true; }); + return self_type::create_proxy(std::forward(range), std::move(iota_to_is_non_missing)); + } + + // range of nullable values + template + template + requires std::is_same_v, nullable> + arrow_proxy primitive_array::create_proxy(R&& range) + { + // split into values and is_non_null ranges + auto values = range | std::views::transform([](const auto& v) { return v.get(); }); + auto is_non_null = range | std::views::transform([](const auto& v) { return v.has_value(); }); + return self_type::create_proxy(values, is_non_null); + } template auto primitive_array::data() -> pointer diff --git a/include/sparrow/layout/run_end_encoded_layout/run_end_encoded_array.hpp b/include/sparrow/layout/run_end_encoded_layout/run_end_encoded_array.hpp index 2c1b1d59..839d3968 100644 --- a/include/sparrow/layout/run_end_encoded_layout/run_end_encoded_array.hpp +++ b/include/sparrow/layout/run_end_encoded_layout/run_end_encoded_array.hpp @@ -19,6 +19,7 @@ #include "sparrow/array_factory.hpp" #include "sparrow/utils/memory.hpp" #include "sparrow/layout/run_end_encoded_layout/run_end_encoded_iterator.hpp" +#include "sparrow/layout/array_access.hpp" namespace sparrow { @@ -78,6 +79,7 @@ namespace sparrow SPARROW_API static acc_length_ptr_variant_type get_acc_lengths_ptr(const array_wrapper& ar); SPARROW_API std::uint64_t get_run_length(std::uint64_t run_index) const; + arrow_proxy extract_arrow_proxy() &&; [[nodiscard]] arrow_proxy& get_arrow_proxy(); [[nodiscard]] const arrow_proxy& get_arrow_proxy() const; @@ -93,6 +95,7 @@ namespace sparrow friend class run_encoded_array_iterator; template friend class array_wrapper_impl; + friend class detail::array_access; }; SPARROW_API @@ -158,6 +161,10 @@ namespace sparrow { return m_proxy; } + inline arrow_proxy run_end_encoded_array::extract_arrow_proxy() && + { + return std::move(m_proxy); + } inline const arrow_proxy& run_end_encoded_array::get_arrow_proxy() const { diff --git a/include/sparrow/layout/union_array.hpp b/include/sparrow/layout/union_array.hpp index 6da9fd0f..1e58b41d 100644 --- a/include/sparrow/layout/union_array.hpp +++ b/include/sparrow/layout/union_array.hpp @@ -23,7 +23,7 @@ #include "sparrow/layout/array_helper.hpp" #include "sparrow/utils/crtp_base.hpp" #include "sparrow/utils/functor_index_iterator.hpp" - +#include "sparrow/layout/array_access.hpp" namespace sparrow { @@ -96,6 +96,7 @@ namespace sparrow union_array_crtp_base(self_type&& rhs) = default; self_type& operator=(self_type&& rhs) = default; + arrow_proxy extract_arrow_proxy() &&; [[nodiscard]] arrow_proxy& get_arrow_proxy(); [[nodiscard]] const arrow_proxy& get_arrow_proxy() const; @@ -108,6 +109,7 @@ namespace sparrow template friend class array_wrapper_impl; + friend class detail::array_access; }; template @@ -172,6 +174,11 @@ namespace sparrow { return m_proxy; } + template + arrow_proxy union_array_crtp_base::extract_arrow_proxy() && + { + return std::move(m_proxy); + } template const arrow_proxy& union_array_crtp_base::get_arrow_proxy() const diff --git a/include/sparrow/utils/mp_utils.hpp b/include/sparrow/utils/mp_utils.hpp index 4774bda6..f0935140 100644 --- a/include/sparrow/utils/mp_utils.hpp +++ b/include/sparrow/utils/mp_utils.hpp @@ -478,6 +478,28 @@ namespace sparrow::mpl template typename Qualifier> concept T_matches_qualifier_if_Y_is = Qualifier::value || !Qualifier::value; + + // helper class to exclude copy and move constructors beeing routed + // to a constructor with variadic arguments / perfect forwarding + template + struct excludes_copy_and_move_ctor + { + constexpr static bool value = true; + }; + template + struct excludes_copy_and_move_ctor + { + constexpr static bool value = true; + }; + template + struct excludes_copy_and_move_ctor + { + constexpr static bool value = !std::is_same_v>; + }; + + template + constexpr bool excludes_copy_and_move_ctor_v = excludes_copy_and_move_ctor::value; + /** * Concept to check if an iterator is of a specific type. * diff --git a/src/array.cpp b/src/array.cpp index e88806d4..41831efb 100644 --- a/src/array.cpp +++ b/src/array.cpp @@ -76,5 +76,10 @@ namespace sparrow { return array_element(*p_array, index); } + + cloning_ptr array::extract_array_wrapper() && + { + return std::move(p_array); + } } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dfa7b23f..fdbbd2da 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -81,6 +81,7 @@ else() test_variable_size_binary_array.cpp test_run_end_encoded_array.cpp test_union_array.cpp + test_high_level_constructors.cpp ) endif() @@ -147,7 +148,6 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "C $<$:-Wduplicated-cond> # warn if if / else chain has duplicated conditions $<$:-Wlogical-op> # warn about logical operations being used where bitwise were probably wanted $<$:-Wno-subobject-linkage> # suppress warnings about subobject linkage - $<$:-Wuseless-cast> # warn if you perform a cast to the same type ) target_compile_options(${test_target} PRIVATE "-ftemplate-backtrace-limit=0") endif() diff --git a/test/test_high_level_constructors.cpp b/test/test_high_level_constructors.cpp new file mode 100644 index 00000000..9be78c80 --- /dev/null +++ b/test/test_high_level_constructors.cpp @@ -0,0 +1,82 @@ +// Copyright 2024 Man Group Operations Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +#include "sparrow/layout/primitive_array.hpp" +#include "sparrow/layout/list_layout/list_array.hpp" +#include "sparrow/array.hpp" +#include "doctest/doctest.h" +#include "test_utils.hpp" + +#include + +namespace sparrow +{ + + TEST_SUITE("high_level_constructors") + { + TEST_CASE("list") + { + // a primite array + std::size_t flat_size = 10; + primitive_array primitive_arr(std::ranges::iota_view{std::size_t(0), std::size_t(10)} | std::views::transform([](auto i){ + return static_cast(i);}) + ); + + // wrap into an detyped array + array arr(std::move(primitive_arr)); + + // create a list array (fixed size) + std::uint64_t list_size = 2; + fixed_sized_list_array list_arr(list_size, std::move(arr)); + CHECK(list_arr.size() == flat_size / list_size); + + std::size_t n = flat_size / static_cast(list_size); + + const auto size = list_arr.size(); + + REQUIRE_EQ(size, n); + + auto flat_i = 0; + for(std::size_t i = 0; i < size; ++i) + { + auto list = list_arr[i].value(); + CHECK_EQ(list.size(), list_size); + + for(std::size_t j = 0; j < list.size(); ++j) + { + auto opt_val_variant = list[j]; + std::visit([&](auto&& opt_val){ + using nullable_type = std::decay_t; + using inner_type = std::decay_t; + if constexpr(std::is_same_v) + { + REQUIRE(opt_val.has_value()); + CHECK_EQ(opt_val.value(), static_cast(flat_i)); + } + else + { + REQUIRE(false); + } + }, opt_val_variant); + ++flat_i; + } + + } + } + } + + +} + diff --git a/test/test_primitive_array.cpp b/test/test_primitive_array.cpp index 4fab4439..09eb3749 100644 --- a/test/test_primitive_array.cpp +++ b/test/test_primitive_array.cpp @@ -611,6 +611,73 @@ namespace sparrow CHECK_EQ(ar[2].get(), values[3]); } } + + TEST_CASE_TEMPLATE("convenience_constructors", T, std::uint8_t) + { + using inner_value_type = T; + + std::vector data = { + static_cast(0), + static_cast(1), + static_cast(2), + static_cast(3) + }; + SUBCASE("range-of-inner-values") { + primitive_array arr(data); + CHECK_EQ(arr.size(), data.size()); + for (std::size_t i = 0; i < data.size(); ++i) { + REQUIRE(arr[i].has_value()); + CHECK_EQ(arr[i].value(), data[i]); + } + } + SUBCASE("range-of-nullables") + { + using nullable_type = nullable; + std::vector nullable_vector{ + nullable_type(data[0]), + nullable_type(data[1]), + nullval, + nullable_type(data[3]) + }; + primitive_array arr(nullable_vector); + REQUIRE(arr.size() == nullable_vector.size()); + REQUIRE(arr[0].has_value()); + REQUIRE(arr[1].has_value()); + REQUIRE(!arr[2].has_value()); + REQUIRE(arr[3].has_value()); + CHECK_EQ(arr[0].value(), data[0]); + CHECK_EQ(arr[1].value(), data[1]); + CHECK_EQ(arr[3].value(), data[3]); + } + + } TEST_CASE_TEMPLATE_APPLY(primitive_array_id, testing_types); + + TEST_CASE("convenience_constructors_from_iota") + { + primitive_array arr(std::ranges::iota_view{std::size_t(0), std::size_t(4)}); + REQUIRE(arr.size() == 4); + for (std::size_t i = 0; i < 4; ++i) { + REQUIRE(arr[i].has_value()); + CHECK_EQ(arr[i].value(), static_cast(i)); + } + } + TEST_CASE("convenience_constructors_index_of_missing") + { + primitive_array arr( + std::ranges::iota_view{std::size_t(0), std::size_t(5)}, + std::vector{1,3} + ); + REQUIRE(arr.size() == 5); + CHECK(arr[0].has_value()); + CHECK(!arr[1].has_value()); + CHECK(arr[2].has_value()); + CHECK(!arr[3].has_value()); + CHECK(arr[4].has_value()); + + CHECK_EQ(arr[0].value(), std::size_t(0)); + CHECK_EQ(arr[2].value(), std::size_t(2)); + CHECK_EQ(arr[4].value(), std::size_t(4)); + } } }