Skip to content

Commit

Permalink
Cleanup of external_array_data_creation (#251)
Browse files Browse the repository at this point in the history
  • Loading branch information
JohanMabille authored Oct 24, 2024
1 parent 56c2dbc commit f5d7a95
Show file tree
Hide file tree
Showing 3 changed files with 168 additions and 138 deletions.
2 changes: 1 addition & 1 deletion test/arrow_array_schema_creation.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ inline std::pair<ArrowArray, ArrowSchema> make_external_arrow_schema_and_array()
std::pair<ArrowArray, ArrowSchema> pair;
constexpr size_t size = 10;
constexpr size_t offset = 1;
sparrow::test::fill_schema_and_array<uint32_t>(pair.second, pair.first, size, offset, {2, 3});
sparrow::test::fill_external_schema_and_array<uint32_t>(pair.second, pair.first, size, offset, {2, 3});
return pair;
}

Expand Down
43 changes: 33 additions & 10 deletions test/external_array_data_creation.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include "sparrow/arrow_interface/arrow_array.hpp"
#include "sparrow/arrow_interface/arrow_schema.hpp"
#include "sparrow/arrow_interface/arrow_array_schema_factory.hpp"
#include "external_array_data_creation.hpp"

#ifdef __GNUC__
Expand All @@ -28,7 +25,7 @@ namespace sparrow::test
namespace detail
{
template <class T>
void release_common_arrow(T* t)
void release_external_common_arrow(T* t)
{
if (t->dictionary)
{
Expand All @@ -48,20 +45,46 @@ namespace sparrow::test
}
}

void release_arrow_schema(ArrowSchema* schema)
void release_external_arrow_schema(ArrowSchema* schema)
{
detail::release_common_arrow(schema);
detail::release_external_common_arrow(schema);
}

void release_arrow_array(ArrowArray* arr)
void release_external_arrow_array(ArrowArray* arr)
{
for (std::int64_t i = 0; i < arr->n_buffers; ++i)
{
delete[] reinterpret_cast<const std::uint8_t*>(arr->buffers[i]);
}
delete[] reinterpret_cast<const std::uint8_t**>(arr->buffers);
arr->buffers = nullptr;
detail::release_common_arrow(arr);
detail::release_external_common_arrow(arr);
}

sparrow::buffer<std::uint8_t> make_offset_buffer_from_sizes(const std::vector<size_t>& sizes, bool big)
{
const auto n = sizes.size() + 1;
const auto buf_size = n * (big ? sizeof(std::uint64_t) : sizeof(std::uint32_t));
auto buf = new std::uint8_t[buf_size];
if (big)
{
auto* ptr = reinterpret_cast<std::uint64_t*>(buf);
ptr[0] = 0;
for (std::size_t i = 0; i < sizes.size(); ++i)
{
ptr[i + 1] = ptr[i] + static_cast<std::uint64_t>(sizes[i]);
}
}
else
{
auto* ptr = reinterpret_cast<std::uint32_t*>(buf);
ptr[0] = 0;
for (std::size_t i = 0; i < sizes.size(); ++i)
{
ptr[i + 1] = ptr[i] + static_cast<std::uint32_t>(sizes[i]);
}
}
return {buf, buf_size};
}

sparrow::buffer<std::uint8_t> make_size_buffer(const std::vector<size_t>& sizes, bool big)
Expand Down Expand Up @@ -113,7 +136,7 @@ namespace sparrow::test
std::vector<buffer_type> arr_buffs =
{
sparrow::make_bitmap_buffer(list_lengths.size(), false_positions),
make_offset_buffer_from_sizes2(list_lengths, big_list)
make_offset_buffer_from_sizes(list_lengths, big_list)
};

ArrowArray** array_children = new ArrowArray*[1];
Expand Down Expand Up @@ -203,7 +226,7 @@ namespace sparrow::test
std::vector<buffer_type> arr_buffs =
{
sparrow::make_bitmap_buffer(list_lengths.size(), false_positions),
make_offset_buffer_from_sizes2(list_lengths, big_list),
make_offset_buffer_from_sizes(list_lengths, big_list),
make_size_buffer(list_lengths, big_list)
};

Expand Down
Loading

0 comments on commit f5d7a95

Please sign in to comment.