[DO NOT MERGE] ORC debug

cpp/CMakeLists.txt

@@ -786,6 +786,21 @@ set_source_files_properties(
   PROPERTIES COMPILE_DEFINITIONS "_FILE_OFFSET_BITS=64"
 )
 
+set_property(
+  SOURCE
+  src/io/orc/dict_enc.cu
+  src/io/orc/reader_impl.cu
+  src/io/orc/reader_impl_chunking.cu
+  src/io/orc/reader_impl_decode.cu
+  src/io/orc/stats_enc.cu
+  src/io/orc/stripe_data.cu
+  src/io/orc/stripe_enc.cu
+  src/io/orc/stripe_init.cu
+  src/io/orc/writer_impl.cu
+  APPEND
+  PROPERTY COMPILE_OPTIONS "-g;-G"
+)
+
 set_property(
   SOURCE src/io/parquet/writer_impl.cu
   APPEND

cpp/examples/build.sh

@@ -7,7 +7,7 @@
 set -euo pipefail
 
 # Parallelism control
-PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
+PARALLEL_LEVEL=${PARALLEL_LEVEL:-8}
 # Installation disabled by default
 INSTALL_EXAMPLES=false
 
@@ -47,7 +47,7 @@ build_example() {
   build_dir="${example_dir}/build"
 
   # Configure
-  cmake -S ${example_dir} -B ${build_dir} -Dcudf_ROOT="${LIB_BUILD_DIR}"
+  cmake -S ${example_dir} -B ${build_dir} -Dcudf_ROOT="${LIB_BUILD_DIR}" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CUDA_ARCHITECTURES=86
   # Build
   cmake --build ${build_dir} -j${PARALLEL_LEVEL}
   # Install if needed
@@ -56,9 +56,10 @@ build_example() {
   fi
 }
 
-build_example basic
-build_example strings
-build_example nested_types
-build_example parquet_io
-build_example billion_rows
-build_example interop
+# build_example basic
+# build_example strings
+# build_example nested_types
+# build_example parquet_io
+build_example orc_io
+# build_example billion_rows
+# build_example interop

cpp/examples/fetch_dependencies.cmake

@@ -26,8 +26,8 @@ include(${CMAKE_BINARY_DIR}/cmake/get_cpm.cmake)
 CPMFindPackage(
   NAME cudf
   FIND_PACKAGE_ARGUMENTS "PATHS ${cudf_ROOT} ${cudf_ROOT}/latest" GIT_REPOSITORY
-                         https://github.com/rapidsai/cudf
-  GIT_TAG ${CUDF_TAG}
+                         https://github.com/kingcrimsontianyu/cudf
+  GIT_TAG "orc-debug"
   GIT_SHALLOW
     TRUE
     SOURCE_SUBDIR

cpp/examples/orc_io/CMakeLists.txt

@@ -0,0 +1,26 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+cmake_minimum_required(VERSION 3.26.4)
+
+include(../set_cuda_architecture.cmake)
+
+# initialize cuda architecture
+rapids_cuda_init_architectures(orc_io)
+
+project(
+  orc_io
+  VERSION 0.0.1
+  LANGUAGES CXX CUDA
+)
+
+include(../fetch_dependencies.cmake)
+
+add_library(orc_io_utils OBJECT common_utils.cpp io_source.cpp)
+target_compile_features(orc_io_utils PRIVATE cxx_std_17)
+target_link_libraries(orc_io_utils PRIVATE cudf::cudf)
+
+# Build and install orc_io
+add_executable(orc_io orc_io.cpp)
+target_link_libraries(orc_io PRIVATE cudf::cudf $<TARGET_OBJECTS:orc_io_utils>)
+target_compile_features(orc_io PRIVATE cxx_std_17)
+install(TARGETS orc_io DESTINATION bin/examples/libcudf)

cpp/examples/orc_io/common_utils.cpp

@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2024, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common_utils.hpp"
+
+#include <cudf/concatenate.hpp>
+#include <cudf/io/types.hpp>
+#include <cudf/join.hpp>
+#include <cudf/table/table_view.hpp>
+
+#include <rmm/mr/device/device_memory_resource.hpp>
+#include <rmm/mr/device/owning_wrapper.hpp>
+#include <rmm/mr/device/pool_memory_resource.hpp>
+
+#include <chrono>
+#include <iomanip>
+#include <string>
+
+/**
+ * @file common_utils.cpp
+ * @brief Definitions for common utilities for `parquet_io` examples
+ *
+ */
+
+std::shared_ptr<rmm::mr::device_memory_resource> create_memory_resource(bool is_pool_used)
+{
+  auto cuda_mr = std::make_shared<rmm::mr::cuda_memory_resource>();
+  if (is_pool_used) {
+    return rmm::mr::make_owning_wrapper<rmm::mr::pool_memory_resource>(
+      cuda_mr, rmm::percent_of_free_device_memory(50));
+  }
+  return cuda_mr;
+}
+
+cudf::io::column_encoding get_encoding_type(std::string name)
+{
+  using encoding_type = cudf::io::column_encoding;
+
+  static std::unordered_map<std::string_view, encoding_type> const map = {
+    {"DEFAULT", encoding_type::USE_DEFAULT},
+    {"DICTIONARY", encoding_type::DICTIONARY},
+    {"PLAIN", encoding_type::PLAIN},
+    {"DELTA_BINARY_PACKED", encoding_type::DELTA_BINARY_PACKED},
+    {"DELTA_LENGTH_BYTE_ARRAY", encoding_type::DELTA_LENGTH_BYTE_ARRAY},
+    {"DELTA_BYTE_ARRAY", encoding_type::DELTA_BYTE_ARRAY},
+  };
+
+  std::transform(name.begin(), name.end(), name.begin(), ::toupper);
+  if (map.find(name) != map.end()) { return map.at(name); }
+  throw std::invalid_argument(name +
+                              " is not a valid encoding type.\n\n"
+                              "Available encoding types: DEFAULT, DICTIONARY, PLAIN,\n"
+                              "DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY,\n"
+                              "DELTA_BYTE_ARRAY\n\n");
+}
+
+cudf::io::compression_type get_compression_type(std::string name)
+{
+  using compression_type = cudf::io::compression_type;
+
+  static std::unordered_map<std::string_view, compression_type> const map = {
+    {"NONE", compression_type::NONE},
+    {"AUTO", compression_type::AUTO},
+    {"SNAPPY", compression_type::SNAPPY},
+    {"LZ4", compression_type::LZ4},
+    {"ZSTD", compression_type::ZSTD}};
+
+  std::transform(name.begin(), name.end(), name.begin(), ::toupper);
+  if (map.find(name) != map.end()) { return map.at(name); }
+  throw std::invalid_argument(name +
+                              " is not a valid compression type.\n\n"
+                              "Available compression types: NONE, AUTO, SNAPPY,\n"
+                              "LZ4, ZSTD\n\n");
+}
+
+bool get_boolean(std::string input)
+{
+  std::transform(input.begin(), input.end(), input.begin(), ::toupper);
+
+  // Check if the input string matches to any of the following
+  return input == "ON" or input == "TRUE" or input == "YES" or input == "Y" or input == "T";
+}
+
+void check_tables_equal(cudf::table_view const& lhs_table, cudf::table_view const& rhs_table)
+{
+  try {
+    // Left anti-join the original and transcoded tables
+    // identical tables should not throw an exception and
+    // return an empty indices vector
+    auto const indices = cudf::left_anti_join(lhs_table, rhs_table, cudf::null_equality::EQUAL);
+
+    // No exception thrown, check indices
+    auto const valid = indices->size() == 0;
+    std::cout << "Tables identical: " << valid << "\n\n";
+  } catch (std::exception& e) {
+    std::cerr << e.what() << std::endl << std::endl;
+    throw std::runtime_error("Tables identical: false\n\n");
+  }
+}
+
+std::unique_ptr<cudf::table> concatenate_tables(std::vector<std::unique_ptr<cudf::table>> tables,
+                                                rmm::cuda_stream_view stream)
+{
+  if (tables.size() == 1) { return std::move(tables[0]); }
+
+  std::vector<cudf::table_view> table_views;
+  table_views.reserve(tables.size());
+  std::transform(
+    tables.begin(), tables.end(), std::back_inserter(table_views), [&](auto const& tbl) {
+      return tbl->view();
+    });
+  // Construct the final table
+  return cudf::concatenate(table_views, stream);
+}
+
+std::string current_date_and_time()
+{
+  auto const time       = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
+  auto const local_time = *std::localtime(&time);
+  // Stringstream to format the date and time
+  std::stringstream ss;
+  ss << std::put_time(&local_time, "%Y-%m-%d-%H-%M-%S");
+  return ss.str();
+}

cpp/examples/orc_io/common_utils.hpp

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2024, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cudf/io/types.hpp>
+#include <cudf/table/table_view.hpp>
+
+#include <rmm/cuda_stream_view.hpp>
+#include <rmm/mr/device/device_memory_resource.hpp>
+
+#include <memory>
+#include <string>
+
+/**
+ * @file common_utils.hpp
+ * @brief Common utilities for `parquet_io` examples
+ *
+ */
+
+/**
+ * @brief Create memory resource for libcudf functions
+ *
+ * @param pool Whether to use a pool memory resource.
+ * @return Memory resource instance
+ */
+std::shared_ptr<rmm::mr::device_memory_resource> create_memory_resource(bool is_pool_used);
+
+/**
+ * @brief Get encoding type from the keyword
+ *
+ * @param name encoding keyword name
+ * @return corresponding column encoding type
+ */
+[[nodiscard]] cudf::io::column_encoding get_encoding_type(std::string name);
+
+/**
+ * @brief Get compression type from the keyword
+ *
+ * @param name compression keyword name
+ * @return corresponding compression type
+ */
+[[nodiscard]] cudf::io::compression_type get_compression_type(std::string name);
+
+/**
+ * @brief Get boolean from they keyword
+ *
+ * @param input keyword affirmation string such as: Y, T, YES, TRUE, ON
+ * @return true or false
+ */
+[[nodiscard]] bool get_boolean(std::string input);
+
+/**
+ * @brief Check if two tables are identical, throw an error otherwise
+ *
+ * @param lhs_table View to lhs table
+ * @param rhs_table View to rhs table
+ */
+void check_tables_equal(cudf::table_view const& lhs_table, cudf::table_view const& rhs_table);
+
+/**
+ * @brief Concatenate a vector of tables and return the resultant table
+ *
+ * @param tables Vector of tables to concatenate
+ * @param stream CUDA stream to use
+ *
+ * @return Unique pointer to the resultant concatenated table.
+ */
+std::unique_ptr<cudf::table> concatenate_tables(std::vector<std::unique_ptr<cudf::table>> tables,
+                                                rmm::cuda_stream_view stream);
+
+/**
+ * @brief Returns a string containing current date and time
+ *
+ */
+std::string current_date_and_time();

cpp/examples/orc_io/debug/breakpoints.txt

@@ -0,0 +1,3 @@
+break /home/coder/cudf/cpp/examples/orc_io/build/_deps/cudf-src/cpp/src/io/orc/stripe_data.cu:1891
+break /home/coder/cudf/cpp/examples/orc_io/build/_deps/cudf-src/cpp/src/io/orc/stripe_data.cu:1384
+

cpp/examples/orc_io/debug/debug.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+root_dir=/home/coder/cudf/cpp/examples/orc_io
+
+example_bin=$root_dir/build/orc_io
+input_file=$root_dir/timestamp_bug.snappy.orc
+output_file=$root_dir/debug/test_output.orc
+
+# DEFAULT, DICTIONARY, PLAIN, DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY
+encoding_type=DEFAULT
+
+# NONE, AUTO, SNAPPY, LZ4, ZSTD
+compression_type=NONE
+
+write_page_stats=yes
+
+export LIBCUDF_LOGGING_LEVEL=INFO
+
+cuda-gdb -ex start --ex 'source breakpoints.txt' --args $example_bin $input_file $output_file $encoding_type $compression_type
+
+# $example_bin $input_file $output_file $encoding_type $compression_type $write_page_stats
+