Merge branch 'branch-24.12' into new-read-parquet-api

ci/test_wheel_cudf_polars.sh

@@ -17,11 +17,11 @@ rapids-logger "Installing cudf_polars and its dependencies"
 # generate constraints (possibly pinning to oldest support versions of dependencies)
 rapids-generate-pip-constraints py_test_cudf_polars ./constraints.txt
 
-# echo to expand wildcard before adding `[test]` requires for pip
+# echo to expand wildcard before adding `[test,experimental]` requires for pip
 python -m pip install \
     -v \
     --constraint ./constraints.txt \
-    "$(echo ./dist/cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
+    "$(echo ./dist/cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,experimental]" \
     "$(echo ./dist/libcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
     "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)"
 

cpp/benchmarks/CMakeLists.txt

@@ -355,8 +355,8 @@ ConfigureNVBench(
 # ##################################################################################################
 # * strings benchmark -------------------------------------------------------------------
 ConfigureBench(
-  STRINGS_BENCH string/factory.cu string/filter.cpp string/repeat_strings.cpp string/replace.cpp
-  string/translate.cpp string/url_decode.cu
+  STRINGS_BENCH string/factory.cu string/repeat_strings.cpp string/replace.cpp string/translate.cpp
+  string/url_decode.cu
 )
 
 ConfigureNVBench(
@@ -374,6 +374,7 @@ ConfigureNVBench(
   string/copy_range.cpp
   string/count.cpp
   string/extract.cpp
+  string/filter.cpp
   string/find.cpp
   string/find_multiple.cpp
   string/join_strings.cpp

cpp/benchmarks/string/filter.cpp

@@ -14,13 +14,7 @@
  * limitations under the License.
  */
 
-#include "string_bench_args.hpp"
-
 #include <benchmarks/common/generate_input.hpp>
-#include <benchmarks/fixture/benchmark_fixture.hpp>
-#include <benchmarks/synchronization/synchronization.hpp>
-
-#include <cudf_test/column_wrapper.hpp>
 
 #include <cudf/scalar/scalar.hpp>
 #include <cudf/strings/char_types/char_types.hpp>
@@ -29,57 +23,58 @@
 #include <cudf/strings/translate.hpp>
 #include <cudf/utilities/default_stream.hpp>
 
-#include <vector>
-
-enum FilterAPI { filter, filter_chars, strip };
+#include <nvbench/nvbench.cuh>
 
-class StringFilterChars : public cudf::benchmark {};
+#include <vector>
 
-static void BM_filter_chars(benchmark::State& state, FilterAPI api)
+static void bench_filter(nvbench::state& state)
 {
-  cudf::size_type const n_rows{static_cast<cudf::size_type>(state.range(0))};
-  cudf::size_type const max_str_length{static_cast<cudf::size_type>(state.range(1))};
+  auto const num_rows  = static_cast<cudf::size_type>(state.get_int64("num_rows"));
+  auto const min_width = static_cast<cudf::size_type>(state.get_int64("min_width"));
+  auto const max_width = static_cast<cudf::size_type>(state.get_int64("max_width"));
+  auto const api       = state.get_string("api");
+
   data_profile const profile = data_profile_builder().distribution(
-    cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length);
-  auto const column = create_random_column(cudf::type_id::STRING, row_count{n_rows}, profile);
-  cudf::strings_column_view input(column->view());
+    cudf::type_id::STRING, distribution_id::NORMAL, min_width, max_width);
+  auto const column = create_random_column(cudf::type_id::STRING, row_count{num_rows}, profile);
+  auto const input  = cudf::strings_column_view(column->view());
 
-  auto const types = cudf::strings::string_character_types::SPACE;
-  std::vector<std::pair<cudf::char_utf8, cudf::char_utf8>> filter_table{
-    {cudf::char_utf8{'a'}, cudf::char_utf8{'c'}}};
+  auto stream = cudf::get_default_stream();
+  state.set_cuda_stream(nvbench::make_cuda_stream_view(stream.value()));
+  auto chars_size = input.chars_size(stream);
+  state.add_global_memory_reads<nvbench::int8_t>(chars_size);
 
-  for (auto _ : state) {
-    cuda_event_timer raii(state, true, cudf::get_default_stream());
-    switch (api) {
-      case filter: cudf::strings::filter_characters_of_type(input, types); break;
-      case filter_chars: cudf::strings::filter_characters(input, filter_table); break;
-      case strip: cudf::strings::strip(input); break;
+  if (api == "filter") {
+    auto const types = cudf::strings::string_character_types::SPACE;
+    {
+      auto result = cudf::strings::filter_characters_of_type(input, types);
+      auto sv     = cudf::strings_column_view(result->view());
+      state.add_global_memory_writes<nvbench::int8_t>(sv.chars_size(stream));
     }
+    state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      cudf::strings::filter_characters_of_type(input, types);
+    });
+  } else if (api == "chars") {
+    state.add_global_memory_writes<nvbench::int8_t>(chars_size);
+    std::vector<std::pair<cudf::char_utf8, cudf::char_utf8>> filter_table{
+      {cudf::char_utf8{'a'}, cudf::char_utf8{'c'}}};
+    state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      cudf::strings::filter_characters(input, filter_table);
+    });
+  } else if (api == "strip") {
+    {
+      auto result = cudf::strings::strip(input);
+      auto sv     = cudf::strings_column_view(result->view());
+      state.add_global_memory_writes<nvbench::int8_t>(sv.chars_size(stream));
+    }
+    state.exec(nvbench::exec_tag::sync,
+               [&](nvbench::launch& launch) { cudf::strings::strip(input); });
   }
-
-  state.SetBytesProcessed(state.iterations() * input.chars_size(cudf::get_default_stream()));
-}
-
-static void generate_bench_args(benchmark::internal::Benchmark* b)
-{
-  int const min_rows          = 1 << 12;
-  int const max_rows          = 1 << 24;
-  int const row_multiplier    = 8;
-  int const min_length        = 1 << 5;
-  int const max_length        = 1 << 13;
-  int const length_multiplier = 2;
-  generate_string_bench_args(
-    b, min_rows, max_rows, row_multiplier, min_length, max_length, length_multiplier);
 }
 
-#define STRINGS_BENCHMARK_DEFINE(name)                                \
-  BENCHMARK_DEFINE_F(StringFilterChars, name)                         \
-  (::benchmark::State & st) { BM_filter_chars(st, FilterAPI::name); } \
-  BENCHMARK_REGISTER_F(StringFilterChars, name)                       \
-    ->Apply(generate_bench_args)                                      \
-    ->UseManualTime()                                                 \
-    ->Unit(benchmark::kMillisecond);
-
-STRINGS_BENCHMARK_DEFINE(filter)
-STRINGS_BENCHMARK_DEFINE(filter_chars)
-STRINGS_BENCHMARK_DEFINE(strip)
+NVBENCH_BENCH(bench_filter)
+  .set_name("filter")
+  .add_int64_axis("min_width", {0})
+  .add_int64_axis("max_width", {32, 64, 128, 256})
+  .add_int64_axis("num_rows", {32768, 262144, 2097152})
+  .add_string_axis("api", {"filter", "chars", "strip"});

dependencies.yaml

@@ -41,9 +41,11 @@ files:
       - test_cpp
       - test_python_common
       - test_python_cudf
+      - test_python_cudf_common
       - test_python_dask_cudf
       - test_python_pylibcudf
       - test_python_cudf_pandas
+      - test_python_cudf_polars
   test_static_build:
     output: none
     includes:
@@ -59,6 +61,7 @@ files:
       - cuda_version
       - py_version
       - test_python_common
+      - test_python_cudf_common
       - test_python_cudf
       - test_python_cudf_pandas
   test_python_cudf:
@@ -67,13 +70,15 @@ files:
       - cuda_version
       - py_version
       - test_python_common
+      - test_python_cudf_common
       - test_python_cudf
   test_python_other:
     output: none
     includes:
       - cuda_version
       - py_version
       - test_python_common
+      - test_python_cudf_common
       - test_python_dask_cudf
   test_java:
     output: none
@@ -152,6 +157,7 @@ files:
       key: test
     includes:
       - test_python_common
+      - test_python_cudf_common
       - test_python_cudf
   py_build_libcudf:
     output: pyproject
@@ -216,6 +222,7 @@ files:
       key: test
     includes:
       - test_python_common
+      - test_python_cudf_common
       - test_python_pylibcudf
   py_test_pandas_cudf:
     output: pyproject
@@ -248,6 +255,14 @@ files:
     includes:
       - run_cudf_polars
       - depends_on_pylibcudf
+  py_run_cudf_polars_experimental:
+    output: pyproject
+    pyproject_dir: python/cudf_polars
+    extras:
+      table: project.optional-dependencies
+      key: experimental
+    includes:
+      - run_cudf_polars_experimental
   py_test_cudf_polars:
     output: pyproject
     pyproject_dir: python/cudf_polars
@@ -256,6 +271,7 @@ files:
       key: test
     includes:
       - test_python_common
+      - test_python_cudf_polars
   py_build_dask_cudf:
     output: pyproject
     pyproject_dir: python/dask_cudf
@@ -281,6 +297,7 @@ files:
       key: test
     includes:
       - test_python_common
+      - test_python_cudf_common
       - test_python_dask_cudf
   py_build_cudf_kafka:
     output: pyproject
@@ -313,6 +330,7 @@ files:
       key: test
     includes:
       - test_python_common
+      - test_python_cudf_common
   py_build_custreamz:
     output: pyproject
     pyproject_dir: python/custreamz
@@ -337,6 +355,7 @@ files:
       key: test
     includes:
       - test_python_common
+      - test_python_cudf_common
 channels:
   - rapidsai
   - rapidsai-nightly
@@ -730,6 +749,11 @@ dependencies:
       - output_types: [conda, requirements, pyproject]
         packages:
           - polars>=1.11,<1.14
+  run_cudf_polars_experimental:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - rapids-dask-dependency==24.12.*,>=0.0.0a0
   run_dask_cudf:
     common:
       - output_types: [conda, requirements, pyproject]
@@ -779,6 +803,7 @@ dependencies:
           - pytest<8
           - pytest-cov
           - pytest-xdist
+  test_python_cudf_common:
     specific:
       # Define additional constraints for testing with oldest dependencies.
       - output_types: [conda, requirements]
@@ -884,6 +909,11 @@ dependencies:
               - pyarrow==14.0.1
           - matrix:
             packages:
+  test_python_cudf_polars:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - *numpy
   depends_on_libcudf:
     common:
       - output_types: conda

docs/cudf/source/user_guide/io/io.md

@@ -194,3 +194,13 @@ If no value is set, behavior will be the same as the "STABLE" option.
     +-----------------------+--------+--------+--------------+--------------+---------+--------+--------------+--------------+--------+
 
 ```
+
+## Low Memory Considerations
+
+By default, cuDF's parquet and json readers will try to read the entire file in one pass. This can cause problems when dealing with large datasets or when running workloads on GPUs with limited memory.
+
+To better support low memory systems, cuDF provides a "low-memory" reader for parquet and json files. This low memory reader processes data in chunks, leading to lower peak memory usage due to the smaller size of intermediate allocations.
+
+To read a parquet or json file in low memory mode, there are [cuDF options](https://docs.rapids.ai/api/cudf/nightly/user_guide/api_docs/options/#api-options) that must be set globally prior to calling the reader. To set those options, call:
+- `cudf.set_option("io.parquet.low_memory", True)` for parquet files, or
+- `cudf.set_option("io.json.low_memory", True)` for json files.

java/src/main/java/ai/rapids/cudf/ColumnView.java

@@ -3332,6 +3332,36 @@ public final ColumnVector stringContains(Scalar compString) {
     return new ColumnVector(stringContains(getNativeView(), compString.getScalarHandle()));
   }
 
+  /**
+   * @brief Searches for the given target strings within each string in the provided column
+   *
+   * Each column in the result table corresponds to the result for the target string at the same
+   * ordinal. i.e. 0th column is the BOOL8 column result for the 0th target string, 1th for 1th,
+   * etc.
+   *
+   * If the target is not found for a string, false is returned for that entry in the output column.
+   * If the target is an empty string, true is returned for all non-null entries in the output column.
+   *
+   * Any null input strings return corresponding null entries in the output columns.
+   *
+   * input = ["a", "b", "c"]
+   * targets = ["a", "c"]
+   * output is a table with two boolean columns:
+   *   column 0: [true, false, false]
+   *   column 1: [false, false, true]
+   *
+   * @param targets UTF-8 encoded strings to search for in each string in `input`
+   * @return BOOL8 columns
+   */
+  public final ColumnVector[] stringContains(ColumnView targets) {
+    assert type.equals(DType.STRING) : "column type must be a String";
+    assert targets.getType().equals(DType.STRING) : "targets type must be a string";
+    assert targets.getNullCount() == 0 : "targets must not contain nulls";
+    assert targets.getRowCount() > 0 : "targets must not be empty";
+    long[] resultPointers = stringContainsMulti(getNativeView(), targets.getNativeView());
+    return Arrays.stream(resultPointers).mapToObj(ColumnVector::new).toArray(ColumnVector[]::new);
+  }
+
   /**
    * Replaces values less than `lo` in `input` with `lo`,
    * and values greater than `hi` with `hi`.
@@ -4437,6 +4467,13 @@ private static native long stringReplaceWithBackrefs(long columnView, String pat
    */
   private static native long stringContains(long cudfViewHandle, long compString) throws CudfException;
 
+  /**
+   * Native method for searching for the given target strings within each string in the provided column.
+   * @param cudfViewHandle native handle of the cudf::column_view being operated on.
+   * @param targetViewHandle handle of the column view containing the strings being searched for.
+   */
+  private static native long[] stringContainsMulti(long cudfViewHandle, long targetViewHandle) throws CudfException;
+
   /**
    * Native method for extracting results from a regex program pattern. Returns a table handle.
    *

java/src/main/native/src/ColumnViewJni.cpp

@@ -64,6 +64,7 @@
 #include <cudf/strings/convert/convert_urls.hpp>
 #include <cudf/strings/extract.hpp>
 #include <cudf/strings/find.hpp>
+#include <cudf/strings/find_multiple.hpp>
 #include <cudf/strings/findall.hpp>
 #include <cudf/strings/padding.hpp>
 #include <cudf/strings/regex/regex_program.hpp>
@@ -2827,4 +2828,23 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_toHex(JNIEnv* env, jclass
   }
   CATCH_STD(env, 0);
 }
+
+JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_ColumnView_stringContainsMulti(
+  JNIEnv* env, jobject j_object, jlong j_view_handle, jlong j_target_view_handle)
+{
+  JNI_NULL_CHECK(env, j_view_handle, "column is null", 0);
+  JNI_NULL_CHECK(env, j_target_view_handle, "targets is null", 0);
+
+  try {
+    cudf::jni::auto_set_device(env);
+    auto* column_view         = reinterpret_cast<cudf::column_view*>(j_view_handle);
+    auto* targets_view        = reinterpret_cast<cudf::column_view*>(j_target_view_handle);
+    auto const strings_column = cudf::strings_column_view(*column_view);
+    auto const targets_column = cudf::strings_column_view(*targets_view);
+    auto contains_results     = cudf::strings::contains_multiple(strings_column, targets_column);
+    return cudf::jni::convert_table_for_return(env, std::move(contains_results));
+  }
+  CATCH_STD(env, 0);
+}
+
 }  // extern "C"