feat: introduce feature flags to select major arrow versions

.github/workflows/build.yml

@@ -37,10 +37,10 @@ jobs:
           cargo install cargo-msrv --locked
       - name: verify-msrv
         run: |
-          cargo msrv --path kernel/ verify --all-features
-          cargo msrv --path derive-macros/ verify --all-features
-          cargo msrv --path ffi/ verify --all-features
-          cargo msrv --path ffi-proc-macros/ verify --all-features
+          cargo msrv --path kernel/ verify --features $(< .github/workflows/default-cargo-features)
+          cargo msrv --path derive-macros/ verify --features $(< .github/workflows/default-cargo-features)
+          cargo msrv --path ffi/ verify --features $(< .github/workflows/default-cargo-features)
+          cargo msrv --path ffi-proc-macros/ verify --features $(< .github/workflows/default-cargo-features)
   msrv-run-tests:
     runs-on: ubuntu-latest
     steps:
@@ -104,7 +104,7 @@ jobs:
       - name: check kernel builds with no-default-features
         run: cargo build -p delta_kernel --no-default-features
       - name: build and lint with clippy
-        run: cargo clippy --benches --tests --all-features -- -D warnings
+        run: cargo clippy --benches --tests --features $(< .github/workflows/default-cargo-features) -- -D warnings
       - name: lint without default features
         run: cargo clippy --no-default-features -- -D warnings
       - name: check kernel builds with default-engine
@@ -129,7 +129,7 @@ jobs:
           override: true
       - uses: Swatinem/rust-cache@v2
       - name: test
-        run: cargo test --workspace --verbose --all-features -- --skip read_table_version_hdfs
+        run: cargo test --workspace --verbose --features $(< .github/workflows/default-cargo-features) -- --skip read_table_version_hdfs
 
   ffi_test:
     runs-on: ${{ matrix.os }}
@@ -229,7 +229,7 @@ jobs:
         uses: taiki-e/install-action@cargo-llvm-cov
       - uses: Swatinem/rust-cache@v2
       - name: Generate code coverage
-        run: cargo llvm-cov --all-features --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs
+        run: cargo llvm-cov --features $(< .github/workflows/default-cargo-features) --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
         with:

.github/workflows/default-cargo-features

@@ -0,0 +1 @@
+integration-test,default-engine,default-engine-rustls,cloud,arrow,sync-engine

Cargo.toml

@@ -20,24 +20,9 @@ license = "Apache-2.0"
 repository = "https://github.com/delta-io/delta-kernel-rs"
 readme = "README.md"
 rust-version = "1.80"
-version = "0.6.1"
+version = "0.7.0"
 
 [workspace.dependencies]
-# When changing the arrow version range, also modify ffi/Cargo.toml which has
-# its own arrow version ranges witeh modified features. Failure to do so will
-# result in compilation errors as two different sets of arrow dependencies may
-# be sourced
-arrow = { version = ">=53, <55" }
-arrow-arith = { version = ">=53, <55" }
-arrow-array = { version = ">=53, <55" }
-arrow-buffer = { version = ">=53, <55" }
-arrow-cast = { version = ">=53, <55" }
-arrow-data = { version = ">=53, <55" }
-arrow-ord = { version = ">=53, <55" }
-arrow-json = { version = ">=53, <55" }
-arrow-select = { version = ">=53, <55" }
-arrow-schema = { version = ">=53, <55" }
-parquet = { version = ">=53, <55", features = ["object_store"] }
 object_store = { version = ">=0.11, <0.12" }
 hdfs-native-object-store = "0.12.0"
 hdfs-native = "0.10.0"

acceptance/Cargo.toml

@@ -14,19 +14,14 @@ rust-version.workspace = true
 release = false
 
 [dependencies]
-arrow-array = { workspace = true }
-arrow-cast = { workspace = true }
-arrow-ord = { workspace = true }
-arrow-select = { workspace = true }
-arrow-schema = { workspace = true }
 delta_kernel = { path = "../kernel", features = [
   "default-engine",
+  "arrow_53",
   "developer-visibility",
 ] }
 futures = "0.3"
 itertools = "0.13"
 object_store = { workspace = true }
-parquet = { workspace = true }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 thiserror = "1"

acceptance/src/data.rs

@@ -1,15 +1,18 @@
 use std::{path::Path, sync::Arc};
 
-use arrow_array::{Array, RecordBatch};
-use arrow_ord::sort::{lexsort_to_indices, SortColumn};
-use arrow_schema::{DataType, Schema};
-use arrow_select::{concat::concat_batches, filter::filter_record_batch, take::take};
+use delta_kernel::arrow::array::{Array, RecordBatch};
+use delta_kernel::arrow::compute::{
+    concat_batches, filter_record_batch, lexsort_to_indices, take, SortColumn,
+};
+use delta_kernel::arrow::datatypes::{DataType, Schema};
 
+use delta_kernel::parquet::arrow::async_reader::{
+    ParquetObjectReader, ParquetRecordBatchStreamBuilder,
+};
 use delta_kernel::{engine::arrow_data::ArrowEngineData, DeltaResult, Engine, Error, Table};
 use futures::{stream::TryStreamExt, StreamExt};
 use itertools::Itertools;
 use object_store::{local::LocalFileSystem, ObjectStore};
-use parquet::arrow::async_reader::{ParquetObjectReader, ParquetRecordBatchStreamBuilder};
 
 use crate::{TestCaseInfo, TestResult};
 
@@ -83,8 +86,11 @@ fn assert_schema_fields_match(schema: &Schema, golden: &Schema) {
 fn normalize_col(col: Arc<dyn Array>) -> Arc<dyn Array> {
     if let DataType::Timestamp(unit, Some(zone)) = col.data_type() {
         if **zone == *"+00:00" {
-            arrow_cast::cast::cast(&col, &DataType::Timestamp(*unit, Some("UTC".into())))
-                .expect("Could not cast to UTC")
+            delta_kernel::arrow::compute::cast(
+                &col,
+                &DataType::Timestamp(*unit, Some("UTC".into())),
+            )
+            .expect("Could not cast to UTC")
         } else {
             col
         }

ffi/Cargo.toml

@@ -24,7 +24,7 @@ url = "2"
 delta_kernel = { path = "../kernel", default-features = false, features = [
   "developer-visibility",
 ] }
-delta_kernel_ffi_macros = { path = "../ffi-proc-macros", version = "0.6.1" }
+delta_kernel_ffi_macros = { path = "../ffi-proc-macros", version = "0.7.0" }
 
 # used if we use the default engine to be able to move arrow data into the c-ffi format
 arrow-schema = { version = ">=53, <55", default-features = false, features = [

kernel/Cargo.toml

@@ -39,27 +39,29 @@ uuid = "1.10.0"
 z85 = "3.0.5"
 
 # bring in our derive macros
-delta_kernel_derive = { path = "../derive-macros", version = "0.6.1" }
+delta_kernel_derive = { path = "../derive-macros", version = "0.7.0" }
 
 # used for developer-visibility
 visibility = "0.1.1"
 
 # Used in the sync engine
 tempfile = { version = "3", optional = true }
+
+# Arrow supported versions
+## 53
 # Used in default engine
-arrow-buffer = { workspace = true, optional = true }
-arrow-array = { workspace = true, optional = true, features = ["chrono-tz"] }
-arrow-select = { workspace = true, optional = true }
-arrow-arith = { workspace = true, optional = true }
-arrow-cast = { workspace = true, optional = true }
-arrow-json = { workspace = true, optional = true }
-arrow-ord = { workspace = true, optional = true }
-arrow-schema = { workspace = true, optional = true }
+arrow_53 = { package = "arrow", version = "53", features = ["chrono-tz", "json", "prettyprint"], optional = true }
+# Used in default and sync engine
+parquet_53 = { package = "parquet", version = "53", features = ["async", "object_store"] , optional = true }
+######
+## 54
+arrow_54 = { package = "arrow", version = "54", features = ["chrono-tz", "json", "prettyprint"], optional = true }
+parquet_54 = { package = "parquet", version = "54", features = ["async", "object_store"] , optional = true }
+######
+
 futures = { version = "0.3", optional = true }
 object_store = { workspace = true, optional = true }
 hdfs-native-object-store = { workspace = true, optional = true }
-# Used in default and sync engine
-parquet = { workspace = true, optional = true }
 # Used for fetching direct urls (like pre-signed urls)
 reqwest = { version = "0.12.8", default-features = false, optional = true }
 strum = { version = "0.26", features = ["derive"] }
@@ -73,8 +75,20 @@ hdfs-native = { workspace = true, optional = true }
 walkdir = { workspace = true, optional = true }
 
 [features]
-arrow-conversion = ["arrow-schema"]
-arrow-expression = ["arrow-arith", "arrow-array", "arrow-buffer", "arrow-ord", "arrow-schema"]
+# The default version to be expected
+arrow = ["arrow_53"]
+# The default version to be expected
+parquet = ["parquet_53"]
+
+arrow_53 = ["dep:arrow_53", "parquet_53"]
+parquet_53 = ["dep:parquet_53"]
+
+arrow_54 = ["dep:arrow_54", "parquet_54"]
+parquet_54 = ["dep:parquet_54"]
+
+arrow-conversion = []
+arrow-expression = []
+
 cloud = [
   "object_store/aws",
   "object_store/azure",
@@ -89,16 +103,8 @@ default = []
 default-engine-base = [
   "arrow-conversion",
   "arrow-expression",
-  "arrow-array",
-  "arrow-buffer",
-  "arrow-cast",
-  "arrow-json",
-  "arrow-schema",
-  "arrow-select",
   "futures",
   "object_store",
-  "parquet/async",
-  "parquet/object_store",
   "tokio",
   "uuid/v4",
   "uuid/fast-rng",
@@ -119,13 +125,6 @@ default-engine-rustls = [
 
 developer-visibility = []
 sync-engine = [
-  "arrow-cast",
-  "arrow-conversion",
-  "arrow-expression",
-  "arrow-array",
-  "arrow-json",
-  "arrow-select",
-  "parquet",
   "tempfile",
 ]
 integration-test = [
@@ -141,7 +140,6 @@ version = "=0.5.9"
 rustc_version = "0.4.1"
 
 [dev-dependencies]
-arrow = { workspace = true, features = ["json", "prettyprint"] }
 delta_kernel = { path = ".", features = ["default-engine", "sync-engine"] }
 test_utils = { path = "../test-utils" }
 paste = "1.0"

kernel/examples/inspect-table/Cargo.toml

@@ -5,11 +5,11 @@ edition = "2021"
 publish = false
 
 [dependencies]
-arrow-array = { workspace = true }
-arrow-schema = { workspace = true }
+arrow = "53"
 clap = { version = "4.5", features = ["derive"] }
 delta_kernel = { path = "../../../kernel", features = [
   "cloud",
+  "arrow_53",
   "default-engine",
   "developer-visibility",
 ] }

kernel/examples/read-table-changes/Cargo.toml

@@ -8,14 +8,13 @@ publish = false
 release = false
 
 [dependencies]
-arrow-array = { workspace = true }
-arrow-schema = { workspace = true }
+arrow = { version = "53", features = ["prettyprint"] }
 clap = { version = "4.5", features = ["derive"] }
 delta_kernel = { path = "../../../kernel", features = [
   "cloud",
+  "arrow_53",
   "default-engine",
 ] }
 env_logger = "0.11.3"
 url = "2"
 itertools = "0.13"
-arrow = { workspace = true, features = ["prettyprint"] }

kernel/examples/read-table-multi-threaded/Cargo.toml

@@ -5,10 +5,11 @@ edition = "2021"
 publish = false
 
 [dependencies]
-arrow = { workspace = true, features = ["prettyprint", "chrono-tz"] }
+arrow = { version = "53", features = ["prettyprint", "chrono-tz"] }
 clap = { version = "4.5", features = ["derive"] }
 delta_kernel = { path = "../../../kernel", features = [
   "cloud",
+  "arrow_53",
   "default-engine",
   "sync-engine",
   "developer-visibility",

kernel/examples/read-table-single-threaded/Cargo.toml

@@ -5,9 +5,10 @@ edition = "2021"
 publish = false
 
 [dependencies]
-arrow = { workspace = true, features = ["prettyprint", "chrono-tz"] }
+arrow = { version = "53", features = ["prettyprint", "chrono-tz"] }
 clap = { version = "4.5", features = ["derive"] }
 delta_kernel = { path = "../../../kernel", features = [
+  "arrow_53",
   "cloud",
   "default-engine",
   "sync-engine",

kernel/src/actions/visitors.rs

@@ -474,8 +474,8 @@ pub(crate) fn visit_deletion_vector_at<'a>(
 mod tests {
     use std::sync::Arc;
 
-    use arrow_array::{RecordBatch, StringArray};
-    use arrow_schema::{DataType, Field, Schema as ArrowSchema};
+    use crate::arrow::array::{RecordBatch, StringArray};
+    use crate::arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
 
     use super::*;
     use crate::{

kernel/src/arrow.rs

@@ -0,0 +1,11 @@
+//! This module exists to help re-export the version of arrow used by default-gengine and other
+//! parts of kernel that need arrow
+
+#[cfg(all(feature = "arrow_53", feature = "arrow_54"))]
+compile_error!("Multiple versions of the arrow cannot be used at the same time!");
+
+#[cfg(feature = "arrow_53")]
+pub use arrow_53::*;
+
+#[cfg(feature = "arrow_54")]
+pub use arrow_54::*;

kernel/src/engine/arrow_conversion.rs

@@ -2,10 +2,11 @@
 
 use std::sync::Arc;
 
-use arrow_schema::{
-    ArrowError, DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema,
+use crate::arrow::datatypes::{
+    DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema,
     SchemaRef as ArrowSchemaRef, TimeUnit,
 };
+use crate::arrow::error::ArrowError;
 use itertools::Itertools;
 
 use crate::error::Error;

kernel/src/engine/arrow_data.rs

@@ -2,11 +2,11 @@ use crate::engine_data::{EngineData, EngineList, EngineMap, GetData, RowVisitor}
 use crate::schema::{ColumnName, DataType};
 use crate::{DeltaResult, Error};
 
-use arrow_array::cast::AsArray;
-use arrow_array::types::{Int32Type, Int64Type};
-use arrow_array::{Array, ArrayRef, GenericListArray, MapArray, OffsetSizeTrait, RecordBatch, StructArray};
-use arrow_schema::{FieldRef, DataType as ArrowDataType};
-use tracing::{debug};
+use crate::arrow::array::cast::AsArray;
+use crate::arrow::array::types::{Int32Type, Int64Type};
+use crate::arrow::array::{Array, ArrayRef, GenericListArray, MapArray, OffsetSizeTrait, RecordBatch, StructArray};
+use crate::arrow::datatypes::{FieldRef, DataType as ArrowDataType};
+use tracing::debug;
 
 use std::collections::{HashMap, HashSet};
 
@@ -269,8 +269,8 @@ impl ArrowEngineData {
 mod tests {
     use std::sync::Arc;
 
-    use arrow_array::{RecordBatch, StringArray};
-    use arrow_schema::{DataType, Field, Schema as ArrowSchema};
+    use crate::arrow::array::{RecordBatch, StringArray};
+    use crate::arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
 
     use crate::{
         actions::{get_log_schema, Metadata, Protocol},