diff --git a/.github/actions/setup-builder/action.yml b/.github/actions/setup-builder/action.yml index 43de1cbaa..476feb9ca 100644 --- a/.github/actions/setup-builder/action.yml +++ b/.github/actions/setup-builder/action.yml @@ -21,18 +21,24 @@ name: Prepare Rust Builder description: 'Prepare Rust Build Environment' inputs: rust-version: - description: 'version of rust to install (e.g. stable)' - required: true - default: 'stable' + description: 'version of rust to install and use' runs: using: "composite" steps: - - name: Setup Rust toolchain + - name: Setup specified Rust toolchain shell: bash + if: ${{ inputs.rust-version != '' }} run: | echo "Installing ${{ inputs.rust-version }}" rustup toolchain install ${{ inputs.rust-version }} - rustup default ${{ inputs.rust-version }} + rustup override set ${{ inputs.rust-version }} + rustup component add rustfmt clippy + - name: Setup Rust toolchain according to rust-toolchain.toml + shell: bash + if: ${{ inputs.rust-version == '' }} + run: | + echo "Installing toolchain according to rust-toolchain.toml" + rustup show rustup component add rustfmt clippy - name: Fixup git permissions # https://github.com/actions/checkout/issues/766 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 873924c29..2b7f8b694 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + - name: Check License Header uses: apache/skywalking-eyes/header@v0.6.0 @@ -89,8 +92,6 @@ jobs: - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - with: - rust-version: ${{ env.rust_msrv }} - name: Cache Rust artifacts uses: Swatinem/rust-cache@v2 @@ -109,6 +110,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + - name: Cache Rust artifacts uses: Swatinem/rust-cache@v2 @@ -122,8 +126,6 @@ jobs: - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - with: - rust-version: ${{ env.rust_msrv }} - name: Cache Rust artifacts uses: Swatinem/rust-cache@v2 @@ -136,3 +138,24 @@ jobs: - name: Doc Test run: cargo test --no-fail-fast --doc --all-features --workspace + + msrv: + name: Verify MSRV + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Nightly Rust toolchain + uses: ./.github/actions/setup-builder + - name: Generate minimal versions lockfile + run: | + cargo generate-lockfile -Z direct-minimal-versions -Z minimal-versions + # Some dependencies don't correctly specify a minimal version for their dependencies and will fail to build. + # So we update these transitive dependencies here. + cargo update tap faststr metainfo linkedbytes + - name: Setup MSRV Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{ env.rust_msrv }} + - name: Check MSRV + run: | + cargo +${{ env.rust_msrv }} check --locked --workspace --exclude iceberg-datafusion --exclude iceberg-catalog-s3tables diff --git a/Cargo.toml b/Cargo.toml index d627af0bd..e4b982e7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,27 +41,27 @@ rust-version = "1.77.1" anyhow = "1.0.72" apache-avro = "0.17" array-init = "2" -arrow-arith = { version = "53" } -arrow-array = { version = "53" } -arrow-cast = { version = "53" } -arrow-ord = { version = "53" } -arrow-schema = { version = "53" } -arrow-select = { version = "53" } -arrow-string = { version = "53" } +arrow-arith = { version = "53.3.0" } +arrow-array = { version = "53.3.0" } +arrow-cast = { version = "53.3.0" } +arrow-ord = { version = "53.3.0" } +arrow-schema = { version = "53.3.0" } +arrow-select = { version = "53.3.0" } +arrow-string = { version = "53.3.0" } async-stream = "0.3.5" -async-trait = "0.1" +async-trait = "0.1.73" async-std = "1.12" -aws-config = "1.5.13" -aws-sdk-glue = "1.76" +aws-config = "1" +aws-sdk-glue = "1.39" bimap = "0.6" bitvec = "1.0.1" -bytes = "1.5" -chrono = "0.4.34" +bytes = "1.6" +chrono = "0.4.38" ctor = "0.2.8" derive_builder = "0.20" either = "1" env_logger = "0.11.0" -fnv = "1" +fnv = "1.0.7" futures = "0.3" iceberg = { version = "0.4.0", path = "./crates/iceberg" } iceberg-catalog-rest = { version = "0.4.0", path = "./crates/catalog/rest" } @@ -69,34 +69,34 @@ iceberg-catalog-hms = { version = "0.4.0", path = "./crates/catalog/hms" } iceberg-catalog-memory = { version = "0.4.0", path = "./crates/catalog/memory" } iceberg-datafusion = { version = "0.4.0", path = "./crates/integrations/datafusion" } itertools = "0.13" -log = "0.4" +log = "0.4.22" mockito = "1" murmur3 = "0.5.2" num-bigint = "0.4.6" -once_cell = "1" +once_cell = "1.19" opendal = "0.51.0" ordered-float = "4" -parquet = "53.1" -paste = "1" +parquet = "53.3.0" +paste = "1.0.15" pilota = "0.11.2" pretty_assertions = "1.4" port_scanner = "0.1.5" -rand = "0.8" +rand = "0.8.5" regex = "1.10.5" -reqwest = { version = "0.12", default-features = false, features = ["json"] } +reqwest = { version = "0.12.2", default-features = false, features = ["json"] } rust_decimal = "1.31" -serde = { version = "1", features = ["rc"] } -serde_bytes = "0.11.8" -serde_derive = "1" -serde_json = "1" +serde = { version = "1.0.204", features = ["rc"] } +serde_bytes = "0.11.15" +serde_derive = "1.0.204" +serde_json = "1.0.120" serde_repr = "0.1.16" serde_with = "3.4" tempfile = "3.15" -tokio = { version = "1", default-features = false } +tokio = { version = "1.36", default-features = false } typed-builder = "0.20" -url = "2" +url = "2.2.2" urlencoding = "2" -uuid = { version = "1.6.1", features = ["v7"] } +uuid = { version = "1.10.0", features = ["v7"] } volo-thrift = "0.10" hive_metastore = "0.1" tera = "1" diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index add57183b..691545493 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -35,7 +35,7 @@ chrono = { workspace = true } http = "1.1.0" iceberg = { workspace = true } itertools = { workspace = true } -log = "0.4.20" +log = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } diff --git a/crates/catalog/s3tables/Cargo.toml b/crates/catalog/s3tables/Cargo.toml index 9ea66a0f9..d0454d12d 100644 --- a/crates/catalog/s3tables/Cargo.toml +++ b/crates/catalog/s3tables/Cargo.toml @@ -20,7 +20,7 @@ name = "iceberg-catalog-s3tables" version = { workspace = true } edition = { workspace = true } homepage = { workspace = true } -rust-version = { workspace = true } +rust-version = "1.81.0" categories = ["database"] description = "Apache Iceberg Rust S3Tables Catalog" diff --git a/crates/catalog/sql/Cargo.toml b/crates/catalog/sql/Cargo.toml index a51671650..71cf37c14 100644 --- a/crates/catalog/sql/Cargo.toml +++ b/crates/catalog/sql/Cargo.toml @@ -40,7 +40,7 @@ uuid = { workspace = true, features = ["v4"] } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } itertools = { workspace = true } regex = "1.10.5" -sqlx = { version = "0.8.0", features = [ +sqlx = { version = "0.8.1", features = [ "tls-rustls", "runtime-tokio", "any", diff --git a/crates/examples/Cargo.toml b/crates/examples/Cargo.toml index 2fb3060c1..2f1dbf858 100644 --- a/crates/examples/Cargo.toml +++ b/crates/examples/Cargo.toml @@ -27,7 +27,7 @@ rust-version = { workspace = true } [dependencies] iceberg = { workspace = true } iceberg-catalog-rest = { workspace = true } -tokio = { version = "1", features = ["full"] } +tokio = { workspace = true, features = ["full"] } [[example]] name = "rest-catalog-namespace" diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index 6ea7594ba..d52f8a6d0 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use std::mem::size_of_val; use std::sync::Arc; use crate::io::FileIO; diff --git a/crates/iceberg/src/utils.rs b/crates/iceberg/src/utils.rs index 70514cccb..00d3e69bd 100644 --- a/crates/iceberg/src/utils.rs +++ b/crates/iceberg/src/utils.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::num::NonZero; +use std::num::NonZeroUsize; // Use a default value of 1 as the safest option. // See https://doc.rust-lang.org/std/thread/fn.available_parallelism.html#limitations @@ -31,12 +31,12 @@ const DEFAULT_PARALLELISM: usize = 1; /// are circumstances where the level of available /// parallelism can change during the lifetime of an executing /// process, but this should not be called in a hot loop. -pub(crate) fn available_parallelism() -> NonZero { +pub(crate) fn available_parallelism() -> NonZeroUsize { std::thread::available_parallelism().unwrap_or_else(|_err| { // Failed to get the level of parallelism. // TODO: log/trace when this fallback occurs. // Using a default value. - NonZero::new(DEFAULT_PARALLELISM).unwrap() + NonZeroUsize::new(DEFAULT_PARALLELISM).unwrap() }) } diff --git a/crates/integrations/datafusion/Cargo.toml b/crates/integrations/datafusion/Cargo.toml index 81a94d839..7a706d0e9 100644 --- a/crates/integrations/datafusion/Cargo.toml +++ b/crates/integrations/datafusion/Cargo.toml @@ -20,7 +20,10 @@ name = "iceberg-datafusion" version = { workspace = true } edition = { workspace = true } homepage = { workspace = true } -rust-version = { workspace = true } +# kept the same as DataFusion's MSRV +# https://github.com/apache/datafusion?tab=readme-ov-file#rust-version-compatibility-policy +# https://github.com/apache/datafusion/blob/main/Cargo.toml#L68 +rust-version = "1.80.1" categories = ["database"] description = "Apache Iceberg DataFusion Integration" diff --git a/crates/test_utils/Cargo.toml b/crates/test_utils/Cargo.toml index d4f6e1696..c98309760 100644 --- a/crates/test_utils/Cargo.toml +++ b/crates/test_utils/Cargo.toml @@ -27,7 +27,7 @@ license = { workspace = true } [dependencies] env_logger = { workspace = true } -log = "0.4.20" +log = { workspace = true } [features] tests = []