Merge branch 'main' into grtlr/re-auth

.github/workflows/contrib_checks.yml

@@ -76,7 +76,8 @@ jobs:
 
   no-codegen-changes:
     name: Check if running codegen would produce any changes
-    runs-on: ubuntu-latest-16-cores
+    # TODO(andreas): setup-vulkan doesn't work on 24.4 right now due to missing .so
+    runs-on: ubuntu-22.04-large
     steps:
       # Note: We explicitly don't override `ref` here. We need to see if changes would be made
       # in a context where we have merged with main. Otherwise we might miss changes such as one
@@ -92,9 +93,12 @@ jobs:
 
   rs-lints:
     name: Rust lints (fmt, check, clippy, tests, doc)
-    runs-on: ubuntu-latest-16-cores
+    # TODO(andreas): setup-vulkan doesn't work on 24.4 right now due to missing .so
+    runs-on: ubuntu-22.04-16core
     steps:
       - uses: actions/checkout@v4
+        with:
+          lfs: true
 
       - uses: prefix-dev/[email protected]
         with:

.github/workflows/reusable_checks_rust.yml

@@ -52,7 +52,8 @@ jobs:
 
   rs-lints:
     name: Rust lints (fmt, check, clippy, tests, doc)
-    runs-on: ubuntu-latest-16-cores
+    # TODO(andreas): setup-vulkan doesn't work on 24.4 right now due to missing .so
+    runs-on: ubuntu-22.04-large
     steps:
       - uses: actions/checkout@v4
         with:

.vscode/settings.json

@@ -46,11 +46,7 @@
     // Uncomment the following options and restart rust-analyzer to get it to check code behind `cfg(target_arch=wasm32)`.
     // Don't forget to put it in a comment again before committing.
     // "rust-analyzer.cargo.target": "wasm32-unknown-unknown",
-    // "rust-analyzer.cargo.cfgs": {
-    //     "web": null,
-    //     "webgl": null,
-    //     "webgpu": null,
-    // },
+    // "rust-analyzer.cargo.cfgs": ["web","webgl","webgpu"],
 
     "C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools", // Use cmake-tools to grab configs.
     "C_Cpp.autoAddFileAssociations": false,

ARCHITECTURE.md

@@ -212,6 +212,7 @@ Update instructions:
 | Crate              | Description                                                                          |
 | ------------------ | ------------------------------------------------------------------------------------ |
 | re_analytics       | Rerun's analytics SDK                                                                |
+| re_arrow_util      | Helpers for working with arrow                                                       |
 | re_auth            | Authentication and authorization helpers                                             |
 | re_byte_size       | Calculate the heap-allocated size of values at runtime                               |
 | re_capabilities    | Capability tokens                                                                    |

Cargo.lock

@@ -365,6 +365,7 @@ dependencies = [
  "arrow-buffer",
  "arrow-cast",
  "arrow-data",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-row",
  "arrow-schema",
@@ -457,6 +458,20 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "arrow-ipc"
+version = "53.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
+ "arrow-data",
+ "arrow-schema",
+ "flatbuffers",
+]
+
 [[package]]
 name = "arrow-ord"
 version = "53.2.0"
@@ -2476,9 +2491,9 @@ checksum = "c1671b620ba6e60c11c62b0ea5fec4f8621991e7b1229fa13c010a2cd04e4342"
 
 [[package]]
 name = "flatbuffers"
-version = "23.5.26"
+version = "24.12.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640"
+checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096"
 dependencies = [
  "bitflags 1.3.2",
  "rustc_version",
@@ -5582,7 +5597,7 @@ dependencies = [
 [[package]]
 name = "re_arrow2"
 version = "0.18.1"
-source = "git+https://github.com/rerun-io/re_arrow2.git?branch=main#82095762a42eed76e4043c82833e391893d5821c"
+source = "git+https://github.com/rerun-io/re_arrow2.git?branch=main#61ac48418df229584155c5f669477bedf76d4505"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -5606,6 +5621,17 @@ dependencies = [
  "simdutf8",
 ]
 
+[[package]]
+name = "re_arrow_util"
+version = "0.22.0-alpha.1+dev"
+dependencies = [
+ "arrow",
+ "itertools 0.13.0",
+ "re_arrow2",
+ "re_log",
+ "re_tracing",
+]
+
 [[package]]
 name = "re_auth"
 version = "0.22.0-alpha.1+dev"
@@ -5701,6 +5727,7 @@ dependencies = [
  "nohash-hasher",
  "rand",
  "re_arrow2",
+ "re_arrow_util",
  "re_byte_size",
  "re_error",
  "re_format",
@@ -5712,6 +5739,7 @@ dependencies = [
  "re_types_core",
  "serde",
  "similar-asserts",
+ "tap",
  "thiserror 1.0.65",
 ]
 
@@ -5733,6 +5761,7 @@ dependencies = [
  "parking_lot",
  "rand",
  "re_arrow2",
+ "re_arrow_util",
  "re_byte_size",
  "re_chunk",
  "re_format",
@@ -5828,6 +5857,7 @@ version = "0.22.0-alpha.1+dev"
 dependencies = [
  "ahash",
  "anyhow",
+ "arrow",
  "crossbeam",
  "image",
  "notify",
@@ -5905,12 +5935,16 @@ name = "re_dataframe"
 version = "0.22.0-alpha.1+dev"
 dependencies = [
  "anyhow",
+ "arrow",
+ "insta",
  "itertools 0.13.0",
  "nohash-hasher",
  "rayon",
  "re_arrow2",
+ "re_arrow_util",
  "re_chunk",
  "re_chunk_store",
+ "re_format_arrow",
  "re_log",
  "re_log_encoding",
  "re_log_types",
@@ -6003,8 +6037,10 @@ dependencies = [
 name = "re_format_arrow"
 version = "0.22.0-alpha.1+dev"
 dependencies = [
+ "arrow",
  "comfy-table",
- "re_arrow2",
+ "itertools 0.13.0",
+ "re_arrow_util",
  "re_tuid",
  "re_types_core",
 ]
@@ -6013,6 +6049,8 @@ dependencies = [
 name = "re_grpc_client"
 version = "0.22.0-alpha.1+dev"
 dependencies = [
+ "arrow",
+ "re_arrow_util",
  "re_chunk",
  "re_error",
  "re_log",
@@ -6079,6 +6117,7 @@ dependencies = [
 name = "re_log_encoding"
 version = "0.22.0-alpha.1+dev"
 dependencies = [
+ "arrow",
  "criterion",
  "ehttp",
  "js-sys",
@@ -6096,6 +6135,7 @@ dependencies = [
  "re_types",
  "rmp-serde",
  "serde_test",
+ "similar-asserts",
  "thiserror 1.0.65",
  "wasm-bindgen",
  "wasm-bindgen-futures",
@@ -6124,6 +6164,7 @@ dependencies = [
  "num-derive",
  "num-traits",
  "re_arrow2",
+ "re_arrow_util",
  "re_build_info",
  "re_byte_size",
  "re_format",
@@ -6216,6 +6257,7 @@ version = "0.22.0-alpha.1+dev"
 dependencies = [
  "ahash",
  "anyhow",
+ "arrow",
  "backtrace",
  "bytemuck",
  "criterion",
@@ -6227,6 +6269,7 @@ dependencies = [
  "paste",
  "rand",
  "re_arrow2",
+ "re_arrow_util",
  "re_byte_size",
  "re_chunk",
  "re_chunk_store",
@@ -6291,6 +6334,7 @@ dependencies = [
  "ordered-float",
  "parking_lot",
  "pathdiff",
+ "pollster 0.4.0",
  "profiling",
  "re_arrow2",
  "re_build_tools",
@@ -6356,7 +6400,6 @@ dependencies = [
  "once_cell",
  "parking_lot",
  "rand",
- "re_arrow2",
  "re_build_info",
  "re_build_tools",
  "re_byte_size",
@@ -6451,7 +6494,6 @@ dependencies = [
  "anyhow",
  "criterion",
  "egui",
- "egui_kittest",
  "itertools 0.13.0",
  "nohash-hasher",
  "once_cell",
@@ -6587,6 +6629,7 @@ dependencies = [
  "nohash-hasher",
  "once_cell",
  "re_arrow2",
+ "re_arrow_util",
  "re_byte_size",
  "re_case",
  "re_error",
@@ -6610,10 +6653,12 @@ dependencies = [
  "egui_extras",
  "egui_kittest",
  "egui_tiles",
+ "getrandom",
  "itertools 0.13.0",
  "once_cell",
  "parking_lot",
  "rand",
+ "re_arrow_util",
  "re_entity_db",
  "re_format",
  "re_log",
@@ -6708,6 +6753,7 @@ dependencies = [
  "egui",
  "egui_table",
  "itertools 0.13.0",
+ "re_arrow_util",
  "re_chunk_store",
  "re_dataframe",
  "re_error",
@@ -7045,7 +7091,6 @@ dependencies = [
  "ahash",
  "egui",
  "egui_tiles",
- "itertools 0.13.0",
  "nohash-hasher",
  "rayon",
  "re_context_menu",
@@ -7301,6 +7346,7 @@ name = "rerun"
 version = "0.22.0-alpha.1+dev"
 dependencies = [
  "anyhow",
+ "arrow",
  "clap",
  "document-features",
  "env_logger",
@@ -7321,6 +7367,7 @@ dependencies = [
  "re_entity_db",
  "re_error",
  "re_format",
+ "re_format_arrow",
  "re_log",
  "re_log_encoding",
  "re_log_types",
@@ -7367,10 +7414,12 @@ name = "rerun_c"
 version = "0.22.0-alpha.1+dev"
 dependencies = [
  "ahash",
+ "arrow",
  "infer",
  "once_cell",
  "parking_lot",
  "re_arrow2",
+ "re_arrow_util",
  "re_log",
  "re_sdk",
  "re_video",
@@ -7393,7 +7442,7 @@ dependencies = [
  "pyo3",
  "pyo3-build-config",
  "rand",
- "re_arrow2",
+ "re_arrow_util",
  "re_build_info",
  "re_build_tools",
  "re_chunk",
@@ -8507,6 +8556,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "tap"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
+
 [[package]]
 name = "target-lexicon"
 version = "0.12.16"