Merge branch 'main' into feature_fix

databendlabs · Nov 8, 2024 · c8b17fe · c8b17fe
2 parents b948224 + f4e599f
commit c8b17fe
Show file tree

Hide file tree

Showing 187 changed files with 4,769 additions and 4,413 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -219,9 +219,9 @@ arrow-ipc = { version = "53" }
 arrow-ord = { version = "53" }
 arrow-schema = { version = "53", features = ["serde"] }
 arrow-select = { version = "53" }
-arrow-udf-js = "0.5.0"
-arrow-udf-python = "0.4.0"
-arrow-udf-wasm = "0.4.0"
+arrow-udf-js = { git = "https://github.com/arrow-udf/arrow-udf", rev = "80b09d6" }
+arrow-udf-python = { git = "https://github.com/arrow-udf/arrow-udf", rev = "80b09d6" }
+arrow-udf-wasm = { git = "https://github.com/arrow-udf/arrow-udf", rev = "80b09d6" }
 async-backtrace = "0.2"
 async-channel = "1.7.1"
 async-compression = { git = "https://github.com/datafuse-extras/async-compression", rev = "dc81082", features = [
@@ -260,6 +260,7 @@ cidr = { version = "0.2.2" }
 clap = { version = "4.4.2", features = ["derive"] }
 comfy-table = "7"
 convert_case = "0.6.0"
+cookie = "0.18.1"
 crc32fast = "1.3.2"
 criterion = "0.5"
 cron = "0.12.0"

diff --git a/src/common/arrow/src/arrow/array/binview/ffi.rs b/src/common/arrow/src/arrow/array/binview/ffi.rs
@@ -64,7 +64,6 @@ unsafe impl<T: ViewType + ?Sized> ToFfi for BinaryViewArrayGeneric<T> {
             validity,
             views: self.views.clone(),
             buffers: self.buffers.clone(),
-            raw_buffers: self.raw_buffers.clone(),
             phantom: Default::default(),
             total_bytes_len: AtomicU64::new(self.total_bytes_len.load(Ordering::Relaxed)),
             total_buffer_len: self.total_buffer_len,

diff --git a/src/common/arrow/src/arrow/array/binview/from.rs b/src/common/arrow/src/arrow/array/binview/from.rs
@@ -12,13 +12,89 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use arrow_data::ArrayData;
+use arrow_data::ArrayDataBuilder;
+use arrow_schema::DataType;
+
+use crate::arrow::array::Arrow2Arrow;
+use crate::arrow::array::BinaryViewArray;
 use crate::arrow::array::BinaryViewArrayGeneric;
 use crate::arrow::array::MutableBinaryViewArray;
+use crate::arrow::array::Utf8ViewArray;
 use crate::arrow::array::ViewType;
+use crate::arrow::bitmap::Bitmap;
 
 impl<T: ViewType + ?Sized, P: AsRef<T>> FromIterator<Option<P>> for BinaryViewArrayGeneric<T> {
     #[inline]
     fn from_iter<I: IntoIterator<Item = Option<P>>>(iter: I) -> Self {
         MutableBinaryViewArray::<T>::from_iter(iter).into()
     }
 }
+
+impl Arrow2Arrow for BinaryViewArray {
+    fn to_data(&self) -> ArrayData {
+        let builder = ArrayDataBuilder::new(DataType::BinaryView)
+            .len(self.len())
+            .add_buffer(self.views.clone().into())
+            .add_buffers(
+                self.buffers
+                    .iter()
+                    .map(|x| x.clone().into())
+                    .collect::<Vec<_>>(),
+            )
+            .nulls(self.validity.clone().map(Into::into));
+        unsafe { builder.build_unchecked() }
+    }
+
+    fn from_data(data: &ArrayData) -> Self {
+        let views = crate::arrow::buffer::Buffer::from(data.buffers()[0].clone());
+        let buffers = data.buffers()[1..]
+            .iter()
+            .map(|x| crate::arrow::buffer::Buffer::from(x.clone()))
+            .collect();
+        let validity = data.nulls().map(|x| Bitmap::from_null_buffer(x.clone()));
+        unsafe {
+            Self::new_unchecked_unknown_md(
+                crate::arrow::datatypes::DataType::BinaryView,
+                views,
+                buffers,
+                validity,
+                None,
+            )
+        }
+    }
+}
+
+impl Arrow2Arrow for Utf8ViewArray {
+    fn to_data(&self) -> ArrayData {
+        let builder = ArrayDataBuilder::new(DataType::Utf8View)
+            .len(self.len())
+            .add_buffer(self.views.clone().into())
+            .add_buffers(
+                self.buffers
+                    .iter()
+                    .map(|x| x.clone().into())
+                    .collect::<Vec<_>>(),
+            )
+            .nulls(self.validity.clone().map(Into::into));
+        unsafe { builder.build_unchecked() }
+    }
+
+    fn from_data(data: &ArrayData) -> Self {
+        let views = crate::arrow::buffer::Buffer::from(data.buffers()[0].clone());
+        let buffers = data.buffers()[1..]
+            .iter()
+            .map(|x| crate::arrow::buffer::Buffer::from(x.clone()))
+            .collect();
+        let validity = data.nulls().map(|x| Bitmap::from_null_buffer(x.clone()));
+        unsafe {
+            Self::new_unchecked_unknown_md(
+                crate::arrow::datatypes::DataType::Utf8View,
+                views,
+                buffers,
+                validity,
+                None,
+            )
+        }
+    }
+}