Modify get_files_by_partitions to use partition values instead of fil…

…e paths. (#362)
delta-io · Aug 8, 2021 · 61e2941 · 61e2941
1 parent b1925f2
commit 61e2941
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 15 deletions.
diff --git a/rust/src/delta.rs b/rust/src/delta.rs
@@ -765,26 +765,15 @@ impl DeltaTable {
         &self,
         filters: &[PartitionFilter<&str>],
     ) -> Result<Vec<String>, DeltaTableError> {
-        let partitions_number = match &self
-            .state
-            .current_metadata
-            .as_ref()
-            .ok_or(DeltaTableError::NoMetadata)?
-            .partition_columns
-        {
-            partitions if !partitions.is_empty() => partitions.len(),
-            _ => return Err(DeltaTableError::LoadPartitions),
-        };
-        let separator = "/";
         let files = self
             .state
             .files
             .iter()
             .filter(|add| {
                 let partitions = add
-                    .path
-                    .splitn(partitions_number + 1, separator)
-                    .filter_map(|p: &str| DeltaTablePartition::try_from(p).ok())
+                    .partition_values
+                    .iter()
+                    .map(|p| DeltaTablePartition::from_partition_value(p, ""))
                     .collect::<Vec<DeltaTablePartition>>();
                 filters
                     .iter()

diff --git a/rust/src/partitions.rs b/rust/src/partitions.rs
@@ -126,3 +126,18 @@ impl<'a> TryFrom<&'a str> for DeltaTablePartition<'a> {
         }
     }
 }
+
+impl<'a> DeltaTablePartition<'a> {
+    /// Try to create a DeltaTable partition from a partition value kv pair.
+    pub fn from_partition_value(
+        partition_value: (&'a String, &'a Option<String>),
+        default_for_null: &'a str,
+    ) -> Self {
+        let (k, v) = partition_value;
+        let v = match v {
+            Some(s) => s,
+            None => default_for_null,
+        };
+        DeltaTablePartition { key: k, value: v }
+    }
+}
diff --git a/rust/tests/read_delta_test.rs b/rust/tests/read_delta_test.rs
@@ -272,7 +272,7 @@ async fn read_delta_8_0_table_with_null_partition() {
 
     let filters = vec![deltalake::PartitionFilter {
         key: "k",
-        value: deltalake::PartitionValue::Equal("__HIVE_DEFAULT_PARTITION__"),
+        value: deltalake::PartitionValue::Equal(""),
     }];
     assert_eq!(
         table.get_files_by_partitions(&filters).unwrap(),