From b4ca0a173836d867734ebe5755b5c2990348d56d Mon Sep 17 00:00:00 2001
From: Sergey Kulik <104143901+zxqfd555-pw@users.noreply.github.com>
Date: Tue, 15 Oct 2024 18:34:04 +0200
Subject: [PATCH] allow more customization in deltalake connector (#7478)

GitOrigin-RevId: 7bde44c2c966ec00aeb5de5bb4108f5970ce41cc
---
 CHANGELOG.md      | 4 ++++
 src/python_api.rs | 9 ++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 669c69b3..ed163098 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
+### Added
+- `pw.io.deltalake.read` now supports custom S3 Delta Lakes with HTTP endpoints.
+- `pw.io.deltalake.read` now supports specifying both a custom endpoint and a custom region for Delta Lakes via `pw.io.s3.AwsS3Settings`.
+
 ## [0.15.1] - 2024-10-04
 
 ### Fixed
diff --git a/src/python_api.rs b/src/python_api.rs
index ea0bbf36..8c8de32d 100644
--- a/src/python_api.rs
+++ b/src/python_api.rs
@@ -4220,7 +4220,7 @@ impl DataStorage {
             storage_options.insert("AWS_PROFILE".to_string(), profile.to_string());
         }
 
-        if let s3::Region::Custom { endpoint, .. } = &s3_settings.region {
+        if let s3::Region::Custom { endpoint, region } = &s3_settings.region {
             if endpoint.starts_with("https://") || endpoint.starts_with("http://") {
                 storage_options.insert("AWS_ENDPOINT_URL".to_string(), endpoint.to_string());
             } else {
@@ -4229,12 +4229,15 @@ impl DataStorage {
                     format!("https://{endpoint}"),
                 );
             }
+            storage_options.insert("AWS_ALLOW_HTTP".to_string(), "True".to_string());
+            storage_options.insert("AWS_STORAGE_ALLOW_HTTP".to_string(), "True".to_string());
+            if region != endpoint {
+                storage_options.insert("AWS_REGION".to_string(), region.to_string());
+            }
         } else {
             storage_options.insert("AWS_REGION".to_string(), s3_settings.region.to_string());
         }
 
-        warn!("{:?}", storage_options);
-
         Ok(storage_options)
     }