feat: support custom endpoints

tonbo-io · Nov 1, 2024 · a97407a · a97407a
1 parent a407ef1
commit a97407a
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 13 deletions.
diff --git a/fusio-dispatch/src/lib.rs b/fusio-dispatch/src/lib.rs
@@ -11,6 +11,7 @@ pub enum FsOptions {
     S3 {
         bucket: String,
         credential: Option<fusio::remotes::aws::AwsCredential>,
+        endpoint: Option<String>,
         region: Option<String>,
         sign_payload: Option<bool>,
         checksum: Option<bool>,
@@ -26,6 +27,7 @@ impl FsOptions {
             FsOptions::S3 {
                 bucket,
                 credential,
+                endpoint,
                 region,
                 sign_payload,
                 checksum,
@@ -44,6 +46,9 @@ impl FsOptions {
                         builder = builder.with_token(token);
                     }
                 }
+                if let Some(endpoint) = endpoint {
+                    builder = builder.with_endpoint(endpoint);
+                }
                 if let Some(region) = region {
                     builder = builder.with_region(region);
                 }
@@ -61,6 +66,7 @@ impl FsOptions {
             FsOptions::S3 {
                 bucket,
                 credential,
+                endpoint,
                 region,
                 sign_payload,
                 checksum,
@@ -72,6 +78,9 @@ impl FsOptions {
                 if let Some(credential) = credential {
                     builder = builder.credential(credential);
                 }
+                if let Some(endpoint) = endpoint {
+                    builder = builder.endpoint(endpoint);
+                }
                 if let Some(region) = region {
                     builder = builder.region(region);
                 }

diff --git a/fusio-object-store/src/lib.rs b/fusio-object-store/src/lib.rs
@@ -141,7 +141,7 @@ mod tests {
         if let Ok((region, bucket_name, access_key_id, secret_access_key)) = fn_env() {
             let path = object_store::path::Path::parse("/test_file").unwrap();
             let s3 = AmazonS3Builder::new()
-                .with_region(region)
+                .with_aws_region(region)
                 .with_bucket_name(bucket_name)
                 .with_access_key_id(access_key_id)
                 .with_secret_access_key(secret_access_key)

diff --git a/fusio/src/impls/remotes/aws/fs.rs b/fusio/src/impls/remotes/aws/fs.rs
@@ -21,6 +21,7 @@ use crate::{
 };
 
 pub struct AmazonS3Builder {
+    endpoint: Option<String>,
     region: String,
     bucket: String,
     credential: Option<AwsCredential>,
@@ -36,6 +37,7 @@ impl AmazonS3Builder {
             if #[cfg(all(feature = "tokio-http", not(feature = "completion-based")))] {
                 let client = Box::new(crate::remotes::http::tokio::TokioClient::new());
                 Self {
+                    endpoint: None,
                     region: "us-east-1".into(),
                     bucket,
                     credential: None,
@@ -56,6 +58,11 @@ impl AmazonS3Builder {
         self
     }
 
+    pub fn endpoint(mut self, endpoint: String) -> Self {
+        self.endpoint = Some(endpoint);
+        self
+    }
+
     pub fn credential(mut self, credential: AwsCredential) -> Self {
         self.credential = Some(credential);
         self
@@ -72,10 +79,21 @@ impl AmazonS3Builder {
     }
 
     pub fn build(self) -> AmazonS3 {
+        let trimmed_bucket = self.bucket.trim_start_matches('/');
+        let endpoint = if let Some(endpoint) = self.endpoint {
+            let trimmed_endpoint = endpoint.trim_end_matches('/');
+            format!("{}/{}", trimmed_endpoint, trimmed_bucket)
+        } else {
+            format!(
+                "https://{}.s3.{}.amazonaws.com",
+                trimmed_bucket, self.region
+            )
+        };
+
         AmazonS3 {
             inner: Arc::new(AmazonS3Inner {
                 options: S3Options {
-                    endpoint: format!("https://{}.s3.{}.amazonaws.com", self.bucket, self.region),
+                    endpoint,
                     region: self.region,
                     credential: self.credential,
                     sign_payload: self.sign_payload,

diff --git a/fusio/src/impls/remotes/aws/s3.rs b/fusio/src/impls/remotes/aws/s3.rs
@@ -253,10 +253,11 @@ impl Write for S3File {
 
 #[cfg(test)]
 mod tests {
+    #[ignore]
     #[cfg(all(feature = "tokio-http", not(feature = "completion-based")))]
     #[tokio::test]
     async fn write_and_read_s3_file() {
-        use std::{env, sync::Arc};
+        use std::sync::Arc;
 
         use crate::{
             remotes::{
@@ -271,17 +272,13 @@ mod tests {
             Read, Write,
         };
 
-        if env::var("AWS_ACCESS_KEY_ID").is_err() {
-            eprintln!("skipping AWS s3 test");
-            return;
-        }
-        let key_id = env::var("AWS_ACCESS_KEY_ID").unwrap();
-        let secret_key = env::var("AWS_SECRET_ACCESS_KEY").unwrap();
+        let key_id = "user".to_string();
+        let secret_key = "password".to_string();
 
         let client = TokioClient::new();
         let region = "ap-southeast-1";
         let options = S3Options {
-            endpoint: "https://fusio-test.s3.ap-southeast-1.amazonaws.com".into(),
+            endpoint: "http://localhost:9000/data".into(),
             credential: Some(AwsCredential {
                 key_id,
                 secret_key,

diff --git a/fusio/src/impls/remotes/aws/writer.rs b/fusio/src/impls/remotes/aws/writer.rs
@@ -139,10 +139,10 @@ mod tests {
 
         let region = "ap-southeast-2";
         let options = S3Options {
-            endpoint: "endpoint".into(),
+            endpoint: "http://localhost:9000/data".into(),
             credential: Some(AwsCredential {
-                key_id: "key".to_string(),
-                secret_key: "secret_key".to_string(),
+                key_id: "user".to_string(),
+                secret_key: "password".to_string(),
                 token: None,
             }),
             region: region.into(),