From a97407aaea1a82fde1cff997e4db13be0f24b517 Mon Sep 17 00:00:00 2001 From: kkould <2435992353@qq.com> Date: Fri, 1 Nov 2024 09:38:47 +0000 Subject: [PATCH] feat: support custom endpoints --- fusio-dispatch/src/lib.rs | 9 +++++++++ fusio-object-store/src/lib.rs | 2 +- fusio/src/impls/remotes/aws/fs.rs | 20 +++++++++++++++++++- fusio/src/impls/remotes/aws/s3.rs | 13 +++++-------- fusio/src/impls/remotes/aws/writer.rs | 6 +++--- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/fusio-dispatch/src/lib.rs b/fusio-dispatch/src/lib.rs index e48d2dc..54b770e 100644 --- a/fusio-dispatch/src/lib.rs +++ b/fusio-dispatch/src/lib.rs @@ -11,6 +11,7 @@ pub enum FsOptions { S3 { bucket: String, credential: Option, + endpoint: Option, region: Option, sign_payload: Option, checksum: Option, @@ -26,6 +27,7 @@ impl FsOptions { FsOptions::S3 { bucket, credential, + endpoint, region, sign_payload, checksum, @@ -44,6 +46,9 @@ impl FsOptions { builder = builder.with_token(token); } } + if let Some(endpoint) = endpoint { + builder = builder.with_endpoint(endpoint); + } if let Some(region) = region { builder = builder.with_region(region); } @@ -61,6 +66,7 @@ impl FsOptions { FsOptions::S3 { bucket, credential, + endpoint, region, sign_payload, checksum, @@ -72,6 +78,9 @@ impl FsOptions { if let Some(credential) = credential { builder = builder.credential(credential); } + if let Some(endpoint) = endpoint { + builder = builder.endpoint(endpoint); + } if let Some(region) = region { builder = builder.region(region); } diff --git a/fusio-object-store/src/lib.rs b/fusio-object-store/src/lib.rs index dd945ef..1c6091f 100644 --- a/fusio-object-store/src/lib.rs +++ b/fusio-object-store/src/lib.rs @@ -141,7 +141,7 @@ mod tests { if let Ok((region, bucket_name, access_key_id, secret_access_key)) = fn_env() { let path = object_store::path::Path::parse("/test_file").unwrap(); let s3 = AmazonS3Builder::new() - .with_region(region) + .with_aws_region(region) .with_bucket_name(bucket_name) .with_access_key_id(access_key_id) .with_secret_access_key(secret_access_key) diff --git a/fusio/src/impls/remotes/aws/fs.rs b/fusio/src/impls/remotes/aws/fs.rs index a469b05..022a968 100644 --- a/fusio/src/impls/remotes/aws/fs.rs +++ b/fusio/src/impls/remotes/aws/fs.rs @@ -21,6 +21,7 @@ use crate::{ }; pub struct AmazonS3Builder { + endpoint: Option, region: String, bucket: String, credential: Option, @@ -36,6 +37,7 @@ impl AmazonS3Builder { if #[cfg(all(feature = "tokio-http", not(feature = "completion-based")))] { let client = Box::new(crate::remotes::http::tokio::TokioClient::new()); Self { + endpoint: None, region: "us-east-1".into(), bucket, credential: None, @@ -56,6 +58,11 @@ impl AmazonS3Builder { self } + pub fn endpoint(mut self, endpoint: String) -> Self { + self.endpoint = Some(endpoint); + self + } + pub fn credential(mut self, credential: AwsCredential) -> Self { self.credential = Some(credential); self @@ -72,10 +79,21 @@ impl AmazonS3Builder { } pub fn build(self) -> AmazonS3 { + let trimmed_bucket = self.bucket.trim_start_matches('/'); + let endpoint = if let Some(endpoint) = self.endpoint { + let trimmed_endpoint = endpoint.trim_end_matches('/'); + format!("{}/{}", trimmed_endpoint, trimmed_bucket) + } else { + format!( + "https://{}.s3.{}.amazonaws.com", + trimmed_bucket, self.region + ) + }; + AmazonS3 { inner: Arc::new(AmazonS3Inner { options: S3Options { - endpoint: format!("https://{}.s3.{}.amazonaws.com", self.bucket, self.region), + endpoint, region: self.region, credential: self.credential, sign_payload: self.sign_payload, diff --git a/fusio/src/impls/remotes/aws/s3.rs b/fusio/src/impls/remotes/aws/s3.rs index 5644e67..3c7aa70 100644 --- a/fusio/src/impls/remotes/aws/s3.rs +++ b/fusio/src/impls/remotes/aws/s3.rs @@ -253,10 +253,11 @@ impl Write for S3File { #[cfg(test)] mod tests { + #[ignore] #[cfg(all(feature = "tokio-http", not(feature = "completion-based")))] #[tokio::test] async fn write_and_read_s3_file() { - use std::{env, sync::Arc}; + use std::sync::Arc; use crate::{ remotes::{ @@ -271,17 +272,13 @@ mod tests { Read, Write, }; - if env::var("AWS_ACCESS_KEY_ID").is_err() { - eprintln!("skipping AWS s3 test"); - return; - } - let key_id = env::var("AWS_ACCESS_KEY_ID").unwrap(); - let secret_key = env::var("AWS_SECRET_ACCESS_KEY").unwrap(); + let key_id = "user".to_string(); + let secret_key = "password".to_string(); let client = TokioClient::new(); let region = "ap-southeast-1"; let options = S3Options { - endpoint: "https://fusio-test.s3.ap-southeast-1.amazonaws.com".into(), + endpoint: "http://localhost:9000/data".into(), credential: Some(AwsCredential { key_id, secret_key, diff --git a/fusio/src/impls/remotes/aws/writer.rs b/fusio/src/impls/remotes/aws/writer.rs index f937337..520a18d 100644 --- a/fusio/src/impls/remotes/aws/writer.rs +++ b/fusio/src/impls/remotes/aws/writer.rs @@ -139,10 +139,10 @@ mod tests { let region = "ap-southeast-2"; let options = S3Options { - endpoint: "endpoint".into(), + endpoint: "http://localhost:9000/data".into(), credential: Some(AwsCredential { - key_id: "key".to_string(), - secret_key: "secret_key".to_string(), + key_id: "user".to_string(), + secret_key: "password".to_string(), token: None, }), region: region.into(),