diff --git a/bigquery/Cargo.toml b/bigquery/Cargo.toml index 62ea4e23..7ed7a1c2 100644 --- a/bigquery/Cargo.toml +++ b/bigquery/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-cloud-bigquery" -version = "0.2.0" +version = "0.3.0" edition = "2021" authors = ["yoshidan "] repository = "https://github.com/yoshidan/google-cloud-rust/tree/main/bigquery" @@ -22,13 +22,13 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio = { version="1.20", features=["macros"] } time = { version = "0.3", features = ["std", "macros", "formatting", "parsing", "serde"] } -arrow = { version="42.0", default-features = false, features = ["ipc"] } +arrow = { version="44.0", default-features = false, features = ["ipc"] } base64 = "0.21" bigdecimal = { version="0.3", features=["serde"] } num-bigint = "0.4" backon = "0.4" -google-cloud-auth = { optional = true, version = "0.11", path="../foundation/auth", default-features=false } +google-cloud-auth = { optional = true, version = "0.12", path="../foundation/auth", default-features=false } [dev-dependencies] tokio = { version="1.20", features=["rt-multi-thread"] } @@ -41,7 +41,8 @@ base64-serde = "0.7" [features] default = ["default-tls", "auth"] -default-tls = ["reqwest/default-tls"] -rustls-tls = ["reqwest/rustls-tls"] +default-tls = ["reqwest/default-tls","google-cloud-auth?/default-tls"] +rustls-tls = ["reqwest/rustls-tls","google-cloud-auth?/rustls-tls"] trace = [] auth = ["google-cloud-auth"] +external-account = ["google-cloud-auth?/external-account"] diff --git a/deny.toml b/deny.toml index 665e7178..6cf9dd52 100644 --- a/deny.toml +++ b/deny.toml @@ -235,8 +235,9 @@ skip = [ { name = "syn", version = "=1.0.109" }, { name = "regex-syntax", version = "=0.6.29" }, { name = "webpki-roots", version = "=0.22.6" }, + { name = "rustls-webpki", version = "=0.100.1" }, + { name = "regex-automata", version = "=0.1.10" }, { name = "hashbrown", version = "=0.14.0" }, - { name = "miniz_oxide", version = "=0.6.2" } ] # Similarly to `skip` allows you to skip certain crates during duplicate # detection. Unlike skip, it also includes the entire tree of transitive diff --git a/foundation/auth/Cargo.toml b/foundation/auth/Cargo.toml index 3606b898..562d8c6e 100644 --- a/foundation/auth/Cargo.toml +++ b/foundation/auth/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-cloud-auth" -version = "0.11.0" +version = "0.12.0" authors = ["yoshidan "] edition = "2021" repository = "https://github.com/yoshidan/google-cloud-rust/tree/main/foundation/auth" @@ -25,6 +25,13 @@ google-cloud-token = { version = "0.1.1", path = "../token" } base64 = "0.21" time = "0.3" +url = { version="2.4", optional = true } +path-clean = { version="1.0", optional = true } +sha2 = {version = "0.10", optional = true} +percent-encoding = { version="2.3", optional = true } +hmac = { version = "0.12", optional = true } +hex = { version = "0.4", optional = true } + [dev-dependencies] tokio = { version = "1.20", features = ["test-util", "rt-multi-thread", "macros"]} tracing-subscriber = {version="0.3", features=["env-filter","std"]} @@ -35,3 +42,4 @@ serial_test = "0.9" default = ["default-tls"] default-tls = ["reqwest/default-tls"] rustls-tls = ["reqwest/rustls-tls"] +external-account = ["sha2", "path-clean", "url", "percent-encoding", "hmac", "hex"] diff --git a/foundation/auth/README.md b/foundation/auth/README.md index 4ce964b7..71d27c9e 100644 --- a/foundation/auth/README.md +++ b/foundation/auth/README.md @@ -61,7 +61,7 @@ preferring the first location found: https://cloud.google.com/iam/docs/workload-identity-federation -- [ ] AWS +- [x] AWS - [ ] Azure Active Directory - [ ] On-premises Active Directory - [ ] Okta diff --git a/foundation/auth/src/credentials.rs b/foundation/auth/src/credentials.rs index 7bd40cbb..bff57cc1 100644 --- a/foundation/auth/src/credentials.rs +++ b/foundation/auth/src/credentials.rs @@ -7,23 +7,42 @@ use crate::error::Error; const CREDENTIALS_FILE: &str = "application_default_credentials.json"; #[allow(dead_code)] -#[derive(Deserialize, Clone)] -pub(crate) struct Format { - tp: String, - subject_token_field_name: String, +#[derive(Deserialize, Clone, Debug)] +pub struct ServiceAccountImpersonationInfo { + pub(crate) token_lifetime_seconds: i32, } #[allow(dead_code)] -#[derive(Deserialize, Clone)] +#[derive(Deserialize, Clone, Debug)] +pub struct ExecutableConfig { + pub(crate) command: String, + pub(crate) timeout_millis: Option, + pub(crate) output_file: String, +} + +#[allow(dead_code)] +#[derive(Deserialize, Clone, Debug)] +pub struct Format { + pub(crate) tp: String, + pub(crate) subject_token_field_name: String, +} + +#[allow(dead_code)] +#[derive(Deserialize, Clone, Debug)] pub struct CredentialSource { - file: String, - url: String, - headers: std::collections::HashMap, - environment_id: String, - region_url: String, - regional_cred_verification_url: String, - cred_verification_url: String, - format: Format, + pub(crate) file: Option, + + pub(crate) url: Option, + pub(crate) headers: Option>, + + pub(crate) executable: Option, + + pub(crate) environment_id: Option, + pub(crate) region_url: Option, + pub(crate) regional_cred_verification_url: Option, + pub(crate) cred_verification_url: Option, + pub(crate) imdsv2_session_token_url: Option, + pub(crate) format: Option, } #[allow(dead_code)] @@ -49,21 +68,15 @@ pub struct CredentialsFile { // External Account fields pub audience: Option, pub subject_token_type: Option, + #[serde(rename = "token_url")] pub token_url_external: Option, pub token_info_url: Option, pub service_account_impersonation_url: Option, + pub service_account_impersonation: Option, + pub delegates: Option>, pub credential_source: Option, pub quota_project_id: Option, -} - -#[allow(dead_code)] -#[derive(Deserialize, Clone)] -pub struct Credentials { - client_id: String, - client_secret: String, - redirect_urls: Vec, - auth_uri: String, - token_uri: String, + pub workforce_pool_user_project: Option, } impl CredentialsFile { diff --git a/foundation/auth/src/error.rs b/foundation/auth/src/error.rs index d424a198..44400854 100644 --- a/foundation/auth/src/error.rs +++ b/foundation/auth/src/error.rs @@ -38,4 +38,14 @@ pub enum Error { #[error("invalid authentication token")] InvalidToken, + + #[error(transparent)] + TimeParse(#[from] time::error::Parse), + + #[cfg(feature = "external-account")] + #[error("external account error : {0}")] + ExternalAccountSource(#[from] crate::token_source::external_account_source::error::Error), + + #[error("unexpected impersonation token response : status={0}, detail={1}")] + UnexpectedImpersonateTokenResponse(u16, String), } diff --git a/foundation/auth/src/project.rs b/foundation/auth/src/project.rs index b5b03d7d..a92c7cc0 100644 --- a/foundation/auth/src/project.rs +++ b/foundation/auth/src/project.rs @@ -12,6 +12,8 @@ use crate::{credentials, error}; pub(crate) const SERVICE_ACCOUNT_KEY: &str = "service_account"; const USER_CREDENTIALS_KEY: &str = "authorized_user"; +#[cfg(feature = "external-account")] +const EXTERNAL_ACCOUNT_KEY: &str = "external_account"; #[derive(Debug, Clone, Default)] pub struct Config<'a> { @@ -82,7 +84,7 @@ pub async fn create_token_source_from_credentials( credentials: &CredentialsFile, config: &Config<'_>, ) -> Result, error::Error> { - let ts = credentials_from_json_with_params(credentials, config)?; + let ts = credentials_from_json_with_params(credentials, config).await?; let token = ts.token().await?; Ok(Box::new(ReuseTokenSource::new(ts, token))) } @@ -109,9 +111,9 @@ pub async fn create_token_source(config: Config<'_>) -> Result, ) -> Result, error::Error> { match credentials.tp.as_str() { SERVICE_ACCOUNT_KEY => { @@ -137,8 +139,34 @@ fn credentials_from_json_with_params( } } USER_CREDENTIALS_KEY => Ok(Box::new(UserAccountTokenSource::new(credentials)?)), - //TODO support GDC https://console.developers.google.com, - //TODO support external account + #[cfg(feature = "external-account")] + EXTERNAL_ACCOUNT_KEY => { + let ts = crate::token_source::external_account_source::ExternalAccountTokenSource::new( + config.scopes_to_string(" "), + credentials.clone(), + ) + .await?; + if let Some(impersonation_url) = &credentials.service_account_impersonation_url { + let url = impersonation_url.clone(); + let mut scopes = config.scopes.map(|v| v.to_vec()).unwrap_or(vec![]); + scopes.push("https://www.googleapis.com/auth/cloud-platform"); + let scopes = scopes.iter().map(|e| e.to_string()).collect(); + let lifetime = credentials + .service_account_impersonation + .clone() + .map(|v| v.token_lifetime_seconds); + let ts = crate::token_source::impersonate_token_source::ImpersonateTokenSource::new( + url, + vec![], + scopes, + lifetime, + Box::new(ts), + ); + Ok(Box::new(ts)) + } else { + Ok(Box::new(ts)) + } + } _ => Err(error::Error::UnsupportedAccountType(credentials.tp.to_string())), } } diff --git a/foundation/auth/src/token_source/external_account_source/aws_subject_token_source.rs b/foundation/auth/src/token_source/external_account_source/aws_subject_token_source.rs new file mode 100644 index 00000000..b08df019 --- /dev/null +++ b/foundation/auth/src/token_source/external_account_source/aws_subject_token_source.rs @@ -0,0 +1,405 @@ +use std::env::var; +use std::fmt::{Debug, Formatter}; +use std::path::PathBuf; + +use async_trait::async_trait; +use hmac::Mac; +use path_clean::PathClean; +use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use time::macros::format_description; +use time::OffsetDateTime; +use url::Url; + +use crate::credentials::CredentialSource; +use crate::misc::UnwrapOrEmpty; +use crate::token_source::default_http_client; +use crate::token_source::external_account_source::error::Error; +use crate::token_source::external_account_source::subject_token_source::SubjectTokenSource; + +const AWS_ALGORITHM: &str = "AWS4-HMAC-SHA256"; +const AWS_REQUEST_TYPE: &str = "aws4_request"; +const AWS_ACCESS_KEY_ID: &str = "AWS_ACCESS_KEY_ID"; +const AWS_DEFAULT_REGION: &str = "AWS_DEFAULT_REGION"; +const AWS_REGION: &str = "AWS_REGION"; +const AWS_SECRET_ACCESS_KEY: &str = "AWS_SECRET_ACCESS_KEY"; +const AWS_SESSION_TOKEN: &str = "AWS_SESSION_TOKEN"; +const AWS_IMDS_V2_SESSION_TOKEN_HEADER: &str = "X-aws-ec2-metadata-token"; + +pub struct AWSSubjectTokenSource { + subject_token_url: Url, + target_resource: Option, + credentials: AWSSecurityCredentials, + region: String, +} + +impl Debug for AWSSubjectTokenSource { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AWSSubjectTokenSource") + .field("target_resource", &self.target_resource) + .field("region", &self.region) + .finish_non_exhaustive() + } +} + +impl AWSSubjectTokenSource { + pub async fn new(audience: Option, value: CredentialSource) -> Result { + if !validate_metadata_server(&value.region_url) { + return Err(Error::InvalidRegionURL(value.region_url.unwrap_or_empty())); + } + // Not value.cred_verification_url but value.url + if !validate_metadata_server(&value.url) { + return Err(Error::InvalidSecurityCredentialsURL(value.url.unwrap_or_empty())); + } + if !validate_metadata_server(&value.imdsv2_session_token_url) { + return Err(Error::InvalidIMDSv2SessionTokenURL( + value.imdsv2_session_token_url.unwrap_or_empty(), + )); + } + + let aws_session_token = if should_use_metadata_server() { + get_aws_session_token(&value.imdsv2_session_token_url).await? + } else { + None + }; + + let credentials = get_security_credentials(&aws_session_token, &value.url).await?; + let region = get_region(&aws_session_token, &value.region_url).await?; + + let url = value + .regional_cred_verification_url + .as_ref() + .ok_or(Error::MissingRegionalCredVerificationURL)?; + let subject_token_url = Url::parse(&url.replace("{region}", ®ion))?; + + Ok(Self { + subject_token_url, + target_resource: audience, + credentials, + region, + }) + } + + fn create_auth_header( + &self, + method: &str, + now: &OffsetDateTime, + headers: &[(&str, &str)], + ) -> Result { + let date_stamp_short = now.format(&format_description!("[year][month][day]"))?; + let service_name: Vec = self + .subject_token_url + .host_str() + .unwrap_or_default() + .split('.') + .map(|v| v.to_string()) + .collect(); + let service_name = service_name[0].as_str(); + let credential_scope = format!("{}/{}/{}/{}", date_stamp_short, &self.region, service_name, AWS_REQUEST_TYPE); + + let (header_keys, header_values) = canonical_headers(headers); + let query = self.subject_token_url.query().unwrap_or_default(); + let path = self.subject_token_url.path(); + let path = if path.is_empty() { + "/".to_string() + } else { + PathBuf::from(path).clean().to_string_lossy().to_string() + }; + + // canonicalize request + let data_hash = hex::encode(Sha256::digest(vec![])); // hash for empty body + let request_string = format!( + "{}\n{}\n{}\n{}\n{}\n{}", + method, path, query, header_values, header_keys, data_hash + ); + let request_hash = hex::encode(Sha256::digest(request_string.as_bytes())); + let date_stamp_long = now.format(&format_description!("[year][month][day]T[hour][minute][second]Z"))?; + let string_to_sign = format!("{}\n{}\n{}\n{}", AWS_ALGORITHM, date_stamp_long, credential_scope, request_hash); + + // sign + let mut signing_key = format!("AWS4{}", self.credentials.secret_access_key).into_bytes(); + for input in [ + date_stamp_short.as_str(), + self.region.as_str(), + service_name, + AWS_REQUEST_TYPE, + string_to_sign.as_str(), + ] { + let mut mac = hmac::Hmac::::new_from_slice(&signing_key)?; + mac.update(input.as_bytes()); + let result = mac.finalize(); + signing_key = result.into_bytes().to_vec(); + } + + Ok(format!( + "{} Credential={}/{}, SignedHeaders={}, Signature={}", + AWS_ALGORITHM, + self.credentials.access_key_id, + credential_scope, + header_keys, + hex::encode(signing_key) + )) + } + + fn create_subject_token(&self, now: OffsetDateTime) -> Result { + let format_date = now.format(&format_description!("[year][month][day]T[hour][minute][second]Z"))?; + let mut sorted_headers: Vec<(&str, &str)> = vec![ + ("host", self.subject_token_url.host_str().unwrap_or("")), + ("x-amz-date", &format_date), + ]; + if let Some(security_token) = &self.credentials.token { + sorted_headers.push(("x-amz-security-token", security_token)); + } + // The full, canonical resource name of the workload identity pool + // provider, with or without the HTTPS prefix. + // Including this header as part of the signature is recommended to + // ensure data integrity. + if let Some(target_resource) = &self.target_resource { + sorted_headers.push(("x-goog-cloud-target-resource", target_resource)); + } + let method = "POST"; + let authorization = self.create_auth_header(method, &now, &sorted_headers)?; + + let mut aws_headers = Vec::with_capacity(sorted_headers.len() + 1); + aws_headers.push(AWSRequestHeader { + key: "Authorization".to_string(), + value: authorization, + }); + for header in sorted_headers { + aws_headers.push(AWSRequestHeader { + key: header.0.to_string(), + value: header.1.to_string(), + }) + } + let aws_request = AWSRequest { + url: self.subject_token_url.to_string(), + method, + headers: aws_headers, + }; + let result = serde_json::to_string(&aws_request)?; + Ok(utf8_percent_encode(&result, NON_ALPHANUMERIC).to_string()) + } +} + +#[async_trait] +impl SubjectTokenSource for AWSSubjectTokenSource { + async fn subject_token(&self) -> Result { + self.create_subject_token(OffsetDateTime::now_utc()) + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct AWSSecurityCredentials { + access_key_id: String, + secret_access_key: String, + token: Option, +} + +#[derive(Serialize)] +struct AWSRequestHeader { + key: String, + value: String, +} + +#[derive(Serialize)] +struct AWSRequest { + url: String, + method: &'static str, + headers: Vec, +} + +const VALID_HOST_NAMES: [&str; 2] = ["169.254.169.254", "fd00:ec2::254"]; + +fn validate_metadata_server(metadata_url: &Option) -> bool { + let metadata_url = metadata_url.unwrap_or_empty(); + if metadata_url.is_empty() { + return true; + } + let host = match Url::parse(&metadata_url) { + Err(_) => return false, + Ok(v) => v, + }; + + VALID_HOST_NAMES.contains(&host.host_str().unwrap_or("")) +} + +fn should_use_metadata_server() -> bool { + !can_retrieve_region_from_environment() || !can_retrieve_security_credential_from_environment() +} + +fn can_retrieve_region_from_environment() -> bool { + var(AWS_REGION).is_ok() || var(AWS_DEFAULT_REGION).is_ok() +} + +fn can_retrieve_security_credential_from_environment() -> bool { + var(AWS_ACCESS_KEY_ID).is_ok() && var(AWS_SECRET_ACCESS_KEY).is_ok() +} + +async fn get_aws_session_token(imds_v2_session_token_url: &Option) -> Result, Error> { + let url = match imds_v2_session_token_url { + Some(url) => url, + None => return Ok(None), + }; + + let client = default_http_client(); + let response = client + .put(url) + .header("X-aws-ec2-metadata-token-ttl-seconds", "300") + .send() + .await?; + if !response.status().is_success() { + return Err(Error::UnexpectedStatusOnGetSessionToken(response.status().as_u16())); + } + Ok(response.text().await.map(Some)?) +} + +async fn get_security_credentials( + temporary_session_token: &Option, + url: &Option, +) -> Result { + if can_retrieve_security_credential_from_environment() { + return Ok(AWSSecurityCredentials { + access_key_id: var(AWS_ACCESS_KEY_ID).unwrap(), + secret_access_key: var(AWS_SECRET_ACCESS_KEY).unwrap(), + token: var(AWS_SESSION_TOKEN).ok(), + }); + } + + // get metadata role name + let url = url.as_ref().ok_or(Error::MissingSecurityCredentialsURL)?; + let client = default_http_client(); + let mut builder = client.get(url); + if let Some(token) = temporary_session_token { + builder = builder.header(AWS_IMDS_V2_SESSION_TOKEN_HEADER, token); + } + let response = builder.send().await?; + if !response.status().is_success() { + return Err(Error::UnexpectedStatusOnGetRoleName(response.status().as_u16())); + } + let role_name = response.text().await?; + + // get metadata security credentials + let url = format!("{}/{}", url, role_name); + let mut builder = client.get(url); + if let Some(token) = temporary_session_token { + builder = builder.header(AWS_IMDS_V2_SESSION_TOKEN_HEADER, token); + } + let response = builder.send().await?; + if !response.status().is_success() { + return Err(Error::UnexpectedStatusOnGetCredentials(response.status().as_u16())); + } + let cred: AWSSecurityCredentials = response.json().await?; + Ok(cred) +} + +async fn get_region(temporary_session_token: &Option, url: &Option) -> Result { + if can_retrieve_region_from_environment() { + if let Ok(region) = var(AWS_REGION) { + return Ok(region); + } + return Ok(var(AWS_DEFAULT_REGION).unwrap()); + } + let url = url.as_ref().ok_or(Error::MissingRegionURL)?; + let client = default_http_client(); + let mut builder = client.get(url); + if let Some(token) = temporary_session_token { + builder = builder.header(AWS_IMDS_V2_SESSION_TOKEN_HEADER, token); + } + let response = builder.send().await?; + if !response.status().is_success() { + return Err(Error::UnexpectedStatusOnGetRegion(response.status().as_u16())); + } + let body = response.bytes().await?; + + // This endpoint will return the region in format: us-east-2b. + // Only the us-east-2 part should be used. + let resp_body_end = if !body.is_empty() { body.len() - 1 } else { 0 }; + Ok(String::from_utf8_lossy(&body[0..resp_body_end]).to_string()) +} + +fn canonical_headers<'a>(sorted_headers: &[(&'a str, &'a str)]) -> (String, String) { + let mut full_headers: Vec = Vec::with_capacity(sorted_headers.len()); + let mut keys = Vec::with_capacity(sorted_headers.len()); + for header in sorted_headers { + keys.push(header.0); + full_headers.push(format!("{}:{}\n", header.0, header.1)); + } + (keys.join(";"), full_headers.join("")) +} + +#[cfg(test)] +mod tests { + use time::macros::{datetime, format_description}; + use url::Url; + + use crate::credentials::CredentialsFile; + use crate::token_source::external_account_source::aws_subject_token_source::{ + AWSSecurityCredentials, AWSSubjectTokenSource, + }; + + fn create_token_source() -> AWSSubjectTokenSource { + let cred = r#"{ + "type": "external_account", + "audience": "//iam.googleapis.com/projects/myprojectnumber/locations/global/workloadIdentityPools/aws-test/providers/aws-test", + "subject_token_type": "urn:ietf:params:aws:token-type:aws4_request", + "service_account_impersonation_url": "https://iamcredentials.googleapis.com/test", + "token_url": "https://sts.googleapis.com/v1/token", + "credential_source": { + "environment_id": "aws1", + "region_url": "http://169.254.169.254/latest/meta-data/placement/availability-zone", + "url": "http://169.254.169.254/latest/meta-data/iam/security-credentials", + "regional_cred_verification_url": "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15" + } + }"#; + let region = "ap-northeast-1b".to_string(); + let cred: CredentialsFile = serde_json::from_str(cred).unwrap(); + let url = cred.credential_source.unwrap().regional_cred_verification_url.unwrap(); + let subject_token_url = Url::parse(&url.replace("{region}", ®ion)).unwrap(); + + AWSSubjectTokenSource { + subject_token_url, + target_resource: cred.audience, + credentials: AWSSecurityCredentials { + access_key_id: "AccessKeyId".to_string(), + secret_access_key: "SecretAccessKey".to_string(), + token: Some("SecurityToken".to_string()), + }, + region, + } + } + #[test] + fn test_create_auth_header() { + let source = create_token_source(); + let now = datetime!(2022-12-31 00:00:00).assume_utc(); + let format_date = now + .format(&format_description!("[year][month][day]T[hour][minute][second]Z")) + .unwrap(); + let sorted_headers: Vec<(&str, &str)> = vec![ + ("host", source.subject_token_url.host_str().unwrap_or("")), + ("x-amz-date", &format_date), + ("x-amz-security-token", source.credentials.token.as_ref().unwrap()), + ("x-goog-cloud-target-resource", source.target_resource.as_ref().unwrap()), + ]; + let actual = source.create_auth_header("POST", &now, &sorted_headers).unwrap(); + let expected = "AWS4-HMAC-SHA256 Credential=AccessKeyId/20221231/ap-northeast-1b/sts/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token;x-goog-cloud-target-resource, Signature=168a40df8b7c11fb0588a13cada1443e31e4736de702232f9a2177b26edda21c"; + assert_eq!(actual, expected); + } + + #[test] + fn test_create_subject_token() { + let source = create_token_source(); + let now = datetime!(2022-12-31 00:00:00).assume_utc(); + match source.create_subject_token(now) { + Ok(token) => { + let expected = "%7B%22url%22%3A%22https%3A%2F%2Fsts%2Eap%2Dnortheast%2D1b%2Eamazonaws%2Ecom%2F%3FAction%3DGetCallerIdentity%26Version%3D2011%2D06%2D15%22%2C%22method%22%3A%22POST%22%2C%22headers%22%3A%5B%7B%22key%22%3A%22Authorization%22%2C%22value%22%3A%22AWS4%2DHMAC%2DSHA256%20Credential%3DAccessKeyId%2F20221231%2Fap%2Dnortheast%2D1b%2Fsts%2Faws4%5Frequest%2C%20SignedHeaders%3Dhost%3Bx%2Damz%2Ddate%3Bx%2Damz%2Dsecurity%2Dtoken%3Bx%2Dgoog%2Dcloud%2Dtarget%2Dresource%2C%20Signature%3D168a40df8b7c11fb0588a13cada1443e31e4736de702232f9a2177b26edda21c%22%7D%2C%7B%22key%22%3A%22host%22%2C%22value%22%3A%22sts%2Eap%2Dnortheast%2D1b%2Eamazonaws%2Ecom%22%7D%2C%7B%22key%22%3A%22x%2Damz%2Ddate%22%2C%22value%22%3A%2220221231T000000Z%22%7D%2C%7B%22key%22%3A%22x%2Damz%2Dsecurity%2Dtoken%22%2C%22value%22%3A%22SecurityToken%22%7D%2C%7B%22key%22%3A%22x%2Dgoog%2Dcloud%2Dtarget%2Dresource%22%2C%22value%22%3A%22%2F%2Fiam%2Egoogleapis%2Ecom%2Fprojects%2Fmyprojectnumber%2Flocations%2Fglobal%2FworkloadIdentityPools%2Faws%2Dtest%2Fproviders%2Faws%2Dtest%22%7D%5D%7D"; + assert_eq!(token, expected); + } + Err(err) => { + tracing::error!("error={},{:?}", err, err); + unreachable!(); + } + } + } +} diff --git a/foundation/auth/src/token_source/external_account_source/error.rs b/foundation/auth/src/token_source/external_account_source/error.rs new file mode 100644 index 00000000..1ecef59a --- /dev/null +++ b/foundation/auth/src/token_source/external_account_source/error.rs @@ -0,0 +1,67 @@ +use url::ParseError; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Invalid Region URL: {0}")] + InvalidRegionURL(String), + + #[error("Invalid Security Credentials URL: {0}")] + InvalidSecurityCredentialsURL(String), + + #[error("Invalid imds v2 session token URL: {0}")] + InvalidIMDSv2SessionTokenURL(String), + + #[error("No Credentials Source ")] + NoCredentialsSource, + + #[error("AWS version {0} is not supported in the current build")] + UnsupportedAWSVersion(String), + + #[error("Unsupported Subject Token Source")] + UnsupportedSubjectTokenSource, + + #[error(transparent)] + HttpError(#[from] reqwest::Error), + + #[error(transparent)] + JsonError(#[from] serde_json::error::Error), + + #[error(transparent)] + URLError(#[from] ParseError), + + #[error(transparent)] + TimeFormatError(#[from] time::error::Format), + + #[error("Missing Region URL")] + MissingRegionURL, + + #[error("Missing Security Credentials URL")] + MissingSecurityCredentialsURL, + + #[error("Missing Regional Cred Verification URL")] + MissingRegionalCredVerificationURL, + + #[error("Missing Token URL")] + MissingTokenURL, + + #[error("Missing Subject Token Type")] + MissingSubjectTokenType, + + #[error(transparent)] + InvalidHashLength(#[from] sha2::digest::InvalidLength), + + #[error("Failed to get role name. No IAM role may be attached to instance : status={0}")] + UnexpectedStatusOnGetRoleName(u16), + + #[error("Failed to get session token : status={0}")] + UnexpectedStatusOnGetSessionToken(u16), + + #[error("Failed to get credentials : status={0}")] + UnexpectedStatusOnGetCredentials(u16), + + #[error("Failed to get region : status={0}")] + UnexpectedStatusOnGetRegion(u16), + + #[error("Failed to get subject token: status={0}, detail={1}")] + UnexpectedStatusOnGetSubjectToken(u16, String), +} diff --git a/foundation/auth/src/token_source/external_account_source/mod.rs b/foundation/auth/src/token_source/external_account_source/mod.rs new file mode 100644 index 00000000..b4f2db12 --- /dev/null +++ b/foundation/auth/src/token_source/external_account_source/mod.rs @@ -0,0 +1,105 @@ +use std::fmt::Debug; + +use async_trait::async_trait; +use base64::prelude::BASE64_STANDARD; +use base64::Engine; +use time::OffsetDateTime; + +use crate::credentials::{CredentialSource, CredentialsFile}; +use crate::misc::UnwrapOrEmpty; +use crate::token::Token; +use crate::token_source::external_account_source::error::Error; +use crate::token_source::external_account_source::subject_token_source::SubjectTokenSource; +use crate::token_source::{default_http_client, InternalToken, TokenSource}; + +mod aws_subject_token_source; +pub mod error; +mod subject_token_source; + +#[derive(Debug)] +pub struct ExternalAccountTokenSource { + source: CredentialSource, + subject_token_type: String, + url: String, + audience: Option, + auth_header: Option, + scopes: String, + client: reqwest::Client, +} + +impl ExternalAccountTokenSource { + pub(crate) async fn new(scopes: String, credentials: CredentialsFile) -> Result { + let auth_header = if credentials.client_id.is_some() && credentials.client_secret.is_some() { + let plain_text = format!( + "{}:{}", + credentials.client_id.as_ref().unwrap(), + credentials.client_secret.as_ref().unwrap() + ); + Some(format!("Basic: {}", BASE64_STANDARD.encode(plain_text))) + } else { + None + }; + let subject_token_type = credentials.subject_token_type.ok_or(Error::MissingSubjectTokenType)?; + Ok(ExternalAccountTokenSource { + source: credentials.credential_source.ok_or(Error::NoCredentialsSource)?, + subject_token_type, + url: credentials.token_url_external.ok_or(Error::MissingTokenURL)?, + audience: credentials.audience, + auth_header, + scopes, + client: default_http_client(), + }) + } +} + +#[async_trait] +impl TokenSource for ExternalAccountTokenSource { + async fn token(&self) -> Result { + let subject_token_source = subject_token_source(self.audience.clone(), self.source.clone()).await?; + + let mut builder = self.client.post(&self.url); + if let Some(auth_header) = &self.auth_header { + builder = builder.header(reqwest::header::AUTHORIZATION, auth_header); + } + + let audience = match self.audience.as_ref() { + Some(audience) => audience.as_ref(), + None => "", + }; + + let subject_token = subject_token_source.subject_token().await?; + let sts_request = vec![ + ("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"), + ("audience", audience), + ("scope", &self.scopes), + ("subject_token_type", &self.subject_token_type), + ("subject_token", &subject_token), + ("requested_token_type", "urn:ietf:params:oauth:token-type:access_token"), + ]; + let response = builder.form(&sts_request).send().await?; + if !response.status().is_success() { + let status = response.status().as_u16(); + let detail = response.text().await?; + return Err(Error::UnexpectedStatusOnGetSubjectToken(status, detail).into()); + } + let it = response.json::().await?; + Ok(it.to_token(OffsetDateTime::now_utc())) + } +} + +async fn subject_token_source( + audience: Option, + source: CredentialSource, +) -> Result { + let environment_id = &source.environment_id.unwrap_or_empty(); + if environment_id.len() > 3 && environment_id.starts_with("aws") { + if environment_id != "aws1" { + return Err(Error::UnsupportedAWSVersion(environment_id.clone())); + } + let ts = aws_subject_token_source::AWSSubjectTokenSource::new(audience, source).await?; + Ok(ts) + } else { + //TODO support file, url and executable + Err(Error::UnsupportedSubjectTokenSource) + } +} diff --git a/foundation/auth/src/token_source/external_account_source/subject_token_source.rs b/foundation/auth/src/token_source/external_account_source/subject_token_source.rs new file mode 100644 index 00000000..1163a7db --- /dev/null +++ b/foundation/auth/src/token_source/external_account_source/subject_token_source.rs @@ -0,0 +1,8 @@ +use std::fmt::Debug; + +use async_trait::async_trait; + +#[async_trait] +pub trait SubjectTokenSource: Send + Sync + Debug { + async fn subject_token(&self) -> Result; +} diff --git a/foundation/auth/src/token_source/impersonate_token_source.rs b/foundation/auth/src/token_source/impersonate_token_source.rs new file mode 100644 index 00000000..ccd5201d --- /dev/null +++ b/foundation/auth/src/token_source/impersonate_token_source.rs @@ -0,0 +1,87 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use time::format_description::well_known::Rfc3339; + +use crate::error::Error; +use crate::token::Token; +use crate::token_source::{default_http_client, TokenSource}; + +#[derive(Debug)] +pub struct ImpersonateTokenSource { + target: Box, + scopes: Vec, + delegates: Vec, + url: String, + lifetime: Option, + client: reqwest::Client, +} + +impl ImpersonateTokenSource { + #[allow(dead_code)] + pub(crate) fn new( + url: String, + delegates: Vec, + scopes: Vec, + lifetime: Option, + target: Box, + ) -> Self { + ImpersonateTokenSource { + target, + scopes, + delegates, + url, + lifetime, + client: default_http_client(), + } + } +} + +#[async_trait] +impl TokenSource for ImpersonateTokenSource { + async fn token(&self) -> Result { + let body = ImpersonateTokenRequest { + lifetime: format!("{}s", self.lifetime.unwrap_or(3600)), + scope: self.scopes.clone(), + delegates: self.delegates.clone(), + }; + + let auth_token = self.target.token().await?; + let response = self + .client + .post(&self.url) + .json(&body) + .header( + "Authorization", + format!("{} {}", auth_token.token_type, auth_token.access_token), + ) + .send() + .await?; + let response = if !response.status().is_success() { + let status = response.status().as_u16(); + return Err(Error::UnexpectedImpersonateTokenResponse(status, response.text().await?)); + } else { + response.json::().await? + }; + + let expiry = time::OffsetDateTime::parse(&response.expire_time, &Rfc3339)?; + Ok(Token { + access_token: response.access_token, + token_type: "Bearer".to_string(), + expiry: Some(expiry), + }) + } +} + +#[derive(Serialize)] +struct ImpersonateTokenRequest { + pub delegates: Vec, + pub lifetime: String, + pub scope: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ImpersonateTokenResponse { + pub access_token: String, + pub expire_time: String, +} diff --git a/foundation/auth/src/token_source/mod.rs b/foundation/auth/src/token_source/mod.rs index 114910a8..21d1d4f2 100644 --- a/foundation/auth/src/token_source/mod.rs +++ b/foundation/auth/src/token_source/mod.rs @@ -10,15 +10,19 @@ use crate::token::Token; pub mod authorized_user_token_source; pub mod compute_identity_source; pub mod compute_token_source; +pub mod impersonate_token_source; pub mod reuse_token_source; pub mod service_account_token_source; +#[cfg(feature = "external-account")] +pub mod external_account_source; + #[async_trait] pub trait TokenSource: Send + Sync + Debug { async fn token(&self) -> Result; } -fn default_http_client() -> reqwest::Client { +pub(crate) fn default_http_client() -> reqwest::Client { reqwest::Client::builder() .timeout(Duration::from_secs(3)) .build() diff --git a/foundation/longrunning/Cargo.toml b/foundation/longrunning/Cargo.toml index a51533da..5a0a9534 100644 --- a/foundation/longrunning/Cargo.toml +++ b/foundation/longrunning/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-cloud-longrunning" -version = "0.15.0" +version = "0.15.1" authors = ["yoshidan "] edition = "2021" repository = "https://github.com/yoshidan/google-cloud-rust/tree/main/foundation/longrunning" diff --git a/foundation/longrunning/src/longrunning.rs b/foundation/longrunning/src/longrunning.rs index cc80917c..d49099b4 100644 --- a/foundation/longrunning/src/longrunning.rs +++ b/foundation/longrunning/src/longrunning.rs @@ -19,7 +19,7 @@ impl Operation { Self { client, inner, - _marker: PhantomData::default(), + _marker: PhantomData, } } diff --git a/pubsub/Cargo.toml b/pubsub/Cargo.toml index 47b96072..2d59239b 100644 --- a/pubsub/Cargo.toml +++ b/pubsub/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-cloud-pubsub" -version = "0.17.0" +version = "0.18.0" authors = ["yoshidan "] edition = "2021" repository = "https://github.com/yoshidan/google-cloud-rust/tree/main/pubsub" @@ -23,7 +23,7 @@ google-cloud-token = { version = "0.1.1", path = "../foundation/token" } google-cloud-gax = { version = "0.15.0", path = "../foundation/gax" } google-cloud-googleapis = { version = "0.10.0", path = "../googleapis", features = ["pubsub"]} -google-cloud-auth = { optional = true, version = "0.11", path="../foundation/auth", default-features=false } +google-cloud-auth = { optional = true, version = "0.12", path="../foundation/auth", default-features=false } [dev-dependencies] tokio = { version="1.20", features=["rt-multi-thread"] } @@ -35,7 +35,10 @@ ctor = "0.1.21" futures-util = "0.3" [features] -default = ["auth"] +default = ["auth", "default-tls"] +default-tls = ["google-cloud-auth?/default-tls"] +rustls-tls = ["google-cloud-auth?/rustls-tls"] +external-account = ["google-cloud-auth?/external-account"] trace = [] bytes = ["google-cloud-googleapis/bytes"] auth = ["google-cloud-auth"] diff --git a/pubsub/src/apiv1/conn_pool.rs b/pubsub/src/apiv1/conn_pool.rs index 9afa1192..787230dc 100644 --- a/pubsub/src/apiv1/conn_pool.rs +++ b/pubsub/src/apiv1/conn_pool.rs @@ -5,7 +5,7 @@ pub const AUDIENCE: &str = "https://pubsub.googleapis.com/"; pub const PUBSUB: &str = "pubsub.googleapis.com"; pub const SCOPES: [&str; 2] = [ "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/pubsub.data", + "https://www.googleapis.com/auth/pubsub", ]; #[derive(Debug)] diff --git a/pubsub/src/client.rs b/pubsub/src/client.rs index 6ac5b759..fda2f5d9 100644 --- a/pubsub/src/client.rs +++ b/pubsub/src/client.rs @@ -86,7 +86,7 @@ impl ClientConfig { pub enum Error { #[error(transparent)] GAX(#[from] google_cloud_gax::conn::Error), - #[error("invalid project_id")] + #[error("Project ID was not found")] ProjectIdNotFound, } diff --git a/spanner/Cargo.toml b/spanner/Cargo.toml index e367cb4c..6efc592e 100644 --- a/spanner/Cargo.toml +++ b/spanner/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-cloud-spanner" -version = "0.22.0" +version = "0.23.0" authors = ["yoshidan "] edition = "2021" repository = "https://github.com/yoshidan/google-cloud-rust/tree/main/spanner" @@ -24,11 +24,11 @@ tokio-util = "0.7" bigdecimal = { version="0.3", features=["serde"] } google-cloud-token = { version = "0.1.1", path = "../foundation/token" } -google-cloud-longrunning= { version = "0.15.0", path = "../foundation/longrunning" } +google-cloud-longrunning = { version = "0.15.1", path = "../foundation/longrunning" } google-cloud-gax = { version = "0.15.0", path = "../foundation/gax" } google-cloud-googleapis = { version = "0.10.0", path = "../googleapis", features = ["spanner"]} -google-cloud-auth = { optional = true, version = "0.11", path="../foundation/auth", default-features=false } +google-cloud-auth = { optional = true, version = "0.12", path="../foundation/auth", default-features=false } [dev-dependencies] tokio = { version="1.20", features=["rt-multi-thread"] } @@ -38,6 +38,9 @@ ctor = "0.1" google-cloud-auth = { path="../foundation/auth", default-features=false, features=["rustls-tls"]} [features] -default = ["serde", "auth"] +default = ["serde", "auth", "default-tls"] trace = [] auth = ["google-cloud-auth"] +default-tls = ["google-cloud-auth?/default-tls"] +rustls-tls = ["google-cloud-auth?/rustls-tls"] +external-account = ["google-cloud-auth?/external-account"] diff --git a/spanner/src/retry.rs b/spanner/src/retry.rs index b3680798..2155d347 100644 --- a/spanner/src/retry.rs +++ b/spanner/src/retry.rs @@ -97,7 +97,7 @@ where fn condition(&self) -> TransactionCondition { TransactionCondition { inner: CodeCondition::new(self.inner.codes.clone()), - _marker: PhantomData::default(), + _marker: PhantomData, } } } diff --git a/storage/Cargo.toml b/storage/Cargo.toml index d7be4bd8..5ef3e746 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-cloud-storage" -version = "0.12.0" +version = "0.13.0" edition = "2021" authors = ["yoshidan "] repository = "https://github.com/yoshidan/google-cloud-rust/tree/main/storage" @@ -22,18 +22,18 @@ ring = "0.16" tokio = { version="1.20", features=["macros"] } async-stream = "0.3" once_cell = "1.13" -hex = "0.4.3" -url = "2.2.2" +hex = "0.4" +url = "2.4" tracing = "0.1" reqwest = { version = "0.11", features = ["json", "stream", "multipart"], default-features = false } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -percent-encoding = "2.1" +percent-encoding = "2.3" futures-util = "0.3" bytes = "1.2" google-cloud-metadata = { optional = true, version = "0.3", path = "../foundation/metadata" } -google-cloud-auth = { optional = true, version = "0.11", path="../foundation/auth", default-features=false } +google-cloud-auth = { optional = true, version = "0.12", path="../foundation/auth", default-features=false } [dev-dependencies] tokio = { version="1.20", features=["rt-multi-thread"] } @@ -45,7 +45,8 @@ google-cloud-auth = { path = "../foundation/auth", default-features=false } [features] default = ["default-tls", "auth"] -default-tls = ["reqwest/default-tls"] -rustls-tls = ["reqwest/rustls-tls"] +default-tls = ["reqwest/default-tls","google-cloud-auth?/default-tls"] +rustls-tls = ["reqwest/rustls-tls","google-cloud-auth?/rustls-tls"] trace = [] auth = ["google-cloud-auth", "google-cloud-metadata"] +external-account = ["google-cloud-auth?/external-account"]