From e62d4b2b73c8e53e18bcfb6c06d4783d9c8dd8f9 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 25 Sep 2024 14:57:55 +1000 Subject: [PATCH 1/7] feat(config): add configuration values for C4GH S3 and Url storage --- htsget-actix/README.md | 2 +- htsget-axum/README.md | 2 +- htsget-config/README.md | 21 +++++---- htsget-config/src/resolver.rs | 6 ++- htsget-config/src/storage/object/c4gh.rs | 55 ++++++++++++++++++++++-- htsget-config/src/storage/s3.rs | 16 ++++++- htsget-config/src/storage/url.rs | 21 +++++++++ htsget-http/README.md | 2 +- htsget-lambda/README.md | 2 +- htsget-search/README.md | 2 +- htsget-storage/README.md | 2 +- htsget-test/README.md | 2 +- 12 files changed, 110 insertions(+), 23 deletions(-) diff --git a/htsget-actix/README.md b/htsget-actix/README.md index 29575f04..90ba9c15 100644 --- a/htsget-actix/README.md +++ b/htsget-actix/README.md @@ -50,7 +50,7 @@ are exposed in the public API. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features like `C4GHStorage`. ## Benchmarks Benchmarks for this crate written using [Criterion.rs][criterion-rs], and aim to compare the performance of this crate with the diff --git a/htsget-axum/README.md b/htsget-axum/README.md index 6330d1c1..73fe6048 100644 --- a/htsget-axum/README.md +++ b/htsget-axum/README.md @@ -171,7 +171,7 @@ htsget-rs. It also contains the data block server which fetches data from a `Loc This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features like `C4GHStorage`. ## License diff --git a/htsget-config/README.md b/htsget-config/README.md index 24d4e7cf..8a646767 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -193,6 +193,7 @@ For example, a `resolvers` value of: [[resolvers]] regex = '^(example_bucket)/(?P.*)$' substitution_string = '$key' + [resolvers.storage] type = 'S3' # Uses the first capture group in the regex as the bucket. @@ -275,6 +276,7 @@ regex = '.*' substitution_string = '$0' [resolvers.storage] +type = 'S3' bucket = 'bucket' [resolvers.allow_guard] @@ -452,6 +454,7 @@ export HTSGET_RESOLVERS="[{ regex=regex, substitution_string=substitution_string, storage={ + type=S3, bucket=bucket }, allow_guard={ @@ -483,6 +486,7 @@ regex = '.*' substitution_string = '$0' [resolvers.storage] +type = 'S3' bucket = 'bucket' endpoint = 'http://127.0.0.1:9000' path_style = true @@ -504,8 +508,7 @@ serving the data, htsget-rs will decrypt the headers of the Crypt4GH files and r them. When the client receives byte ranges from htsget-rs and concatenates them, the output bytes will be Crypt4GH encrypted, and will need to be decrypted before they can be read. All file formats (BAM, CRAM, VCF, and BCF) are supported using Crypt4GH. -To use this feature, an additional config option called `object_type` under `resolvers.storage` is required, -which allows specifying the private and public keys: +To use this feature, additional config under `resolvers.storage` is required to specify the private and public keys: | Option | Description | Type | Default | |------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------| @@ -516,17 +519,17 @@ For example: ```toml [[resolvers]] -regex = ".*" -substitution_string = "$0" +regex = '.*' +substitution_string = '$0' [resolvers.storage] -object_type = { private_key = "data/c4gh/keys/bob.sec", recipient_public_key = "data/c4gh/keys/alice.pub" } # pragma: allowlist secret +type = 'Local' +private_key = 'data/c4gh/keys/bob.sec' # pragma: allowlist secret +recipient_public_key = 'data/c4gh/keys/alice.pub' ``` The htsget-rs server expects the Crypt4GH file to end with `.c4gh`, and the index file to be unencrypted. See the [`data/c4gh`][data-c4gh] for examples of file structure. - -> [!NOTE] -> This option is currently only supported for `LocalStorage`. The `object_type` will not have an effect if using `S3Storage` or `UrlStorage`. +Any of the storage types are supported, i.e. `Local`, `S3`, or `Url`. ### As a library @@ -541,7 +544,7 @@ regex, and changing it by using a substitution string. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features like `C4GHStorage`. ## License diff --git a/htsget-config/src/resolver.rs b/htsget-config/src/resolver.rs index 28c7cb5c..dae7bb12 100644 --- a/htsget-config/src/resolver.rs +++ b/htsget-config/src/resolver.rs @@ -499,7 +499,7 @@ mod tests { #[cfg(feature = "s3-storage")] #[tokio::test] async fn resolver_resolve_s3_request_tagged() { - let s3_storage = S3Storage::new("id".to_string(), None, false); + let s3_storage = S3Storage::new("id".to_string(), None, false, Default::default()); let resolver = Resolver::new( Storage::S3(s3_storage), "(id)-1", @@ -538,6 +538,7 @@ mod tests { }), true, vec![], + Default::default(), client, ); @@ -692,7 +693,8 @@ mod tests { Interval::new(Some(100), Some(1000)), ); let resolver = config.resolvers().first().unwrap(); - let expected_storage = S3Storage::new("bucket".to_string(), None, false); + let expected_storage = + S3Storage::new("bucket".to_string(), None, false, Default::default()); assert_eq!(resolver.regex().to_string(), "regex"); assert_eq!(resolver.substitution_string(), "substitution_string"); diff --git a/htsget-config/src/storage/object/c4gh.rs b/htsget-config/src/storage/object/c4gh.rs index 93001cd6..3639acbf 100644 --- a/htsget-config/src/storage/object/c4gh.rs +++ b/htsget-config/src/storage/object/c4gh.rs @@ -64,13 +64,16 @@ impl From for Error { #[cfg(test)] mod tests { use crate::config::tests::test_config_from_file; + use crate::config::Config; use crate::storage::Storage; use std::fs::copy; use std::path::PathBuf; use tempfile::TempDir; - #[test] - fn config_storage_c4gh() { + fn test_c4gh_storage_config(storage_config: &str, test_fn: F) + where + F: Fn(Config), + { let tmp = TempDir::new().unwrap(); let private_key = tmp.path().join("bob.sec"); let recipient_public_key = tmp.path().join("alice.pub"); @@ -94,18 +97,62 @@ mod tests { regex = "regex" [resolvers.storage] - type = "Local" + {} private_key = "{}" recipient_public_key = "{}" "#, + storage_config, private_key.to_string_lossy(), recipient_public_key.to_string_lossy() ), |config| { println!("{:?}", config.resolvers().first().unwrap().storage()); - assert!(matches!( + test_fn(config); + }, + ); + } + + #[test] + fn config_local_storage_c4gh() { + test_c4gh_storage_config(r#"type = "Local""#, |config| { + assert!(matches!( config.resolvers().first().unwrap().storage(), Storage::Local(local_storage) if local_storage.object_type().keys().is_some() + )); + }); + } + + #[cfg(feature = "s3-storage")] + #[test] + fn config_s3_storage_c4gh() { + test_c4gh_storage_config( + r#" + type = "S3" + bucket = "bucket" + "#, + |config| { + assert!(matches!( + config.resolvers().first().unwrap().storage(), + Storage::S3(s3_storage) if s3_storage.object_type().keys().is_some() + )); + }, + ); + } + + #[cfg(feature = "url-storage")] + #[test] + fn config_url_storage_c4gh() { + test_c4gh_storage_config( + r#" + type = "Url" + url = "https://example.com/" + response_url = "https://example.com/" + forward_headers = false + "#, + |config| { + assert!(matches!( + config.resolvers().first().unwrap().storage(), + Storage::Url(url_storage) if url_storage.object_type().keys().is_some() )); }, ); diff --git a/htsget-config/src/storage/s3.rs b/htsget-config/src/storage/s3.rs index cea0e846..6edd15af 100644 --- a/htsget-config/src/storage/s3.rs +++ b/htsget-config/src/storage/s3.rs @@ -1,3 +1,4 @@ +use crate::storage::object::ObjectType; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] @@ -6,15 +7,23 @@ pub struct S3Storage { pub(crate) bucket: String, pub(crate) endpoint: Option, pub(crate) path_style: bool, + #[serde(flatten)] + pub(crate) object_type: ObjectType, } impl S3Storage { /// Create a new S3 storage. - pub fn new(bucket: String, endpoint: Option, path_style: bool) -> Self { + pub fn new( + bucket: String, + endpoint: Option, + path_style: bool, + object_type: ObjectType, + ) -> Self { Self { bucket, endpoint, path_style, + object_type, } } @@ -32,6 +41,11 @@ impl S3Storage { pub fn path_style(self) -> bool { self.path_style } + + /// Get the object type. + pub fn object_type(&self) -> &ObjectType { + &self.object_type + } } #[cfg(test)] diff --git a/htsget-config/src/storage/url.rs b/htsget-config/src/storage/url.rs index 6310142c..a2906aae 100644 --- a/htsget-config/src/storage/url.rs +++ b/htsget-config/src/storage/url.rs @@ -8,6 +8,7 @@ use serde_with::with_prefix; use crate::error::Error::ParseError; use crate::error::{Error, Result}; use crate::storage::local::default_authority; +use crate::storage::object::ObjectType; use crate::tls::client::TlsClientConfig; fn default_url() -> ValidatedUrl { @@ -28,6 +29,8 @@ pub struct UrlStorage { header_blacklist: Vec, #[serde(skip_serializing)] tls: TlsClientConfig, + #[serde(flatten)] + object_type: ObjectType, } #[derive(Deserialize, Debug, Clone)] @@ -37,6 +40,7 @@ pub struct UrlStorageClient { response_url: ValidatedUrl, forward_headers: bool, header_blacklist: Vec, + object_type: ObjectType, client: Client, } @@ -66,6 +70,7 @@ impl TryFrom for UrlStorageClient { storage.response_url, storage.forward_headers, storage.header_blacklist, + storage.object_type, client, )) } @@ -78,6 +83,7 @@ impl UrlStorageClient { response_url: ValidatedUrl, forward_headers: bool, header_blacklist: Vec, + object_type: ObjectType, client: Client, ) -> Self { Self { @@ -85,6 +91,7 @@ impl UrlStorageClient { response_url, forward_headers, header_blacklist, + object_type, client, } } @@ -113,6 +120,11 @@ impl UrlStorageClient { pub fn client_cloned(&self) -> Client { self.client.clone() } + + /// Get the object type. + pub fn object_type(&self) -> &ObjectType { + &self.object_type + } } /// A wrapper around `http::Uri` type which implements serialize and deserialize. @@ -154,6 +166,7 @@ impl UrlStorage { forward_headers: bool, header_blacklist: Vec, tls: TlsClientConfig, + object_type: ObjectType, ) -> Self { Self { url: ValidatedUrl(Url { inner: url }), @@ -163,6 +176,7 @@ impl UrlStorage { forward_headers, header_blacklist, tls, + object_type, } } @@ -186,6 +200,11 @@ impl UrlStorage { pub fn tls(&self) -> &TlsClientConfig { &self.tls } + + /// Get the object type. + pub fn object_type(&self) -> &ObjectType { + &self.object_type + } } impl Default for UrlStorage { @@ -196,6 +215,7 @@ impl Default for UrlStorage { forward_headers: true, header_blacklist: vec![], tls: TlsClientConfig::default(), + object_type: Default::default(), } } } @@ -221,6 +241,7 @@ mod tests { true, vec![], client_config, + Default::default(), )); assert!(url_storage.is_ok()); diff --git a/htsget-http/README.md b/htsget-http/README.md index b7969e3b..2cb6be0d 100644 --- a/htsget-http/README.md +++ b/htsget-http/README.md @@ -38,7 +38,7 @@ These functions take query and endpoint information, and process it using [htsge This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features like `C4GHStorage`. [warp]: https://github.com/seanmonstar/warp [htsget-search]: ../htsget-search diff --git a/htsget-lambda/README.md b/htsget-lambda/README.md index 16806fd1..cd498d9f 100644 --- a/htsget-lambda/README.md +++ b/htsget-lambda/README.md @@ -46,7 +46,7 @@ library code, and it instead uses `htsget-axum`. Please use that crate for funct This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features like `C4GHStorage`. ## License diff --git a/htsget-search/README.md b/htsget-search/README.md index 3d179331..30d49473 100644 --- a/htsget-search/README.md +++ b/htsget-search/README.md @@ -59,7 +59,7 @@ used to process requests. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features like `C4GHStorage`. ## Minimising Byte Ranges diff --git a/htsget-storage/README.md b/htsget-storage/README.md index 23bf7bed..bb23e577 100644 --- a/htsget-storage/README.md +++ b/htsget-storage/README.md @@ -49,7 +49,7 @@ and [url] modules implement the `Storage` functionality. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features like `C4GHStorage`. [local]: src/local.rs [s3]: src/s3.rs diff --git a/htsget-test/README.md b/htsget-test/README.md index e7af1ff8..a9bdab4e 100644 --- a/htsget-test/README.md +++ b/htsget-test/README.md @@ -40,7 +40,7 @@ This crate has the following features: * `aws-mocks`: used to enable AWS mocking for tests. * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features like `C4GHStorage`. [dev-dependencies]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#development-dependencies From 45ec44734caeba587af3973501c2bfca7da597d5 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 25 Sep 2024 17:24:43 +1000 Subject: [PATCH 2/7] feat(storage): allow S3 and Url storage to use C4GHStorage --- htsget-storage/src/c4gh/storage.rs | 389 ++++++++++++++++++++--------- htsget-storage/src/lib.rs | 34 ++- htsget-storage/src/local.rs | 30 ++- htsget-storage/src/s3.rs | 28 ++- htsget-storage/src/url.rs | 83 +++--- 5 files changed, 357 insertions(+), 207 deletions(-) diff --git a/htsget-storage/src/c4gh/storage.rs b/htsget-storage/src/c4gh/storage.rs index 5ce46b3b..6eeea139 100644 --- a/htsget-storage/src/c4gh/storage.rs +++ b/htsget-storage/src/c4gh/storage.rs @@ -288,167 +288,328 @@ impl From for StorageError { mod tests { use super::*; use crate::local::tests::with_local_storage; + #[cfg(feature = "s3-storage")] + use crate::s3::tests::with_aws_s3_storage; + #[cfg(feature = "url-storage")] + use crate::url::tests::{test_headers, with_url_test_server}; use htsget_config::types::Headers; use htsget_test::c4gh::{encrypt_data, get_decryption_keys}; + use http::HeaderMap; use std::future::Future; - use tokio::fs::File; + use std::path::Path; + #[cfg(feature = "s3-storage")] + use std::sync::Arc; + use tokio::fs::{read, File}; use tokio::io::AsyncWriteExt; #[tokio::test] - async fn test_preprocess() { - with_c4gh_storage(|mut storage| async move { - storage - .preprocess( - "key", - GetOptions::new_with_default_range(&Default::default()), - ) - .await - .unwrap(); + async fn test_preprocess_local_storage() { + with_local_c4gh_storage(|mut storage| async move { + test_preprocess(&mut storage, "folder/key", &Default::default()).await + }) + .await; + } - let state = storage.state.get("key.c4gh").unwrap(); + #[cfg(feature = "s3-storage")] + #[tokio::test] + async fn test_preprocess_s3_storage() { + with_s3_c4gh_storage(|mut storage| async move { + test_preprocess(&mut storage, "key", &Default::default()).await + }) + .await; + } - assert_eq!(state.unencrypted_file_size, 6); - assert_eq!(state.encrypted_file_size, 158); - assert_eq!(state.deserialized_header.header_info.packets_count, 1); + #[cfg(feature = "url-storage")] + #[tokio::test] + async fn test_preprocess_url_storage() { + with_url_c4gh_storage(|mut storage, _| async move { + let mut headers = HeaderMap::default(); + let headers = test_headers(&mut headers); + test_preprocess(&mut storage, "assets/folder/key", headers).await }) .await; } #[tokio::test] - async fn test_get() { - with_c4gh_storage(|mut storage| async move { - let headers = Default::default(); - let options = GetOptions::new_with_default_range(&headers); - storage.preprocess("key", options.clone()).await.unwrap(); - let mut object = vec![]; - - storage - .get("key", options) - .await - .unwrap() - .read_to_end(&mut object) - .await - .unwrap(); - assert_eq!(object, b"value1"); + async fn test_get_local_storage() { + with_local_c4gh_storage(|mut storage| async move { + test_get(&mut storage, "folder/key", &Default::default()).await }) .await; } + #[cfg(feature = "s3-storage")] #[tokio::test] - async fn test_head() { - with_c4gh_storage(|mut storage| async move { - let headers = Default::default(); - let options = GetOptions::new_with_default_range(&headers); - storage.preprocess("key", options.clone()).await.unwrap(); - - let size = storage - .head("key", HeadOptions::new(&headers)) - .await - .unwrap(); - assert_eq!(size, 6); + async fn test_get_s3_storage() { + with_s3_c4gh_storage(|mut storage| async move { + test_get(&mut storage, "key", &Default::default()).await }) .await; } + #[cfg(feature = "url-storage")] #[tokio::test] - async fn test_postprocess() { - with_c4gh_storage(|mut storage| async move { - let headers = Default::default(); - let options = GetOptions::new_with_default_range(&headers); - storage.preprocess("key", options.clone()).await.unwrap(); - - let blocks = storage - .postprocess( - "key", - BytesPositionOptions::new( - vec![BytesPosition::default().with_start(0).with_end(6)], - &headers, - ), - ) - .await - .unwrap(); + async fn test_get_url_storage() { + with_url_c4gh_storage(|mut storage, _| async move { + let mut headers = HeaderMap::default(); + let headers = test_headers(&mut headers); + test_get(&mut storage, "assets/folder/key", headers).await + }) + .await; + } - assert_eq!( - blocks[0], - DataBlock::Data( - vec![99, 114, 121, 112, 116, 52, 103, 104, 1, 0, 0, 0, 3, 0, 0, 0], - Some(Class::Header) - ) - ); - assert_eq!( - blocks[1], - DataBlock::Range(BytesPosition::new(Some(16), Some(124), None)) - ); - assert_eq!( - blocks[3], - DataBlock::Range(BytesPosition::new(Some(124), Some(158), None)) - ); + #[tokio::test] + async fn test_head_local_storage() { + with_local_c4gh_storage(|mut storage| async move { + test_head(&mut storage, "folder/key", &Default::default()).await }) .await; } + #[cfg(feature = "s3-storage")] #[tokio::test] - async fn test_range_url() { - with_c4gh_storage(|mut storage| async move { - let headers = Default::default(); - let options = GetOptions::new_with_default_range(&headers); - storage.preprocess("key", options.clone()).await.unwrap(); - - let blocks = storage - .postprocess( - "key", - BytesPositionOptions::new( - vec![BytesPosition::default().with_start(0).with_end(6)], - &headers, - ), - ) - .await - .unwrap(); + async fn test_head_s3_storage() { + with_s3_c4gh_storage(|mut storage| async move { + test_head(&mut storage, "key", &Default::default()).await + }) + .await; + } - if let DataBlock::Range(range) = blocks.last().unwrap() { - let url = storage - .range_url("key", RangeUrlOptions::new(range.clone(), &headers)) - .await - .unwrap(); - let expected = Url::new("http://127.0.0.1:8081/data/key.c4gh") - .with_headers(Headers::default().with_header("Range", "bytes=124-157")); + #[cfg(feature = "url-storage")] + #[tokio::test] + async fn test_head_url_storage() { + with_url_c4gh_storage(|mut storage, _| async move { + let mut headers = HeaderMap::default(); + let headers = test_headers(&mut headers); + test_head(&mut storage, "assets/folder/key", headers).await + }) + .await; + } - assert_eq!(url, expected); - } + #[tokio::test] + async fn test_postprocess_local_storage() { + with_local_c4gh_storage(|mut storage| async move { + test_postprocess(&mut storage, "folder/key", &Default::default()).await }) .await; } - pub(crate) async fn with_c4gh_storage(test: F) - where - F: FnOnce(C4GHStorage) -> Fut, - Fut: Future, - { - with_local_storage(|storage| async move { - let mut data = vec![]; - StorageTrait::get( - &storage, - "folder/../key1", - GetOptions::new_with_default_range(&Default::default()), + #[cfg(feature = "s3-storage")] + #[tokio::test] + async fn test_postprocess_s3_storage() { + with_s3_c4gh_storage(|mut storage| async move { + test_postprocess(&mut storage, "key", &Default::default()).await + }) + .await; + } + + #[cfg(feature = "url-storage")] + #[tokio::test] + async fn test_postprocess_url_storage() { + with_url_c4gh_storage(|mut storage, _| async move { + let mut headers = HeaderMap::default(); + let headers = test_headers(&mut headers); + test_postprocess(&mut storage, "assets/folder/key", headers).await + }) + .await; + } + + #[tokio::test] + async fn test_range_local_storage() { + with_local_c4gh_storage(|mut storage| async move { + test_range_url( + &mut storage, + "http://127.0.0.1:8081/data/folder/key.c4gh", + "folder/key", + &Default::default(), ) .await + }) + .await; + } + + #[cfg(feature = "s3-storage")] + #[tokio::test] + async fn test_range_s3_storage() { + with_s3_c4gh_storage(|mut storage| async move { + test_range_url( + &mut storage, + "http://folder.localhost:8014/key.c4gh", + "key", + &Default::default(), + ) + .await + }) + .await; + } + + #[cfg(feature = "url-storage")] + #[tokio::test] + async fn test_range_url_storage() { + with_url_c4gh_storage(|mut storage, url| async move { + let mut headers = HeaderMap::default(); + let headers = test_headers(&mut headers); + test_range_url( + &mut storage, + &format!("{}/assets/folder/key.c4gh", url), + "assets/folder/key", + headers, + ) + .await + }) + .await; + } + + async fn test_preprocess(storage: &mut C4GHStorage, key: &str, headers: &HeaderMap) { + storage + .preprocess(key, GetOptions::new_with_default_range(headers)) + .await + .unwrap(); + + let state = storage.state.get(&format!("{}.c4gh", key)).unwrap(); + + assert_eq!(state.unencrypted_file_size, 6); + assert_eq!(state.encrypted_file_size, 158); + assert_eq!(state.deserialized_header.header_info.packets_count, 1); + } + + async fn test_get(storage: &mut C4GHStorage, key: &str, headers: &HeaderMap) { + let options = GetOptions::new_with_default_range(headers); + storage.preprocess(key, options.clone()).await.unwrap(); + let mut object = vec![]; + + storage + .get(key, options) + .await .unwrap() - .read_to_end(&mut data) + .read_to_end(&mut object) .await .unwrap(); + assert_eq!(object, b"value1"); + } - let data = encrypt_data(&data); + async fn test_head(storage: &mut C4GHStorage, key: &str, headers: &HeaderMap) { + let options = GetOptions::new_with_default_range(headers); + storage.preprocess(key, options.clone()).await.unwrap(); - let key = "key.c4gh"; - File::create(storage.base_path().join(key)) - .await - .unwrap() - .write_all(&data) + let size = storage.head(key, HeadOptions::new(headers)).await.unwrap(); + assert_eq!(size, 6); + } + + async fn test_postprocess(storage: &mut C4GHStorage, key: &str, headers: &HeaderMap) { + let options = GetOptions::new_with_default_range(headers); + storage.preprocess(key, options.clone()).await.unwrap(); + + let blocks = storage + .postprocess( + key, + BytesPositionOptions::new( + vec![BytesPosition::default().with_start(0).with_end(6)], + headers, + ), + ) + .await + .unwrap(); + + assert_eq!( + blocks[0], + DataBlock::Data( + vec![99, 114, 121, 112, 116, 52, 103, 104, 1, 0, 0, 0, 3, 0, 0, 0], + Some(Class::Header) + ) + ); + assert_eq!( + blocks[1], + DataBlock::Range(BytesPosition::new(Some(16), Some(124), None)) + ); + assert_eq!( + blocks[3], + DataBlock::Range(BytesPosition::new(Some(124), Some(158), None)) + ); + } + + async fn test_range_url(storage: &mut C4GHStorage, url: &str, key: &str, headers: &HeaderMap) { + let options = GetOptions::new_with_default_range(headers); + storage.preprocess(key, options.clone()).await.unwrap(); + + let blocks = storage + .postprocess( + key, + BytesPositionOptions::new( + vec![BytesPosition::default().with_start(0).with_end(6)], + headers, + ), + ) + .await + .unwrap(); + + if let DataBlock::Range(range) = blocks.last().unwrap() { + let range = storage + .range_url(key, RangeUrlOptions::new(range.clone(), headers)) .await .unwrap(); + println!("{:?}", range); + assert!(range.url.starts_with(url)); + assert_eq!( + range.headers, + Some(Headers::default().with_header("Range", "bytes=124-157")) + ); + assert_eq!(range.class, None); + } + } + + async fn create_encrypted_files(base_path: &Path) { + let data = read(base_path.join("folder/../key1")).await.unwrap(); + let data = encrypt_data(&data); + + File::create(base_path.join("folder/key.c4gh")) + .await + .unwrap() + .write_all(&data) + .await + .unwrap(); + } + + pub(crate) async fn with_local_c4gh_storage(test: F) + where + F: FnOnce(C4GHStorage) -> Fut, + Fut: Future, + { + with_local_storage(|storage, base_path| async move { + create_encrypted_files(&base_path).await; test(C4GHStorage::new(get_decryption_keys(), storage)).await; }) .await; } + + #[cfg(feature = "s3-storage")] + pub(crate) async fn with_s3_c4gh_storage(test: F) + where + F: FnOnce(C4GHStorage) -> Fut, + Fut: Future, + { + with_aws_s3_storage(|storage, base_path| async move { + create_encrypted_files(&base_path).await; + test(C4GHStorage::new( + get_decryption_keys(), + Arc::try_unwrap(storage).unwrap(), + )) + .await; + }) + .await; + } + + #[cfg(feature = "url-storage")] + pub(crate) async fn with_url_c4gh_storage(test: F) + where + F: FnOnce(C4GHStorage, String) -> Fut, + Fut: Future, + { + with_url_test_server(|storage, url, base_path| async move { + create_encrypted_files(&base_path).await; + test(C4GHStorage::new(get_decryption_keys(), storage), url).await; + }) + .await; + } } diff --git a/htsget-storage/src/lib.rs b/htsget-storage/src/lib.rs index ad2c7c52..dbab8fc2 100644 --- a/htsget-storage/src/lib.rs +++ b/htsget-storage/src/lib.rs @@ -155,34 +155,42 @@ impl Storage { } /// Create from local storage config. - pub async fn from_local(config: &LocalStorageConfig) -> Result { - let storage = Storage::new(LocalStorage::new(config.local_path(), config.clone())?); - Ok(Storage::from_object_type(config.object_type(), storage)) + pub async fn from_local(local_storage: &LocalStorageConfig) -> Result { + let storage = Storage::new(LocalStorage::new( + local_storage.local_path(), + local_storage.clone(), + )?); + Ok(Storage::from_object_type( + local_storage.object_type(), + storage, + )) } /// Create from s3 config. #[cfg(feature = "s3-storage")] pub async fn from_s3(s3_storage: &S3StorageConfig) -> Storage { - Storage::new( + let storage = Storage::new( S3Storage::new_with_default_config( s3_storage.bucket().to_string(), s3_storage.clone().endpoint(), s3_storage.clone().path_style(), ) .await, - ) + ); + Storage::from_object_type(s3_storage.object_type(), storage) } /// Create from url config. #[cfg(feature = "url-storage")] - pub async fn from_url(url_storage_config: &UrlStorageConfig) -> Storage { - Storage::new(UrlStorage::new( - url_storage_config.client_cloned(), - url_storage_config.url().clone(), - url_storage_config.response_url().clone(), - url_storage_config.forward_headers(), - url_storage_config.header_blacklist().to_vec(), - )) + pub async fn from_url(url_storage: &UrlStorageConfig) -> Storage { + let storage = Storage::new(UrlStorage::new( + url_storage.client_cloned(), + url_storage.url().clone(), + url_storage.response_url().clone(), + url_storage.forward_headers(), + url_storage.header_blacklist().to_vec(), + )); + Storage::from_object_type(url_storage.object_type(), storage) } pub fn new(inner: impl StorageTrait + Send + Sync + 'static) -> Self { diff --git a/htsget-storage/src/local.rs b/htsget-storage/src/local.rs index d79132a3..71757459 100644 --- a/htsget-storage/src/local.rs +++ b/htsget-storage/src/local.rs @@ -150,7 +150,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_non_existing_key() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = storage.get("non-existing-key").await; assert!(matches!(result, Err(StorageError::KeyNotFound(msg)) if msg == "non-existing-key")); }) @@ -159,7 +159,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_folder() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::get( &storage, "folder", @@ -173,7 +173,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_forbidden_path() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::get( &storage, "folder/../../passwords", @@ -189,7 +189,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_existing_key() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::get( &storage, "folder/../key1", @@ -203,7 +203,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_non_existing_key() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::range_url( &storage, "non-existing-key", @@ -217,7 +217,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_folder() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::range_url( &storage, "folder", @@ -231,7 +231,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_with_forbidden_path() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::range_url( &storage, "folder/../../passwords", @@ -247,7 +247,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_existing_key() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::range_url( &storage, "folder/../key1", @@ -262,7 +262,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_existing_key_with_specified_range() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::range_url( &storage, "folder/../key1", @@ -281,7 +281,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_existing_key_with_specified_open_ended_range() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::range_url( &storage, "folder/../key1", @@ -297,7 +297,7 @@ pub(crate) mod tests { #[tokio::test] async fn file_size() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::head( &storage, "folder/../key1", @@ -354,10 +354,14 @@ pub(crate) mod tests { pub(crate) async fn with_local_storage(test: F) where - F: FnOnce(LocalStorage) -> Fut, + F: FnOnce(LocalStorage, PathBuf) -> Fut, Fut: Future, { let (_, base_path) = create_local_test_files().await; - test(test_local_storage(base_path.path())).await + test( + test_local_storage(base_path.path()), + base_path.path().to_path_buf(), + ) + .await } } diff --git a/htsget-storage/src/s3.rs b/htsget-storage/src/s3.rs index 5fd9bae7..79d9fff5 100644 --- a/htsget-storage/src/s3.rs +++ b/htsget-storage/src/s3.rs @@ -291,7 +291,7 @@ impl StorageTrait for S3Storage { #[cfg(test)] pub(crate) mod tests { use std::future::Future; - use std::path::Path; + use std::path::{Path, PathBuf}; use std::sync::Arc; use htsget_test::aws_mocks::with_s3_test_server; @@ -305,18 +305,22 @@ pub(crate) mod tests { pub(crate) async fn with_aws_s3_storage_fn(test: F, folder_name: String, base_path: &Path) where - F: FnOnce(Arc) -> Fut, + F: FnOnce(Arc, PathBuf) -> Fut, Fut: Future, { with_s3_test_server(base_path, |client| async move { - test(Arc::new(S3Storage::new(client, folder_name))).await; + test( + Arc::new(S3Storage::new(client, folder_name)), + base_path.to_path_buf(), + ) + .await; }) .await; } - async fn with_aws_s3_storage(test: F) + pub(crate) async fn with_aws_s3_storage(test: F) where - F: FnOnce(Arc) -> Fut, + F: FnOnce(Arc, PathBuf) -> Fut, Fut: Future, { let (folder_name, base_path) = create_local_test_files().await; @@ -325,7 +329,7 @@ pub(crate) mod tests { #[tokio::test] async fn existing_key() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage .get( "key2", @@ -339,7 +343,7 @@ pub(crate) mod tests { #[tokio::test] async fn non_existing_key() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage .get( "non-existing-key", @@ -353,7 +357,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_existing_key() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage .range_url( "key2", @@ -372,7 +376,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_with_specified_range() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage .range_url( "key2", @@ -399,7 +403,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_with_specified_open_ended_range() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage .range_url( "key2", @@ -423,7 +427,7 @@ pub(crate) mod tests { #[tokio::test] async fn file_size() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage .head("key2", HeadOptions::new(&Default::default())) .await; @@ -435,7 +439,7 @@ pub(crate) mod tests { #[tokio::test] async fn retrieval_type() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage.get_retrieval_type("key2").await; println!("{result:?}"); }) diff --git a/htsget-storage/src/url.rs b/htsget-storage/src/url.rs index 10e9f9d0..5178f91a 100644 --- a/htsget-storage/src/url.rs +++ b/htsget-storage/src/url.rs @@ -99,6 +99,7 @@ impl UrlStorage { let key = key.as_ref(); let url = self.get_url_from_key(key)?; + println!("url: {:?}", url); let request = Request::builder().method(method).uri(&url); let request = headers @@ -245,9 +246,9 @@ impl StorageTrait for UrlStorage { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::future::Future; - use std::path::Path; + use std::path::{Path, PathBuf}; use std::str::FromStr; use std::{result, vec}; @@ -326,15 +327,7 @@ mod tests { #[tokio::test] async fn send_request() { - with_url_test_server(|url| async move { - let storage = UrlStorage::new( - test_client(), - Uri::from_str(&url).unwrap(), - Uri::from_str(&url).unwrap(), - true, - vec![], - ); - + with_url_test_server(|storage, _, _| async move { let mut headers = HeaderMap::default(); let headers = test_headers(&mut headers); @@ -356,15 +349,7 @@ mod tests { #[tokio::test] async fn get_key() { - with_url_test_server(|url| async move { - let storage = UrlStorage::new( - test_client(), - Uri::from_str(&url).unwrap(), - Uri::from_str(&url).unwrap(), - true, - vec![], - ); - + with_url_test_server(|storage, _, _| async move { let mut headers = HeaderMap::default(); let headers = test_headers(&mut headers); @@ -386,15 +371,7 @@ mod tests { #[tokio::test] async fn head_key() { - with_url_test_server(|url| async move { - let storage = UrlStorage::new( - test_client(), - Uri::from_str(&url).unwrap(), - Uri::from_str(&url).unwrap(), - true, - vec![], - ); - + with_url_test_server(|storage, _, _| async move { let mut headers = HeaderMap::default(); let headers = test_headers(&mut headers); @@ -416,15 +393,7 @@ mod tests { #[tokio::test] async fn get_storage() { - with_url_test_server(|url| async move { - let storage = UrlStorage::new( - test_client(), - Uri::from_str(&url).unwrap(), - Uri::from_str(&url).unwrap(), - true, - vec![], - ); - + with_url_test_server(|storage, _, _| async move { let mut headers = HeaderMap::default(); let headers = test_headers(&mut headers); let options = GetOptions::new_with_default_range(headers); @@ -441,7 +410,7 @@ mod tests { #[tokio::test] async fn range_url_storage() { - with_url_test_server(|url| async move { + with_url_test_server(|_, url, _| async move { let storage = UrlStorage::new( test_client(), Uri::from_str(&url).unwrap(), @@ -449,7 +418,6 @@ mod tests { true, vec![], ); - let mut headers = HeaderMap::default(); let options = test_range_options(&mut headers); @@ -464,7 +432,7 @@ mod tests { #[tokio::test] async fn range_url_storage_blacklisted_headers() { - with_url_test_server(|url| async move { + with_url_test_server(|_, url, _| async move { let storage = UrlStorage::new( test_client(), Uri::from_str(&url).unwrap(), @@ -492,15 +460,7 @@ mod tests { #[tokio::test] async fn head_storage() { - with_url_test_server(|url| async move { - let storage = UrlStorage::new( - test_client(), - Uri::from_str(&url).unwrap(), - Uri::from_str(&url).unwrap(), - true, - vec![], - ); - + with_url_test_server(|storage, _, _| async move { let mut headers = HeaderMap::default(); let headers = test_headers(&mut headers); let options = HeadOptions::new(headers); @@ -575,7 +535,7 @@ mod tests { pub(crate) async fn with_url_test_server(test: F) where - F: FnOnce(String) -> Fut, + F: FnOnce(UrlStorage, String, PathBuf) -> Fut, Fut: Future, { let (_, base_path) = create_local_test_files().await; @@ -596,11 +556,12 @@ mod tests { pub(crate) async fn with_test_server(server_base_path: &Path, test: F) where - F: FnOnce(String) -> Fut, + F: FnOnce(UrlStorage, String, PathBuf) -> Fut, Fut: Future, { + let path = server_base_path.to_str().unwrap(); let router = Router::new() - .nest_service("/assets", ServeDir::new(server_base_path.to_str().unwrap())) + .nest_service("/assets", ServeDir::new(path)) .route_layer(middleware::from_fn(test_auth)); // TODO fix this in htsget-test to bind and return tcp listener. @@ -609,10 +570,22 @@ mod tests { tokio::spawn(async move { axum::serve(listener, router.into_make_service()).await }); - test(format!("http://{}", addr)).await; + let url = format!("http://{}", addr); + test( + UrlStorage::new( + test_client(), + Uri::from_str(&url).unwrap(), + Uri::from_str(&url).unwrap(), + false, + vec![], + ), + url, + server_base_path.to_path_buf(), + ) + .await; } - fn test_headers(headers: &mut HeaderMap) -> &HeaderMap { + pub(crate) fn test_headers(headers: &mut HeaderMap) -> &HeaderMap { headers.append( HeaderName::from_str(AUTHORIZATION.as_str()).unwrap(), HeaderValue::from_str("secret").unwrap(), From eb5b226c62924675daf1fb7f5a2161f54dc86572 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 25 Sep 2024 17:34:02 +1000 Subject: [PATCH 3/7] ci: bump action versions, mozilla-actions/sccache-action@v0.0.5, docker/build-push-action@v6 --- .github/workflows/build.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4a6ad01..71dffe7d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ jobs: os: [ubuntu-latest] steps: - name: Cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.5 - name: Check out uses: actions/checkout@v4 - name: Install Rust diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 87b9466b..8eb55fb1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker GitHub release - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . file: deploy/Dockerfile diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2d15bbbf..6e1bd50a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: os: [ubuntu-latest] steps: - name: Cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.5 - name: Check out uses: actions/checkout@v4 - name: Install Rust From aae1b2c6d24292e49bb29c75847ecc567b485b62 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 30 Sep 2024 12:08:23 +1000 Subject: [PATCH 4/7] refactor: rename type to backend and clarify experimental feature flag --- deploy/config/dev_umccr.toml | 16 +++++++------- deploy/config/example_deploy.toml | 2 +- deploy/config/prod_umccr.toml | 6 ++--- deploy/config/public_umccr.toml | 4 ++-- htsget-actix/README.md | 2 +- htsget-axum/README.md | 2 +- htsget-config/README.md | 22 +++++++++---------- htsget-config/examples/config-files/c4gh.toml | 2 +- .../examples/config-files/s3_storage.toml | 4 ++-- .../config-files/tls_data_server.toml | 2 +- .../config-files/tls_ticket_server.toml | 2 +- .../examples/config-files/url_storage.toml | 2 +- htsget-config/src/config/mod.rs | 2 +- htsget-config/src/resolver.rs | 2 +- htsget-config/src/storage/local.rs | 2 +- htsget-config/src/storage/mod.rs | 10 ++++----- htsget-config/src/storage/object/c4gh.rs | 6 ++--- htsget-config/src/storage/s3.rs | 2 +- htsget-config/src/storage/url.rs | 2 +- htsget-http/README.md | 2 +- htsget-lambda/README.md | 2 +- htsget-search/README.md | 2 +- htsget-storage/README.md | 2 +- htsget-test/README.md | 2 +- 24 files changed, 51 insertions(+), 51 deletions(-) diff --git a/deploy/config/dev_umccr.toml b/deploy/config/dev_umccr.toml index a64c8449..2a9303fd 100644 --- a/deploy/config/dev_umccr.toml +++ b/deploy/config/dev_umccr.toml @@ -26,39 +26,39 @@ environment = "dev" [[resolvers]] regex = '^(org.umccr.dev.htsget-rs-test-data)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-10c-data-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-10f-data-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-10g-data-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-agha-test-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-research-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-primary-data-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-validation-prod)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' diff --git a/deploy/config/example_deploy.toml b/deploy/config/example_deploy.toml index abd7ec0d..39007774 100644 --- a/deploy/config/example_deploy.toml +++ b/deploy/config/example_deploy.toml @@ -17,4 +17,4 @@ environment = "dev" [[resolvers]] regex = '^(?P.*?)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' diff --git a/deploy/config/prod_umccr.toml b/deploy/config/prod_umccr.toml index 2a164a38..5a6899f7 100644 --- a/deploy/config/prod_umccr.toml +++ b/deploy/config/prod_umccr.toml @@ -22,14 +22,14 @@ environment = "prod" [[resolvers]] regex = '^(umccr-research-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-validation-prod)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-primary-data-prod)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' diff --git a/deploy/config/public_umccr.toml b/deploy/config/public_umccr.toml index 4f388e92..6648f2ce 100644 --- a/deploy/config/public_umccr.toml +++ b/deploy/config/public_umccr.toml @@ -11,9 +11,9 @@ environment = 'public' [[resolvers]] regex = '^(org.umccr.demo.sbeacon-data)/CINECA_UK1/(?P.*)$' substitution_string = 'CINECA_UK1/$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(org.umccr.demo.htsget-rs-data)/(?Pbam|cram|vcf|bcf|crypt4gh|mixed)/(?P.*)$' substitution_string = '$type/$key' -storage.type = 'S3' +storage.backend = 'S3' diff --git a/htsget-actix/README.md b/htsget-actix/README.md index 90ba9c15..3dc87b92 100644 --- a/htsget-actix/README.md +++ b/htsget-actix/README.md @@ -50,7 +50,7 @@ are exposed in the public API. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable experimental features like `C4GHStorage`. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## Benchmarks Benchmarks for this crate written using [Criterion.rs][criterion-rs], and aim to compare the performance of this crate with the diff --git a/htsget-axum/README.md b/htsget-axum/README.md index 73fe6048..9e370394 100644 --- a/htsget-axum/README.md +++ b/htsget-axum/README.md @@ -171,7 +171,7 @@ htsget-rs. It also contains the data block server which fetches data from a `Loc This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable experimental features like `C4GHStorage`. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-config/README.md b/htsget-config/README.md index 8a646767..f88bc89c 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -151,7 +151,7 @@ For more information about regex options see the [regex crate](https://docs.rs/r Each resolver also maps to a certain storage backend. This storage backend can be used to set query IDs which are served from local storage, from S3-style bucket storage, or from HTTP URLs. To set the storage backend for a resolver, add a `[resolvers.storage]` table. Some storage backends require feature flags to be set when compiling htsget-rs. -To use `LocalStorage`, set `type = 'Local'` under `[resolvers.storage]`, and specify any additional options from below: +To use `LocalStorage`, set `backend = 'Local'` under `[resolvers.storage]`, and specify any additional options from below: | Option | Description | Type | Default | |--------------------------|-------------------------------------------------------------------------------------------------------------------------------------|------------------------------|--------------------| @@ -161,7 +161,7 @@ To use `LocalStorage`, set `type = 'Local'` under `[resolvers.storage]`, and spe | `path_prefix` | The path prefix which the URL tickets will have. This should likely match the `data_server_serve_at` path. | URL path | `''` | | `use_data_server_config` | Whether to use the data server config to fill in the above values. This overrides any other options specified from this table. | Boolean | `false` | -To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, set `type = 'S3'` under `[resolvers.storage]`, and specify any additional options from below: +To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, set `backend = 'S3'` under `[resolvers.storage]`, and specify any additional options from below: | Option | Description | Type | Default | |--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------| @@ -170,7 +170,7 @@ To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, set ` | `path_style` | The S3 path style to request from the storage backend. If `true`, "path style" is used, e.g. `host.com/bucket/object.bam`, otherwise `bucket.host.com/object` style is used. | Boolean | `false` | `UrlStorage` is another storage backend which can be used to serve data from a remote HTTP URL. When using this storage backend, htsget-rs will fetch data from a `url` which is set in the config. It will also forward any headers received with the initial query, which is useful for authentication. -To use `UrlStorage`, build htsget-rs with the `url-storage` feature enabled, set `type = 'Url'` under `[resolvers.storage]`, and specify any additional options from below: +To use `UrlStorage`, build htsget-rs with the `url-storage` feature enabled, set `backend = 'Url'` under `[resolvers.storage]`, and specify any additional options from below: | Option | Description | Type | Default | |--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------| @@ -195,7 +195,7 @@ regex = '^(example_bucket)/(?P.*)$' substitution_string = '$key' [resolvers.storage] -type = 'S3' +backend = 'S3' # Uses the first capture group in the regex as the bucket. ``` Will use "example_bucket" as the S3 bucket if that resolver matches, because this is the first capture group in the `regex`. @@ -210,7 +210,7 @@ regex = '.*' substitution_string = '$0' [resolvers.storage] -type = 'Local' +backend = 'Local' scheme = 'Http' authority = '127.0.0.1:8081' local_path = './' @@ -225,7 +225,7 @@ regex = '.*' substitution_string = '$0' [resolvers.storage] -type = 'S3' +backend = 'S3' bucket = 'bucket' ``` @@ -237,7 +237,7 @@ regex = ".*" substitution_string = "$0" [resolvers.storage] -type = 'Url' +backend = 'Url' url = "http://localhost:8080" response_url = "https://example.com" forward_headers = true @@ -276,7 +276,7 @@ regex = '.*' substitution_string = '$0' [resolvers.storage] -type = 'S3' +backend = 'S3' bucket = 'bucket' [resolvers.allow_guard] @@ -486,7 +486,7 @@ regex = '.*' substitution_string = '$0' [resolvers.storage] -type = 'S3' +backend = 'S3' bucket = 'bucket' endpoint = 'http://127.0.0.1:9000' path_style = true @@ -523,7 +523,7 @@ regex = '.*' substitution_string = '$0' [resolvers.storage] -type = 'Local' +backend = 'Local' private_key = 'data/c4gh/keys/bob.sec' # pragma: allowlist secret recipient_public_key = 'data/c4gh/keys/alice.pub' ``` @@ -544,7 +544,7 @@ regex, and changing it by using a substitution string. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable experimental features like `C4GHStorage`. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-config/examples/config-files/c4gh.toml b/htsget-config/examples/config-files/c4gh.toml index ce260931..c17c36d3 100644 --- a/htsget-config/examples/config-files/c4gh.toml +++ b/htsget-config/examples/config-files/c4gh.toml @@ -9,7 +9,7 @@ regex = ".*" substitution_string = "$0" [resolvers.storage] -type = 'Local' +backend = 'Local' private_key = "data/c4gh/keys/bob.sec" # pragma: allowlist secret recipient_public_key = "data/c4gh/keys/alice.pub" diff --git a/htsget-config/examples/config-files/s3_storage.toml b/htsget-config/examples/config-files/s3_storage.toml index 66828a80..5cde4dff 100644 --- a/htsget-config/examples/config-files/s3_storage.toml +++ b/htsget-config/examples/config-files/s3_storage.toml @@ -11,9 +11,9 @@ data_server_enabled = false [[resolvers]] regex = '^(bucket)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' # Or, set the bucket manually #[resolvers.storage] -#type = 'S3' +#backend = 'S3' #bucket = 'bucket' diff --git a/htsget-config/examples/config-files/tls_data_server.toml b/htsget-config/examples/config-files/tls_data_server.toml index 649710bb..d2e4316e 100644 --- a/htsget-config/examples/config-files/tls_data_server.toml +++ b/htsget-config/examples/config-files/tls_data_server.toml @@ -12,5 +12,5 @@ regex = ".*" substitution_string = "$0" [resolvers.storage] -type = 'Local' +backend = 'Local' use_data_server_config = true diff --git a/htsget-config/examples/config-files/tls_ticket_server.toml b/htsget-config/examples/config-files/tls_ticket_server.toml index 4ef72c6e..9bd196ff 100644 --- a/htsget-config/examples/config-files/tls_ticket_server.toml +++ b/htsget-config/examples/config-files/tls_ticket_server.toml @@ -12,5 +12,5 @@ regex = ".*" substitution_string = "$0" [resolvers.storage] -type = 'S3' +backend = 'S3' bucket = "bucket" diff --git a/htsget-config/examples/config-files/url_storage.toml b/htsget-config/examples/config-files/url_storage.toml index cfda737f..372b0080 100644 --- a/htsget-config/examples/config-files/url_storage.toml +++ b/htsget-config/examples/config-files/url_storage.toml @@ -16,7 +16,7 @@ regex = ".*" substitution_string = "$0" [resolvers.storage] -type = 'Url' +backend = 'Url' url = "http://127.0.0.1:8081" response_url = "https://127.0.0.1:8081" forward_headers = true diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index cfa44354..71750fe9 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -775,7 +775,7 @@ pub(crate) mod tests { [[resolvers]] [resolvers.storage] - type = "Local" + backend = "Local" use_data_server_config = true "#, |config| { diff --git a/htsget-config/src/resolver.rs b/htsget-config/src/resolver.rs index dae7bb12..d25192f0 100644 --- a/htsget-config/src/resolver.rs +++ b/htsget-config/src/resolver.rs @@ -678,7 +678,7 @@ mod tests { vec![( "HTSGET_RESOLVERS", "[{ regex=regex, substitution_string=substitution_string, \ - storage={ type=S3, bucket=bucket }, \ + storage={ backend=S3, bucket=bucket }, \ allow_guard={ allow_reference_names=[chr1], allow_fields=[QNAME], allow_tags=[RG], \ allow_formats=[BAM], allow_classes=[body], allow_interval_start=100, \ allow_interval_end=1000 } }]", diff --git a/htsget-config/src/storage/local.rs b/htsget-config/src/storage/local.rs index b6b60818..e853351a 100644 --- a/htsget-config/src/storage/local.rs +++ b/htsget-config/src/storage/local.rs @@ -126,7 +126,7 @@ mod tests { regex = "regex" [resolvers.storage] - type = "Local" + backend = "Local" local_path = "path" scheme = "HTTPS" path_prefix = "path" diff --git a/htsget-config/src/storage/mod.rs b/htsget-config/src/storage/mod.rs index aa0cac92..c562ce60 100644 --- a/htsget-config/src/storage/mod.rs +++ b/htsget-config/src/storage/mod.rs @@ -30,7 +30,7 @@ impl ResolvedId { /// Specify the storage backend to use as config values. #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(tag = "type")] +#[serde(tag = "backend")] #[non_exhaustive] pub enum Storage { #[serde(alias = "local", alias = "LOCAL")] @@ -61,7 +61,7 @@ pub(crate) mod tests { r#" [[resolvers]] [resolvers.storage] - type = "Local" + backend = "Local" regex = "regex" "#, |config| { @@ -79,7 +79,7 @@ pub(crate) mod tests { test_config_from_env( vec![( "HTSGET_RESOLVERS", - "[{storage={ type=Local, use_data_server_config=true}}]", + "[{storage={ backend=Local, use_data_server_config=true}}]", )], |config| { assert!(matches!( @@ -97,7 +97,7 @@ pub(crate) mod tests { r#" [[resolvers]] [resolvers.storage] - type = "S3" + backend = "S3" regex = "regex" "#, |config| { @@ -114,7 +114,7 @@ pub(crate) mod tests { #[test] fn config_storage_tagged_s3_env() { test_config_from_env( - vec![("HTSGET_RESOLVERS", "[{storage={ type=S3 }}]")], + vec![("HTSGET_RESOLVERS", "[{storage={ backend=S3 }}]")], |config| { assert!(matches!( config.resolvers().first().unwrap().storage(), diff --git a/htsget-config/src/storage/object/c4gh.rs b/htsget-config/src/storage/object/c4gh.rs index 3639acbf..191a458e 100644 --- a/htsget-config/src/storage/object/c4gh.rs +++ b/htsget-config/src/storage/object/c4gh.rs @@ -114,7 +114,7 @@ mod tests { #[test] fn config_local_storage_c4gh() { - test_c4gh_storage_config(r#"type = "Local""#, |config| { + test_c4gh_storage_config(r#"backend = "Local""#, |config| { assert!(matches!( config.resolvers().first().unwrap().storage(), Storage::Local(local_storage) if local_storage.object_type().keys().is_some() @@ -127,7 +127,7 @@ mod tests { fn config_s3_storage_c4gh() { test_c4gh_storage_config( r#" - type = "S3" + backend = "S3" bucket = "bucket" "#, |config| { @@ -144,7 +144,7 @@ mod tests { fn config_url_storage_c4gh() { test_c4gh_storage_config( r#" - type = "Url" + backend = "Url" url = "https://example.com/" response_url = "https://example.com/" forward_headers = false diff --git a/htsget-config/src/storage/s3.rs b/htsget-config/src/storage/s3.rs index 6edd15af..22f95184 100644 --- a/htsget-config/src/storage/s3.rs +++ b/htsget-config/src/storage/s3.rs @@ -61,7 +61,7 @@ mod tests { regex = "regex" [resolvers.storage] - type = "S3" + backend = "S3" bucket = "bucket" "#, |config| { diff --git a/htsget-config/src/storage/url.rs b/htsget-config/src/storage/url.rs index a2906aae..49204599 100644 --- a/htsget-config/src/storage/url.rs +++ b/htsget-config/src/storage/url.rs @@ -261,7 +261,7 @@ mod tests { regex = "regex" [resolvers.storage] - type = "Url" + backend = "Url" url = "https://example.com/" response_url = "https://example.com/" forward_headers = false diff --git a/htsget-http/README.md b/htsget-http/README.md index 2cb6be0d..0eae7c78 100644 --- a/htsget-http/README.md +++ b/htsget-http/README.md @@ -38,7 +38,7 @@ These functions take query and endpoint information, and process it using [htsge This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable experimental features like `C4GHStorage`. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [warp]: https://github.com/seanmonstar/warp [htsget-search]: ../htsget-search diff --git a/htsget-lambda/README.md b/htsget-lambda/README.md index cd498d9f..e24d9395 100644 --- a/htsget-lambda/README.md +++ b/htsget-lambda/README.md @@ -46,7 +46,7 @@ library code, and it instead uses `htsget-axum`. Please use that crate for funct This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable experimental features like `C4GHStorage`. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-search/README.md b/htsget-search/README.md index 30d49473..7623f8c1 100644 --- a/htsget-search/README.md +++ b/htsget-search/README.md @@ -59,7 +59,7 @@ used to process requests. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable experimental features like `C4GHStorage`. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## Minimising Byte Ranges diff --git a/htsget-storage/README.md b/htsget-storage/README.md index bb23e577..75923e8c 100644 --- a/htsget-storage/README.md +++ b/htsget-storage/README.md @@ -49,7 +49,7 @@ and [url] modules implement the `Storage` functionality. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable experimental features like `C4GHStorage`. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [local]: src/local.rs [s3]: src/s3.rs diff --git a/htsget-test/README.md b/htsget-test/README.md index a9bdab4e..93af3453 100644 --- a/htsget-test/README.md +++ b/htsget-test/README.md @@ -40,7 +40,7 @@ This crate has the following features: * `aws-mocks`: used to enable AWS mocking for tests. * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable experimental features like `C4GHStorage`. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [dev-dependencies]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#development-dependencies From 20c46721578e282a648ccfb6b540f3fa1705fa56 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 30 Sep 2024 14:02:07 +1000 Subject: [PATCH 5/7] refactor: remove `ObjectType` in favour of a more flattened config hierarchy --- Cargo.lock | 1 + htsget-config/Cargo.toml | 3 +- htsget-config/src/resolver.rs | 25 +++--- .../src/storage/{object => }/c4gh.rs | 6 +- htsget-config/src/storage/local.rs | 61 +++++++------ htsget-config/src/storage/mod.rs | 11 +-- htsget-config/src/storage/object/mod.rs | 25 ------ htsget-config/src/storage/s3.rs | 36 ++++---- htsget-config/src/storage/url.rs | 87 ++++++++++++------- htsget-http/src/lib.rs | 3 +- htsget-search/benches/search_benchmarks.rs | 3 +- htsget-search/src/bam_search.rs | 2 +- htsget-search/src/bcf_search.rs | 2 +- htsget-search/src/cram_search.rs | 2 +- htsget-search/src/from_storage.rs | 5 +- htsget-search/src/vcf_search.rs | 2 +- htsget-storage/src/lib.rs | 68 +++++++++------ htsget-storage/src/local.rs | 3 +- htsget-test/src/c4gh.rs | 2 +- htsget-test/src/http/mod.rs | 5 +- 20 files changed, 184 insertions(+), 168 deletions(-) rename htsget-config/src/storage/{object => }/c4gh.rs (93%) delete mode 100644 htsget-config/src/storage/object/mod.rs diff --git a/Cargo.lock b/Cargo.lock index fafcba16..044cd848 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2275,6 +2275,7 @@ name = "htsget-config" version = "0.10.1" dependencies = [ "async-trait", + "cfg-if", "clap", "crypt4gh", "figment", diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index 49dda224..4b083f5e 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -12,7 +12,7 @@ repository = "https://github.com/umccr/htsget-rs" [features] s3-storage = [] -url-storage = ["dep:reqwest"] +url-storage = ["dep:reqwest", "dep:cfg-if"] experimental = ["dep:crypt4gh"] default = [] @@ -37,6 +37,7 @@ rustls-pki-types = "1" # url-storage reqwest = { version = "0.12", features = ["rustls-tls"], default-features = false, optional = true } +cfg-if = { version = "1", optional = true } # Crypt4GH crypt4gh = { version = "0.4", git = "https://github.com/EGA-archive/crypt4gh-rust", optional = true } diff --git a/htsget-config/src/resolver.rs b/htsget-config/src/resolver.rs index d25192f0..93720595 100644 --- a/htsget-config/src/resolver.rs +++ b/htsget-config/src/resolver.rs @@ -8,9 +8,9 @@ use serde_with::with_prefix; use tracing::instrument; use crate::config::DataServerConfig; -use crate::storage::local::LocalStorage; +use crate::storage::local::Local; #[cfg(feature = "s3-storage")] -use crate::storage::s3::S3Storage; +use crate::storage::s3::S3; #[cfg(feature = "url-storage")] use crate::storage::url::UrlStorageClient; use crate::storage::{ResolvedId, Storage}; @@ -27,11 +27,11 @@ pub trait IdResolver { #[async_trait] pub trait ResolveResponse { /// Convert from `LocalStorage`. - async fn from_local(local_storage: &LocalStorage, query: &Query) -> Result; + async fn from_local(local_storage: &Local, query: &Query) -> Result; /// Convert from `S3Storage`. #[cfg(feature = "s3-storage")] - async fn from_s3(s3_storage: &S3Storage, query: &Query) -> Result; + async fn from_s3(s3_storage: &S3, query: &Query) -> Result; /// Convert from `UrlStorage`. #[cfg(feature = "url-storage")] @@ -444,7 +444,7 @@ mod tests { use crate::config::tests::{test_config_from_env, test_config_from_file}; #[cfg(feature = "s3-storage")] - use crate::storage::s3::S3Storage; + use crate::storage::s3::S3; use crate::types::Scheme::Http; use crate::types::Url; @@ -454,7 +454,7 @@ mod tests { #[async_trait] impl ResolveResponse for TestResolveResponse { - async fn from_local(local_storage: &LocalStorage, _: &Query) -> Result { + async fn from_local(local_storage: &Local, _: &Query) -> Result { Ok(Response::new( Bam, vec![Url::new(local_storage.authority().to_string())], @@ -462,7 +462,7 @@ mod tests { } #[cfg(feature = "s3-storage")] - async fn from_s3(s3_storage: &S3Storage, _: &Query) -> Result { + async fn from_s3(s3_storage: &S3, _: &Query) -> Result { Ok(Response::new(Bam, vec![Url::new(s3_storage.bucket())])) } @@ -477,12 +477,11 @@ mod tests { #[tokio::test] async fn resolver_resolve_local_request() { - let local_storage = LocalStorage::new( + let local_storage = Local::new( Http, Authority::from_static("127.0.0.1:8080"), "data".to_string(), "/data".to_string(), - Default::default(), false, ); let resolver = Resolver::new( @@ -499,7 +498,7 @@ mod tests { #[cfg(feature = "s3-storage")] #[tokio::test] async fn resolver_resolve_s3_request_tagged() { - let s3_storage = S3Storage::new("id".to_string(), None, false, Default::default()); + let s3_storage = S3::new("id".to_string(), None, false); let resolver = Resolver::new( Storage::S3(s3_storage), "(id)-1", @@ -515,7 +514,7 @@ mod tests { #[tokio::test] async fn resolver_resolve_s3_request() { let resolver = Resolver::new( - Storage::S3(S3Storage::default()), + Storage::S3(S3::default()), "(id)-1", "$1-test", AllowGuard::default(), @@ -538,7 +537,6 @@ mod tests { }), true, vec![], - Default::default(), client, ); @@ -693,8 +691,7 @@ mod tests { Interval::new(Some(100), Some(1000)), ); let resolver = config.resolvers().first().unwrap(); - let expected_storage = - S3Storage::new("bucket".to_string(), None, false, Default::default()); + let expected_storage = S3::new("bucket".to_string(), None, false); assert_eq!(resolver.regex().to_string(), "regex"); assert_eq!(resolver.substitution_string(), "substitution_string"); diff --git a/htsget-config/src/storage/object/c4gh.rs b/htsget-config/src/storage/c4gh.rs similarity index 93% rename from htsget-config/src/storage/object/c4gh.rs rename to htsget-config/src/storage/c4gh.rs index 191a458e..3f300e8c 100644 --- a/htsget-config/src/storage/object/c4gh.rs +++ b/htsget-config/src/storage/c4gh.rs @@ -117,7 +117,7 @@ mod tests { test_c4gh_storage_config(r#"backend = "Local""#, |config| { assert!(matches!( config.resolvers().first().unwrap().storage(), - Storage::Local(local_storage) if local_storage.object_type().keys().is_some() + Storage::Local(local_storage) if local_storage.keys().is_some() )); }); } @@ -133,7 +133,7 @@ mod tests { |config| { assert!(matches!( config.resolvers().first().unwrap().storage(), - Storage::S3(s3_storage) if s3_storage.object_type().keys().is_some() + Storage::S3(s3_storage) if s3_storage.keys().is_some() )); }, ); @@ -152,7 +152,7 @@ mod tests { |config| { assert!(matches!( config.resolvers().first().unwrap().storage(), - Storage::Url(url_storage) if url_storage.object_type().keys().is_some() + Storage::Url(url_storage) if url_storage.keys().is_some() )); }, ); diff --git a/htsget-config/src/storage/local.rs b/htsget-config/src/storage/local.rs index e853351a..3a8007bc 100644 --- a/htsget-config/src/storage/local.rs +++ b/htsget-config/src/storage/local.rs @@ -4,7 +4,8 @@ use http::uri::Authority; use serde::{Deserialize, Serialize}; use crate::config::{default_localstorage_addr, default_path, DataServerConfig}; -use crate::storage::object::ObjectType; +#[cfg(feature = "experimental")] +use crate::storage::c4gh::C4GHKeys; use crate::tls::KeyPairScheme; use crate::types::Scheme; @@ -18,25 +19,25 @@ fn default_local_path() -> String { #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(default)] -pub struct LocalStorage { +pub struct Local { scheme: Scheme, #[serde(with = "http_serde::authority")] authority: Authority, local_path: String, path_prefix: String, - #[serde(flatten)] - object_type: ObjectType, use_data_server_config: bool, + #[serde(skip_serializing, flatten)] + #[cfg(feature = "experimental")] + keys: Option, } -impl LocalStorage { +impl Local { /// Create a new local storage. pub fn new( scheme: Scheme, authority: Authority, local_path: String, path_prefix: String, - object_type: ObjectType, use_data_server_config: bool, ) -> Self { Self { @@ -44,8 +45,9 @@ impl LocalStorage { authority, local_path, path_prefix, - object_type, use_data_server_config, + #[cfg(feature = "experimental")] + keys: None, } } @@ -69,38 +71,44 @@ impl LocalStorage { &self.path_prefix } - /// Get the object type. - pub fn object_type(&self) -> &ObjectType { - &self.object_type - } - /// Get whether config should be inherited from the data server config. pub fn use_data_server_config(&self) -> bool { self.use_data_server_config } + + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() + } } -impl Default for LocalStorage { +impl Default for Local { fn default() -> Self { - Self { - scheme: Scheme::Http, - authority: default_authority(), - local_path: default_local_path(), - path_prefix: Default::default(), - object_type: Default::default(), - use_data_server_config: false, - } + Self::new( + Scheme::Http, + default_authority(), + default_local_path(), + Default::default(), + false, + ) } } -impl From<&DataServerConfig> for LocalStorage { +impl From<&DataServerConfig> for Local { fn from(config: &DataServerConfig) -> Self { - LocalStorage::new( + Self::new( config.tls().get_scheme(), Authority::from_str(&config.addr().to_string()).expect("expected valid authority"), config.local_path().to_string_lossy().to_string(), config.serve_at().to_string(), - Default::default(), true, ) } @@ -151,13 +159,12 @@ mod tests { None, CorsConfig::default(), ); - let result: LocalStorage = (&data_server_config).into(); - let expected = LocalStorage::new( + let result: Local = (&data_server_config).into(); + let expected = Local::new( Http, Authority::from_static("127.0.0.1:8080"), "data".to_string(), "/data".to_string(), - Default::default(), true, ); diff --git a/htsget-config/src/storage/mod.rs b/htsget-config/src/storage/mod.rs index c562ce60..d8d277dc 100644 --- a/htsget-config/src/storage/mod.rs +++ b/htsget-config/src/storage/mod.rs @@ -1,12 +1,13 @@ -use crate::storage::local::LocalStorage; +use crate::storage::local::Local; #[cfg(feature = "s3-storage")] -use crate::storage::s3::S3Storage; +use crate::storage::s3::S3; #[cfg(feature = "url-storage")] use crate::storage::url::UrlStorageClient; use serde::{Deserialize, Serialize}; +#[cfg(feature = "experimental")] +pub mod c4gh; pub mod local; -pub mod object; #[cfg(feature = "s3-storage")] pub mod s3; #[cfg(feature = "url-storage")] @@ -34,10 +35,10 @@ impl ResolvedId { #[non_exhaustive] pub enum Storage { #[serde(alias = "local", alias = "LOCAL")] - Local(LocalStorage), + Local(Local), #[cfg(feature = "s3-storage")] #[serde(alias = "s3")] - S3(S3Storage), + S3(S3), #[cfg(feature = "url-storage")] #[serde(alias = "url", alias = "URL")] Url(#[serde(skip_serializing)] UrlStorageClient), diff --git a/htsget-config/src/storage/object/mod.rs b/htsget-config/src/storage/object/mod.rs deleted file mode 100644 index c076d452..00000000 --- a/htsget-config/src/storage/object/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! Defines the type of object used by storage. -//! - -#[cfg(feature = "experimental")] -pub mod c4gh; - -#[cfg(feature = "experimental")] -use crate::storage::object::c4gh::C4GHKeys; -use serde::{Deserialize, Serialize}; - -/// An object type, can be regular or Crypt4GH encrypted. -#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)] -pub struct ObjectType { - #[serde(skip_serializing, flatten)] - #[cfg(feature = "experimental")] - keys: Option, -} - -impl ObjectType { - #[cfg(feature = "experimental")] - /// Get the C4GH keys. - pub fn keys(&self) -> Option<&C4GHKeys> { - self.keys.as_ref() - } -} diff --git a/htsget-config/src/storage/s3.rs b/htsget-config/src/storage/s3.rs index 22f95184..9098fa37 100644 --- a/htsget-config/src/storage/s3.rs +++ b/htsget-config/src/storage/s3.rs @@ -1,29 +1,27 @@ -use crate::storage::object::ObjectType; +#[cfg(feature = "experimental")] +use crate::storage::c4gh::C4GHKeys; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] #[serde(default)] -pub struct S3Storage { +pub struct S3 { pub(crate) bucket: String, pub(crate) endpoint: Option, pub(crate) path_style: bool, - #[serde(flatten)] - pub(crate) object_type: ObjectType, + #[serde(skip_serializing, flatten)] + #[cfg(feature = "experimental")] + pub(crate) keys: Option, } -impl S3Storage { +impl S3 { /// Create a new S3 storage. - pub fn new( - bucket: String, - endpoint: Option, - path_style: bool, - object_type: ObjectType, - ) -> Self { + pub fn new(bucket: String, endpoint: Option, path_style: bool) -> Self { Self { bucket, endpoint, path_style, - object_type, + #[cfg(feature = "experimental")] + keys: None, } } @@ -42,9 +40,17 @@ impl S3Storage { self.path_style } - /// Get the object type. - pub fn object_type(&self) -> &ObjectType { - &self.object_type + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() } } diff --git a/htsget-config/src/storage/url.rs b/htsget-config/src/storage/url.rs index 49204599..d5396dca 100644 --- a/htsget-config/src/storage/url.rs +++ b/htsget-config/src/storage/url.rs @@ -1,21 +1,19 @@ -use std::str::FromStr; - +use cfg_if::cfg_if; use http::Uri as InnerUrl; use reqwest::Client; use serde::{Deserialize, Serialize}; use serde_with::with_prefix; +use std::str::FromStr; use crate::error::Error::ParseError; use crate::error::{Error, Result}; +#[cfg(feature = "experimental")] +use crate::storage::c4gh::C4GHKeys; use crate::storage::local::default_authority; -use crate::storage::object::ObjectType; use crate::tls::client::TlsClientConfig; -fn default_url() -> ValidatedUrl { - ValidatedUrl(Url { - inner: InnerUrl::from_str(&format!("https://{}", default_authority())) - .expect("expected valid url"), - }) +fn default_url() -> InnerUrl { + InnerUrl::from_str(&format!("https://{}", default_authority())).expect("expected valid url") } with_prefix!(client_auth_prefix "client_"); @@ -29,8 +27,9 @@ pub struct UrlStorage { header_blacklist: Vec, #[serde(skip_serializing)] tls: TlsClientConfig, - #[serde(flatten)] - object_type: ObjectType, + #[serde(skip_serializing, flatten)] + #[cfg(feature = "experimental")] + keys: Option, } #[derive(Deserialize, Debug, Clone)] @@ -40,8 +39,9 @@ pub struct UrlStorageClient { response_url: ValidatedUrl, forward_headers: bool, header_blacklist: Vec, - object_type: ObjectType, client: Client, + #[cfg(feature = "experimental")] + keys: Option, } impl TryFrom for UrlStorageClient { @@ -65,14 +65,21 @@ impl TryFrom for UrlStorageClient { .build() .map_err(|err| ParseError(format!("building url storage client: {}", err)))?; - Ok(Self::new( + let url_storage = Self::new( storage.url, storage.response_url, storage.forward_headers, storage.header_blacklist, - storage.object_type, client, - )) + ); + + cfg_if! { + if #[cfg(feature = "experimental")] { + Ok(url_storage.set_keys(storage.keys)) + } else { + Ok(url_storage) + } + } } } @@ -83,7 +90,6 @@ impl UrlStorageClient { response_url: ValidatedUrl, forward_headers: bool, header_blacklist: Vec, - object_type: ObjectType, client: Client, ) -> Self { Self { @@ -91,8 +97,9 @@ impl UrlStorageClient { response_url, forward_headers, header_blacklist, - object_type, client, + #[cfg(feature = "experimental")] + keys: None, } } @@ -121,9 +128,17 @@ impl UrlStorageClient { self.client.clone() } - /// Get the object type. - pub fn object_type(&self) -> &ObjectType { - &self.object_type + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() } } @@ -166,7 +181,6 @@ impl UrlStorage { forward_headers: bool, header_blacklist: Vec, tls: TlsClientConfig, - object_type: ObjectType, ) -> Self { Self { url: ValidatedUrl(Url { inner: url }), @@ -176,7 +190,8 @@ impl UrlStorage { forward_headers, header_blacklist, tls, - object_type, + #[cfg(feature = "experimental")] + keys: None, } } @@ -201,22 +216,29 @@ impl UrlStorage { &self.tls } - /// Get the object type. - pub fn object_type(&self) -> &ObjectType { - &self.object_type + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() } } impl Default for UrlStorage { fn default() -> Self { - Self { - url: default_url(), - response_url: default_url(), - forward_headers: true, - header_blacklist: vec![], - tls: TlsClientConfig::default(), - object_type: Default::default(), - } + Self::new( + default_url(), + default_url(), + true, + vec![], + TlsClientConfig::default(), + ) } } @@ -241,7 +263,6 @@ mod tests { true, vec![], client_config, - Default::default(), )); assert!(url_storage.is_ok()); diff --git a/htsget-http/src/lib.rs b/htsget-http/src/lib.rs index 5000c857..3ef99f1c 100644 --- a/htsget-http/src/lib.rs +++ b/htsget-http/src/lib.rs @@ -86,7 +86,7 @@ mod tests { use http::uri::Authority; - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_config::types::{Headers, JsonResponse, Request, Scheme, Url}; use htsget_search::from_storage::HtsGetFromStorage; use htsget_search::HtsGet; @@ -278,7 +278,6 @@ mod tests { Authority::from_static("127.0.0.1:8081"), "data".to_string(), "/data".to_string(), - Default::default(), false, ), ) diff --git a/htsget-search/benches/search_benchmarks.rs b/htsget-search/benches/search_benchmarks.rs index b0b3c88f..aab8c499 100644 --- a/htsget-search/benches/search_benchmarks.rs +++ b/htsget-search/benches/search_benchmarks.rs @@ -6,7 +6,7 @@ use http::uri::Authority; use tokio::runtime::Runtime; use htsget_config::resolver::ResolveResponse; -use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; +use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_config::types::Class::Header; use htsget_config::types::Format::{Bam, Bcf, Cram, Vcf}; use htsget_config::types::{HtsGetError, Query, Scheme}; @@ -22,7 +22,6 @@ async fn perform_query(query: Query) -> Result<(), HtsGetError> { Authority::from_static("127.0.0.1:8081"), "../data".to_string(), "/data".to_string(), - Default::default(), false, ), &query, diff --git a/htsget-search/src/bam_search.rs b/htsget-search/src/bam_search.rs index 53e46c87..1fd83e3f 100644 --- a/htsget-search/src/bam_search.rs +++ b/htsget-search/src/bam_search.rs @@ -151,7 +151,7 @@ pub(crate) mod tests { use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::{Class::Body, Class::Header, Headers, HtsGetError::NotFound, Response, Url}; - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_storage::local::LocalStorage; use htsget_test::http::concat::ConcatResponse; use std::future::Future; diff --git a/htsget-search/src/bcf_search.rs b/htsget-search/src/bcf_search.rs index d3a4a668..605c12b1 100644 --- a/htsget-search/src/bcf_search.rs +++ b/htsget-search/src/bcf_search.rs @@ -108,7 +108,7 @@ impl BcfSearch { #[cfg(test)] mod tests { - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_config::types::Class::Body; use htsget_test::http::concat::ConcatResponse; use std::future::Future; diff --git a/htsget-search/src/cram_search.rs b/htsget-search/src/cram_search.rs index babe3638..10aeaf0d 100644 --- a/htsget-search/src/cram_search.rs +++ b/htsget-search/src/cram_search.rs @@ -274,7 +274,7 @@ impl CramSearch { mod tests { use std::future::Future; - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_test::http::concat::ConcatResponse; use super::*; diff --git a/htsget-search/src/from_storage.rs b/htsget-search/src/from_storage.rs index 419b4a2b..c05b9ef4 100644 --- a/htsget-search/src/from_storage.rs +++ b/htsget-search/src/from_storage.rs @@ -6,9 +6,9 @@ use tracing::debug; use tracing::instrument; use htsget_config::resolver::{ResolveResponse, StorageResolver}; -use htsget_config::storage::local::LocalStorage as LocalStorageConfig; +use htsget_config::storage::local::Local as LocalStorageConfig; #[cfg(feature = "s3-storage")] -use htsget_config::storage::s3::S3Storage as S3StorageConfig; +use htsget_config::storage::s3::S3 as S3StorageConfig; #[cfg(feature = "url-storage")] use htsget_config::storage::url::UrlStorageClient as UrlStorageConfig; @@ -271,7 +271,6 @@ pub(crate) mod tests { Authority::from_static("127.0.0.1:8081"), base_path.to_str().unwrap().to_string(), "/data".to_string(), - Default::default(), false, ), ) diff --git a/htsget-search/src/vcf_search.rs b/htsget-search/src/vcf_search.rs index d5c3acf6..8508c067 100644 --- a/htsget-search/src/vcf_search.rs +++ b/htsget-search/src/vcf_search.rs @@ -119,7 +119,7 @@ impl VcfSearch { #[cfg(test)] pub(crate) mod tests { - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_config::types::Class::Body; use htsget_test::http::concat::ConcatResponse; use std::future::Future; diff --git a/htsget-storage/src/lib.rs b/htsget-storage/src/lib.rs index dbab8fc2..c3470086 100644 --- a/htsget-storage/src/lib.rs +++ b/htsget-storage/src/lib.rs @@ -23,10 +23,11 @@ use async_trait::async_trait; use base64::engine::general_purpose; use base64::Engine; use cfg_if::cfg_if; -use htsget_config::storage::local::LocalStorage as LocalStorageConfig; -use htsget_config::storage::object::ObjectType; +#[cfg(feature = "experimental")] +use htsget_config::storage::c4gh::C4GHKeys; +use htsget_config::storage::local::Local as LocalStorageConfig; #[cfg(feature = "s3-storage")] -use htsget_config::storage::s3::S3Storage as S3StorageConfig; +use htsget_config::storage::s3::S3 as S3StorageConfig; #[cfg(feature = "url-storage")] use htsget_config::storage::url::UrlStorageClient as UrlStorageConfig; use htsget_config::types::Scheme; @@ -136,21 +137,16 @@ impl StorageTrait for Storage { } impl Storage { - /// Wrap an existing storage with the object type storage. - pub fn from_object_type(_object_type: &ObjectType, storage: Storage) -> Storage { - cfg_if! { - if #[cfg(feature = "experimental")] { - if let Some(keys) = _object_type.keys() { - Storage::new(C4GHStorage::new_box( - keys.clone().into_inner(), - storage.into_inner(), - )) - } else { - storage - } - } else { - storage - } + #[cfg(feature = "experimental")] + /// Wrap an existing storage with C4GH storage + pub fn from_c4gh_keys(keys: Option<&C4GHKeys>, storage: Storage) -> Storage { + if let Some(keys) = keys { + Storage::new(C4GHStorage::new_box( + keys.clone().into_inner(), + storage.into_inner(), + )) + } else { + storage } } @@ -160,10 +156,14 @@ impl Storage { local_storage.local_path(), local_storage.clone(), )?); - Ok(Storage::from_object_type( - local_storage.object_type(), - storage, - )) + + cfg_if! { + if #[cfg(feature = "experimental")] { + Ok(Self::from_c4gh_keys(local_storage.keys(), storage)) + } else { + Ok(storage) + } + } } /// Create from s3 config. @@ -177,7 +177,14 @@ impl Storage { ) .await, ); - Storage::from_object_type(s3_storage.object_type(), storage) + + cfg_if! { + if #[cfg(feature = "experimental")] { + Self::from_c4gh_keys(s3_storage.keys(), storage) + } else { + storage + } + } } /// Create from url config. @@ -190,7 +197,14 @@ impl Storage { url_storage.forward_headers(), url_storage.header_blacklist().to_vec(), )); - Storage::from_object_type(url_storage.object_type(), storage) + + cfg_if! { + if #[cfg(feature = "experimental")] { + Self::from_c4gh_keys(url_storage.keys(), storage) + } else { + storage + } + } } pub fn new(inner: impl StorageTrait + Send + Sync + 'static) -> Self { @@ -265,7 +279,7 @@ pub trait UrlFormatter { fn format_url>(&self, key: K) -> Result; } -impl UrlFormatter for htsget_config::storage::local::LocalStorage { +impl UrlFormatter for htsget_config::storage::local::Local { fn format_url>(&self, key: K) -> Result { uri::Builder::new() .scheme(match self.scheme() { @@ -285,7 +299,7 @@ mod tests { use http::uri::Authority; use crate::local::LocalStorage; - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_test::util::default_dir; use super::*; @@ -310,7 +324,6 @@ mod tests { Authority::from_static("127.0.0.1:8080"), "data".to_string(), "/data".to_string(), - Default::default(), false, ); test_formatter_authority(formatter, "http"); @@ -323,7 +336,6 @@ mod tests { Authority::from_static("127.0.0.1:8080"), "data".to_string(), "/data".to_string(), - Default::default(), false, ); test_formatter_authority(formatter, "https"); diff --git a/htsget-storage/src/local.rs b/htsget-storage/src/local.rs index 71757459..f94e8e32 100644 --- a/htsget-storage/src/local.rs +++ b/htsget-storage/src/local.rs @@ -140,7 +140,7 @@ pub(crate) mod tests { use tokio::fs::{create_dir, File}; use tokio::io::AsyncWriteExt; - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_config::types::Scheme; use super::*; @@ -345,7 +345,6 @@ pub(crate) mod tests { Authority::from_static("127.0.0.1:8081"), "data".to_string(), "/data".to_string(), - Default::default(), false, ), ) diff --git a/htsget-test/src/c4gh.rs b/htsget-test/src/c4gh.rs index 8f1de6e5..cb402764 100644 --- a/htsget-test/src/c4gh.rs +++ b/htsget-test/src/c4gh.rs @@ -1,7 +1,7 @@ use crate::util::default_dir; use crypt4gh::keys::{get_private_key, get_public_key}; use crypt4gh::{decrypt, encrypt, Keys}; -use htsget_config::storage::object::c4gh::{C4GHKeys, C4GHPath}; +use htsget_config::storage::c4gh::{C4GHKeys, C4GHPath}; use std::collections::HashSet; use std::io::{BufReader, BufWriter, Cursor}; diff --git a/htsget-test/src/http/mod.rs b/htsget-test/src/http/mod.rs index 4d643f99..43e9b4ef 100644 --- a/htsget-test/src/http/mod.rs +++ b/htsget-test/src/http/mod.rs @@ -18,7 +18,7 @@ use serde::de; use htsget_config::config::cors::{AllowType, CorsConfig}; use htsget_config::config::{DataServerConfig, TicketServerConfig}; use htsget_config::resolver::Resolver; -use htsget_config::storage::{local::LocalStorage, Storage}; +use htsget_config::storage::{local::Local, Storage}; use htsget_config::tls::{ load_certs, load_key, tls_server_config, CertificateKeyPair, TlsServerConfig, }; @@ -95,12 +95,11 @@ pub trait TestServer { /// Get the default test storage. pub fn default_test_resolver(addr: SocketAddr, scheme: Scheme) -> Vec { - let local_storage = LocalStorage::new( + let local_storage = Local::new( scheme, Authority::from_str(&addr.to_string()).unwrap(), default_dir_data().to_str().unwrap().to_string(), "/data".to_string(), - Default::default(), false, ); vec![ From ece5c4574e457c8edb6e26e8e7252fac22b316df Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 30 Sep 2024 14:44:28 +1000 Subject: [PATCH 6/7] fix(storage): make overflow handling more robust --- htsget-storage/src/c4gh/mod.rs | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/htsget-storage/src/c4gh/mod.rs b/htsget-storage/src/c4gh/mod.rs index 44a1c893..48857a7b 100644 --- a/htsget-storage/src/c4gh/mod.rs +++ b/htsget-storage/src/c4gh/mod.rs @@ -141,13 +141,14 @@ impl DeserializedHeader { data, )) } - - /// Decrypt the - pub fn decrypt_stream() {} } /// Convert an encrypted file position to an unencrypted position if the header length is known. pub fn to_unencrypted(encrypted_position: u64, header_length: u64) -> u64 { + if encrypted_position < header_length + NONCE_SIZE { + return 0; + } + let number_data_blocks = encrypted_position / DATA_BLOCK_SIZE; let mut additional_bytes = number_data_blocks * (NONCE_SIZE + MAC_SIZE); @@ -161,6 +162,10 @@ pub fn to_unencrypted(encrypted_position: u64, header_length: u64) -> u64 { /// Convert an encrypted file size to an unencrypted file size if the header length is known. pub fn to_unencrypted_file_size(encrypted_file_size: u64, header_length: u64) -> u64 { + if encrypted_file_size < header_length + NONCE_SIZE + MAC_SIZE { + return 0; + } + to_unencrypted(encrypted_file_size, header_length) - MAC_SIZE } @@ -272,6 +277,26 @@ mod tests { assert_eq!(result, expected); } + #[test] + fn test_to_unencrypted() { + let result = to_unencrypted(124, 124); + assert_eq!(result, 0); + let result = to_unencrypted(124 + 12, 124); + assert_eq!(result, 0); + let result = to_unencrypted(124 + 12 + 12, 124); + assert_eq!(result, 12); + } + + #[test] + fn test_to_unencrypted_file_size() { + let result = to_unencrypted_file_size(124, 124); + assert_eq!(result, 0); + let result = to_unencrypted_file_size(124 + 12 + 16, 124); + assert_eq!(result, 0); + let result = to_unencrypted_file_size(124 + 12 + 16 + 12, 124); + assert_eq!(result, 12); + } + #[test] fn test_unencrypted_clamp() { let pos = 0; From 7aa2fe5a8c892090f2cc09aebbb2933f20f4ebb9 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 30 Sep 2024 15:07:55 +1000 Subject: [PATCH 7/7] build(deps): bump noodles and tower --- Cargo.lock | 294 ++++++++++++++++++++------------------ htsget-axum/Cargo.toml | 2 +- htsget-config/Cargo.toml | 2 +- htsget-lambda/Cargo.toml | 2 +- htsget-search/Cargo.toml | 2 +- htsget-storage/Cargo.toml | 2 +- htsget-test/Cargo.toml | 2 +- 7 files changed, 157 insertions(+), 149 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 044cd848..7ad1fb26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,7 +81,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -218,7 +218,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -419,18 +419,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] name = "async-trait" -version = "0.1.82" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -459,15 +459,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.6" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848d7b9b605720989929279fa644ce8f244d0ce3146fcca5b70e4eb7b3c020fc" +checksum = "8191fb3091fa0561d1379ef80333c3c7191c6f0435d986e85821bcf7acbd1126" dependencies = [ "aws-credential-types", "aws-runtime", @@ -560,9 +560,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.51.0" +version = "1.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c09fd4b5c7ed75f52b913b4f3ff0501dae7f8cb9125f6d45db4553980cbc0528" +checksum = "f571deb0a80c20d21d9f3e8418c1712af9ff4bf399d057e5549a934eca4844e2" dependencies = [ "ahash", "aws-credential-types", @@ -595,9 +595,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.43.0" +version = "1.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a9d27ed1c12b1140c47daf1bc541606c43fdafd918c4797d520db0043ceef2" +checksum = "0b90cfe6504115e13c41d3ea90286ede5aa14da294f3fe077027a6e83850843c" dependencies = [ "aws-credential-types", "aws-runtime", @@ -617,9 +617,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.44.0" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44514a6ca967686cde1e2a1b81df6ef1883d0e3e570da8d8bc5c491dcb6fc29b" +checksum = "167c0fad1f212952084137308359e8e4c4724d1c643038ce163f06de9662c1d0" dependencies = [ "aws-credential-types", "aws-runtime", @@ -639,9 +639,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.43.0" +version = "1.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7a4d279762a35b9df97209f6808b95d4fe78547fe2316b4d200a0283960c5a" +checksum = "2cb5f98188ec1435b68097daa2a37d74b9d17c9caa799466338a8d1544e71b9d" dependencies = [ "aws-credential-types", "aws-runtime", @@ -842,9 +842,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.6" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03701449087215b5369c7ea17fef0dd5d24cb93439ec5af0c7615f58c3f22605" +checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" dependencies = [ "base64-simd", "bytes", @@ -917,9 +917,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.5" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" dependencies = [ "async-trait", "axum-core", @@ -943,7 +943,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower 0.4.13", + "tower 0.5.1", "tower-layer", "tower-service", "tracing", @@ -951,9 +951,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", @@ -964,7 +964,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tower-layer", "tower-service", "tracing", @@ -972,9 +972,9 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733" +checksum = "73c3220b188aea709cf1b6c5f9b01c3bd936bb08bd2b5184a12b35ac8131b1f9" dependencies = [ "axum", "axum-core", @@ -987,7 +987,7 @@ dependencies = [ "pin-project-lite", "serde", "serde_json", - "tower 0.4.13", + "tower 0.5.1", "tower-layer", "tower-service", "tracing", @@ -1081,7 +1081,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.77", + "syn 2.0.79", "which", ] @@ -1277,9 +1277,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.21" +version = "1.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" +checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" dependencies = [ "jobserver", "libc", @@ -1389,9 +1389,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" dependencies = [ "clap_builder", "clap_derive", @@ -1399,9 +1399,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" dependencies = [ "anstream", "anstyle", @@ -1411,14 +1411,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1691,7 +1691,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1715,7 +1715,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1726,7 +1726,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1771,7 +1771,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1931,9 +1931,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.33" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", "miniz_oxide", @@ -2016,7 +2016,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -2589,9 +2589,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", "futures-channel", @@ -2602,7 +2602,6 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower 0.4.13", "tower-service", "tracing", ] @@ -2860,9 +2859,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lexical-core" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -2873,9 +2872,9 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0" dependencies = [ "lexical-parse-integer", "lexical-util", @@ -2884,9 +2883,9 @@ dependencies = [ [[package]] name = "lexical-parse-integer" -version = "0.8.6" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61" dependencies = [ "lexical-util", "static_assertions", @@ -2894,18 +2893,18 @@ dependencies = [ [[package]] name = "lexical-util" -version = "0.8.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0" dependencies = [ "static_assertions", ] [[package]] name = "lexical-write-float" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809" dependencies = [ "lexical-util", "lexical-write-integer", @@ -2914,9 +2913,9 @@ dependencies = [ [[package]] name = "lexical-write-integer" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162" dependencies = [ "lexical-util", "static_assertions", @@ -2924,9 +2923,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libloading" @@ -3080,9 +3079,9 @@ checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" [[package]] name = "mutually_exclusive_features" -version = "0.0.3" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d02c0b00610773bb7fc61d85e13d86c7858cbdf00e1a120bfc41bc055dbaa0e" +checksum = "e94e1e6445d314f972ff7395df2de295fe51b71821694f0b0e1e79c4f12c8577" [[package]] name = "nom" @@ -3096,9 +3095,9 @@ dependencies = [ [[package]] name = "noodles" -version = "0.82.0" +version = "0.83.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5855c944a96d41a76b13609b9696b2fb7adbc248048f10a4df836edabddd65" +checksum = "9f156eba509c563e509897ff9b540389eb4744bc4dbe35e5e464c9ea298cb466" dependencies = [ "noodles-bam", "noodles-bcf", @@ -3116,9 +3115,9 @@ dependencies = [ [[package]] name = "noodles-bam" -version = "0.67.0" +version = "0.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bc69bd00891e3e1c5faffe4f55d00c94d9e53d4bdbe63ec8c7e2b881f3bc85" +checksum = "1b41154a96c7c65e06ea09eb9784ecec00b04d05cfcd8f0e56add7810b2e7ad2" dependencies = [ "bstr", "byteorder", @@ -3134,9 +3133,9 @@ dependencies = [ [[package]] name = "noodles-bcf" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df27249855797f89f13750ff58a914acd4ba97714b3818dff5cb476cef3907c7" +checksum = "e465d9d332ee4f5366cb1861aae94177767504f4a490277ed33a16ffb372b47e" dependencies = [ "byteorder", "futures", @@ -3175,9 +3174,9 @@ dependencies = [ [[package]] name = "noodles-cram" -version = "0.68.0" +version = "0.69.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68d1ea4ae713b5e17321ebb20e2f5ca99cadbe2b9e08501a7043458f3a66406e" +checksum = "4e3664097393d4468de20d57e23a274e5ab267faf7af75c906ed0393bd164d04" dependencies = [ "async-compression", "bitflags", @@ -3200,9 +3199,9 @@ dependencies = [ [[package]] name = "noodles-csi" -version = "0.38.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69e79dbc09bd0cb86d29469ed29066e9a163bce6640527b343bdea458144618" +checksum = "a0f41004636fb4232155421cbf4706565073623838a8252875085fa670b8185c" dependencies = [ "bit-vec", "byteorder", @@ -3214,9 +3213,9 @@ dependencies = [ [[package]] name = "noodles-fasta" -version = "0.43.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0634eec06d20e899a5d99922c40fa5186064d8c675c78690a461ccbb2edc60d1" +checksum = "0769070ca53f7b5bcc19db0a465d10328da8dc996418bcde317682f05a236f46" dependencies = [ "bstr", "bytes", @@ -3228,9 +3227,9 @@ dependencies = [ [[package]] name = "noodles-fastq" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c596792c857f37e6a85e2cf1e68578f5b70f867cad028bf95c1e2b5d7c9c84eb" +checksum = "7985cba9ac68f13795fc2294851bca2711362e80c0c6ab4d8df7c110591f69d7" dependencies = [ "bstr", "futures", @@ -3240,9 +3239,9 @@ dependencies = [ [[package]] name = "noodles-gff" -version = "0.37.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8750c06d43f2066ea511a874ed5736470c883f2e39cd0f60f28dd4530d4591b4" +checksum = "ca42cb034ccc595d963de2ff2fd9cf2c3f563bcb65c1337e90f9d73724743d84" dependencies = [ "futures", "indexmap 2.5.0", @@ -3255,9 +3254,9 @@ dependencies = [ [[package]] name = "noodles-sam" -version = "0.64.0" +version = "0.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72da678e9332b32a916f8c5d5a7c4324da11891bc7148077744566981acaf00c" +checksum = "40c4289f74645d4a7b3da5a4d779cb0e53ac221b7958b56778b3f21baf771c95" dependencies = [ "bitflags", "bstr", @@ -3273,9 +3272,9 @@ dependencies = [ [[package]] name = "noodles-tabix" -version = "0.44.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "263e63f58871224d0cd30ffc4a8531fb4f8f8ec686807febde8e19a69673fe01" +checksum = "fb5c5ed1fa0b9ae083c2e1e1cedc07715861400fdd943fdb1ed6c593719be187" dependencies = [ "byteorder", "indexmap 2.5.0", @@ -3287,9 +3286,9 @@ dependencies = [ [[package]] name = "noodles-vcf" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37459769faa61c655a2f0cfc1f068c69dbaf92eb648250bd7a7e544b6c6664d0" +checksum = "5be568315676d38294c4b00bd608bd19d04394e6106c2db08c4b351b275741df" dependencies = [ "futures", "indexmap 2.5.0", @@ -3384,9 +3383,12 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" +dependencies = [ + "portable-atomic", +] [[package]] name = "oorandom" @@ -3517,7 +3519,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3553,7 +3555,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3580,9 +3582,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "plotters" @@ -3623,6 +3625,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + [[package]] name = "powerfmt" version = "0.2.0" @@ -3665,7 +3673,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" dependencies = [ "proc-macro2", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3709,7 +3717,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "version_check", "yansi", ] @@ -3727,9 +3735,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.36.1" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" dependencies = [ "memchr", "serde", @@ -3857,23 +3865,23 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" +checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -3887,13 +3895,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -3910,9 +3918,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" @@ -4116,9 +4124,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" [[package]] name = "rustls-webpki" @@ -4332,9 +4340,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -4379,7 +4387,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4417,9 +4425,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -4463,7 +4471,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4524,9 +4532,9 @@ dependencies = [ [[package]] name = "simdutf8" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "slab" @@ -4600,9 +4608,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.77" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -4626,9 +4634,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", @@ -4648,22 +4656,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4758,7 +4766,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4830,9 +4838,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.21" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap 2.5.0", "serde", @@ -4851,7 +4859,6 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio", "tower-layer", "tower-service", "tracing", @@ -4870,13 +4877,14 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] name = "tower-http" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" dependencies = [ "bitflags", "bytes", @@ -4923,9 +4931,9 @@ dependencies = [ [[package]] name = "tracing-actix-web" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284586dc201db407be8c9d721abad1b3a6dacbbce5cccecd4fd15a37db95ab0d" +checksum = "15bc0cd5f72e837e310f4d978a90abf202a7f7d8ef3272246bae381d0086d3bf" dependencies = [ "actix-web", "mutually_exclusive_features", @@ -4942,7 +4950,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -5180,7 +5188,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "wasm-bindgen-shared", ] @@ -5214,7 +5222,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5227,9 +5235,9 @@ checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-streams" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" dependencies = [ "futures-util", "js-sys", @@ -5489,9 +5497,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] @@ -5544,7 +5552,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] diff --git a/htsget-axum/Cargo.toml b/htsget-axum/Cargo.toml index 15e20263..9ab58397 100644 --- a/htsget-axum/Cargo.toml +++ b/htsget-axum/Cargo.toml @@ -37,7 +37,7 @@ default = [] hyper = { version = "1", features = ["http1", "http2", "server"] } rustls = "0.23" hyper-util = "0.1" -tower-http = { version = "0.5", features = ["trace", "cors", "fs"] } +tower-http = { version = "0.6", features = ["trace", "cors", "fs"] } http = "1" axum = { version = "0.7", features = ["http2"] } axum-extra = { version = "0.9", features = ["erased-json"] } diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index 4b083f5e..2977ee96 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -19,7 +19,7 @@ default = [] [dependencies] thiserror = "1" async-trait = "0.1" -noodles = { version = "0.82", features = ["core"] } +noodles = { version = "0.83", features = ["core"] } serde = { version = "1", features = ["derive"] } serde_with = "3" serde_regex = "1" diff --git a/htsget-lambda/Cargo.toml b/htsget-lambda/Cargo.toml index 85ca4430..48b3b78e 100644 --- a/htsget-lambda/Cargo.toml +++ b/htsget-lambda/Cargo.toml @@ -24,7 +24,7 @@ default = [] [dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -tower-http = { version = "0.5", features = ["cors"] } +tower-http = { version = "0.6", features = ["cors"] } rustls = "0.23" lambda_http = { version = "0.13" } lambda_runtime = { version = "0.13" } diff --git a/htsget-search/Cargo.toml b/htsget-search/Cargo.toml index 1aa6e5ad..33fe4881 100644 --- a/htsget-search/Cargo.toml +++ b/htsget-search/Cargo.toml @@ -37,7 +37,7 @@ futures-util = "0.3" async-trait = "0.1" # Noodles -noodles = { version = "0.82", features = ["async", "core", "bgzf", "bam", "bcf", "cram", "csi", "sam", "tabix", "vcf"] } +noodles = { version = "0.83", features = ["async", "core", "bgzf", "bam", "bcf", "cram", "csi", "sam", "tabix", "vcf"] } # Error control, tracing, config thiserror = "1" diff --git a/htsget-storage/Cargo.toml b/htsget-storage/Cargo.toml index df56aab2..b828a883 100644 --- a/htsget-storage/Cargo.toml +++ b/htsget-storage/Cargo.toml @@ -62,7 +62,7 @@ htsget-config = { version = "0.10.1", path = "../htsget-config", default-feature htsget-test = { version = "0.6.2", path = "../htsget-test", features = ["http"], default-features = false } [dev-dependencies] -tower-http = { version = "0.5", features = ["fs"] } +tower-http = { version = "0.6", features = ["fs"] } axum = "0.7" tempfile = "3" data-url = "0.3" diff --git a/htsget-test/Cargo.toml b/htsget-test/Cargo.toml index 3d518451..1844a162 100644 --- a/htsget-test/Cargo.toml +++ b/htsget-test/Cargo.toml @@ -43,7 +43,7 @@ default = [] # Server tests dependencies htsget-config = { version = "0.10.1", path = "../htsget-config", default-features = false, optional = true } -noodles = { version = "0.82", optional = true, features = ["async", "bgzf", "vcf", "cram", "bcf", "bam", "fasta"] } +noodles = { version = "0.83", optional = true, features = ["async", "bgzf", "vcf", "cram", "bcf", "bam", "fasta"] } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"], optional = true } tokio = { version = "1", features = ["rt-multi-thread", "fs"], optional = true }