diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4a6ad01..71dffe7d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ jobs: os: [ubuntu-latest] steps: - name: Cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.5 - name: Check out uses: actions/checkout@v4 - name: Install Rust diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 87b9466b..8eb55fb1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker GitHub release - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . file: deploy/Dockerfile diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2d15bbbf..6e1bd50a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: os: [ubuntu-latest] steps: - name: Cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.5 - name: Check out uses: actions/checkout@v4 - name: Install Rust diff --git a/Cargo.lock b/Cargo.lock index fafcba16..7ad1fb26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,7 +81,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -218,7 +218,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -419,18 +419,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] name = "async-trait" -version = "0.1.82" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -459,15 +459,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.6" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848d7b9b605720989929279fa644ce8f244d0ce3146fcca5b70e4eb7b3c020fc" +checksum = "8191fb3091fa0561d1379ef80333c3c7191c6f0435d986e85821bcf7acbd1126" dependencies = [ "aws-credential-types", "aws-runtime", @@ -560,9 +560,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.51.0" +version = "1.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c09fd4b5c7ed75f52b913b4f3ff0501dae7f8cb9125f6d45db4553980cbc0528" +checksum = "f571deb0a80c20d21d9f3e8418c1712af9ff4bf399d057e5549a934eca4844e2" dependencies = [ "ahash", "aws-credential-types", @@ -595,9 +595,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.43.0" +version = "1.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a9d27ed1c12b1140c47daf1bc541606c43fdafd918c4797d520db0043ceef2" +checksum = "0b90cfe6504115e13c41d3ea90286ede5aa14da294f3fe077027a6e83850843c" dependencies = [ "aws-credential-types", "aws-runtime", @@ -617,9 +617,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.44.0" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44514a6ca967686cde1e2a1b81df6ef1883d0e3e570da8d8bc5c491dcb6fc29b" +checksum = "167c0fad1f212952084137308359e8e4c4724d1c643038ce163f06de9662c1d0" dependencies = [ "aws-credential-types", "aws-runtime", @@ -639,9 +639,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.43.0" +version = "1.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7a4d279762a35b9df97209f6808b95d4fe78547fe2316b4d200a0283960c5a" +checksum = "2cb5f98188ec1435b68097daa2a37d74b9d17c9caa799466338a8d1544e71b9d" dependencies = [ "aws-credential-types", "aws-runtime", @@ -842,9 +842,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.6" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03701449087215b5369c7ea17fef0dd5d24cb93439ec5af0c7615f58c3f22605" +checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" dependencies = [ "base64-simd", "bytes", @@ -917,9 +917,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.5" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" dependencies = [ "async-trait", "axum-core", @@ -943,7 +943,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower 0.4.13", + "tower 0.5.1", "tower-layer", "tower-service", "tracing", @@ -951,9 +951,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", @@ -964,7 +964,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tower-layer", "tower-service", "tracing", @@ -972,9 +972,9 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733" +checksum = "73c3220b188aea709cf1b6c5f9b01c3bd936bb08bd2b5184a12b35ac8131b1f9" dependencies = [ "axum", "axum-core", @@ -987,7 +987,7 @@ dependencies = [ "pin-project-lite", "serde", "serde_json", - "tower 0.4.13", + "tower 0.5.1", "tower-layer", "tower-service", "tracing", @@ -1081,7 +1081,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.77", + "syn 2.0.79", "which", ] @@ -1277,9 +1277,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.21" +version = "1.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" +checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" dependencies = [ "jobserver", "libc", @@ -1389,9 +1389,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" dependencies = [ "clap_builder", "clap_derive", @@ -1399,9 +1399,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" dependencies = [ "anstream", "anstyle", @@ -1411,14 +1411,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1691,7 +1691,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1715,7 +1715,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1726,7 +1726,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1771,7 +1771,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1931,9 +1931,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.33" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", "miniz_oxide", @@ -2016,7 +2016,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -2275,6 +2275,7 @@ name = "htsget-config" version = "0.10.1" dependencies = [ "async-trait", + "cfg-if", "clap", "crypt4gh", "figment", @@ -2588,9 +2589,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", "futures-channel", @@ -2601,7 +2602,6 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower 0.4.13", "tower-service", "tracing", ] @@ -2859,9 +2859,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lexical-core" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -2872,9 +2872,9 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0" dependencies = [ "lexical-parse-integer", "lexical-util", @@ -2883,9 +2883,9 @@ dependencies = [ [[package]] name = "lexical-parse-integer" -version = "0.8.6" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61" dependencies = [ "lexical-util", "static_assertions", @@ -2893,18 +2893,18 @@ dependencies = [ [[package]] name = "lexical-util" -version = "0.8.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0" dependencies = [ "static_assertions", ] [[package]] name = "lexical-write-float" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809" dependencies = [ "lexical-util", "lexical-write-integer", @@ -2913,9 +2913,9 @@ dependencies = [ [[package]] name = "lexical-write-integer" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162" dependencies = [ "lexical-util", "static_assertions", @@ -2923,9 +2923,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libloading" @@ -3079,9 +3079,9 @@ checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" [[package]] name = "mutually_exclusive_features" -version = "0.0.3" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d02c0b00610773bb7fc61d85e13d86c7858cbdf00e1a120bfc41bc055dbaa0e" +checksum = "e94e1e6445d314f972ff7395df2de295fe51b71821694f0b0e1e79c4f12c8577" [[package]] name = "nom" @@ -3095,9 +3095,9 @@ dependencies = [ [[package]] name = "noodles" -version = "0.82.0" +version = "0.83.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5855c944a96d41a76b13609b9696b2fb7adbc248048f10a4df836edabddd65" +checksum = "9f156eba509c563e509897ff9b540389eb4744bc4dbe35e5e464c9ea298cb466" dependencies = [ "noodles-bam", "noodles-bcf", @@ -3115,9 +3115,9 @@ dependencies = [ [[package]] name = "noodles-bam" -version = "0.67.0" +version = "0.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bc69bd00891e3e1c5faffe4f55d00c94d9e53d4bdbe63ec8c7e2b881f3bc85" +checksum = "1b41154a96c7c65e06ea09eb9784ecec00b04d05cfcd8f0e56add7810b2e7ad2" dependencies = [ "bstr", "byteorder", @@ -3133,9 +3133,9 @@ dependencies = [ [[package]] name = "noodles-bcf" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df27249855797f89f13750ff58a914acd4ba97714b3818dff5cb476cef3907c7" +checksum = "e465d9d332ee4f5366cb1861aae94177767504f4a490277ed33a16ffb372b47e" dependencies = [ "byteorder", "futures", @@ -3174,9 +3174,9 @@ dependencies = [ [[package]] name = "noodles-cram" -version = "0.68.0" +version = "0.69.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68d1ea4ae713b5e17321ebb20e2f5ca99cadbe2b9e08501a7043458f3a66406e" +checksum = "4e3664097393d4468de20d57e23a274e5ab267faf7af75c906ed0393bd164d04" dependencies = [ "async-compression", "bitflags", @@ -3199,9 +3199,9 @@ dependencies = [ [[package]] name = "noodles-csi" -version = "0.38.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69e79dbc09bd0cb86d29469ed29066e9a163bce6640527b343bdea458144618" +checksum = "a0f41004636fb4232155421cbf4706565073623838a8252875085fa670b8185c" dependencies = [ "bit-vec", "byteorder", @@ -3213,9 +3213,9 @@ dependencies = [ [[package]] name = "noodles-fasta" -version = "0.43.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0634eec06d20e899a5d99922c40fa5186064d8c675c78690a461ccbb2edc60d1" +checksum = "0769070ca53f7b5bcc19db0a465d10328da8dc996418bcde317682f05a236f46" dependencies = [ "bstr", "bytes", @@ -3227,9 +3227,9 @@ dependencies = [ [[package]] name = "noodles-fastq" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c596792c857f37e6a85e2cf1e68578f5b70f867cad028bf95c1e2b5d7c9c84eb" +checksum = "7985cba9ac68f13795fc2294851bca2711362e80c0c6ab4d8df7c110591f69d7" dependencies = [ "bstr", "futures", @@ -3239,9 +3239,9 @@ dependencies = [ [[package]] name = "noodles-gff" -version = "0.37.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8750c06d43f2066ea511a874ed5736470c883f2e39cd0f60f28dd4530d4591b4" +checksum = "ca42cb034ccc595d963de2ff2fd9cf2c3f563bcb65c1337e90f9d73724743d84" dependencies = [ "futures", "indexmap 2.5.0", @@ -3254,9 +3254,9 @@ dependencies = [ [[package]] name = "noodles-sam" -version = "0.64.0" +version = "0.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72da678e9332b32a916f8c5d5a7c4324da11891bc7148077744566981acaf00c" +checksum = "40c4289f74645d4a7b3da5a4d779cb0e53ac221b7958b56778b3f21baf771c95" dependencies = [ "bitflags", "bstr", @@ -3272,9 +3272,9 @@ dependencies = [ [[package]] name = "noodles-tabix" -version = "0.44.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "263e63f58871224d0cd30ffc4a8531fb4f8f8ec686807febde8e19a69673fe01" +checksum = "fb5c5ed1fa0b9ae083c2e1e1cedc07715861400fdd943fdb1ed6c593719be187" dependencies = [ "byteorder", "indexmap 2.5.0", @@ -3286,9 +3286,9 @@ dependencies = [ [[package]] name = "noodles-vcf" -version = "0.65.0" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37459769faa61c655a2f0cfc1f068c69dbaf92eb648250bd7a7e544b6c6664d0" +checksum = "5be568315676d38294c4b00bd608bd19d04394e6106c2db08c4b351b275741df" dependencies = [ "futures", "indexmap 2.5.0", @@ -3383,9 +3383,12 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" +dependencies = [ + "portable-atomic", +] [[package]] name = "oorandom" @@ -3516,7 +3519,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3552,7 +3555,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3579,9 +3582,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "plotters" @@ -3622,6 +3625,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + [[package]] name = "powerfmt" version = "0.2.0" @@ -3664,7 +3673,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" dependencies = [ "proc-macro2", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3708,7 +3717,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "version_check", "yansi", ] @@ -3726,9 +3735,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.36.1" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" dependencies = [ "memchr", "serde", @@ -3856,23 +3865,23 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" +checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -3886,13 +3895,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -3909,9 +3918,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" @@ -4115,9 +4124,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" [[package]] name = "rustls-webpki" @@ -4331,9 +4340,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -4378,7 +4387,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4416,9 +4425,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -4462,7 +4471,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4523,9 +4532,9 @@ dependencies = [ [[package]] name = "simdutf8" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "slab" @@ -4599,9 +4608,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.77" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -4625,9 +4634,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", @@ -4647,22 +4656,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4757,7 +4766,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4829,9 +4838,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.21" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap 2.5.0", "serde", @@ -4850,7 +4859,6 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio", "tower-layer", "tower-service", "tracing", @@ -4869,13 +4877,14 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] name = "tower-http" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" dependencies = [ "bitflags", "bytes", @@ -4922,9 +4931,9 @@ dependencies = [ [[package]] name = "tracing-actix-web" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284586dc201db407be8c9d721abad1b3a6dacbbce5cccecd4fd15a37db95ab0d" +checksum = "15bc0cd5f72e837e310f4d978a90abf202a7f7d8ef3272246bae381d0086d3bf" dependencies = [ "actix-web", "mutually_exclusive_features", @@ -4941,7 +4950,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -5179,7 +5188,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "wasm-bindgen-shared", ] @@ -5213,7 +5222,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5226,9 +5235,9 @@ checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-streams" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" dependencies = [ "futures-util", "js-sys", @@ -5488,9 +5497,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] @@ -5543,7 +5552,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] diff --git a/deploy/config/dev_umccr.toml b/deploy/config/dev_umccr.toml index a64c8449..2a9303fd 100644 --- a/deploy/config/dev_umccr.toml +++ b/deploy/config/dev_umccr.toml @@ -26,39 +26,39 @@ environment = "dev" [[resolvers]] regex = '^(org.umccr.dev.htsget-rs-test-data)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-10c-data-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-10f-data-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-10g-data-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-agha-test-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-research-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-primary-data-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-validation-prod)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' diff --git a/deploy/config/example_deploy.toml b/deploy/config/example_deploy.toml index abd7ec0d..39007774 100644 --- a/deploy/config/example_deploy.toml +++ b/deploy/config/example_deploy.toml @@ -17,4 +17,4 @@ environment = "dev" [[resolvers]] regex = '^(?P.*?)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' diff --git a/deploy/config/prod_umccr.toml b/deploy/config/prod_umccr.toml index 2a164a38..5a6899f7 100644 --- a/deploy/config/prod_umccr.toml +++ b/deploy/config/prod_umccr.toml @@ -22,14 +22,14 @@ environment = "prod" [[resolvers]] regex = '^(umccr-research-dev)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-validation-prod)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(umccr-primary-data-prod)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' diff --git a/deploy/config/public_umccr.toml b/deploy/config/public_umccr.toml index 4f388e92..6648f2ce 100644 --- a/deploy/config/public_umccr.toml +++ b/deploy/config/public_umccr.toml @@ -11,9 +11,9 @@ environment = 'public' [[resolvers]] regex = '^(org.umccr.demo.sbeacon-data)/CINECA_UK1/(?P.*)$' substitution_string = 'CINECA_UK1/$key' -storage.type = 'S3' +storage.backend = 'S3' [[resolvers]] regex = '^(org.umccr.demo.htsget-rs-data)/(?Pbam|cram|vcf|bcf|crypt4gh|mixed)/(?P.*)$' substitution_string = '$type/$key' -storage.type = 'S3' +storage.backend = 'S3' diff --git a/htsget-actix/README.md b/htsget-actix/README.md index 29575f04..3dc87b92 100644 --- a/htsget-actix/README.md +++ b/htsget-actix/README.md @@ -50,7 +50,7 @@ are exposed in the public API. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## Benchmarks Benchmarks for this crate written using [Criterion.rs][criterion-rs], and aim to compare the performance of this crate with the diff --git a/htsget-axum/Cargo.toml b/htsget-axum/Cargo.toml index 15e20263..9ab58397 100644 --- a/htsget-axum/Cargo.toml +++ b/htsget-axum/Cargo.toml @@ -37,7 +37,7 @@ default = [] hyper = { version = "1", features = ["http1", "http2", "server"] } rustls = "0.23" hyper-util = "0.1" -tower-http = { version = "0.5", features = ["trace", "cors", "fs"] } +tower-http = { version = "0.6", features = ["trace", "cors", "fs"] } http = "1" axum = { version = "0.7", features = ["http2"] } axum-extra = { version = "0.9", features = ["erased-json"] } diff --git a/htsget-axum/README.md b/htsget-axum/README.md index 6330d1c1..9e370394 100644 --- a/htsget-axum/README.md +++ b/htsget-axum/README.md @@ -171,7 +171,7 @@ htsget-rs. It also contains the data block server which fetches data from a `Loc This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index 49dda224..2977ee96 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -12,14 +12,14 @@ repository = "https://github.com/umccr/htsget-rs" [features] s3-storage = [] -url-storage = ["dep:reqwest"] +url-storage = ["dep:reqwest", "dep:cfg-if"] experimental = ["dep:crypt4gh"] default = [] [dependencies] thiserror = "1" async-trait = "0.1" -noodles = { version = "0.82", features = ["core"] } +noodles = { version = "0.83", features = ["core"] } serde = { version = "1", features = ["derive"] } serde_with = "3" serde_regex = "1" @@ -37,6 +37,7 @@ rustls-pki-types = "1" # url-storage reqwest = { version = "0.12", features = ["rustls-tls"], default-features = false, optional = true } +cfg-if = { version = "1", optional = true } # Crypt4GH crypt4gh = { version = "0.4", git = "https://github.com/EGA-archive/crypt4gh-rust", optional = true } diff --git a/htsget-config/README.md b/htsget-config/README.md index 24d4e7cf..f88bc89c 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -151,7 +151,7 @@ For more information about regex options see the [regex crate](https://docs.rs/r Each resolver also maps to a certain storage backend. This storage backend can be used to set query IDs which are served from local storage, from S3-style bucket storage, or from HTTP URLs. To set the storage backend for a resolver, add a `[resolvers.storage]` table. Some storage backends require feature flags to be set when compiling htsget-rs. -To use `LocalStorage`, set `type = 'Local'` under `[resolvers.storage]`, and specify any additional options from below: +To use `LocalStorage`, set `backend = 'Local'` under `[resolvers.storage]`, and specify any additional options from below: | Option | Description | Type | Default | |--------------------------|-------------------------------------------------------------------------------------------------------------------------------------|------------------------------|--------------------| @@ -161,7 +161,7 @@ To use `LocalStorage`, set `type = 'Local'` under `[resolvers.storage]`, and spe | `path_prefix` | The path prefix which the URL tickets will have. This should likely match the `data_server_serve_at` path. | URL path | `''` | | `use_data_server_config` | Whether to use the data server config to fill in the above values. This overrides any other options specified from this table. | Boolean | `false` | -To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, set `type = 'S3'` under `[resolvers.storage]`, and specify any additional options from below: +To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, set `backend = 'S3'` under `[resolvers.storage]`, and specify any additional options from below: | Option | Description | Type | Default | |--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|---------------------------------------------------------------------------------------------------------------------------| @@ -170,7 +170,7 @@ To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, set ` | `path_style` | The S3 path style to request from the storage backend. If `true`, "path style" is used, e.g. `host.com/bucket/object.bam`, otherwise `bucket.host.com/object` style is used. | Boolean | `false` | `UrlStorage` is another storage backend which can be used to serve data from a remote HTTP URL. When using this storage backend, htsget-rs will fetch data from a `url` which is set in the config. It will also forward any headers received with the initial query, which is useful for authentication. -To use `UrlStorage`, build htsget-rs with the `url-storage` feature enabled, set `type = 'Url'` under `[resolvers.storage]`, and specify any additional options from below: +To use `UrlStorage`, build htsget-rs with the `url-storage` feature enabled, set `backend = 'Url'` under `[resolvers.storage]`, and specify any additional options from below: | Option | Description | Type | Default | |--------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|--------------------------|-----------------------------------------------------------------------------------------------------------------| @@ -193,8 +193,9 @@ For example, a `resolvers` value of: [[resolvers]] regex = '^(example_bucket)/(?P.*)$' substitution_string = '$key' + [resolvers.storage] -type = 'S3' +backend = 'S3' # Uses the first capture group in the regex as the bucket. ``` Will use "example_bucket" as the S3 bucket if that resolver matches, because this is the first capture group in the `regex`. @@ -209,7 +210,7 @@ regex = '.*' substitution_string = '$0' [resolvers.storage] -type = 'Local' +backend = 'Local' scheme = 'Http' authority = '127.0.0.1:8081' local_path = './' @@ -224,7 +225,7 @@ regex = '.*' substitution_string = '$0' [resolvers.storage] -type = 'S3' +backend = 'S3' bucket = 'bucket' ``` @@ -236,7 +237,7 @@ regex = ".*" substitution_string = "$0" [resolvers.storage] -type = 'Url' +backend = 'Url' url = "http://localhost:8080" response_url = "https://example.com" forward_headers = true @@ -275,6 +276,7 @@ regex = '.*' substitution_string = '$0' [resolvers.storage] +backend = 'S3' bucket = 'bucket' [resolvers.allow_guard] @@ -452,6 +454,7 @@ export HTSGET_RESOLVERS="[{ regex=regex, substitution_string=substitution_string, storage={ + type=S3, bucket=bucket }, allow_guard={ @@ -483,6 +486,7 @@ regex = '.*' substitution_string = '$0' [resolvers.storage] +backend = 'S3' bucket = 'bucket' endpoint = 'http://127.0.0.1:9000' path_style = true @@ -504,8 +508,7 @@ serving the data, htsget-rs will decrypt the headers of the Crypt4GH files and r them. When the client receives byte ranges from htsget-rs and concatenates them, the output bytes will be Crypt4GH encrypted, and will need to be decrypted before they can be read. All file formats (BAM, CRAM, VCF, and BCF) are supported using Crypt4GH. -To use this feature, an additional config option called `object_type` under `resolvers.storage` is required, -which allows specifying the private and public keys: +To use this feature, additional config under `resolvers.storage` is required to specify the private and public keys: | Option | Description | Type | Default | |------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------| @@ -516,17 +519,17 @@ For example: ```toml [[resolvers]] -regex = ".*" -substitution_string = "$0" +regex = '.*' +substitution_string = '$0' [resolvers.storage] -object_type = { private_key = "data/c4gh/keys/bob.sec", recipient_public_key = "data/c4gh/keys/alice.pub" } # pragma: allowlist secret +backend = 'Local' +private_key = 'data/c4gh/keys/bob.sec' # pragma: allowlist secret +recipient_public_key = 'data/c4gh/keys/alice.pub' ``` The htsget-rs server expects the Crypt4GH file to end with `.c4gh`, and the index file to be unencrypted. See the [`data/c4gh`][data-c4gh] for examples of file structure. - -> [!NOTE] -> This option is currently only supported for `LocalStorage`. The `object_type` will not have an effect if using `S3Storage` or `UrlStorage`. +Any of the storage types are supported, i.e. `Local`, `S3`, or `Url`. ### As a library @@ -541,7 +544,7 @@ regex, and changing it by using a substitution string. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-config/examples/config-files/c4gh.toml b/htsget-config/examples/config-files/c4gh.toml index ce260931..c17c36d3 100644 --- a/htsget-config/examples/config-files/c4gh.toml +++ b/htsget-config/examples/config-files/c4gh.toml @@ -9,7 +9,7 @@ regex = ".*" substitution_string = "$0" [resolvers.storage] -type = 'Local' +backend = 'Local' private_key = "data/c4gh/keys/bob.sec" # pragma: allowlist secret recipient_public_key = "data/c4gh/keys/alice.pub" diff --git a/htsget-config/examples/config-files/s3_storage.toml b/htsget-config/examples/config-files/s3_storage.toml index 66828a80..5cde4dff 100644 --- a/htsget-config/examples/config-files/s3_storage.toml +++ b/htsget-config/examples/config-files/s3_storage.toml @@ -11,9 +11,9 @@ data_server_enabled = false [[resolvers]] regex = '^(bucket)/(?P.*)$' substitution_string = '$key' -storage.type = 'S3' +storage.backend = 'S3' # Or, set the bucket manually #[resolvers.storage] -#type = 'S3' +#backend = 'S3' #bucket = 'bucket' diff --git a/htsget-config/examples/config-files/tls_data_server.toml b/htsget-config/examples/config-files/tls_data_server.toml index 649710bb..d2e4316e 100644 --- a/htsget-config/examples/config-files/tls_data_server.toml +++ b/htsget-config/examples/config-files/tls_data_server.toml @@ -12,5 +12,5 @@ regex = ".*" substitution_string = "$0" [resolvers.storage] -type = 'Local' +backend = 'Local' use_data_server_config = true diff --git a/htsget-config/examples/config-files/tls_ticket_server.toml b/htsget-config/examples/config-files/tls_ticket_server.toml index 4ef72c6e..9bd196ff 100644 --- a/htsget-config/examples/config-files/tls_ticket_server.toml +++ b/htsget-config/examples/config-files/tls_ticket_server.toml @@ -12,5 +12,5 @@ regex = ".*" substitution_string = "$0" [resolvers.storage] -type = 'S3' +backend = 'S3' bucket = "bucket" diff --git a/htsget-config/examples/config-files/url_storage.toml b/htsget-config/examples/config-files/url_storage.toml index cfda737f..372b0080 100644 --- a/htsget-config/examples/config-files/url_storage.toml +++ b/htsget-config/examples/config-files/url_storage.toml @@ -16,7 +16,7 @@ regex = ".*" substitution_string = "$0" [resolvers.storage] -type = 'Url' +backend = 'Url' url = "http://127.0.0.1:8081" response_url = "https://127.0.0.1:8081" forward_headers = true diff --git a/htsget-config/src/config/mod.rs b/htsget-config/src/config/mod.rs index cfa44354..71750fe9 100644 --- a/htsget-config/src/config/mod.rs +++ b/htsget-config/src/config/mod.rs @@ -775,7 +775,7 @@ pub(crate) mod tests { [[resolvers]] [resolvers.storage] - type = "Local" + backend = "Local" use_data_server_config = true "#, |config| { diff --git a/htsget-config/src/resolver.rs b/htsget-config/src/resolver.rs index 28c7cb5c..93720595 100644 --- a/htsget-config/src/resolver.rs +++ b/htsget-config/src/resolver.rs @@ -8,9 +8,9 @@ use serde_with::with_prefix; use tracing::instrument; use crate::config::DataServerConfig; -use crate::storage::local::LocalStorage; +use crate::storage::local::Local; #[cfg(feature = "s3-storage")] -use crate::storage::s3::S3Storage; +use crate::storage::s3::S3; #[cfg(feature = "url-storage")] use crate::storage::url::UrlStorageClient; use crate::storage::{ResolvedId, Storage}; @@ -27,11 +27,11 @@ pub trait IdResolver { #[async_trait] pub trait ResolveResponse { /// Convert from `LocalStorage`. - async fn from_local(local_storage: &LocalStorage, query: &Query) -> Result; + async fn from_local(local_storage: &Local, query: &Query) -> Result; /// Convert from `S3Storage`. #[cfg(feature = "s3-storage")] - async fn from_s3(s3_storage: &S3Storage, query: &Query) -> Result; + async fn from_s3(s3_storage: &S3, query: &Query) -> Result; /// Convert from `UrlStorage`. #[cfg(feature = "url-storage")] @@ -444,7 +444,7 @@ mod tests { use crate::config::tests::{test_config_from_env, test_config_from_file}; #[cfg(feature = "s3-storage")] - use crate::storage::s3::S3Storage; + use crate::storage::s3::S3; use crate::types::Scheme::Http; use crate::types::Url; @@ -454,7 +454,7 @@ mod tests { #[async_trait] impl ResolveResponse for TestResolveResponse { - async fn from_local(local_storage: &LocalStorage, _: &Query) -> Result { + async fn from_local(local_storage: &Local, _: &Query) -> Result { Ok(Response::new( Bam, vec![Url::new(local_storage.authority().to_string())], @@ -462,7 +462,7 @@ mod tests { } #[cfg(feature = "s3-storage")] - async fn from_s3(s3_storage: &S3Storage, _: &Query) -> Result { + async fn from_s3(s3_storage: &S3, _: &Query) -> Result { Ok(Response::new(Bam, vec![Url::new(s3_storage.bucket())])) } @@ -477,12 +477,11 @@ mod tests { #[tokio::test] async fn resolver_resolve_local_request() { - let local_storage = LocalStorage::new( + let local_storage = Local::new( Http, Authority::from_static("127.0.0.1:8080"), "data".to_string(), "/data".to_string(), - Default::default(), false, ); let resolver = Resolver::new( @@ -499,7 +498,7 @@ mod tests { #[cfg(feature = "s3-storage")] #[tokio::test] async fn resolver_resolve_s3_request_tagged() { - let s3_storage = S3Storage::new("id".to_string(), None, false); + let s3_storage = S3::new("id".to_string(), None, false); let resolver = Resolver::new( Storage::S3(s3_storage), "(id)-1", @@ -515,7 +514,7 @@ mod tests { #[tokio::test] async fn resolver_resolve_s3_request() { let resolver = Resolver::new( - Storage::S3(S3Storage::default()), + Storage::S3(S3::default()), "(id)-1", "$1-test", AllowGuard::default(), @@ -677,7 +676,7 @@ mod tests { vec![( "HTSGET_RESOLVERS", "[{ regex=regex, substitution_string=substitution_string, \ - storage={ type=S3, bucket=bucket }, \ + storage={ backend=S3, bucket=bucket }, \ allow_guard={ allow_reference_names=[chr1], allow_fields=[QNAME], allow_tags=[RG], \ allow_formats=[BAM], allow_classes=[body], allow_interval_start=100, \ allow_interval_end=1000 } }]", @@ -692,7 +691,7 @@ mod tests { Interval::new(Some(100), Some(1000)), ); let resolver = config.resolvers().first().unwrap(); - let expected_storage = S3Storage::new("bucket".to_string(), None, false); + let expected_storage = S3::new("bucket".to_string(), None, false); assert_eq!(resolver.regex().to_string(), "regex"); assert_eq!(resolver.substitution_string(), "substitution_string"); diff --git a/htsget-config/src/storage/object/c4gh.rs b/htsget-config/src/storage/c4gh.rs similarity index 67% rename from htsget-config/src/storage/object/c4gh.rs rename to htsget-config/src/storage/c4gh.rs index 93001cd6..3f300e8c 100644 --- a/htsget-config/src/storage/object/c4gh.rs +++ b/htsget-config/src/storage/c4gh.rs @@ -64,13 +64,16 @@ impl From for Error { #[cfg(test)] mod tests { use crate::config::tests::test_config_from_file; + use crate::config::Config; use crate::storage::Storage; use std::fs::copy; use std::path::PathBuf; use tempfile::TempDir; - #[test] - fn config_storage_c4gh() { + fn test_c4gh_storage_config(storage_config: &str, test_fn: F) + where + F: Fn(Config), + { let tmp = TempDir::new().unwrap(); let private_key = tmp.path().join("bob.sec"); let recipient_public_key = tmp.path().join("alice.pub"); @@ -94,18 +97,62 @@ mod tests { regex = "regex" [resolvers.storage] - type = "Local" + {} private_key = "{}" recipient_public_key = "{}" "#, + storage_config, private_key.to_string_lossy(), recipient_public_key.to_string_lossy() ), |config| { println!("{:?}", config.resolvers().first().unwrap().storage()); - assert!(matches!( + test_fn(config); + }, + ); + } + + #[test] + fn config_local_storage_c4gh() { + test_c4gh_storage_config(r#"backend = "Local""#, |config| { + assert!(matches!( config.resolvers().first().unwrap().storage(), - Storage::Local(local_storage) if local_storage.object_type().keys().is_some() + Storage::Local(local_storage) if local_storage.keys().is_some() + )); + }); + } + + #[cfg(feature = "s3-storage")] + #[test] + fn config_s3_storage_c4gh() { + test_c4gh_storage_config( + r#" + backend = "S3" + bucket = "bucket" + "#, + |config| { + assert!(matches!( + config.resolvers().first().unwrap().storage(), + Storage::S3(s3_storage) if s3_storage.keys().is_some() + )); + }, + ); + } + + #[cfg(feature = "url-storage")] + #[test] + fn config_url_storage_c4gh() { + test_c4gh_storage_config( + r#" + backend = "Url" + url = "https://example.com/" + response_url = "https://example.com/" + forward_headers = false + "#, + |config| { + assert!(matches!( + config.resolvers().first().unwrap().storage(), + Storage::Url(url_storage) if url_storage.keys().is_some() )); }, ); diff --git a/htsget-config/src/storage/local.rs b/htsget-config/src/storage/local.rs index b6b60818..3a8007bc 100644 --- a/htsget-config/src/storage/local.rs +++ b/htsget-config/src/storage/local.rs @@ -4,7 +4,8 @@ use http::uri::Authority; use serde::{Deserialize, Serialize}; use crate::config::{default_localstorage_addr, default_path, DataServerConfig}; -use crate::storage::object::ObjectType; +#[cfg(feature = "experimental")] +use crate::storage::c4gh::C4GHKeys; use crate::tls::KeyPairScheme; use crate::types::Scheme; @@ -18,25 +19,25 @@ fn default_local_path() -> String { #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(default)] -pub struct LocalStorage { +pub struct Local { scheme: Scheme, #[serde(with = "http_serde::authority")] authority: Authority, local_path: String, path_prefix: String, - #[serde(flatten)] - object_type: ObjectType, use_data_server_config: bool, + #[serde(skip_serializing, flatten)] + #[cfg(feature = "experimental")] + keys: Option, } -impl LocalStorage { +impl Local { /// Create a new local storage. pub fn new( scheme: Scheme, authority: Authority, local_path: String, path_prefix: String, - object_type: ObjectType, use_data_server_config: bool, ) -> Self { Self { @@ -44,8 +45,9 @@ impl LocalStorage { authority, local_path, path_prefix, - object_type, use_data_server_config, + #[cfg(feature = "experimental")] + keys: None, } } @@ -69,38 +71,44 @@ impl LocalStorage { &self.path_prefix } - /// Get the object type. - pub fn object_type(&self) -> &ObjectType { - &self.object_type - } - /// Get whether config should be inherited from the data server config. pub fn use_data_server_config(&self) -> bool { self.use_data_server_config } + + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() + } } -impl Default for LocalStorage { +impl Default for Local { fn default() -> Self { - Self { - scheme: Scheme::Http, - authority: default_authority(), - local_path: default_local_path(), - path_prefix: Default::default(), - object_type: Default::default(), - use_data_server_config: false, - } + Self::new( + Scheme::Http, + default_authority(), + default_local_path(), + Default::default(), + false, + ) } } -impl From<&DataServerConfig> for LocalStorage { +impl From<&DataServerConfig> for Local { fn from(config: &DataServerConfig) -> Self { - LocalStorage::new( + Self::new( config.tls().get_scheme(), Authority::from_str(&config.addr().to_string()).expect("expected valid authority"), config.local_path().to_string_lossy().to_string(), config.serve_at().to_string(), - Default::default(), true, ) } @@ -126,7 +134,7 @@ mod tests { regex = "regex" [resolvers.storage] - type = "Local" + backend = "Local" local_path = "path" scheme = "HTTPS" path_prefix = "path" @@ -151,13 +159,12 @@ mod tests { None, CorsConfig::default(), ); - let result: LocalStorage = (&data_server_config).into(); - let expected = LocalStorage::new( + let result: Local = (&data_server_config).into(); + let expected = Local::new( Http, Authority::from_static("127.0.0.1:8080"), "data".to_string(), "/data".to_string(), - Default::default(), true, ); diff --git a/htsget-config/src/storage/mod.rs b/htsget-config/src/storage/mod.rs index aa0cac92..d8d277dc 100644 --- a/htsget-config/src/storage/mod.rs +++ b/htsget-config/src/storage/mod.rs @@ -1,12 +1,13 @@ -use crate::storage::local::LocalStorage; +use crate::storage::local::Local; #[cfg(feature = "s3-storage")] -use crate::storage::s3::S3Storage; +use crate::storage::s3::S3; #[cfg(feature = "url-storage")] use crate::storage::url::UrlStorageClient; use serde::{Deserialize, Serialize}; +#[cfg(feature = "experimental")] +pub mod c4gh; pub mod local; -pub mod object; #[cfg(feature = "s3-storage")] pub mod s3; #[cfg(feature = "url-storage")] @@ -30,14 +31,14 @@ impl ResolvedId { /// Specify the storage backend to use as config values. #[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(tag = "type")] +#[serde(tag = "backend")] #[non_exhaustive] pub enum Storage { #[serde(alias = "local", alias = "LOCAL")] - Local(LocalStorage), + Local(Local), #[cfg(feature = "s3-storage")] #[serde(alias = "s3")] - S3(S3Storage), + S3(S3), #[cfg(feature = "url-storage")] #[serde(alias = "url", alias = "URL")] Url(#[serde(skip_serializing)] UrlStorageClient), @@ -61,7 +62,7 @@ pub(crate) mod tests { r#" [[resolvers]] [resolvers.storage] - type = "Local" + backend = "Local" regex = "regex" "#, |config| { @@ -79,7 +80,7 @@ pub(crate) mod tests { test_config_from_env( vec![( "HTSGET_RESOLVERS", - "[{storage={ type=Local, use_data_server_config=true}}]", + "[{storage={ backend=Local, use_data_server_config=true}}]", )], |config| { assert!(matches!( @@ -97,7 +98,7 @@ pub(crate) mod tests { r#" [[resolvers]] [resolvers.storage] - type = "S3" + backend = "S3" regex = "regex" "#, |config| { @@ -114,7 +115,7 @@ pub(crate) mod tests { #[test] fn config_storage_tagged_s3_env() { test_config_from_env( - vec![("HTSGET_RESOLVERS", "[{storage={ type=S3 }}]")], + vec![("HTSGET_RESOLVERS", "[{storage={ backend=S3 }}]")], |config| { assert!(matches!( config.resolvers().first().unwrap().storage(), diff --git a/htsget-config/src/storage/object/mod.rs b/htsget-config/src/storage/object/mod.rs deleted file mode 100644 index c076d452..00000000 --- a/htsget-config/src/storage/object/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! Defines the type of object used by storage. -//! - -#[cfg(feature = "experimental")] -pub mod c4gh; - -#[cfg(feature = "experimental")] -use crate::storage::object::c4gh::C4GHKeys; -use serde::{Deserialize, Serialize}; - -/// An object type, can be regular or Crypt4GH encrypted. -#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq)] -pub struct ObjectType { - #[serde(skip_serializing, flatten)] - #[cfg(feature = "experimental")] - keys: Option, -} - -impl ObjectType { - #[cfg(feature = "experimental")] - /// Get the C4GH keys. - pub fn keys(&self) -> Option<&C4GHKeys> { - self.keys.as_ref() - } -} diff --git a/htsget-config/src/storage/s3.rs b/htsget-config/src/storage/s3.rs index cea0e846..9098fa37 100644 --- a/htsget-config/src/storage/s3.rs +++ b/htsget-config/src/storage/s3.rs @@ -1,20 +1,27 @@ +#[cfg(feature = "experimental")] +use crate::storage::c4gh::C4GHKeys; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq)] #[serde(default)] -pub struct S3Storage { +pub struct S3 { pub(crate) bucket: String, pub(crate) endpoint: Option, pub(crate) path_style: bool, + #[serde(skip_serializing, flatten)] + #[cfg(feature = "experimental")] + pub(crate) keys: Option, } -impl S3Storage { +impl S3 { /// Create a new S3 storage. pub fn new(bucket: String, endpoint: Option, path_style: bool) -> Self { Self { bucket, endpoint, path_style, + #[cfg(feature = "experimental")] + keys: None, } } @@ -32,6 +39,19 @@ impl S3Storage { pub fn path_style(self) -> bool { self.path_style } + + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() + } } #[cfg(test)] @@ -47,7 +67,7 @@ mod tests { regex = "regex" [resolvers.storage] - type = "S3" + backend = "S3" bucket = "bucket" "#, |config| { diff --git a/htsget-config/src/storage/url.rs b/htsget-config/src/storage/url.rs index 6310142c..d5396dca 100644 --- a/htsget-config/src/storage/url.rs +++ b/htsget-config/src/storage/url.rs @@ -1,20 +1,19 @@ -use std::str::FromStr; - +use cfg_if::cfg_if; use http::Uri as InnerUrl; use reqwest::Client; use serde::{Deserialize, Serialize}; use serde_with::with_prefix; +use std::str::FromStr; use crate::error::Error::ParseError; use crate::error::{Error, Result}; +#[cfg(feature = "experimental")] +use crate::storage::c4gh::C4GHKeys; use crate::storage::local::default_authority; use crate::tls::client::TlsClientConfig; -fn default_url() -> ValidatedUrl { - ValidatedUrl(Url { - inner: InnerUrl::from_str(&format!("https://{}", default_authority())) - .expect("expected valid url"), - }) +fn default_url() -> InnerUrl { + InnerUrl::from_str(&format!("https://{}", default_authority())).expect("expected valid url") } with_prefix!(client_auth_prefix "client_"); @@ -28,6 +27,9 @@ pub struct UrlStorage { header_blacklist: Vec, #[serde(skip_serializing)] tls: TlsClientConfig, + #[serde(skip_serializing, flatten)] + #[cfg(feature = "experimental")] + keys: Option, } #[derive(Deserialize, Debug, Clone)] @@ -38,6 +40,8 @@ pub struct UrlStorageClient { forward_headers: bool, header_blacklist: Vec, client: Client, + #[cfg(feature = "experimental")] + keys: Option, } impl TryFrom for UrlStorageClient { @@ -61,13 +65,21 @@ impl TryFrom for UrlStorageClient { .build() .map_err(|err| ParseError(format!("building url storage client: {}", err)))?; - Ok(Self::new( + let url_storage = Self::new( storage.url, storage.response_url, storage.forward_headers, storage.header_blacklist, client, - )) + ); + + cfg_if! { + if #[cfg(feature = "experimental")] { + Ok(url_storage.set_keys(storage.keys)) + } else { + Ok(url_storage) + } + } } } @@ -86,6 +98,8 @@ impl UrlStorageClient { forward_headers, header_blacklist, client, + #[cfg(feature = "experimental")] + keys: None, } } @@ -113,6 +127,19 @@ impl UrlStorageClient { pub fn client_cloned(&self) -> Client { self.client.clone() } + + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() + } } /// A wrapper around `http::Uri` type which implements serialize and deserialize. @@ -163,6 +190,8 @@ impl UrlStorage { forward_headers, header_blacklist, tls, + #[cfg(feature = "experimental")] + keys: None, } } @@ -186,17 +215,30 @@ impl UrlStorage { pub fn tls(&self) -> &TlsClientConfig { &self.tls } + + #[cfg(feature = "experimental")] + /// Set the C4GH keys. + pub fn set_keys(mut self, keys: Option) -> Self { + self.keys = keys; + self + } + + #[cfg(feature = "experimental")] + /// Get the C4GH keys. + pub fn keys(&self) -> Option<&C4GHKeys> { + self.keys.as_ref() + } } impl Default for UrlStorage { fn default() -> Self { - Self { - url: default_url(), - response_url: default_url(), - forward_headers: true, - header_blacklist: vec![], - tls: TlsClientConfig::default(), - } + Self::new( + default_url(), + default_url(), + true, + vec![], + TlsClientConfig::default(), + ) } } @@ -240,7 +282,7 @@ mod tests { regex = "regex" [resolvers.storage] - type = "Url" + backend = "Url" url = "https://example.com/" response_url = "https://example.com/" forward_headers = false diff --git a/htsget-http/README.md b/htsget-http/README.md index b7969e3b..0eae7c78 100644 --- a/htsget-http/README.md +++ b/htsget-http/README.md @@ -38,7 +38,7 @@ These functions take query and endpoint information, and process it using [htsge This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [warp]: https://github.com/seanmonstar/warp [htsget-search]: ../htsget-search diff --git a/htsget-http/src/lib.rs b/htsget-http/src/lib.rs index 5000c857..3ef99f1c 100644 --- a/htsget-http/src/lib.rs +++ b/htsget-http/src/lib.rs @@ -86,7 +86,7 @@ mod tests { use http::uri::Authority; - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_config::types::{Headers, JsonResponse, Request, Scheme, Url}; use htsget_search::from_storage::HtsGetFromStorage; use htsget_search::HtsGet; @@ -278,7 +278,6 @@ mod tests { Authority::from_static("127.0.0.1:8081"), "data".to_string(), "/data".to_string(), - Default::default(), false, ), ) diff --git a/htsget-lambda/Cargo.toml b/htsget-lambda/Cargo.toml index 85ca4430..48b3b78e 100644 --- a/htsget-lambda/Cargo.toml +++ b/htsget-lambda/Cargo.toml @@ -24,7 +24,7 @@ default = [] [dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } -tower-http = { version = "0.5", features = ["cors"] } +tower-http = { version = "0.6", features = ["cors"] } rustls = "0.23" lambda_http = { version = "0.13" } lambda_runtime = { version = "0.13" } diff --git a/htsget-lambda/README.md b/htsget-lambda/README.md index 16806fd1..e24d9395 100644 --- a/htsget-lambda/README.md +++ b/htsget-lambda/README.md @@ -46,7 +46,7 @@ library code, and it instead uses `htsget-axum`. Please use that crate for funct This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## License diff --git a/htsget-search/Cargo.toml b/htsget-search/Cargo.toml index 1aa6e5ad..33fe4881 100644 --- a/htsget-search/Cargo.toml +++ b/htsget-search/Cargo.toml @@ -37,7 +37,7 @@ futures-util = "0.3" async-trait = "0.1" # Noodles -noodles = { version = "0.82", features = ["async", "core", "bgzf", "bam", "bcf", "cram", "csi", "sam", "tabix", "vcf"] } +noodles = { version = "0.83", features = ["async", "core", "bgzf", "bam", "bcf", "cram", "csi", "sam", "tabix", "vcf"] } # Error control, tracing, config thiserror = "1" diff --git a/htsget-search/README.md b/htsget-search/README.md index 3d179331..7623f8c1 100644 --- a/htsget-search/README.md +++ b/htsget-search/README.md @@ -59,7 +59,7 @@ used to process requests. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. ## Minimising Byte Ranges diff --git a/htsget-search/benches/search_benchmarks.rs b/htsget-search/benches/search_benchmarks.rs index b0b3c88f..aab8c499 100644 --- a/htsget-search/benches/search_benchmarks.rs +++ b/htsget-search/benches/search_benchmarks.rs @@ -6,7 +6,7 @@ use http::uri::Authority; use tokio::runtime::Runtime; use htsget_config::resolver::ResolveResponse; -use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; +use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_config::types::Class::Header; use htsget_config::types::Format::{Bam, Bcf, Cram, Vcf}; use htsget_config::types::{HtsGetError, Query, Scheme}; @@ -22,7 +22,6 @@ async fn perform_query(query: Query) -> Result<(), HtsGetError> { Authority::from_static("127.0.0.1:8081"), "../data".to_string(), "/data".to_string(), - Default::default(), false, ), &query, diff --git a/htsget-search/src/bam_search.rs b/htsget-search/src/bam_search.rs index 53e46c87..1fd83e3f 100644 --- a/htsget-search/src/bam_search.rs +++ b/htsget-search/src/bam_search.rs @@ -151,7 +151,7 @@ pub(crate) mod tests { use crate::from_storage::tests::with_aws_storage_fn; use crate::from_storage::tests::with_local_storage_fn; use crate::{Class::Body, Class::Header, Headers, HtsGetError::NotFound, Response, Url}; - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_storage::local::LocalStorage; use htsget_test::http::concat::ConcatResponse; use std::future::Future; diff --git a/htsget-search/src/bcf_search.rs b/htsget-search/src/bcf_search.rs index d3a4a668..605c12b1 100644 --- a/htsget-search/src/bcf_search.rs +++ b/htsget-search/src/bcf_search.rs @@ -108,7 +108,7 @@ impl BcfSearch { #[cfg(test)] mod tests { - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_config::types::Class::Body; use htsget_test::http::concat::ConcatResponse; use std::future::Future; diff --git a/htsget-search/src/cram_search.rs b/htsget-search/src/cram_search.rs index babe3638..10aeaf0d 100644 --- a/htsget-search/src/cram_search.rs +++ b/htsget-search/src/cram_search.rs @@ -274,7 +274,7 @@ impl CramSearch { mod tests { use std::future::Future; - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_test::http::concat::ConcatResponse; use super::*; diff --git a/htsget-search/src/from_storage.rs b/htsget-search/src/from_storage.rs index 419b4a2b..c05b9ef4 100644 --- a/htsget-search/src/from_storage.rs +++ b/htsget-search/src/from_storage.rs @@ -6,9 +6,9 @@ use tracing::debug; use tracing::instrument; use htsget_config::resolver::{ResolveResponse, StorageResolver}; -use htsget_config::storage::local::LocalStorage as LocalStorageConfig; +use htsget_config::storage::local::Local as LocalStorageConfig; #[cfg(feature = "s3-storage")] -use htsget_config::storage::s3::S3Storage as S3StorageConfig; +use htsget_config::storage::s3::S3 as S3StorageConfig; #[cfg(feature = "url-storage")] use htsget_config::storage::url::UrlStorageClient as UrlStorageConfig; @@ -271,7 +271,6 @@ pub(crate) mod tests { Authority::from_static("127.0.0.1:8081"), base_path.to_str().unwrap().to_string(), "/data".to_string(), - Default::default(), false, ), ) diff --git a/htsget-search/src/vcf_search.rs b/htsget-search/src/vcf_search.rs index d5c3acf6..8508c067 100644 --- a/htsget-search/src/vcf_search.rs +++ b/htsget-search/src/vcf_search.rs @@ -119,7 +119,7 @@ impl VcfSearch { #[cfg(test)] pub(crate) mod tests { - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_config::types::Class::Body; use htsget_test::http::concat::ConcatResponse; use std::future::Future; diff --git a/htsget-storage/Cargo.toml b/htsget-storage/Cargo.toml index df56aab2..b828a883 100644 --- a/htsget-storage/Cargo.toml +++ b/htsget-storage/Cargo.toml @@ -62,7 +62,7 @@ htsget-config = { version = "0.10.1", path = "../htsget-config", default-feature htsget-test = { version = "0.6.2", path = "../htsget-test", features = ["http"], default-features = false } [dev-dependencies] -tower-http = { version = "0.5", features = ["fs"] } +tower-http = { version = "0.6", features = ["fs"] } axum = "0.7" tempfile = "3" data-url = "0.3" diff --git a/htsget-storage/README.md b/htsget-storage/README.md index 23bf7bed..75923e8c 100644 --- a/htsget-storage/README.md +++ b/htsget-storage/README.md @@ -49,7 +49,7 @@ and [url] modules implement the `Storage` functionality. This crate has the following features: * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [local]: src/local.rs [s3]: src/s3.rs diff --git a/htsget-storage/src/c4gh/mod.rs b/htsget-storage/src/c4gh/mod.rs index 44a1c893..48857a7b 100644 --- a/htsget-storage/src/c4gh/mod.rs +++ b/htsget-storage/src/c4gh/mod.rs @@ -141,13 +141,14 @@ impl DeserializedHeader { data, )) } - - /// Decrypt the - pub fn decrypt_stream() {} } /// Convert an encrypted file position to an unencrypted position if the header length is known. pub fn to_unencrypted(encrypted_position: u64, header_length: u64) -> u64 { + if encrypted_position < header_length + NONCE_SIZE { + return 0; + } + let number_data_blocks = encrypted_position / DATA_BLOCK_SIZE; let mut additional_bytes = number_data_blocks * (NONCE_SIZE + MAC_SIZE); @@ -161,6 +162,10 @@ pub fn to_unencrypted(encrypted_position: u64, header_length: u64) -> u64 { /// Convert an encrypted file size to an unencrypted file size if the header length is known. pub fn to_unencrypted_file_size(encrypted_file_size: u64, header_length: u64) -> u64 { + if encrypted_file_size < header_length + NONCE_SIZE + MAC_SIZE { + return 0; + } + to_unencrypted(encrypted_file_size, header_length) - MAC_SIZE } @@ -272,6 +277,26 @@ mod tests { assert_eq!(result, expected); } + #[test] + fn test_to_unencrypted() { + let result = to_unencrypted(124, 124); + assert_eq!(result, 0); + let result = to_unencrypted(124 + 12, 124); + assert_eq!(result, 0); + let result = to_unencrypted(124 + 12 + 12, 124); + assert_eq!(result, 12); + } + + #[test] + fn test_to_unencrypted_file_size() { + let result = to_unencrypted_file_size(124, 124); + assert_eq!(result, 0); + let result = to_unencrypted_file_size(124 + 12 + 16, 124); + assert_eq!(result, 0); + let result = to_unencrypted_file_size(124 + 12 + 16 + 12, 124); + assert_eq!(result, 12); + } + #[test] fn test_unencrypted_clamp() { let pos = 0; diff --git a/htsget-storage/src/c4gh/storage.rs b/htsget-storage/src/c4gh/storage.rs index 5ce46b3b..6eeea139 100644 --- a/htsget-storage/src/c4gh/storage.rs +++ b/htsget-storage/src/c4gh/storage.rs @@ -288,167 +288,328 @@ impl From for StorageError { mod tests { use super::*; use crate::local::tests::with_local_storage; + #[cfg(feature = "s3-storage")] + use crate::s3::tests::with_aws_s3_storage; + #[cfg(feature = "url-storage")] + use crate::url::tests::{test_headers, with_url_test_server}; use htsget_config::types::Headers; use htsget_test::c4gh::{encrypt_data, get_decryption_keys}; + use http::HeaderMap; use std::future::Future; - use tokio::fs::File; + use std::path::Path; + #[cfg(feature = "s3-storage")] + use std::sync::Arc; + use tokio::fs::{read, File}; use tokio::io::AsyncWriteExt; #[tokio::test] - async fn test_preprocess() { - with_c4gh_storage(|mut storage| async move { - storage - .preprocess( - "key", - GetOptions::new_with_default_range(&Default::default()), - ) - .await - .unwrap(); + async fn test_preprocess_local_storage() { + with_local_c4gh_storage(|mut storage| async move { + test_preprocess(&mut storage, "folder/key", &Default::default()).await + }) + .await; + } - let state = storage.state.get("key.c4gh").unwrap(); + #[cfg(feature = "s3-storage")] + #[tokio::test] + async fn test_preprocess_s3_storage() { + with_s3_c4gh_storage(|mut storage| async move { + test_preprocess(&mut storage, "key", &Default::default()).await + }) + .await; + } - assert_eq!(state.unencrypted_file_size, 6); - assert_eq!(state.encrypted_file_size, 158); - assert_eq!(state.deserialized_header.header_info.packets_count, 1); + #[cfg(feature = "url-storage")] + #[tokio::test] + async fn test_preprocess_url_storage() { + with_url_c4gh_storage(|mut storage, _| async move { + let mut headers = HeaderMap::default(); + let headers = test_headers(&mut headers); + test_preprocess(&mut storage, "assets/folder/key", headers).await }) .await; } #[tokio::test] - async fn test_get() { - with_c4gh_storage(|mut storage| async move { - let headers = Default::default(); - let options = GetOptions::new_with_default_range(&headers); - storage.preprocess("key", options.clone()).await.unwrap(); - let mut object = vec![]; - - storage - .get("key", options) - .await - .unwrap() - .read_to_end(&mut object) - .await - .unwrap(); - assert_eq!(object, b"value1"); + async fn test_get_local_storage() { + with_local_c4gh_storage(|mut storage| async move { + test_get(&mut storage, "folder/key", &Default::default()).await }) .await; } + #[cfg(feature = "s3-storage")] #[tokio::test] - async fn test_head() { - with_c4gh_storage(|mut storage| async move { - let headers = Default::default(); - let options = GetOptions::new_with_default_range(&headers); - storage.preprocess("key", options.clone()).await.unwrap(); - - let size = storage - .head("key", HeadOptions::new(&headers)) - .await - .unwrap(); - assert_eq!(size, 6); + async fn test_get_s3_storage() { + with_s3_c4gh_storage(|mut storage| async move { + test_get(&mut storage, "key", &Default::default()).await }) .await; } + #[cfg(feature = "url-storage")] #[tokio::test] - async fn test_postprocess() { - with_c4gh_storage(|mut storage| async move { - let headers = Default::default(); - let options = GetOptions::new_with_default_range(&headers); - storage.preprocess("key", options.clone()).await.unwrap(); - - let blocks = storage - .postprocess( - "key", - BytesPositionOptions::new( - vec![BytesPosition::default().with_start(0).with_end(6)], - &headers, - ), - ) - .await - .unwrap(); + async fn test_get_url_storage() { + with_url_c4gh_storage(|mut storage, _| async move { + let mut headers = HeaderMap::default(); + let headers = test_headers(&mut headers); + test_get(&mut storage, "assets/folder/key", headers).await + }) + .await; + } - assert_eq!( - blocks[0], - DataBlock::Data( - vec![99, 114, 121, 112, 116, 52, 103, 104, 1, 0, 0, 0, 3, 0, 0, 0], - Some(Class::Header) - ) - ); - assert_eq!( - blocks[1], - DataBlock::Range(BytesPosition::new(Some(16), Some(124), None)) - ); - assert_eq!( - blocks[3], - DataBlock::Range(BytesPosition::new(Some(124), Some(158), None)) - ); + #[tokio::test] + async fn test_head_local_storage() { + with_local_c4gh_storage(|mut storage| async move { + test_head(&mut storage, "folder/key", &Default::default()).await }) .await; } + #[cfg(feature = "s3-storage")] #[tokio::test] - async fn test_range_url() { - with_c4gh_storage(|mut storage| async move { - let headers = Default::default(); - let options = GetOptions::new_with_default_range(&headers); - storage.preprocess("key", options.clone()).await.unwrap(); - - let blocks = storage - .postprocess( - "key", - BytesPositionOptions::new( - vec![BytesPosition::default().with_start(0).with_end(6)], - &headers, - ), - ) - .await - .unwrap(); + async fn test_head_s3_storage() { + with_s3_c4gh_storage(|mut storage| async move { + test_head(&mut storage, "key", &Default::default()).await + }) + .await; + } - if let DataBlock::Range(range) = blocks.last().unwrap() { - let url = storage - .range_url("key", RangeUrlOptions::new(range.clone(), &headers)) - .await - .unwrap(); - let expected = Url::new("http://127.0.0.1:8081/data/key.c4gh") - .with_headers(Headers::default().with_header("Range", "bytes=124-157")); + #[cfg(feature = "url-storage")] + #[tokio::test] + async fn test_head_url_storage() { + with_url_c4gh_storage(|mut storage, _| async move { + let mut headers = HeaderMap::default(); + let headers = test_headers(&mut headers); + test_head(&mut storage, "assets/folder/key", headers).await + }) + .await; + } - assert_eq!(url, expected); - } + #[tokio::test] + async fn test_postprocess_local_storage() { + with_local_c4gh_storage(|mut storage| async move { + test_postprocess(&mut storage, "folder/key", &Default::default()).await }) .await; } - pub(crate) async fn with_c4gh_storage(test: F) - where - F: FnOnce(C4GHStorage) -> Fut, - Fut: Future, - { - with_local_storage(|storage| async move { - let mut data = vec![]; - StorageTrait::get( - &storage, - "folder/../key1", - GetOptions::new_with_default_range(&Default::default()), + #[cfg(feature = "s3-storage")] + #[tokio::test] + async fn test_postprocess_s3_storage() { + with_s3_c4gh_storage(|mut storage| async move { + test_postprocess(&mut storage, "key", &Default::default()).await + }) + .await; + } + + #[cfg(feature = "url-storage")] + #[tokio::test] + async fn test_postprocess_url_storage() { + with_url_c4gh_storage(|mut storage, _| async move { + let mut headers = HeaderMap::default(); + let headers = test_headers(&mut headers); + test_postprocess(&mut storage, "assets/folder/key", headers).await + }) + .await; + } + + #[tokio::test] + async fn test_range_local_storage() { + with_local_c4gh_storage(|mut storage| async move { + test_range_url( + &mut storage, + "http://127.0.0.1:8081/data/folder/key.c4gh", + "folder/key", + &Default::default(), ) .await + }) + .await; + } + + #[cfg(feature = "s3-storage")] + #[tokio::test] + async fn test_range_s3_storage() { + with_s3_c4gh_storage(|mut storage| async move { + test_range_url( + &mut storage, + "http://folder.localhost:8014/key.c4gh", + "key", + &Default::default(), + ) + .await + }) + .await; + } + + #[cfg(feature = "url-storage")] + #[tokio::test] + async fn test_range_url_storage() { + with_url_c4gh_storage(|mut storage, url| async move { + let mut headers = HeaderMap::default(); + let headers = test_headers(&mut headers); + test_range_url( + &mut storage, + &format!("{}/assets/folder/key.c4gh", url), + "assets/folder/key", + headers, + ) + .await + }) + .await; + } + + async fn test_preprocess(storage: &mut C4GHStorage, key: &str, headers: &HeaderMap) { + storage + .preprocess(key, GetOptions::new_with_default_range(headers)) + .await + .unwrap(); + + let state = storage.state.get(&format!("{}.c4gh", key)).unwrap(); + + assert_eq!(state.unencrypted_file_size, 6); + assert_eq!(state.encrypted_file_size, 158); + assert_eq!(state.deserialized_header.header_info.packets_count, 1); + } + + async fn test_get(storage: &mut C4GHStorage, key: &str, headers: &HeaderMap) { + let options = GetOptions::new_with_default_range(headers); + storage.preprocess(key, options.clone()).await.unwrap(); + let mut object = vec![]; + + storage + .get(key, options) + .await .unwrap() - .read_to_end(&mut data) + .read_to_end(&mut object) .await .unwrap(); + assert_eq!(object, b"value1"); + } - let data = encrypt_data(&data); + async fn test_head(storage: &mut C4GHStorage, key: &str, headers: &HeaderMap) { + let options = GetOptions::new_with_default_range(headers); + storage.preprocess(key, options.clone()).await.unwrap(); - let key = "key.c4gh"; - File::create(storage.base_path().join(key)) - .await - .unwrap() - .write_all(&data) + let size = storage.head(key, HeadOptions::new(headers)).await.unwrap(); + assert_eq!(size, 6); + } + + async fn test_postprocess(storage: &mut C4GHStorage, key: &str, headers: &HeaderMap) { + let options = GetOptions::new_with_default_range(headers); + storage.preprocess(key, options.clone()).await.unwrap(); + + let blocks = storage + .postprocess( + key, + BytesPositionOptions::new( + vec![BytesPosition::default().with_start(0).with_end(6)], + headers, + ), + ) + .await + .unwrap(); + + assert_eq!( + blocks[0], + DataBlock::Data( + vec![99, 114, 121, 112, 116, 52, 103, 104, 1, 0, 0, 0, 3, 0, 0, 0], + Some(Class::Header) + ) + ); + assert_eq!( + blocks[1], + DataBlock::Range(BytesPosition::new(Some(16), Some(124), None)) + ); + assert_eq!( + blocks[3], + DataBlock::Range(BytesPosition::new(Some(124), Some(158), None)) + ); + } + + async fn test_range_url(storage: &mut C4GHStorage, url: &str, key: &str, headers: &HeaderMap) { + let options = GetOptions::new_with_default_range(headers); + storage.preprocess(key, options.clone()).await.unwrap(); + + let blocks = storage + .postprocess( + key, + BytesPositionOptions::new( + vec![BytesPosition::default().with_start(0).with_end(6)], + headers, + ), + ) + .await + .unwrap(); + + if let DataBlock::Range(range) = blocks.last().unwrap() { + let range = storage + .range_url(key, RangeUrlOptions::new(range.clone(), headers)) .await .unwrap(); + println!("{:?}", range); + assert!(range.url.starts_with(url)); + assert_eq!( + range.headers, + Some(Headers::default().with_header("Range", "bytes=124-157")) + ); + assert_eq!(range.class, None); + } + } + + async fn create_encrypted_files(base_path: &Path) { + let data = read(base_path.join("folder/../key1")).await.unwrap(); + let data = encrypt_data(&data); + + File::create(base_path.join("folder/key.c4gh")) + .await + .unwrap() + .write_all(&data) + .await + .unwrap(); + } + + pub(crate) async fn with_local_c4gh_storage(test: F) + where + F: FnOnce(C4GHStorage) -> Fut, + Fut: Future, + { + with_local_storage(|storage, base_path| async move { + create_encrypted_files(&base_path).await; test(C4GHStorage::new(get_decryption_keys(), storage)).await; }) .await; } + + #[cfg(feature = "s3-storage")] + pub(crate) async fn with_s3_c4gh_storage(test: F) + where + F: FnOnce(C4GHStorage) -> Fut, + Fut: Future, + { + with_aws_s3_storage(|storage, base_path| async move { + create_encrypted_files(&base_path).await; + test(C4GHStorage::new( + get_decryption_keys(), + Arc::try_unwrap(storage).unwrap(), + )) + .await; + }) + .await; + } + + #[cfg(feature = "url-storage")] + pub(crate) async fn with_url_c4gh_storage(test: F) + where + F: FnOnce(C4GHStorage, String) -> Fut, + Fut: Future, + { + with_url_test_server(|storage, url, base_path| async move { + create_encrypted_files(&base_path).await; + test(C4GHStorage::new(get_decryption_keys(), storage), url).await; + }) + .await; + } } diff --git a/htsget-storage/src/lib.rs b/htsget-storage/src/lib.rs index ad2c7c52..c3470086 100644 --- a/htsget-storage/src/lib.rs +++ b/htsget-storage/src/lib.rs @@ -23,10 +23,11 @@ use async_trait::async_trait; use base64::engine::general_purpose; use base64::Engine; use cfg_if::cfg_if; -use htsget_config::storage::local::LocalStorage as LocalStorageConfig; -use htsget_config::storage::object::ObjectType; +#[cfg(feature = "experimental")] +use htsget_config::storage::c4gh::C4GHKeys; +use htsget_config::storage::local::Local as LocalStorageConfig; #[cfg(feature = "s3-storage")] -use htsget_config::storage::s3::S3Storage as S3StorageConfig; +use htsget_config::storage::s3::S3 as S3StorageConfig; #[cfg(feature = "url-storage")] use htsget_config::storage::url::UrlStorageClient as UrlStorageConfig; use htsget_config::types::Scheme; @@ -136,53 +137,74 @@ impl StorageTrait for Storage { } impl Storage { - /// Wrap an existing storage with the object type storage. - pub fn from_object_type(_object_type: &ObjectType, storage: Storage) -> Storage { + #[cfg(feature = "experimental")] + /// Wrap an existing storage with C4GH storage + pub fn from_c4gh_keys(keys: Option<&C4GHKeys>, storage: Storage) -> Storage { + if let Some(keys) = keys { + Storage::new(C4GHStorage::new_box( + keys.clone().into_inner(), + storage.into_inner(), + )) + } else { + storage + } + } + + /// Create from local storage config. + pub async fn from_local(local_storage: &LocalStorageConfig) -> Result { + let storage = Storage::new(LocalStorage::new( + local_storage.local_path(), + local_storage.clone(), + )?); + cfg_if! { if #[cfg(feature = "experimental")] { - if let Some(keys) = _object_type.keys() { - Storage::new(C4GHStorage::new_box( - keys.clone().into_inner(), - storage.into_inner(), - )) - } else { - storage - } + Ok(Self::from_c4gh_keys(local_storage.keys(), storage)) } else { - storage + Ok(storage) } } } - /// Create from local storage config. - pub async fn from_local(config: &LocalStorageConfig) -> Result { - let storage = Storage::new(LocalStorage::new(config.local_path(), config.clone())?); - Ok(Storage::from_object_type(config.object_type(), storage)) - } - /// Create from s3 config. #[cfg(feature = "s3-storage")] pub async fn from_s3(s3_storage: &S3StorageConfig) -> Storage { - Storage::new( + let storage = Storage::new( S3Storage::new_with_default_config( s3_storage.bucket().to_string(), s3_storage.clone().endpoint(), s3_storage.clone().path_style(), ) .await, - ) + ); + + cfg_if! { + if #[cfg(feature = "experimental")] { + Self::from_c4gh_keys(s3_storage.keys(), storage) + } else { + storage + } + } } /// Create from url config. #[cfg(feature = "url-storage")] - pub async fn from_url(url_storage_config: &UrlStorageConfig) -> Storage { - Storage::new(UrlStorage::new( - url_storage_config.client_cloned(), - url_storage_config.url().clone(), - url_storage_config.response_url().clone(), - url_storage_config.forward_headers(), - url_storage_config.header_blacklist().to_vec(), - )) + pub async fn from_url(url_storage: &UrlStorageConfig) -> Storage { + let storage = Storage::new(UrlStorage::new( + url_storage.client_cloned(), + url_storage.url().clone(), + url_storage.response_url().clone(), + url_storage.forward_headers(), + url_storage.header_blacklist().to_vec(), + )); + + cfg_if! { + if #[cfg(feature = "experimental")] { + Self::from_c4gh_keys(url_storage.keys(), storage) + } else { + storage + } + } } pub fn new(inner: impl StorageTrait + Send + Sync + 'static) -> Self { @@ -257,7 +279,7 @@ pub trait UrlFormatter { fn format_url>(&self, key: K) -> Result; } -impl UrlFormatter for htsget_config::storage::local::LocalStorage { +impl UrlFormatter for htsget_config::storage::local::Local { fn format_url>(&self, key: K) -> Result { uri::Builder::new() .scheme(match self.scheme() { @@ -277,7 +299,7 @@ mod tests { use http::uri::Authority; use crate::local::LocalStorage; - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_test::util::default_dir; use super::*; @@ -302,7 +324,6 @@ mod tests { Authority::from_static("127.0.0.1:8080"), "data".to_string(), "/data".to_string(), - Default::default(), false, ); test_formatter_authority(formatter, "http"); @@ -315,7 +336,6 @@ mod tests { Authority::from_static("127.0.0.1:8080"), "data".to_string(), "/data".to_string(), - Default::default(), false, ); test_formatter_authority(formatter, "https"); diff --git a/htsget-storage/src/local.rs b/htsget-storage/src/local.rs index d79132a3..f94e8e32 100644 --- a/htsget-storage/src/local.rs +++ b/htsget-storage/src/local.rs @@ -140,7 +140,7 @@ pub(crate) mod tests { use tokio::fs::{create_dir, File}; use tokio::io::AsyncWriteExt; - use htsget_config::storage::local::LocalStorage as ConfigLocalStorage; + use htsget_config::storage::local::Local as ConfigLocalStorage; use htsget_config::types::Scheme; use super::*; @@ -150,7 +150,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_non_existing_key() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = storage.get("non-existing-key").await; assert!(matches!(result, Err(StorageError::KeyNotFound(msg)) if msg == "non-existing-key")); }) @@ -159,7 +159,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_folder() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::get( &storage, "folder", @@ -173,7 +173,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_forbidden_path() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::get( &storage, "folder/../../passwords", @@ -189,7 +189,7 @@ pub(crate) mod tests { #[tokio::test] async fn get_existing_key() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::get( &storage, "folder/../key1", @@ -203,7 +203,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_non_existing_key() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::range_url( &storage, "non-existing-key", @@ -217,7 +217,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_folder() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::range_url( &storage, "folder", @@ -231,7 +231,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_with_forbidden_path() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::range_url( &storage, "folder/../../passwords", @@ -247,7 +247,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_existing_key() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::range_url( &storage, "folder/../key1", @@ -262,7 +262,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_existing_key_with_specified_range() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::range_url( &storage, "folder/../key1", @@ -281,7 +281,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_existing_key_with_specified_open_ended_range() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::range_url( &storage, "folder/../key1", @@ -297,7 +297,7 @@ pub(crate) mod tests { #[tokio::test] async fn file_size() { - with_local_storage(|storage| async move { + with_local_storage(|storage, _| async move { let result = StorageTrait::head( &storage, "folder/../key1", @@ -345,7 +345,6 @@ pub(crate) mod tests { Authority::from_static("127.0.0.1:8081"), "data".to_string(), "/data".to_string(), - Default::default(), false, ), ) @@ -354,10 +353,14 @@ pub(crate) mod tests { pub(crate) async fn with_local_storage(test: F) where - F: FnOnce(LocalStorage) -> Fut, + F: FnOnce(LocalStorage, PathBuf) -> Fut, Fut: Future, { let (_, base_path) = create_local_test_files().await; - test(test_local_storage(base_path.path())).await + test( + test_local_storage(base_path.path()), + base_path.path().to_path_buf(), + ) + .await } } diff --git a/htsget-storage/src/s3.rs b/htsget-storage/src/s3.rs index 5fd9bae7..79d9fff5 100644 --- a/htsget-storage/src/s3.rs +++ b/htsget-storage/src/s3.rs @@ -291,7 +291,7 @@ impl StorageTrait for S3Storage { #[cfg(test)] pub(crate) mod tests { use std::future::Future; - use std::path::Path; + use std::path::{Path, PathBuf}; use std::sync::Arc; use htsget_test::aws_mocks::with_s3_test_server; @@ -305,18 +305,22 @@ pub(crate) mod tests { pub(crate) async fn with_aws_s3_storage_fn(test: F, folder_name: String, base_path: &Path) where - F: FnOnce(Arc) -> Fut, + F: FnOnce(Arc, PathBuf) -> Fut, Fut: Future, { with_s3_test_server(base_path, |client| async move { - test(Arc::new(S3Storage::new(client, folder_name))).await; + test( + Arc::new(S3Storage::new(client, folder_name)), + base_path.to_path_buf(), + ) + .await; }) .await; } - async fn with_aws_s3_storage(test: F) + pub(crate) async fn with_aws_s3_storage(test: F) where - F: FnOnce(Arc) -> Fut, + F: FnOnce(Arc, PathBuf) -> Fut, Fut: Future, { let (folder_name, base_path) = create_local_test_files().await; @@ -325,7 +329,7 @@ pub(crate) mod tests { #[tokio::test] async fn existing_key() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage .get( "key2", @@ -339,7 +343,7 @@ pub(crate) mod tests { #[tokio::test] async fn non_existing_key() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage .get( "non-existing-key", @@ -353,7 +357,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_of_existing_key() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage .range_url( "key2", @@ -372,7 +376,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_with_specified_range() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage .range_url( "key2", @@ -399,7 +403,7 @@ pub(crate) mod tests { #[tokio::test] async fn url_with_specified_open_ended_range() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage .range_url( "key2", @@ -423,7 +427,7 @@ pub(crate) mod tests { #[tokio::test] async fn file_size() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage .head("key2", HeadOptions::new(&Default::default())) .await; @@ -435,7 +439,7 @@ pub(crate) mod tests { #[tokio::test] async fn retrieval_type() { - with_aws_s3_storage(|storage| async move { + with_aws_s3_storage(|storage, _| async move { let result = storage.get_retrieval_type("key2").await; println!("{result:?}"); }) diff --git a/htsget-storage/src/url.rs b/htsget-storage/src/url.rs index 10e9f9d0..5178f91a 100644 --- a/htsget-storage/src/url.rs +++ b/htsget-storage/src/url.rs @@ -99,6 +99,7 @@ impl UrlStorage { let key = key.as_ref(); let url = self.get_url_from_key(key)?; + println!("url: {:?}", url); let request = Request::builder().method(method).uri(&url); let request = headers @@ -245,9 +246,9 @@ impl StorageTrait for UrlStorage { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::future::Future; - use std::path::Path; + use std::path::{Path, PathBuf}; use std::str::FromStr; use std::{result, vec}; @@ -326,15 +327,7 @@ mod tests { #[tokio::test] async fn send_request() { - with_url_test_server(|url| async move { - let storage = UrlStorage::new( - test_client(), - Uri::from_str(&url).unwrap(), - Uri::from_str(&url).unwrap(), - true, - vec![], - ); - + with_url_test_server(|storage, _, _| async move { let mut headers = HeaderMap::default(); let headers = test_headers(&mut headers); @@ -356,15 +349,7 @@ mod tests { #[tokio::test] async fn get_key() { - with_url_test_server(|url| async move { - let storage = UrlStorage::new( - test_client(), - Uri::from_str(&url).unwrap(), - Uri::from_str(&url).unwrap(), - true, - vec![], - ); - + with_url_test_server(|storage, _, _| async move { let mut headers = HeaderMap::default(); let headers = test_headers(&mut headers); @@ -386,15 +371,7 @@ mod tests { #[tokio::test] async fn head_key() { - with_url_test_server(|url| async move { - let storage = UrlStorage::new( - test_client(), - Uri::from_str(&url).unwrap(), - Uri::from_str(&url).unwrap(), - true, - vec![], - ); - + with_url_test_server(|storage, _, _| async move { let mut headers = HeaderMap::default(); let headers = test_headers(&mut headers); @@ -416,15 +393,7 @@ mod tests { #[tokio::test] async fn get_storage() { - with_url_test_server(|url| async move { - let storage = UrlStorage::new( - test_client(), - Uri::from_str(&url).unwrap(), - Uri::from_str(&url).unwrap(), - true, - vec![], - ); - + with_url_test_server(|storage, _, _| async move { let mut headers = HeaderMap::default(); let headers = test_headers(&mut headers); let options = GetOptions::new_with_default_range(headers); @@ -441,7 +410,7 @@ mod tests { #[tokio::test] async fn range_url_storage() { - with_url_test_server(|url| async move { + with_url_test_server(|_, url, _| async move { let storage = UrlStorage::new( test_client(), Uri::from_str(&url).unwrap(), @@ -449,7 +418,6 @@ mod tests { true, vec![], ); - let mut headers = HeaderMap::default(); let options = test_range_options(&mut headers); @@ -464,7 +432,7 @@ mod tests { #[tokio::test] async fn range_url_storage_blacklisted_headers() { - with_url_test_server(|url| async move { + with_url_test_server(|_, url, _| async move { let storage = UrlStorage::new( test_client(), Uri::from_str(&url).unwrap(), @@ -492,15 +460,7 @@ mod tests { #[tokio::test] async fn head_storage() { - with_url_test_server(|url| async move { - let storage = UrlStorage::new( - test_client(), - Uri::from_str(&url).unwrap(), - Uri::from_str(&url).unwrap(), - true, - vec![], - ); - + with_url_test_server(|storage, _, _| async move { let mut headers = HeaderMap::default(); let headers = test_headers(&mut headers); let options = HeadOptions::new(headers); @@ -575,7 +535,7 @@ mod tests { pub(crate) async fn with_url_test_server(test: F) where - F: FnOnce(String) -> Fut, + F: FnOnce(UrlStorage, String, PathBuf) -> Fut, Fut: Future, { let (_, base_path) = create_local_test_files().await; @@ -596,11 +556,12 @@ mod tests { pub(crate) async fn with_test_server(server_base_path: &Path, test: F) where - F: FnOnce(String) -> Fut, + F: FnOnce(UrlStorage, String, PathBuf) -> Fut, Fut: Future, { + let path = server_base_path.to_str().unwrap(); let router = Router::new() - .nest_service("/assets", ServeDir::new(server_base_path.to_str().unwrap())) + .nest_service("/assets", ServeDir::new(path)) .route_layer(middleware::from_fn(test_auth)); // TODO fix this in htsget-test to bind and return tcp listener. @@ -609,10 +570,22 @@ mod tests { tokio::spawn(async move { axum::serve(listener, router.into_make_service()).await }); - test(format!("http://{}", addr)).await; + let url = format!("http://{}", addr); + test( + UrlStorage::new( + test_client(), + Uri::from_str(&url).unwrap(), + Uri::from_str(&url).unwrap(), + false, + vec![], + ), + url, + server_base_path.to_path_buf(), + ) + .await; } - fn test_headers(headers: &mut HeaderMap) -> &HeaderMap { + pub(crate) fn test_headers(headers: &mut HeaderMap) -> &HeaderMap { headers.append( HeaderName::from_str(AUTHORIZATION.as_str()).unwrap(), HeaderValue::from_str("secret").unwrap(), diff --git a/htsget-test/Cargo.toml b/htsget-test/Cargo.toml index 3d518451..1844a162 100644 --- a/htsget-test/Cargo.toml +++ b/htsget-test/Cargo.toml @@ -43,7 +43,7 @@ default = [] # Server tests dependencies htsget-config = { version = "0.10.1", path = "../htsget-config", default-features = false, optional = true } -noodles = { version = "0.82", optional = true, features = ["async", "bgzf", "vcf", "cram", "bcf", "bam", "fasta"] } +noodles = { version = "0.83", optional = true, features = ["async", "bgzf", "vcf", "cram", "bcf", "bam", "fasta"] } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"], optional = true } tokio = { version = "1", features = ["rt-multi-thread", "fs"], optional = true } diff --git a/htsget-test/README.md b/htsget-test/README.md index e7af1ff8..93af3453 100644 --- a/htsget-test/README.md +++ b/htsget-test/README.md @@ -40,7 +40,7 @@ This crate has the following features: * `aws-mocks`: used to enable AWS mocking for tests. * `s3-storage`: used to enable `S3Storage` functionality. * `url-storage`: used to enable `UrlStorage` functionality. -* `experimental`: used to enable `C4GHStorage` functionality. +* `experimental`: used to enable experimental features that aren't necessarily part of the htsget spec, such as Crypt4GH support through `C4GHStorage`. [dev-dependencies]: https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html#development-dependencies diff --git a/htsget-test/src/c4gh.rs b/htsget-test/src/c4gh.rs index 8f1de6e5..cb402764 100644 --- a/htsget-test/src/c4gh.rs +++ b/htsget-test/src/c4gh.rs @@ -1,7 +1,7 @@ use crate::util::default_dir; use crypt4gh::keys::{get_private_key, get_public_key}; use crypt4gh::{decrypt, encrypt, Keys}; -use htsget_config::storage::object::c4gh::{C4GHKeys, C4GHPath}; +use htsget_config::storage::c4gh::{C4GHKeys, C4GHPath}; use std::collections::HashSet; use std::io::{BufReader, BufWriter, Cursor}; diff --git a/htsget-test/src/http/mod.rs b/htsget-test/src/http/mod.rs index 4d643f99..43e9b4ef 100644 --- a/htsget-test/src/http/mod.rs +++ b/htsget-test/src/http/mod.rs @@ -18,7 +18,7 @@ use serde::de; use htsget_config::config::cors::{AllowType, CorsConfig}; use htsget_config::config::{DataServerConfig, TicketServerConfig}; use htsget_config::resolver::Resolver; -use htsget_config::storage::{local::LocalStorage, Storage}; +use htsget_config::storage::{local::Local, Storage}; use htsget_config::tls::{ load_certs, load_key, tls_server_config, CertificateKeyPair, TlsServerConfig, }; @@ -95,12 +95,11 @@ pub trait TestServer { /// Get the default test storage. pub fn default_test_resolver(addr: SocketAddr, scheme: Scheme) -> Vec { - let local_storage = LocalStorage::new( + let local_storage = Local::new( scheme, Authority::from_str(&addr.to_string()).unwrap(), default_dir_data().to_str().unwrap().to_string(), "/data".to_string(), - Default::default(), false, ); vec![