From 36440c1d7277ea6a9a7daea9946c4091b47ea657 Mon Sep 17 00:00:00 2001 From: Jens Reimann Date: Thu, 4 Apr 2024 14:39:21 +0200 Subject: [PATCH] feat: add search --- Cargo.lock | 115 +++++++++++++++++- Cargo.toml | 3 + entity/src/advisory.rs | 7 +- modules/graph/Cargo.toml | 1 + modules/graph/src/graph/advisory/mod.rs | 55 +++++++-- modules/graph/src/graph/package/mod.rs | 37 +++++- .../src/graph/package/package_version.rs | 9 +- .../src/graph/package/qualified_package.rs | 1 + modules/graph/src/graph/sbom/mod.rs | 1 + modules/graph/src/graph/vulnerability/mod.rs | 24 +++- modules/importer/README.md | 7 ++ modules/ingestor/Cargo.toml | 1 + .../src/service/advisory/csaf/loader.rs | 24 +++- .../src/service/advisory/osv/loader.rs | 24 +++- .../src/service/advisory/osv/schema.rs | 23 ++-- modules/ingestor/src/service/cve/loader.rs | 6 +- modules/search/Cargo.toml | 27 ++++ modules/search/README.md | 5 + modules/search/src/endpoints.rs | 40 ++++++ modules/search/src/lib.rs | 4 + modules/search/src/model.rs | 52 ++++++++ modules/search/src/service.rs | 105 ++++++++++++++++ modules/search/test.sql | 1 + server/Cargo.toml | 1 + server/src/lib.rs | 26 ++-- server/src/openapi.rs | 1 + 26 files changed, 551 insertions(+), 49 deletions(-) create mode 100644 modules/search/Cargo.toml create mode 100644 modules/search/README.md create mode 100644 modules/search/src/endpoints.rs create mode 100644 modules/search/src/lib.rs create mode 100644 modules/search/src/model.rs create mode 100644 modules/search/src/service.rs create mode 100644 modules/search/test.sql diff --git a/Cargo.lock b/Cargo.lock index d36e79169..37436f695 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -994,6 +994,19 @@ dependencies = [ "windows-targets 0.52.4", ] +[[package]] +name = "chumsky" +version = "1.0.0-alpha.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c28d4e5dd9a9262a38b231153591da6ce1471b818233f4727985d3dd0ed93c" +dependencies = [ + "hashbrown 0.14.3", + "regex-automata 0.3.9", + "serde", + "stacker", + "unicode-ident", +] + [[package]] name = "clap" version = "4.5.2" @@ -1074,6 +1087,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.16.2" @@ -1468,7 +1490,7 @@ version = "0.99.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ - "convert_case", + "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version", @@ -3452,6 +3474,15 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "psm" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +dependencies = [ + "cc", +] + [[package]] name = "ptr_meta" version = "0.1.4" @@ -3587,6 +3618,17 @@ dependencies = [ "regex-syntax 0.6.29", ] +[[package]] +name = "regex-automata" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.5", +] + [[package]] name = "regex-automata" version = "0.4.6" @@ -3604,6 +3646,12 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + [[package]] name = "regex-syntax" version = "0.8.2" @@ -4463,6 +4511,32 @@ dependencies = [ "rand_core", ] +[[package]] +name = "sikula" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953a740b3abee4fe9f4d6f755ab8a79eb49be7df0221d57122f08b53e3702adb" +dependencies = [ + "chumsky", + "sea-orm", + "sikula-macros", + "thiserror", + "time", +] + +[[package]] +name = "sikula-macros" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "211dfb262c3b509999eba79fc970a577a81bdc4090eda15b7953d098307f4c52" +dependencies = [ + "convert_case 0.6.0", + "darling 0.20.8", + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "simdutf8" version = "0.1.4" @@ -4809,6 +4883,19 @@ dependencies = [ "uuid", ] +[[package]] +name = "stacker" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "winapi", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -5628,6 +5715,7 @@ dependencies = [ "tempfile", "test-log", "thiserror", + "time", "tokio", "trustify-auth", "trustify-common", @@ -5698,6 +5786,7 @@ dependencies = [ "sha2", "test-log", "thiserror", + "time", "tokio", "trustify-common", "trustify-cvss", @@ -5708,6 +5797,29 @@ dependencies = [ "utoipa", ] +[[package]] +name = "trustify-module-search" +version = "0.1.0" +dependencies = [ + "actix-web", + "anyhow", + "csaf", + "log", + "reqwest", + "sea-orm", + "serde", + "serde_json", + "sikula", + "test-log", + "thiserror", + "time", + "tokio", + "trustify-common", + "trustify-entity", + "url-escape", + "utoipa", +] + [[package]] name = "trustify-module-storage" version = "0.1.0" @@ -5762,6 +5874,7 @@ dependencies = [ "trustify-module-graph", "trustify-module-importer", "trustify-module-ingestor", + "trustify-module-search", "trustify-module-storage", "url", "url-escape", diff --git a/Cargo.toml b/Cargo.toml index b45fb6f78..7a4020744 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "modules/graph", "modules/importer", "modules/ingestor", + "modules/search", "modules/storage", "entity", "importer", @@ -73,6 +74,7 @@ serde = "1.0.183" serde_json = "1.0.114" serde_yaml = "0.9" sha2 = "0.10.8" +sikula = "0.4.4" spdx-expression = "0.5.2" spdx-rs = "0.5.3" sqlx = "0.7" @@ -105,6 +107,7 @@ trustify-entity = { path = "entity" } trustify-module-graph = { path = "modules/graph" } trustify-module-ingestor = { path = "modules/ingestor" } trustify-module-importer = { path = "modules/importer" } +trustify-module-search = { path = "modules/search" } trustify-module-storage = { path = "modules/storage" } trustify-infrastructure = { path = "common/infrastructure" } diff --git a/entity/src/advisory.rs b/entity/src/advisory.rs index 6fef4849f..9a3738cad 100644 --- a/entity/src/advisory.rs +++ b/entity/src/advisory.rs @@ -1,4 +1,5 @@ use sea_orm::entity::prelude::*; +use time::OffsetDateTime; #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] #[sea_orm(table_name = "advisory")] @@ -8,9 +9,9 @@ pub struct Model { pub identifier: String, pub location: String, pub sha256: String, - pub published: Option, - pub modified: Option, - pub withdrawn: Option, + pub published: Option, + pub modified: Option, + pub withdrawn: Option, pub title: Option, } diff --git a/modules/graph/Cargo.toml b/modules/graph/Cargo.toml index cf57eb9b5..bc41787fc 100644 --- a/modules/graph/Cargo.toml +++ b/modules/graph/Cargo.toml @@ -28,6 +28,7 @@ spdx-expression = { workspace = true } spdx-rs = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } +time = { workspace = true } tokio = { workspace = true, features = ["full"] } utoipa = { workspace = true, features = ["actix_extras"] } diff --git a/modules/graph/src/graph/advisory/mod.rs b/modules/graph/src/graph/advisory/mod.rs index e570b8ad6..a0336a67f 100644 --- a/modules/graph/src/graph/advisory/mod.rs +++ b/modules/graph/src/graph/advisory/mod.rs @@ -3,6 +3,7 @@ use crate::graph::advisory::advisory_vulnerability::AdvisoryVulnerabilityContext; use crate::graph::error::Error; use crate::graph::Graph; +use csaf::Csaf; use sea_orm::prelude::DateTimeUtc; use sea_orm::ActiveValue::Set; use sea_orm::{ActiveModelTrait, EntityTrait, FromQueryResult, IntoActiveModel, QueryFilter}; @@ -11,6 +12,7 @@ use sea_query::{Condition, JoinType}; use std::cmp::min; use std::collections::HashMap; use std::fmt::{Debug, Formatter}; +use time::OffsetDateTime; use trustify_common::advisory::{AdvisoryVulnerabilityAssertions, Assertion}; use trustify_common::db::Transactional; use trustify_common::purl::Purl; @@ -23,6 +25,19 @@ pub mod affected_package_version_range; pub mod fixed_package_version; pub mod not_affected_package_version; +#[derive(Clone, Default)] +pub struct AdvisoryInformation { + pub title: Option, + pub published: Option, + pub modified: Option, +} + +impl From<()> for AdvisoryInformation { + fn from(value: ()) -> Self { + Self::default() + } +} + impl Graph { pub async fn get_advisory_by_id>( &self, @@ -35,17 +50,18 @@ impl Graph { .map(|advisory| (self, advisory).into())) } - pub async fn get_advisory( + pub async fn get_advisory>( &self, identifier: &str, location: &str, sha256: &str, + tx: TX, ) -> Result, Error> { Ok(entity::advisory::Entity::find() .filter(Condition::all().add(entity::advisory::Column::Identifier.eq(identifier))) .filter(Condition::all().add(entity::advisory::Column::Location.eq(location))) .filter(Condition::all().add(entity::advisory::Column::Sha256.eq(sha256.to_string()))) - .one(&self.db) + .one(&self.connection(&tx)) .await? .map(|sbom| (self, sbom).into())) } @@ -55,20 +71,33 @@ impl Graph { identifier: impl Into, location: impl Into, sha256: impl Into, + information: impl Into, tx: TX, ) -> Result { let identifier = identifier.into(); let location = location.into(); let sha256 = sha256.into(); - if let Some(found) = self.get_advisory(&identifier, &location, &sha256).await? { + if let Some(found) = self + .get_advisory(&identifier, &location, &sha256, tx) + .await? + { return Ok(found); } + let AdvisoryInformation { + title, + published, + modified, + } = information.into(); + let model = entity::advisory::ActiveModel { identifier: Set(identifier), location: Set(location), sha256: Set(sha256), + title: Set(title), + published: Set(published), + modified: Set(modified), ..Default::default() }; @@ -103,7 +132,7 @@ impl<'g> From<(&'g Graph, entity::advisory::Model)> for AdvisoryContext<'g> { impl<'g> AdvisoryContext<'g> { pub async fn set_published_at>( &self, - published_at: DateTimeUtc, + published_at: time::OffsetDateTime, tx: TX, ) -> Result<(), Error> { let mut entity = self.advisory.clone().into_active_model(); @@ -112,13 +141,13 @@ impl<'g> AdvisoryContext<'g> { Ok(()) } - pub fn published_at(&self) -> Option { + pub fn published_at(&self) -> Option { self.advisory.published } pub async fn set_modified_at>( &self, - modified_at: DateTimeUtc, + modified_at: time::OffsetDateTime, tx: TX, ) -> Result<(), Error> { let mut entity = self.advisory.clone().into_active_model(); @@ -127,13 +156,13 @@ impl<'g> AdvisoryContext<'g> { Ok(()) } - pub fn modified_at(&self) -> Option { + pub fn modified_at(&self) -> Option { self.advisory.modified } pub async fn set_withdrawn_at>( &self, - withdrawn_at: DateTimeUtc, + withdrawn_at: time::OffsetDateTime, tx: TX, ) -> Result<(), Error> { let mut entity = self.advisory.clone().into_active_model(); @@ -142,7 +171,7 @@ impl<'g> AdvisoryContext<'g> { Ok(()) } - pub fn withdrawn_at(&self) -> Option { + pub fn withdrawn_at(&self) -> Option { self.advisory.withdrawn } @@ -474,6 +503,7 @@ mod test { "RHSA-GHSA-1", "http://db.com/rhsa-ghsa-2", "2", + (), Transactional::None, ) .await?; @@ -483,6 +513,7 @@ mod test { "RHSA-GHSA-1", "http://db.com/rhsa-ghsa-2", "2", + (), Transactional::None, ) .await?; @@ -492,6 +523,7 @@ mod test { "RHSA-GHSA-1", "http://db.com/rhsa-ghsa-2", "89", + (), Transactional::None, ) .await?; @@ -512,6 +544,7 @@ mod test { "RHSA-GHSA-1", "http://db.com/rhsa-ghsa-2", "2", + (), Transactional::None, ) .await?; @@ -569,6 +602,7 @@ mod test { "RHSA-GHSA-1", "http://db.com/rhsa-ghsa-2", "2", + (), Transactional::None, ) .await?; @@ -629,6 +663,7 @@ mod test { "RHSA-GHSA-1", "http://db.com/rhsa-ghsa-2", "2", + (), Transactional::None, ) .await?; @@ -656,6 +691,7 @@ mod test { "RHSA-GHSA-1", "http://db.com/rhsa-ghsa-2", "2", + (), Transactional::None, ) .await?; @@ -697,6 +733,7 @@ mod test { "RHSA-GHSA-1", "http://db.com/rhsa-ghsa-2", "2", + (), Transactional::None, ) .await?; diff --git a/modules/graph/src/graph/package/mod.rs b/modules/graph/src/graph/package/mod.rs index e7239567b..c896f3435 100644 --- a/modules/graph/src/graph/package/mod.rs +++ b/modules/graph/src/graph/package/mod.rs @@ -923,6 +923,7 @@ mod tests { "RHSA-1", "http://redhat.com/rhsa-1", "2", + (), Transactional::None, ) .await?; @@ -950,7 +951,13 @@ mod tests { .await?; let ghsa_advisory = system - .ingest_advisory("GHSA-1", "http://ghsa.com/ghsa-1", "2", Transactional::None) + .ingest_advisory( + "GHSA-1", + "http://ghsa.com/ghsa-1", + "2", + (), + Transactional::None, + ) .await?; let ghsa_advisory_vulnerability = ghsa_advisory @@ -1016,6 +1023,7 @@ mod tests { "RHSA-1", "http://redhat.com/rhsa-1", "2", + (), Transactional::None, ) .await?; @@ -1032,7 +1040,13 @@ mod tests { .await?; let ghsa_advisory = system - .ingest_advisory("GHSA-1", "http://ghsa.com/ghsa-1", "2", Transactional::None) + .ingest_advisory( + "GHSA-1", + "http://ghsa.com/ghsa-1", + "2", + (), + Transactional::None, + ) .await?; let ghsa_advisory_vulnerability = ghsa_advisory @@ -1071,6 +1085,7 @@ mod tests { "RHSA-1", "http://redhat.com/rhsa-1", "2", + (), Transactional::None, ) .await?; @@ -1096,7 +1111,13 @@ mod tests { .await?; let ghsa_advisory = system - .ingest_advisory("GHSA-1", "http://ghsa.com/ghsa-1", "2", Transactional::None) + .ingest_advisory( + "GHSA-1", + "http://ghsa.com/ghsa-1", + "2", + (), + Transactional::None, + ) .await?; let ghsa_advisory_vulnerability = ghsa_advisory @@ -1135,6 +1156,7 @@ mod tests { "RHSA-1", "http://redhat.com/rhsa-1", "2", + (), Transactional::None, ) .await?; @@ -1153,7 +1175,13 @@ mod tests { .await?; let ghsa_advisory = system - .ingest_advisory("GHSA-1", "http://ghsa.gov/GHSA-1", "3", Transactional::None) + .ingest_advisory( + "GHSA-1", + "http://ghsa.gov/GHSA-1", + "3", + (), + Transactional::None, + ) .await?; let ghsa_advisory_vulnerability = ghsa_advisory @@ -1172,6 +1200,7 @@ mod tests { "RHSA-299", "http://redhat.com/rhsa-299", "17", + (), Transactional::None, ) .await?; diff --git a/modules/graph/src/graph/package/package_version.rs b/modules/graph/src/graph/package/package_version.rs index d56479290..85c08f21f 100644 --- a/modules/graph/src/graph/package/package_version.rs +++ b/modules/graph/src/graph/package/package_version.rs @@ -223,6 +223,7 @@ mod tests { "RHSA-1", "http://redhat.com/rhsa-1", "2", + (), Transactional::None, ) .await?; @@ -239,7 +240,13 @@ mod tests { .await?; let ghsa_advisory = system - .ingest_advisory("GHSA-1", "http://ghsa.com/ghsa-1", "2", Transactional::None) + .ingest_advisory( + "GHSA-1", + "http://ghsa.com/ghsa-1", + "2", + (), + Transactional::None, + ) .await?; let ghsa_advisory_vulnerability = ghsa_advisory diff --git a/modules/graph/src/graph/package/qualified_package.rs b/modules/graph/src/graph/package/qualified_package.rs index 7016ad84d..1c6fe629c 100644 --- a/modules/graph/src/graph/package/qualified_package.rs +++ b/modules/graph/src/graph/package/qualified_package.rs @@ -147,6 +147,7 @@ mod tests { "RHSA-GHSA-1", "http://db.com/rhsa-ghsa-2", "2", + (), Transactional::None, ) .await?; diff --git a/modules/graph/src/graph/sbom/mod.rs b/modules/graph/src/graph/sbom/mod.rs index ecae58200..cb7ae2abd 100644 --- a/modules/graph/src/graph/sbom/mod.rs +++ b/modules/graph/src/graph/sbom/mod.rs @@ -998,6 +998,7 @@ mod tests { "RHSA-1", "http://redhat.com/secdata/RHSA-1", "7", + (), Transactional::None, ) .await?; diff --git a/modules/graph/src/graph/vulnerability/mod.rs b/modules/graph/src/graph/vulnerability/mod.rs index 2882cb8c1..d66efe7f3 100644 --- a/modules/graph/src/graph/vulnerability/mod.rs +++ b/modules/graph/src/graph/vulnerability/mod.rs @@ -174,15 +174,33 @@ mod tests { let system = Graph::new(db); let advisory1 = system - .ingest_advisory("GHSA-1", "http://ghsa.io/GHSA-1", "7", Transactional::None) + .ingest_advisory( + "GHSA-1", + "http://ghsa.io/GHSA-1", + "7", + (), + Transactional::None, + ) .await?; let advisory2 = system - .ingest_advisory("RHSA-1", "http://rhsa.io/RHSA-1", "8", Transactional::None) + .ingest_advisory( + "RHSA-1", + "http://rhsa.io/RHSA-1", + "8", + (), + Transactional::None, + ) .await?; let advisory3 = system - .ingest_advisory("SNYK-1", "http://snyk.io/SNYK-1", "9", Transactional::None) + .ingest_advisory( + "SNYK-1", + "http://snyk.io/SNYK-1", + "9", + (), + Transactional::None, + ) .await?; advisory1 diff --git a/modules/importer/README.md b/modules/importer/README.md index 6aaed75e2..c32ce1a6a 100644 --- a/modules/importer/README.md +++ b/modules/importer/README.md @@ -29,3 +29,10 @@ Get reports: http GET localhost:8080/api/v1/importer/redhat-csaf/report http GET localhost:8080/api/v1/importer/redhat-sbom/report ``` + +Delete an importer: + +```bash +http DELETE localhost:8080/api/v1/importer/redhat-csaf +http DELETE localhost:8080/api/v1/importer/redhat-sbom +``` diff --git a/modules/ingestor/Cargo.toml b/modules/ingestor/Cargo.toml index 65715df0d..c75c99c69 100644 --- a/modules/ingestor/Cargo.toml +++ b/modules/ingestor/Cargo.toml @@ -28,6 +28,7 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } sha2 = { workspace = true } thiserror = { workspace = true } +time = { workspace = true, features = ["serde-well-known"] } tokio = { workspace = true, features = ["full"] } utoipa = { workspace = true, features = ["actix_extras"] } packageurl = { workspace = true } diff --git a/modules/ingestor/src/service/advisory/csaf/loader.rs b/modules/ingestor/src/service/advisory/csaf/loader.rs index 1c81d5d43..8f6016e53 100644 --- a/modules/ingestor/src/service/advisory/csaf/loader.rs +++ b/modules/ingestor/src/service/advisory/csaf/loader.rs @@ -4,12 +4,32 @@ use crate::service::Error; use csaf::vulnerability::{ProductStatus, Vulnerability}; use csaf::Csaf; use std::io::Read; +use time::OffsetDateTime; use trustify_common::db::Transactional; use trustify_common::purl::Purl; use trustify_module_graph::graph::advisory::advisory_vulnerability::AdvisoryVulnerabilityContext; -use trustify_module_graph::graph::advisory::AdvisoryContext; +use trustify_module_graph::graph::advisory::{AdvisoryContext, AdvisoryInformation}; use trustify_module_graph::graph::Graph; +struct Information<'a>(&'a Csaf); + +impl<'a> From> for AdvisoryInformation { + fn from(value: Information<'a>) -> Self { + let value = value.0; + Self { + title: Some(value.document.title.clone()), + published: OffsetDateTime::from_unix_timestamp( + value.document.tracking.initial_release_date.timestamp(), + ) + .ok(), + modified: OffsetDateTime::from_unix_timestamp( + value.document.tracking.current_release_date.timestamp(), + ) + .ok(), + } + } +} + pub struct CsafLoader<'g> { graph: &'g Graph, } @@ -43,7 +63,7 @@ impl<'g> CsafLoader<'g> { let advisory = self .graph - .ingest_advisory(&advisory_id, location, sha256, &tx) + .ingest_advisory(&advisory_id, location, sha256, Information(&csaf), &tx) .await?; for vuln in csaf.vulnerabilities.iter().flatten() { diff --git a/modules/ingestor/src/service/advisory/osv/loader.rs b/modules/ingestor/src/service/advisory/osv/loader.rs index 7191699c3..5a2304213 100644 --- a/modules/ingestor/src/service/advisory/osv/loader.rs +++ b/modules/ingestor/src/service/advisory/osv/loader.rs @@ -7,6 +7,7 @@ use std::io::Read; use std::str::FromStr; use trustify_common::purl::Purl; use trustify_cvss::cvss3::Cvss3Base; +use trustify_module_graph::graph::advisory::AdvisoryInformation; use trustify_module_graph::graph::Graph; pub struct OsvLoader<'g> { @@ -46,13 +47,16 @@ impl<'g> OsvLoader<'g> { ))); } + let information = AdvisoryInformation { + title: osv.summary.clone(), + published: Some(osv.published), + modified: Some(osv.modified), + }; let advisory = self .graph - .ingest_advisory(osv.id, location, sha256, &tx) + .ingest_advisory(&osv.id, location, sha256, information, &tx) .await?; - advisory.set_published_at(osv.published, &tx).await?; - advisory.set_modified_at(osv.modified, &tx).await?; if let Some(withdrawn) = osv.withdrawn { advisory.set_withdrawn_at(withdrawn, &tx).await?; } @@ -168,7 +172,12 @@ mod test { assert!(loaded_vulnerability.is_none()); let loaded_advisory = graph - .get_advisory("RUSTSEC-2021-0079", "RUSTSEC-2021-0079.json", checksum) + .get_advisory( + "RUSTSEC-2021-0079", + "RUSTSEC-2021-0079.json", + checksum, + Transactional::None, + ) .await?; assert!(loaded_advisory.is_none()); @@ -186,7 +195,12 @@ mod test { assert!(loaded_vulnerability.is_some()); let loaded_advisory = graph - .get_advisory("RUSTSEC-2021-0079", "RUSTSEC-2021-0079.json", checksum) + .get_advisory( + "RUSTSEC-2021-0079", + "RUSTSEC-2021-0079.json", + checksum, + Transactional::None, + ) .await?; assert!(loaded_advisory.is_some()); diff --git a/modules/ingestor/src/service/advisory/osv/schema.rs b/modules/ingestor/src/service/advisory/osv/schema.rs index 2338fdb43..e658d8f98 100644 --- a/modules/ingestor/src/service/advisory/osv/schema.rs +++ b/modules/ingestor/src/service/advisory/osv/schema.rs @@ -1,7 +1,5 @@ -use std::fmt::{Display, Formatter}; - -use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use std::fmt::{Display, Formatter}; /// Package identifies the code library or command that /// is potentially affected by a particular vulnerability. @@ -362,7 +360,8 @@ pub struct BatchVulnerability { /// The modified field gives the time the entry was last modified, as an RFC3339-formatted /// timestamptime stamp in UTC (ending in “Z”). - pub modified: DateTime, + #[serde(with = "time::serde::rfc3339")] + pub modified: time::OffsetDateTime, } /// A vulnerability is the standard exchange format that is @@ -383,18 +382,21 @@ pub struct Vulnerability { /// The published field gives the time the entry should be considered to have been published, /// as an RFC3339-formatted time stamp in UTC (ending in “Z”). - pub published: DateTime, + #[serde(with = "time::serde::rfc3339")] + pub published: time::OffsetDateTime, /// The modified field gives the time the entry was last modified, as an RFC3339-formatted /// timestamptime stamp in UTC (ending in “Z”). - pub modified: DateTime, + #[serde(with = "time::serde::rfc3339")] + pub modified: time::OffsetDateTime, /// The withdrawn field gives the time the entry should be considered to have been withdrawn, /// as an RFC3339-formatted timestamp in UTC (ending in “Z”). If the field is missing, then the /// entry has not been withdrawn. Any rationale for why the vulnerability has been withdrawn /// should go into the summary text. - #[serde(skip_serializing_if = "Option::is_none")] - pub withdrawn: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(with = "time::serde::rfc3339::option")] + pub withdrawn: Option, /// The aliases field gives a list of IDs of the same vulnerability in other databases, in the /// form of the id field. This allows one database to claim that its own entry describes the @@ -452,6 +454,7 @@ pub struct Vulnerability { mod tests { use serde_json::json; use test_log::test; + use time::OffsetDateTime; use super::*; @@ -491,8 +494,8 @@ mod tests { let vuln = Vulnerability { schema_version: Some("1.3.0".to_string()), id: "OSV-2020-484".to_string(), - published: chrono::Utc::now(), - modified: chrono::Utc::now(), + published: OffsetDateTime::now_utc(), + modified: OffsetDateTime::now_utc(), withdrawn: None, aliases: None, related: None, diff --git a/modules/ingestor/src/service/cve/loader.rs b/modules/ingestor/src/service/cve/loader.rs index 5941ea32c..49c06dceb 100644 --- a/modules/ingestor/src/service/cve/loader.rs +++ b/modules/ingestor/src/service/cve/loader.rs @@ -49,7 +49,7 @@ impl<'g> CveLoader<'g> { let advisory = self .graph - .ingest_advisory(cve.cve_metadata.cve_id(), location, sha256, &tx) + .ingest_advisory(cve.cve_metadata.cve_id(), location, sha256, (), &tx) .await?; // Link the advisory to the backing vulnerability @@ -70,7 +70,7 @@ mod test { use std::str::FromStr; use test_log::test; - use trustify_common::db::Database; + use trustify_common::db::{Database, Transactional}; use trustify_module_graph::graph::Graph; use crate::service::cve::loader::CveLoader; @@ -95,6 +95,7 @@ mod test { "CVE-2024-28111", "CVE-2024-28111.json", "06908108e8097f2a56e628e7814a7bd54a5fc95f645b7c9fab02c1eb8dd9cc0c", + Transactional::None, ) .await?; @@ -113,6 +114,7 @@ mod test { "CVE-2024-28111", "CVE-2024-28111.json", "06908108e8097f2a56e628e7814a7bd54a5fc95f645b7c9fab02c1eb8dd9cc0c", + Transactional::None, ) .await?; diff --git a/modules/search/Cargo.toml b/modules/search/Cargo.toml new file mode 100644 index 000000000..08fec0739 --- /dev/null +++ b/modules/search/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "trustify-module-search" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +trustify-common = { path = "../../common" } +trustify-entity = { path = "../../entity" } + +actix-web = { workspace = true } +anyhow = { workspace = true } +csaf = { workspace = true } +log = { workspace = true } +reqwest = { workspace = true } +sea-orm = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +sikula = { workspace = true, features = ["sea-orm"] } +thiserror = { workspace = true } +time = { workspace = true } +tokio = { workspace = true, features = ["full"] } +utoipa = { workspace = true, features = ["actix_extras"] } + +[dev-dependencies] +test-log = { workspace = true, features = ["env_logger", "trace"] } +url-escape = { workspace = true } diff --git a/modules/search/README.md b/modules/search/README.md new file mode 100644 index 000000000..9c14a5659 --- /dev/null +++ b/modules/search/README.md @@ -0,0 +1,5 @@ +Basic advisory search: + +```bash +http GET localhost:8080/api/v1/search/advisory limit==5 q=='ssl AND NOT time' +``` diff --git a/modules/search/src/endpoints.rs b/modules/search/src/endpoints.rs new file mode 100644 index 000000000..cb428975b --- /dev/null +++ b/modules/search/src/endpoints.rs @@ -0,0 +1,40 @@ +use crate::model::{AdvisorySearch, SearchOptions}; +use crate::service::{Error, SearchService}; +use actix_web::{get, web, Responder}; +use sikula::prelude::Search; +use trustify_common::db::Database; +use trustify_common::model::Paginated; +use utoipa::OpenApi; + +/// mount the "search" module +pub fn configure(svc: &mut web::ServiceConfig, db: Database) { + svc.app_data(web::Data::new(SearchService::new(db))); + svc.service(web::scope("/api/v1/search").service(search_advisories)); +} + +#[derive(OpenApi)] +#[openapi(paths(search_advisories), components(schemas()), tags())] +pub struct ApiDoc; + +#[utoipa::path( + context_path = "/api/v1/search/advisory", + tag = "search", + params( + ("q", Query, description = "The query expression"), + ), + responses( + (status = 200, description = "Advisory search result", body = [crate::model::PaginatedAdvisories]) + ) +)] +#[get("/advisory")] +/// Search for advisories +async fn search_advisories( + web::Query(SearchOptions { q }): web::Query, + web::Query(paginated): web::Query, + service: web::Data, +) -> Result { + let search = AdvisorySearch::parse(&q)?; + Ok(web::Json( + service.search_advisories(search, paginated).await?, + )) +} diff --git a/modules/search/src/lib.rs b/modules/search/src/lib.rs new file mode 100644 index 000000000..b06c85217 --- /dev/null +++ b/modules/search/src/lib.rs @@ -0,0 +1,4 @@ +//! Document search +pub mod endpoints; +pub mod model; +pub mod service; diff --git a/modules/search/src/model.rs b/modules/search/src/model.rs new file mode 100644 index 000000000..541b3d12d --- /dev/null +++ b/modules/search/src/model.rs @@ -0,0 +1,52 @@ +use sikula::prelude::*; +use time::OffsetDateTime; +use trustify_common::model::PaginatedResults; +use trustify_entity::advisory; + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, utoipa::ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct SearchOptions { + #[serde(default)] + pub q: String, +} + +#[derive(Debug, sikula::Search)] +pub enum AdvisorySearch<'a> { + #[search(scope, default)] + Title(Primary<'a>), + + #[search(sort)] + Modified(Ordered), + #[search(sort)] + Published(Ordered), +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, utoipa::ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct FoundAdvisory { + pub id: i32, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub title: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(with = "time::serde::rfc3339::option")] + pub published: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(with = "time::serde::rfc3339::option")] + pub modified: Option, +} + +impl From for FoundAdvisory { + fn from(value: advisory::Model) -> Self { + Self { + id: value.id, + title: value.title, + published: value.published, + modified: value.modified, + } + } +} + +pub struct PaginatedAdvisories(pub PaginatedResults); diff --git a/modules/search/src/service.rs b/modules/search/src/service.rs new file mode 100644 index 000000000..317495f8e --- /dev/null +++ b/modules/search/src/service.rs @@ -0,0 +1,105 @@ +use crate::model::{AdvisorySearch, AdvisorySearchSortable, FoundAdvisory}; +use actix_web::{body::BoxBody, HttpResponse, ResponseError}; +use sea_orm::sea_query::extension::postgres::PgExpr; +use sea_orm::sea_query::IntoCondition; +use sea_orm::{ColumnTrait, Condition, EntityTrait, QueryFilter, QueryOrder}; +use sikula::prelude::*; +use sikula::sea_orm::{translate_term, TranslateOrdered}; +use trustify_common::{ + db::{limiter::LimiterTrait, Database}, + error::ErrorInformation, + model::{Paginated, PaginatedResults}, +}; +use trustify_entity::advisory; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("query syntax error: {0}")] + SearchSyntax(String), + #[error("database error: {0}")] + Database(#[from] sea_orm::DbErr), +} + +impl From> for Error { + fn from(value: sikula::prelude::Error) -> Self { + Self::SearchSyntax(value.to_string()) + } +} + +impl ResponseError for Error { + fn error_response(&self) -> HttpResponse { + match self { + Self::SearchSyntax(_) => HttpResponse::BadRequest().json(ErrorInformation { + error: "SearchSyntax".into(), + message: self.to_string(), + details: None, + }), + _ => HttpResponse::InternalServerError().json(ErrorInformation { + error: "Internal".into(), + message: self.to_string(), + details: None, + }), + } + } +} + +pub struct SearchService { + db: Database, +} + +impl SearchService { + pub fn new(db: Database) -> Self { + Self { db } + } + + pub async fn search_advisories<'a>( + &self, + Query { term, sorting }: Query<'a, AdvisorySearch<'a>>, + paginated: Paginated, + ) -> Result, Error> { + let mut select = advisory::Entity::find(); + + select = select.filter(translate_term(term, &translate)); + + for sort in sorting { + let col = match sort.qualifier { + AdvisorySearchSortable::Modified => advisory::Column::Modified, + AdvisorySearchSortable::Published => advisory::Column::Published, + }; + select = select.order_by(col, sort.direction.into()); + } + + // we always sort by ID last, so that we have a stable order for pagination + + select = select.order_by_desc(advisory::Column::Id); + + let limiting = select.limiting(&self.db, paginated.offset, paginated.limit); + + Ok(PaginatedResults { + total: limiting.total().await?, + items: limiting + .fetch() + .await? + .into_iter() + .map(FoundAdvisory::from) + .collect(), + }) + } +} + +fn translate(term: AdvisorySearch) -> Condition { + match term { + AdvisorySearch::Title(Primary::Equal(value)) => { + advisory::Column::Title.eq(value).into_condition() + } + AdvisorySearch::Title(Primary::Partial(value)) => advisory::Column::Title + .into_expr() + .ilike(format!( + "%{}%", + value.replace('%', "\\%").replace('_', "\\_") + )) + .into_condition(), + AdvisorySearch::Published(value) => value.translate(advisory::Column::Published), + AdvisorySearch::Modified(value) => value.translate(advisory::Column::Modified), + } +} diff --git a/modules/search/test.sql b/modules/search/test.sql new file mode 100644 index 000000000..b8198b3fa --- /dev/null +++ b/modules/search/test.sql @@ -0,0 +1 @@ +create text search configuration trustify_en (copy=english); diff --git a/server/Cargo.toml b/server/Cargo.toml index daad10a96..b122a2a62 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -11,6 +11,7 @@ trustify-entity = { workspace = true } trustify-module-graph = { workspace = true } trustify-module-ingestor = { workspace = true } trustify-module-importer = { workspace = true } +trustify-module-search = { workspace = true } trustify-module-storage = { workspace = true } actix-web = { workspace = true } anyhow = { workspace = true } diff --git a/server/src/lib.rs b/server/src/lib.rs index d2e3f9801..0338f2e18 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -2,10 +2,14 @@ mod openapi; -use actix_web::dev::{ConnectionInfo, Url}; -use actix_web::error::UrlGenerationError; -use actix_web::web::Json; -use actix_web::{body::MessageBody, get, web, HttpRequest, HttpResponse, Responder}; +use actix_web::{ + body::MessageBody, + dev::{ConnectionInfo, Url}, + error::UrlGenerationError, + get, web, + web::Json, + HttpRequest, HttpResponse, Responder, +}; use anyhow::Context; use futures::FutureExt; use std::fmt::Display; @@ -14,13 +18,16 @@ use std::path::PathBuf; use std::process::ExitCode; use std::sync::Arc; use std::time::Duration; -use trustify_auth::swagger_ui::{swagger_ui_with_auth, SwaggerUiOidc}; use trustify_auth::{ - auth::AuthConfigArguments, authenticator::Authenticator, authorizer::Authorizer, - swagger_ui::SwaggerUiOidcConfig, + auth::AuthConfigArguments, + authenticator::Authenticator, + authorizer::Authorizer, + swagger_ui::{swagger_ui_with_auth, SwaggerUiOidc, SwaggerUiOidcConfig}, +}; +use trustify_common::{ + config::{Database, StorageConfig}, + db, }; -use trustify_common::config::StorageConfig; -use trustify_common::{config::Database, db}; use trustify_infrastructure::{ app::http::{HttpServerBuilder, HttpServerConfig}, endpoint::Trustify, @@ -176,6 +183,7 @@ impl InitData { db.clone(), storage.clone(), ); + trustify_module_search::endpoints::configure(svc, db.clone()); }); }) }; diff --git a/server/src/openapi.rs b/server/src/openapi.rs index 37267a64f..1a2567801 100644 --- a/server/src/openapi.rs +++ b/server/src/openapi.rs @@ -11,6 +11,7 @@ pub fn openapi() -> utoipa::openapi::OpenApi { doc.merge(trustify_module_graph::endpoints::ApiDoc::openapi()); doc.merge(trustify_module_importer::endpoints::ApiDoc::openapi()); doc.merge(trustify_module_ingestor::endpoints::ApiDoc::openapi()); + doc.merge(trustify_module_search::endpoints::ApiDoc::openapi()); doc }