diff --git a/Cargo.lock b/Cargo.lock index b5b6a8093..d361e1d6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -367,6 +367,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "ahash" version = "0.7.8" @@ -1168,6 +1179,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.13" @@ -1701,6 +1722,12 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +[[package]] +name = "deflate64" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" + [[package]] name = "der" version = "0.7.9" @@ -3133,6 +3160,15 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array 0.14.7", +] + [[package]] name = "instant" version = "0.1.13" @@ -3564,6 +3600,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.22" @@ -3579,6 +3621,16 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -4326,6 +4378,16 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498a099351efa4becc6a19c72aa9270598e8fd274ca47052e37455241c88b696" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "peak_alloc" version = "0.2.1" @@ -5214,6 +5276,12 @@ dependencies = [ "serde", ] +[[package]] +name = "roxmltree" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" + [[package]] name = "rsa" version = "0.9.6" @@ -6072,6 +6140,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "simdutf8" version = "0.1.4" @@ -7276,6 +7350,7 @@ dependencies = [ "jsonpath-rust", "log", "packageurl", + "roxmltree", "sea-orm", "sea-query", "serde", @@ -7302,6 +7377,7 @@ dependencies = [ "urlencoding", "utoipa", "uuid", + "zip 2.2.0", ] [[package]] @@ -7404,6 +7480,7 @@ dependencies = [ "parking_lot 0.12.3", "rand", "ring", + "roxmltree", "rstest", "sbom-walker", "sea-orm", @@ -7414,6 +7491,7 @@ dependencies = [ "spdx", "spdx-expression", "spdx-rs", + "strum 0.26.3", "test-context", "test-log", "thiserror", @@ -7428,6 +7506,7 @@ dependencies = [ "trustify-test-context", "utoipa", "uuid", + "zip 2.2.0", ] [[package]] @@ -7811,7 +7890,7 @@ dependencies = [ "serde_json", "url", "utoipa", - "zip", + "zip 1.1.4", ] [[package]] @@ -8444,6 +8523,20 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] [[package]] name = "zerovec" @@ -8483,6 +8576,49 @@ dependencies = [ "thiserror", ] +[[package]] +name = "zip" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "hmac", + "indexmap 2.3.0", + "lzma-rs", + "memchr", + "pbkdf2", + "rand", + "sha1", + "thiserror", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.2" diff --git a/Cargo.toml b/Cargo.toml index 52fd68d47..dd5dc7449 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,6 +92,7 @@ rand = "0.8.5" regex = "1.10.3" reqwest = "0.12" ring = "0.17.8" +roxmltree = "0.20.0" rstest = "0.22" rust-s3 = "0.34" sbom-walker = { version = "0.9.0", default-features = false, features = ["crypto-openssl", "cyclonedx-bom", "spdx-rs"] } @@ -132,6 +133,7 @@ uuid = "1.7.0" walkdir = "2.5" walker-common = "0.9.0" walker-extras = "0.9.0" +zip = "2.2.0" trustify-auth = { path = "common/auth", features = ["actix", "swagger"] } trustify-common = { path = "common" } diff --git a/entity/src/lib.rs b/entity/src/lib.rs index 75ae26518..00fefdba8 100644 --- a/entity/src/lib.rs +++ b/entity/src/lib.rs @@ -30,3 +30,4 @@ pub mod version_scheme; pub mod versioned_purl; pub mod vulnerability; pub mod vulnerability_description; +pub mod weakness; diff --git a/entity/src/weakness.rs b/entity/src/weakness.rs new file mode 100644 index 000000000..51e94d0b1 --- /dev/null +++ b/entity/src/weakness.rs @@ -0,0 +1,24 @@ +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] +#[sea_orm(table_name = "weakness")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: String, + pub description: Option, + pub extended_description: Option, + pub child_of: Option>, + pub parent_of: Option>, + pub starts_with: Option>, + pub can_follow: Option>, + pub can_precede: Option>, + pub required_by: Option>, + pub requires: Option>, + pub can_also_be: Option>, + pub peer_of: Option>, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/etc/test-data/cwec_latest.xml.zip b/etc/test-data/cwec_latest.xml.zip new file mode 100644 index 000000000..34bf27b8b Binary files /dev/null and b/etc/test-data/cwec_latest.xml.zip differ diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 39282085f..93f43de5c 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -8,7 +8,6 @@ mod m0000040_create_vulnerability; mod m0000050_create_vulnerability_description; mod m0000060_create_advisory; mod m0000070_create_cvss3; -mod m0000070_create_cwe; mod m0000080_create_cvss4; pub mod m0000090_create_advisory_vulnerability; mod m0000100_create_package; @@ -70,6 +69,7 @@ mod m0000550_create_cpe_license_assertion; mod m0000560_alter_vulnerability_cwe_column; mod m0000565_alter_advisory_vulnerability_cwe_column; mod m0000570_add_import_progress; +mod m0000575_create_weakness; pub struct Migrator; @@ -146,6 +146,7 @@ impl MigratorTrait for Migrator { Box::new(m0000560_alter_vulnerability_cwe_column::Migration), Box::new(m0000565_alter_advisory_vulnerability_cwe_column::Migration), Box::new(m0000570_add_import_progress::Migration), + Box::new(m0000575_create_weakness::Migration), ] } } diff --git a/migration/src/m0000070_create_cwe.rs b/migration/src/m0000070_create_cwe.rs deleted file mode 100644 index 8b4d7c6a4..000000000 --- a/migration/src/m0000070_create_cwe.rs +++ /dev/null @@ -1,49 +0,0 @@ -use sea_orm_migration::prelude::*; - -use crate::{Now, UuidV4}; - -#[derive(DeriveMigrationName)] -pub struct Migration; - -#[async_trait::async_trait] -impl MigrationTrait for Migration { - async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - // Replace the sample below with your own migration scripts - manager - .create_table( - Table::create() - .table(Cwe::Table) - .if_not_exists() - .col( - ColumnDef::new(Cwe::Id) - .uuid() - .not_null() - .default(Func::cust(UuidV4)) - .primary_key(), - ) - .col( - ColumnDef::new(Cwe::Timestamp) - .timestamp_with_time_zone() - .default(Func::cust(Now)), - ) - .col(ColumnDef::new(Cwe::Identifier).string().not_null()) - .to_owned(), - ) - .await - } - - async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { - manager - .drop_table(Table::drop().table(Cwe::Table).to_owned()) - .await - } -} - -#[derive(DeriveIden)] -pub enum Cwe { - Table, - Id, - Timestamp, - // -- - Identifier, -} diff --git a/migration/src/m0000575_create_weakness.rs b/migration/src/m0000575_create_weakness.rs new file mode 100644 index 000000000..feebe1b1c --- /dev/null +++ b/migration/src/m0000575_create_weakness.rs @@ -0,0 +1,54 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(Weakness::Table) + .col(ColumnDef::new(Weakness::Id).text().not_null().primary_key()) + .col(ColumnDef::new(Weakness::Description).text().not_null()) + .col(ColumnDef::new(Weakness::ExtendedDescription).text()) + .col(ColumnDef::new(Weakness::ChildOf).array(ColumnType::Text)) + .col(ColumnDef::new(Weakness::ParentOf).array(ColumnType::Text)) + .col(ColumnDef::new(Weakness::StartsWith).array(ColumnType::Text)) + .col(ColumnDef::new(Weakness::CanFollow).array(ColumnType::Text)) + .col(ColumnDef::new(Weakness::CanPrecede).array(ColumnType::Text)) + .col(ColumnDef::new(Weakness::RequiredBy).array(ColumnType::Text)) + .col(ColumnDef::new(Weakness::Requires).array(ColumnType::Text)) + .col(ColumnDef::new(Weakness::CanAlsoBe).array(ColumnType::Text)) + .col(ColumnDef::new(Weakness::PeerOf).array(ColumnType::Text)) + .to_owned(), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(Weakness::Table).to_owned()) + .await + } +} + +#[derive(DeriveIden)] +enum Weakness { + Table, + Id, + Description, + ExtendedDescription, + ChildOf, + ParentOf, + StartsWith, + CanFollow, + CanPrecede, + RequiredBy, + Requires, + CanAlsoBe, + PeerOf, +} diff --git a/modules/fundamental/Cargo.toml b/modules/fundamental/Cargo.toml index 39626fcb8..d4efa8b05 100644 --- a/modules/fundamental/Cargo.toml +++ b/modules/fundamental/Cargo.toml @@ -22,6 +22,7 @@ cpe = { workspace = true } futures-util = { workspace = true } itertools = { workspace = true } log = { workspace = true } +roxmltree = { workspace = true } sea-orm = { workspace = true } sea-query = { workspace = true } serde = { workspace = true } @@ -58,6 +59,7 @@ urlencoding = { workspace = true } criterion = { workspace = true, features = ["html_reports", "async_tokio"] } csaf = { workspace = true } packageurl = { workspace = true } +zip = { workspace = true } [[bench]] name = "bench" diff --git a/modules/fundamental/src/endpoints.rs b/modules/fundamental/src/endpoints.rs index 10f79ad2c..07e5361f9 100644 --- a/modules/fundamental/src/endpoints.rs +++ b/modules/fundamental/src/endpoints.rs @@ -26,5 +26,7 @@ pub fn configure( crate::sbom::endpoints::configure(config, db.clone()); - crate::vulnerability::endpoints::configure(config, db); + crate::vulnerability::endpoints::configure(config, db.clone()); + + crate::weakness::endpoints::configure(config, db.clone()); } diff --git a/modules/fundamental/src/lib.rs b/modules/fundamental/src/lib.rs index 0185f7fa2..3d6ee9783 100644 --- a/modules/fundamental/src/lib.rs +++ b/modules/fundamental/src/lib.rs @@ -8,6 +8,8 @@ pub mod purl; pub mod sbom; pub mod vulnerability; +pub mod weakness; + pub mod openapi; pub use openapi::openapi; diff --git a/modules/fundamental/src/openapi.rs b/modules/fundamental/src/openapi.rs index cc46e2e13..07ac49e0f 100644 --- a/modules/fundamental/src/openapi.rs +++ b/modules/fundamental/src/openapi.rs @@ -15,6 +15,7 @@ pub fn openapi() -> utoipa::openapi::OpenApi { doc.merge(crate::product::endpoints::ApiDoc::openapi()); doc.merge(crate::sbom::endpoints::ApiDoc::openapi()); doc.merge(crate::vulnerability::endpoints::ApiDoc::openapi()); + doc.merge(crate::weakness::endpoints::ApiDoc::openapi()); if let Some(components) = doc.components.as_mut() { let mut obj = Object::with_type(SchemaType::String); diff --git a/modules/fundamental/src/weakness/endpoints/mod.rs b/modules/fundamental/src/weakness/endpoints/mod.rs new file mode 100644 index 000000000..bcb7d7a9b --- /dev/null +++ b/modules/fundamental/src/weakness/endpoints/mod.rs @@ -0,0 +1,68 @@ +use crate::weakness::service::WeaknessService; +use actix_web::{get, web, HttpResponse, Responder}; +use trustify_common::db::query::Query; +use trustify_common::db::Database; +use trustify_common::model::Paginated; +use utoipa::OpenApi; + +pub fn configure(config: &mut web::ServiceConfig, db: Database) { + let weakness_service = WeaknessService::new(db); + + config + .app_data(web::Data::new(weakness_service)) + .service(list_weaknesses) + .service(get_weakness); +} + +#[derive(OpenApi)] +#[openapi( + paths(list_weaknesses, get_weakness,), + components(schemas( + crate::weakness::model::PaginatedWeaknessSummary, + crate::weakness::model::WeaknessSummary, + crate::weakness::model::WeaknessDetails, + crate::weakness::model::WeaknessHead, + )), + tags() +)] +pub struct ApiDoc; + +#[utoipa::path( + tag = "weakness", + operation_id = "listWeaknesses", + context_path = "/api", + params( + Query, + Paginated, + ), + responses( + (status = 200, description = "Matching weaknesses", body = PaginatedLicenseSummary), + ), +)] +#[get("/v1/weakness")] +pub async fn list_weaknesses( + state: web::Data, + web::Query(search): web::Query, + web::Query(paginated): web::Query, +) -> actix_web::Result { + Ok(HttpResponse::Ok().json(state.list_weaknesses(search, paginated).await?)) +} + +#[utoipa::path( + tag = "weakness", + operation_id = "getWeakness", + context_path = "/api", + responses( + (status = 200, description = "The weakness", body = LicenseSummary), + ), +)] +#[get("/v1/weakness/{id}")] +pub async fn get_weakness( + state: web::Data, + id: web::Path, +) -> actix_web::Result { + Ok(HttpResponse::Ok().json(state.get_weakness(&id).await?)) +} + +#[cfg(test)] +mod test; diff --git a/modules/fundamental/src/weakness/endpoints/test.rs b/modules/fundamental/src/weakness/endpoints/test.rs new file mode 100644 index 000000000..2addd022b --- /dev/null +++ b/modules/fundamental/src/weakness/endpoints/test.rs @@ -0,0 +1,96 @@ +use crate::test::caller; +use crate::test::CallService; +use crate::weakness::model::{WeaknessDetails, WeaknessSummary}; +use actix_web::test::TestRequest; +use test_context::test_context; +use test_log::test; +use trustify_common::model::PaginatedResults; +use trustify_test_context::{document_read, TrustifyContext}; +use zip::ZipArchive; + +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn list_weaknesses(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + let zip = document_read("cwec_latest.xml.zip").await?; + + let mut archive = ZipArchive::new(zip)?; + + let entry = archive.by_index(0)?; + + ctx.ingest_read(entry).await?; + + let app = caller(ctx).await?; + + let uri = "/api/v1/weakness"; + + let request = TestRequest::get().uri(uri).to_request(); + + let response: PaginatedResults = app.call_and_read_body_json(request).await; + + assert!(response.total > 900); + + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn query_weaknesses(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + let zip = document_read("cwec_latest.xml.zip").await?; + + let mut archive = ZipArchive::new(zip)?; + + let entry = archive.by_index(0)?; + + ctx.ingest_read(entry).await?; + + let app = caller(ctx).await?; + + let uri = "/api/v1/weakness?q=struts"; + + let request = TestRequest::get().uri(uri).to_request(); + + let response: PaginatedResults = app.call_and_read_body_json(request).await; + + assert_eq!(response.total, 4); + + Ok(()) +} + +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn get_weakness(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + let zip = document_read("cwec_latest.xml.zip").await?; + + let mut archive = ZipArchive::new(zip)?; + + let entry = archive.by_index(0)?; + + ctx.ingest_read(entry).await?; + + let app = caller(ctx).await?; + + let uri = "/api/v1/weakness/CWE-1004"; + + let request = TestRequest::get().uri(uri).to_request(); + + let response: WeaknessDetails = app.call_and_read_body_json(request).await; + + assert_eq!(response.head.id, "CWE-1004"); + assert!(response.head.description.is_some()); + + let desc = response.head.description.unwrap(); + assert!(desc.starts_with("The product uses a cookie to store")); + + assert!(response.extended_description.is_some()); + let ext_desc = response.extended_description.unwrap(); + assert!(ext_desc.starts_with("The HttpOnly flag directs compatible browsers")); + + assert!(response.child_of.is_some()); + + let child_of = response.child_of.unwrap(); + + assert_eq!(1, child_of.len()); + assert!(child_of.contains(&"CWE-732".to_string())); + + Ok(()) +} diff --git a/modules/fundamental/src/weakness/mod.rs b/modules/fundamental/src/weakness/mod.rs new file mode 100644 index 000000000..4931989e6 --- /dev/null +++ b/modules/fundamental/src/weakness/mod.rs @@ -0,0 +1,5 @@ +pub mod endpoints; + +pub mod model; + +pub mod service; diff --git a/modules/fundamental/src/weakness/model.rs b/modules/fundamental/src/weakness/model.rs new file mode 100644 index 000000000..c243b94e8 --- /dev/null +++ b/modules/fundamental/src/weakness/model.rs @@ -0,0 +1,86 @@ +use crate::Error; +use serde::{Deserialize, Serialize}; +use trustify_common::db::ConnectionOrTransaction; +use trustify_common::paginated; +use trustify_entity::weakness; +use utoipa::ToSchema; + +#[derive(Serialize, Deserialize, ToSchema, Debug, Clone)] +pub struct WeaknessHead { + pub id: String, + pub description: Option, +} + +#[derive(Serialize, Deserialize, ToSchema, Debug, Clone)] +pub struct WeaknessSummary { + #[serde(flatten)] + pub head: WeaknessHead, +} + +paginated!(WeaknessSummary); + +impl WeaknessSummary { + pub async fn from_entities( + entities: &[weakness::Model], + tx: &ConnectionOrTransaction<'_>, + ) -> Result, Error> { + let mut summaries = Vec::new(); + for each in entities { + summaries.push(Self::from_entity(each, tx).await?) + } + + Ok(summaries) + } + + pub async fn from_entity( + entity: &weakness::Model, + _tx: &ConnectionOrTransaction<'_>, + ) -> Result { + Ok(Self { + head: WeaknessHead { + id: entity.id.clone(), + description: entity.description.clone(), + }, + }) + } +} + +#[derive(Serialize, Deserialize, ToSchema, Debug, Clone)] +pub struct WeaknessDetails { + #[serde(flatten)] + pub head: WeaknessHead, + pub extended_description: Option, + pub child_of: Option>, + pub parent_of: Option>, + pub starts_with: Option>, + pub can_follow: Option>, + pub can_precede: Option>, + pub required_by: Option>, + pub requires: Option>, + pub can_also_be: Option>, + pub peer_of: Option>, +} + +impl WeaknessDetails { + pub async fn from_entity( + entity: &weakness::Model, + _tx: &ConnectionOrTransaction<'_>, + ) -> Result { + Ok(Self { + head: WeaknessHead { + id: entity.id.clone(), + description: entity.description.clone(), + }, + extended_description: entity.extended_description.clone(), + child_of: entity.child_of.clone(), + parent_of: entity.parent_of.clone(), + starts_with: entity.starts_with.clone(), + can_follow: entity.can_follow.clone(), + can_precede: entity.can_precede.clone(), + required_by: entity.required_by.clone(), + requires: entity.requires.clone(), + can_also_be: entity.can_also_be.clone(), + peer_of: entity.peer_of.clone(), + }) + } +} diff --git a/modules/fundamental/src/weakness/service/mod.rs b/modules/fundamental/src/weakness/service/mod.rs new file mode 100644 index 000000000..bb512563f --- /dev/null +++ b/modules/fundamental/src/weakness/service/mod.rs @@ -0,0 +1,52 @@ +use crate::weakness::model::{PaginatedWeaknessSummary, WeaknessDetails, WeaknessSummary}; +use crate::Error; +use sea_orm::{EntityTrait, TransactionTrait}; +use trustify_common::db::limiter::LimiterTrait; +use trustify_common::db::query::{Filtering, Query}; +use trustify_common::db::Database; +use trustify_common::model::Paginated; +use trustify_entity::weakness; + +pub struct WeaknessService { + db: Database, +} + +impl WeaknessService { + pub fn new(db: Database) -> Self { + Self { db } + } + + pub async fn list_weaknesses( + &self, + query: Query, + paginated: Paginated, + ) -> Result { + let tx = self.db.begin().await?; + let tx = (&tx).into(); + + let limiter = weakness::Entity::find().filtering(query)?.limiting( + &self.db, + paginated.offset, + paginated.limit, + ); + + let total = limiter.total().await?; + let items = limiter.fetch().await?; + + Ok(PaginatedWeaknessSummary { + items: WeaknessSummary::from_entities(&items, &tx).await?, + total, + }) + } + + pub async fn get_weakness(&self, id: &str) -> Result, Error> { + let tx = self.db.begin().await?; + let tx = (&tx).into(); + + if let Some(found) = weakness::Entity::find_by_id(id).one(&self.db).await? { + Ok(Some(WeaknessDetails::from_entity(&found, &tx).await?)) + } else { + Ok(None) + } + } +} diff --git a/modules/ingestor/Cargo.toml b/modules/ingestor/Cargo.toml index 0281fcc34..2284d0cb9 100644 --- a/modules/ingestor/Cargo.toml +++ b/modules/ingestor/Cargo.toml @@ -29,6 +29,7 @@ osv = { workspace = true, features = ["schema"] } packageurl = { workspace = true } parking_lot = { workspace = true } ring = { workspace = true } +roxmltree = { workspace = true } sbom-walker = { workspace = true } sea-orm = { workspace = true } sea-query = { workspace = true } @@ -38,6 +39,7 @@ serde_yml = { workspace = true } spdx = { workspace = true } spdx-expression = { workspace = true } spdx-rs = { workspace = true } +strum = { workspace = true } thiserror = { workspace = true } time = { workspace = true, features = ["serde-well-known"] } tokio = { workspace = true, features = ["full"] } @@ -48,9 +50,9 @@ uuid = { workspace = true, features = ["v7"] } [dev-dependencies] trustify-test-context = { workspace = true } - +zip = { workspace = true } rand = { workspace = true } -rstest = {workspace = true } +rstest = { workspace = true } serde_yml = { workspace = true } test-context = { workspace = true } test-log = { workspace = true, features = ["log", "trace"] } diff --git a/modules/ingestor/src/graph/mod.rs b/modules/ingestor/src/graph/mod.rs index 2dcc9e66b..21f5eb7b3 100644 --- a/modules/ingestor/src/graph/mod.rs +++ b/modules/ingestor/src/graph/mod.rs @@ -14,7 +14,7 @@ use trustify_common::db::{ConnectionOrTransaction, Transactional}; #[derive(Debug, Clone)] pub struct Graph { - db: trustify_common::db::Database, + pub(crate) db: trustify_common::db::Database, } #[derive(Debug, thiserror::Error)] diff --git a/modules/ingestor/src/service/format.rs b/modules/ingestor/src/service/format.rs index 7f1b5abba..ddc870767 100644 --- a/modules/ingestor/src/service/format.rs +++ b/modules/ingestor/src/service/format.rs @@ -1,5 +1,6 @@ use crate::graph::sbom::clearly_defined::Curation; use crate::service::sbom::clearly_defined::ClearlyDefinedLoader; +use crate::service::weakness::CweCatalogLoader; use crate::{ graph::Graph, model::IngestResult, @@ -17,7 +18,9 @@ use futures::Stream; use futures::TryStreamExt; use jsn::{mask::*, Format as JsnFormat, TokenReader}; use osv::schema::Vulnerability; +use roxmltree::Document; use serde_json::Value; +use std::str::from_utf8; use std::{ io::{self}, pin::pin, @@ -36,7 +39,7 @@ pub enum Format { SPDX, CycloneDX, ClearlyDefined, - + CweCatalog, // These should be resolved to one of the above before loading Advisory, SBOM, @@ -98,6 +101,10 @@ impl<'g> Format { let curation: Curation = serde_yml::from_slice(&buffer)?; loader.load(labels, curation, digests).await } + Format::CweCatalog => { + let loader = CweCatalogLoader::new(graph); + loader.load_bytes(labels, &buffer, digests).await + } f => Err(Error::UnsupportedFormat(format!( "Must resolve {f:?} to an actual format" ))), @@ -108,9 +115,10 @@ impl<'g> Format { pub fn from_bytes(bytes: &[u8]) -> Result { match Self::advisory_from_bytes(bytes) { Err(Error::UnsupportedFormat(ea)) => match Self::sbom_from_bytes(bytes) { - Err(Error::UnsupportedFormat(es)) => { - Err(Error::UnsupportedFormat(format!("{ea}\n{es}"))) - } + Err(Error::UnsupportedFormat(es)) => match Self::is_cwe_catalog(bytes) { + Ok(_) => Ok(Self::CweCatalog), + Err(_) => Err(Error::UnsupportedFormat(format!("{ea}\n{es}"))), + }, x => x, }, x => x, @@ -200,6 +208,19 @@ impl<'g> Format { Ok(false) } + + pub fn is_cwe_catalog(bytes: &[u8]) -> Result { + if let Ok(utf8) = from_utf8(bytes) { + if let Ok(candidate) = Document::parse(utf8) { + let root = candidate.root(); + if let Some(catalog) = root.first_element_child() { + return Ok(catalog.has_tag_name("Weakness_Catalog")); + } + } + } + + Ok(false) + } } fn masked(mask: N, bytes: &[u8]) -> Result, Error> { diff --git a/modules/ingestor/src/service/mod.rs b/modules/ingestor/src/service/mod.rs index 9600d9a52..34b05160d 100644 --- a/modules/ingestor/src/service/mod.rs +++ b/modules/ingestor/src/service/mod.rs @@ -1,6 +1,8 @@ pub mod advisory; pub mod sbom; +pub mod weakness; + mod format; pub use format::Format; use tokio::task::JoinError; @@ -26,8 +28,12 @@ pub enum Error { #[error(transparent)] Io(#[from] std::io::Error), #[error(transparent)] + Utf8(#[from] std::str::Utf8Error), + #[error(transparent)] Json(#[from] serde_json::Error), #[error(transparent)] + Xml(#[from] roxmltree::Error), + #[error(transparent)] Yaml(#[from] serde_yml::Error), #[error(transparent)] Graph(#[from] crate::graph::error::Error), @@ -46,11 +52,6 @@ pub enum Error { impl ResponseError for Error { fn error_response(&self) -> HttpResponse { match self { - Self::Io(err) => HttpResponse::BadRequest().json(ErrorInformation { - error: "Io".into(), - message: err.to_string(), - details: None, - }), Self::Json(err) => HttpResponse::BadRequest().json(ErrorInformation { error: "JsonParse".into(), message: err.to_string(), @@ -61,6 +62,21 @@ impl ResponseError for Error { message: err.to_string(), details: None, }), + Self::Xml(err) => HttpResponse::BadRequest().json(ErrorInformation { + error: "XmlParse".into(), + message: err.to_string(), + details: None, + }), + Self::Io(err) => HttpResponse::BadRequest().json(ErrorInformation { + error: "I/O".into(), + message: err.to_string(), + details: None, + }), + Self::Utf8(err) => HttpResponse::BadRequest().json(ErrorInformation { + error: "UTF-8".into(), + message: err.to_string(), + details: None, + }), Self::Storage(err) => HttpResponse::InternalServerError().json(ErrorInformation { error: "Storage".into(), message: err.to_string(), diff --git a/modules/ingestor/src/service/weakness/mod.rs b/modules/ingestor/src/service/weakness/mod.rs new file mode 100644 index 000000000..62b744113 --- /dev/null +++ b/modules/ingestor/src/service/weakness/mod.rs @@ -0,0 +1,229 @@ +use crate::graph::Graph; +use crate::model::IngestResult; +use crate::service::Error; +use hex::ToHex; +use roxmltree::{Document, Node}; +use sea_orm::{EntityTrait, Iterable, Set, TransactionTrait}; +use sea_query::OnConflict; +use std::str::from_utf8; +use trustify_common::db::chunk::EntityChunkedIter; +use trustify_common::hashing::Digests; +use trustify_common::id::Id; +use trustify_entity::labels::Labels; +use trustify_entity::weakness; + +pub struct CweCatalogLoader<'d> { + graph: &'d Graph, +} + +impl<'d> CweCatalogLoader<'d> { + pub fn new(graph: &'d Graph) -> Self { + Self { graph } + } + + pub async fn load_bytes( + &self, + labels: Labels, + buffer: &[u8], + digests: &Digests, + ) -> Result { + let xml = from_utf8(buffer)?; + + let document = Document::parse(xml)?; + + self.load(labels, &document, digests).await + } + + pub async fn load<'x>( + &self, + _labels: Labels, + doc: &Document<'x>, + digests: &Digests, + ) -> Result { + let root = doc.root(); + + let catalog = root.first_element_child(); + if let Some(catalog) = catalog { + let weaknesses = catalog.first_element_child(); + let mut batch = Vec::new(); + + if let Some(weaknesses) = weaknesses { + let tx = self.graph.db.begin().await?; + for weakness in weaknesses.children() { + if weakness.is_element() { + let mut child_of = Vec::new(); + let mut parent_of = Vec::new(); + let mut starts_with = Vec::new(); + let mut can_follow = Vec::new(); + let mut can_precede = Vec::new(); + let mut required_by = Vec::new(); + let mut requires = Vec::new(); + let mut can_also_be = Vec::new(); + let mut peer_of = Vec::new(); + + if let Some(id) = weakness.attribute("ID").map(|id| format!("CWE-{id}")) { + let mut description = None; + let mut extended_description = None; + + if let Some(description_node) = + weakness.children().find(|e| e.has_tag_name("Description")) + { + description = description_node.text().map(|e| e.to_string()); + } + + if let Some(extended_description_node) = weakness + .children() + .find(|e| e.has_tag_name("Extended_Description")) + { + extended_description + .replace(gather_content(&extended_description_node)); + } + + if let Some(related_weaknesses) = weakness + .children() + .find(|e| e.has_tag_name("Related_Weaknesses")) + { + for related in related_weaknesses + .children() + .filter(|e| e.has_tag_name("Related_Weakness")) + { + if let Some(target) = related.attribute("CWE_ID") { + if let Some(nature) = related.attribute("Nature") { + if let Some(dest) = match nature { + "ChildOf" => Some(&mut child_of), + "ParentOf" => Some(&mut parent_of), + "StartsWith" => Some(&mut starts_with), + "CanFollow" => Some(&mut can_follow), + "CanPrecede" => Some(&mut can_precede), + "RequiredBy" => Some(&mut required_by), + "Requires" => Some(&mut requires), + "CanAlsoBe" => Some(&mut can_also_be), + "PeerOf" => Some(&mut peer_of), + _ => None, + } { + dest.push(target.to_string()); + } + } + } + } + } + + batch.push(weakness::ActiveModel { + id: Set(id), + description: Set(description), + extended_description: Set(extended_description), + child_of: Set(normalize(child_of)), + parent_of: Set(normalize(parent_of)), + starts_with: Set(normalize(starts_with)), + can_follow: Set(normalize(can_follow)), + can_precede: Set(normalize(can_precede)), + required_by: Set(normalize(required_by)), + requires: Set(normalize(requires)), + can_also_be: Set(normalize(can_also_be)), + peer_of: Set(normalize(peer_of)), + }); + } + } + } + + for chunk in &batch.chunked() { + weakness::Entity::insert_many(chunk) + .on_conflict( + OnConflict::column(weakness::Column::Id) + .update_columns(weakness::Column::iter()) + .to_owned(), + ) + .exec(&tx) + .await?; + } + tx.commit().await?; + } + } + + Ok(IngestResult { + id: Id::Sha512(digests.sha512.encode_hex()), + document_id: "CWE".to_string(), + warnings: vec![], + }) + } +} + +fn normalize(vec: Vec) -> Option> { + if vec.is_empty() { + None + } else { + Some(canonicalize(vec)) + } +} + +fn canonicalize(vec: Vec) -> Vec { + vec.iter().map(|e| format!("CWE-{e}")).collect() +} + +fn gather_content(node: &Node) -> String { + let mut dest = String::new(); + + let children = node.children(); + + for child in children { + gather_content_inner(&child, &mut dest); + } + + let dest = dest.trim().to_string(); + + dest +} + +fn gather_content_inner(node: &Node, dest: &mut String) { + if node.is_element() { + dest.push_str(&format!("<{}>", node.tag_name().name())); + for child in node.children() { + gather_content_inner(&child, dest); + } + dest.push_str(&format!("", node.tag_name().name())); + } else if node.is_text() { + if let Some(text) = node.text() { + dest.push_str(text); + } + } +} + +#[cfg(test)] +mod test { + use crate::graph::Graph; + use crate::service::weakness::CweCatalogLoader; + use roxmltree::Document; + use std::io::Read; + use test_context::test_context; + use test_log::test; + use trustify_common::hashing::HashingRead; + use trustify_entity::labels::Labels; + use trustify_test_context::document_read; + use trustify_test_context::TrustifyContext; + use zip::ZipArchive; + + #[test_context(TrustifyContext)] + #[test(tokio::test)] + async fn test(ctx: &TrustifyContext) -> Result<(), anyhow::Error> { + let graph = Graph::new(ctx.db.clone()); + let loader = CweCatalogLoader::new(&graph); + + let zip = document_read("cwec_latest.xml.zip").await?; + + let mut archive = ZipArchive::new(zip)?; + + let entry = archive.by_index(0)?; + + let mut hashing = HashingRead::new(entry); + let mut xml = String::new(); + hashing.read_to_string(&mut xml)?; + let digests = hashing.finish()?; + let doc = Document::parse(&xml)?; + + // should work twice without error/conflict. + loader.load(Labels::default(), &doc, &digests).await?; + loader.load(Labels::default(), &doc, &digests).await?; + + Ok(()) + } +} diff --git a/openapi.yaml b/openapi.yaml index f4778025e..01cc0010d 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1465,6 +1465,69 @@ paths: $ref: '#/components/schemas/VulnerabilityDetails' '404': description: Specified vulnerability not found + /api/v1/weakness: + get: + tags: + - weakness + operationId: listWeaknesses + parameters: + - name: q + in: query + required: false + schema: + type: string + - name: sort + in: query + required: false + schema: + type: string + - name: offset + in: query + description: |- + The first item to return, skipping all that come before it. + + NOTE: The order of items is defined by the API being called. + required: false + schema: + type: integer + format: int64 + minimum: 0 + - name: limit + in: query + description: |- + The maximum number of entries to return. + + Zero means: no limit + required: false + schema: + type: integer + format: int64 + minimum: 0 + responses: + '200': + description: Matching weaknesses + content: + application/json: + schema: + $ref: '#/components/schemas/PaginatedLicenseSummary' + /api/v1/weakness/{id}: + get: + tags: + - weakness + operationId: getWeakness + parameters: + - name: id + in: path + required: true + schema: + type: string + responses: + '200': + description: The weakness + content: + application/json: + schema: + $ref: '#/components/schemas/LicenseSummary' components: schemas: AdvisoryDetails: @@ -2172,6 +2235,23 @@ components: format: int64 description: Total number of items found minimum: 0 + PaginatedWeaknessSummary: + type: object + description: Paginated returned items + required: + - items + - total + properties: + items: + type: array + items: + $ref: '#/components/schemas/WeaknessSummary' + description: Returned items + total: + type: integer + format: int64 + description: Total number of items found + minimum: 0 ProductDetails: allOf: - $ref: '#/components/schemas/ProductHead' @@ -2896,6 +2976,73 @@ components: allOf: - $ref: '#/components/schemas/Severity' nullable: true + WeaknessDetails: + allOf: + - $ref: '#/components/schemas/WeaknessHead' + - type: object + properties: + can_also_be: + type: array + items: + type: string + nullable: true + can_follow: + type: array + items: + type: string + nullable: true + can_precede: + type: array + items: + type: string + nullable: true + child_of: + type: array + items: + type: string + nullable: true + extended_description: + type: string + nullable: true + parent_of: + type: array + items: + type: string + nullable: true + peer_of: + type: array + items: + type: string + nullable: true + required_by: + type: array + items: + type: string + nullable: true + requires: + type: array + items: + type: string + nullable: true + starts_with: + type: array + items: + type: string + nullable: true + WeaknessHead: + type: object + required: + - id + properties: + description: + type: string + nullable: true + id: + type: string + WeaknessSummary: + allOf: + - $ref: '#/components/schemas/WeaknessHead' + - type: object Which: type: string enum: diff --git a/test-context/src/lib.rs b/test-context/src/lib.rs index cd86d8b9f..f9f705bf0 100644 --- a/test-context/src/lib.rs +++ b/test-context/src/lib.rs @@ -5,7 +5,7 @@ use peak_alloc::PeakAlloc; use postgresql_embedded::PostgreSQL; use std::env; use std::env::current_dir; -use std::io::ErrorKind; +use std::io::{ErrorKind, Read, Seek}; use std::path::PathBuf; use test_context::AsyncTestContext; use tokio::io::AsyncReadExt; @@ -73,6 +73,16 @@ impl TrustifyContext { .ingest(&bytes, Format::Unknown, ("source", "TrustifyContext"), None) .await?) } + + pub async fn ingest_read(&self, mut read: R) -> Result { + let mut bytes = Vec::new(); + read.read_to_end(&mut bytes)?; + + Ok(self + .ingestor + .ingest(&bytes, Format::Unknown, ("source", "TrustifyContext"), None) + .await?) + } } impl AsyncTestContext for TrustifyContext { @@ -144,6 +154,10 @@ pub async fn document_stream( Ok(ReaderStream::new(file)) } +pub async fn document_read(path: &str) -> Result { + Ok(std::fs::File::open(absolute(path)?)?) +} + pub async fn document(path: &str) -> Result<(T, Digests), anyhow::Error> where T: serde::de::DeserializeOwned + Send + 'static,