fix: disable format detection for importers

Fixes trustification#715 We only auto-detect formats for the actix handlers providing our API. Signed-off-by: Jim Crossley <[email protected]>
ctron · Aug 27, 2024 · 26eda5d · 26eda5d
1 parent d644b78
commit 26eda5d
Show file tree

Hide file tree

Showing 14 changed files with 73 additions and 30 deletions.
diff --git a/modules/fundamental/src/advisory/endpoints/mod.rs b/modules/fundamental/src/advisory/endpoints/mod.rs
@@ -10,7 +10,7 @@ use futures_util::TryStreamExt;
 use std::str::FromStr;
 use trustify_common::{db::query::Query, db::Database, id::Id, model::Paginated};
 use trustify_entity::labels::Labels;
-use trustify_module_ingestor::service::IngestorService;
+use trustify_module_ingestor::service::{Format, IngestorService};
 use trustify_module_storage::service::StorageBackend;
 use utoipa::{IntoParams, OpenApi};
 
@@ -165,7 +165,9 @@ pub async fn upload(
     web::Query(UploadParams { issuer, labels }): web::Query<UploadParams>,
     bytes: web::Bytes,
 ) -> Result<impl Responder, Error> {
-    let result = service.ingest(labels, issuer, &bytes).await?;
+    let result = service
+        .ingest(&bytes, Format::Advisory, labels, issuer)
+        .await?;
     log::info!("Uploaded Advisory: {}", result.id);
     Ok(HttpResponse::Created().json(result))
 }

diff --git a/modules/fundamental/src/sbom/endpoints/mod.rs b/modules/fundamental/src/sbom/endpoints/mod.rs
@@ -23,7 +23,7 @@ use trustify_common::{
     purl::Purl,
 };
 use trustify_entity::{labels::Labels, relationship::Relationship};
-use trustify_module_ingestor::service::IngestorService;
+use trustify_module_ingestor::service::{Format, IngestorService};
 use trustify_module_storage::service::StorageBackend;
 use utoipa::OpenApi;
 
@@ -355,7 +355,7 @@ pub async fn upload(
     web::Query(UploadQuery { labels }): web::Query<UploadQuery>,
     bytes: web::Bytes,
 ) -> Result<impl Responder, Error> {
-    let result = service.ingest(labels, None, &bytes).await?;
+    let result = service.ingest(&bytes, Format::SBOM, labels, None).await?;
     log::info!("Uploaded SBOM: {}", result.id);
     Ok(HttpResponse::Created().json(result))
 }

diff --git a/modules/fundamental/tests/sbom/reingest.rs b/modules/fundamental/tests/sbom/reingest.rs
@@ -8,6 +8,7 @@ use trustify_common::db::query::Query;
 use trustify_common::model::Paginated;
 use trustify_common::purl::Purl;
 use trustify_module_fundamental::sbom::{model::details::SbomDetails, service::SbomService};
+use trustify_module_ingestor::service::Format;
 use trustify_test_context::{document_bytes, TrustifyContext};
 
 fn assert_sboms(sbom1: &SbomDetails, sbom2: &SbomDetails) {
@@ -223,12 +224,17 @@ async fn nhc_same_content(ctx: &TrustifyContext) -> Result<(), anyhow::Error> {
     // ingest the second version
     let result2 = ctx
         .ingestor
-        .ingest(("source", "test"), None, {
-            // re-serialize file (non-pretty)
-            let json: Value =
-                serde_json::from_slice(&document_bytes("nhc/v1/nhc-0.4.z.json.xz").await?)?;
-            &serde_json::to_vec(&json).map(Bytes::from)?
-        })
+        .ingest(
+            {
+                // re-serialize file (non-pretty)
+                let json: Value =
+                    serde_json::from_slice(&document_bytes("nhc/v1/nhc-0.4.z.json.xz").await?)?;
+                &serde_json::to_vec(&json).map(Bytes::from)?
+            },
+            Format::SBOM,
+            ("source", "test"),
+            None,
+        )
         .await?;
 
     assert_eq!(

diff --git a/modules/importer/src/runner/clearly_defined/mod.rs b/modules/importer/src/runner/clearly_defined/mod.rs
@@ -15,7 +15,10 @@ use std::{path::Path, path::PathBuf, sync::Arc};
 use tokio::runtime::Handle;
 use tracing::instrument;
 use trustify_entity::labels::Labels;
-use trustify_module_ingestor::{graph::Graph, service::IngestorService};
+use trustify_module_ingestor::{
+    graph::Graph,
+    service::{Format, IngestorService},
+};
 
 struct Context<C: RunContext + 'static> {
     context: C,
@@ -32,13 +35,14 @@ impl<C: RunContext> Context<C> {
         Handle::current().block_on(async {
             self.ingestor
                 .ingest(
+                    &data,
+                    Format::ClearlyDefined,
                     Labels::new()
                         .add("source", &self.source)
                         .add("importer", self.context.name())
                         .add("file", path.to_string_lossy())
                         .extend(&self.labels.0),
                     None,
-                    &data,
                 )
                 .await
         })?;

diff --git a/modules/importer/src/runner/csaf/storage.rs b/modules/importer/src/runner/csaf/storage.rs
@@ -5,7 +5,7 @@ use csaf_walker::validation::{
 use parking_lot::Mutex;
 use std::sync::Arc;
 use trustify_entity::labels::Labels;
-use trustify_module_ingestor::service::IngestorService;
+use trustify_module_ingestor::service::{Format, IngestorService};
 use walker_common::utils::url::Urlify;
 
 pub struct StorageVisitor<C: RunContext> {
@@ -35,13 +35,14 @@ impl<C: RunContext> ValidatedVisitor for StorageVisitor<C> {
 
         self.ingestor
             .ingest(
+                &doc.data,
+                Format::CSAF,
                 Labels::new()
                     .add("source", &location)
                     .add("importer", self.context.name())
                     .add("file", file)
                     .extend(&self.labels.0),
                 None, /* CSAF tracks issuer internally */
-                &doc.data,
             )
             .await
             .map_err(StorageError::Storage)?;

diff --git a/modules/importer/src/runner/cve/mod.rs b/modules/importer/src/runner/cve/mod.rs
@@ -15,7 +15,10 @@ use std::{path::Path, path::PathBuf, sync::Arc};
 use tokio::runtime::Handle;
 use tracing::instrument;
 use trustify_entity::labels::Labels;
-use trustify_module_ingestor::{graph::Graph, service::IngestorService};
+use trustify_module_ingestor::{
+    graph::Graph,
+    service::{Format, IngestorService},
+};
 
 struct Context<C: RunContext + 'static> {
     context: C,
@@ -32,13 +35,14 @@ impl<C: RunContext> Context<C> {
         Handle::current().block_on(async {
             self.ingestor
                 .ingest(
+                    &data,
+                    Format::CVE,
                     Labels::new()
                         .add("source", &self.source)
                         .add("importer", self.context.name())
                         .add("file", path.to_string_lossy())
                         .extend(&self.labels.0),
                     None,
-                    &data,
                 )
                 .await
         })?;

diff --git a/modules/importer/src/runner/osv/mod.rs b/modules/importer/src/runner/osv/mod.rs
@@ -15,7 +15,10 @@ use std::{path::Path, path::PathBuf, sync::Arc};
 use tokio::runtime::Handle;
 use tracing::instrument;
 use trustify_entity::labels::Labels;
-use trustify_module_ingestor::{graph::Graph, service::IngestorService};
+use trustify_module_ingestor::{
+    graph::Graph,
+    service::{Format, IngestorService},
+};
 
 struct Context<C: RunContext + 'static> {
     context: C,
@@ -32,13 +35,14 @@ impl<C: RunContext> Context<C> {
         Handle::current().block_on(async {
             self.ingestor
                 .ingest(
+                    &data,
+                    Format::OSV,
                     Labels::new()
                         .add("source", &self.source)
                         .add("importer", self.context.name())
                         .add("file", path.to_string_lossy())
                         .extend(&self.labels.0),
                     None,
-                    &data,
                 )
                 .await
         })?;

diff --git a/modules/importer/src/runner/sbom/storage.rs b/modules/importer/src/runner/sbom/storage.rs
@@ -10,7 +10,7 @@ use sbom_walker::validation::{
 };
 use std::sync::Arc;
 use trustify_entity::labels::Labels;
-use trustify_module_ingestor::service::IngestorService;
+use trustify_module_ingestor::service::{Format, IngestorService};
 use walker_common::{compression::decompress_opt, utils::url::Urlify};
 
 pub struct StorageVisitor<C: RunContext> {
@@ -68,13 +68,14 @@ impl<C: RunContext> ValidatedVisitor for StorageVisitor<C> {
         let result = self
             .ingestor
             .ingest(
+                &data,
+                Format::SBOM,
                 Labels::new()
                     .add("source", &self.source)
                     .add("importer", self.context.name())
                     .add("file", &file)
                     .extend(&self.labels.0),
                 None,
-                &data,
             )
             .await
             .map_err(StorageError::Storage)?;

diff --git a/modules/ingestor/src/service/format.rs b/modules/ingestor/src/service/format.rs
@@ -36,6 +36,11 @@ pub enum Format {
     SPDX,
     CycloneDX,
     ClearlyDefined,
+
+    // These should be resolved to one of the above before loading
+    Advisory,
+    SBOM,
+    Unknown,
 }
 
 impl<'g> Format {
@@ -93,6 +98,9 @@ impl<'g> Format {
                 let curation: Curation = serde_yml::from_slice(&buffer)?;
                 loader.load(labels, curation, digests).await
             }
+            f => Err(Error::UnsupportedFormat(format!(
+                "Must resolve {f:?} to an actual format"
+            ))),
         }
     }
 

diff --git a/modules/ingestor/src/service/mod.rs b/modules/ingestor/src/service/mod.rs
@@ -125,12 +125,22 @@ impl IngestorService {
     #[instrument(skip(self, bytes), err)]
     pub async fn ingest(
         &self,
+        bytes: &[u8],
+        format: Format,
         labels: impl Into<Labels> + Debug,
         issuer: Option<String>,
-        bytes: &[u8],
     ) -> Result<IngestResult, Error> {
         let start = Instant::now();
-        let fmt = Format::from_bytes(bytes)?;
+
+        // We want to resolve the format first to avoid storing a
+        // document that we can't subsequently retrieve and load into
+        // the database.
+        let fmt = match format {
+            Format::Advisory => Format::advisory_from_bytes(bytes)?,
+            Format::SBOM => Format::sbom_from_bytes(bytes)?,
+            Format::Unknown => Format::from_bytes(bytes)?,
+            v => v,
+        };
         let stream = ReaderStream::new(bytes);
 
         let result = self

diff --git a/modules/ingestor/src/service/sbom/clearly_defined.rs b/modules/ingestor/src/service/sbom/clearly_defined.rs
@@ -43,7 +43,7 @@ impl<'g> ClearlyDefinedLoader<'g> {
 #[cfg(test)]
 mod test {
     use crate::graph::Graph;
-    use crate::service::IngestorService;
+    use crate::service::{Format, IngestorService};
     use test_context::test_context;
     use test_log::test;
     use trustify_test_context::document_bytes;
@@ -58,7 +58,7 @@ mod test {
         let data = document_bytes("clearly-defined/chrono.yaml").await?;
 
         ingestor
-            .ingest(("source", "test"), None, &data)
+            .ingest(&data, Format::ClearlyDefined, ("source", "test"), None)
             .await
             .expect("must ingest");
 

diff --git a/modules/ingestor/src/service/sbom/cyclonedx.rs b/modules/ingestor/src/service/sbom/cyclonedx.rs
@@ -65,8 +65,8 @@ impl<'g> CyclonedxLoader<'g> {
 
 #[cfg(test)]
 mod test {
-    use crate::graph::Graph;
     use crate::service::IngestorService;
+    use crate::{graph::Graph, service::Format};
     use test_context::test_context;
     use test_log::test;
     use trustify_test_context::{document_bytes, TrustifyContext};
@@ -81,7 +81,7 @@ mod test {
         let ingestor = IngestorService::new(graph, ctx.storage.clone());
 
         ingestor
-            .ingest(("source", "test"), None, &data)
+            .ingest(&data, Format::CycloneDX, ("source", "test"), None)
             .await
             .expect("must ingest");
 

diff --git a/modules/ingestor/src/service/sbom/spdx.rs b/modules/ingestor/src/service/sbom/spdx.rs
@@ -63,8 +63,8 @@ impl<'g> SpdxLoader<'g> {
 
 #[cfg(test)]
 mod test {
-    use crate::graph::Graph;
     use crate::service::IngestorService;
+    use crate::{graph::Graph, service::Format};
     use test_context::test_context;
     use test_log::test;
     use trustify_test_context::{document_bytes, TrustifyContext};
@@ -78,7 +78,7 @@ mod test {
         let ingestor = IngestorService::new(graph, ctx.storage.clone());
 
         ingestor
-            .ingest(("source", "test"), None, &data)
+            .ingest(&data, Format::SPDX, ("source", "test"), None)
             .await
             .expect("must ingest");
 

diff --git a/test-context/src/lib.rs b/test-context/src/lib.rs
@@ -17,7 +17,7 @@ use trustify_common::db;
 use trustify_common::hashing::{Digests, HashingRead};
 use trustify_module_ingestor::graph::Graph;
 use trustify_module_ingestor::model::IngestResult;
-use trustify_module_ingestor::service::IngestorService;
+use trustify_module_ingestor::service::{Format, IngestorService};
 use trustify_module_storage::service::fs::FileSystemBackend;
 
 #[allow(dead_code)]
@@ -68,7 +68,10 @@ impl TrustifyContext {
 
     pub async fn ingest_document(&self, path: &str) -> Result<IngestResult, anyhow::Error> {
         let bytes = document_bytes(path).await?;
-        Ok(self.ingestor.ingest((), None, &bytes).await?)
+        Ok(self
+            .ingestor
+            .ingest(&bytes, Format::Unknown, ("source", "TrustifyContext"), None)
+            .await?)
     }
 }