diff --git a/.github/scripts/excluded_from_copyright b/.github/scripts/excluded_from_copyright index a546bd184c..46feaf0ff8 100644 --- a/.github/scripts/excluded_from_copyright +++ b/.github/scripts/excluded_from_copyright @@ -242,6 +242,7 @@ ./pkg/assembler/graphql/generated/prelude.generated.go ./pkg/assembler/graphql/generated/root_.generated.go ./pkg/assembler/graphql/generated/schema.generated.go +./pkg/assembler/graphql/generated/search.generated.go ./pkg/assembler/graphql/generated/source.generated.go ./pkg/assembler/graphql/generated/vulnEqual.generated.go ./pkg/assembler/graphql/generated/vulnMetadata.generated.go diff --git a/internal/testing/testdata/testdata.go b/internal/testing/testdata/testdata.go index 9a452e3211..ace47f55a1 100644 --- a/internal/testing/testdata/testdata.go +++ b/internal/testing/testdata/testdata.go @@ -822,6 +822,17 @@ var ( Collector: "GUAC", }, }, + { + Pkg: baselayoutPack, + PkgMatchFlag: generated.MatchFlags{Pkg: "SPECIFIC_VERSION"}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:guac/spdx/gcr.io/google-containers/alpine-latest", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, { Pkg: baselayoutdataPack, PkgMatchFlag: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, @@ -844,6 +855,17 @@ var ( Collector: "GUAC", }, }, + { + Pkg: baselayoutdataPack, + PkgMatchFlag: generated.MatchFlags{Pkg: "SPECIFIC_VERSION"}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:guac/spdx/gcr.io/google-containers/alpine-latest", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, { Pkg: keysPack, PkgMatchFlag: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, @@ -888,6 +910,17 @@ var ( Collector: "GUAC", }, }, + { + Pkg: keysPack, + PkgMatchFlag: generated.MatchFlags{Pkg: "SPECIFIC_VERSION"}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:guac/spdx/gcr.io/google-containers/alpine-latest", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, } SpdxIngestionPredicates = assembler.IngestPredicates{ diff --git a/pkg/assembler/backends/ent/backend/certifyVEXStatement.go b/pkg/assembler/backends/ent/backend/certifyVEXStatement.go index 1c4e88cc59..fedb7fbc64 100644 --- a/pkg/assembler/backends/ent/backend/certifyVEXStatement.go +++ b/pkg/assembler/backends/ent/backend/certifyVEXStatement.go @@ -20,6 +20,7 @@ import ( stdsql "database/sql" "entgo.io/ent/dialect/sql" + "github.com/guacsec/guac/internal/testing/ptrfrom" "github.com/guacsec/guac/pkg/assembler/backends/ent" "github.com/guacsec/guac/pkg/assembler/backends/ent/certifyvex" "github.com/guacsec/guac/pkg/assembler/backends/ent/predicate" @@ -29,6 +30,7 @@ import ( "github.com/guacsec/guac/pkg/assembler/graphql/model" "github.com/pkg/errors" "github.com/vektah/gqlparser/v2/gqlerror" + "golang.org/x/sync/errgroup" ) func (b *EntBackend) IngestVEXStatement(ctx context.Context, subject model.PackageOrArtifactInput, vulnerability model.VulnerabilityInputSpec, vexStatement model.VexStatementInputSpec) (*model.CertifyVEXStatement, error) { @@ -63,12 +65,14 @@ func (b *EntBackend) IngestVEXStatement(ctx context.Context, subject model.Packa var conflictWhere *sql.Predicate // manage package or artifact + var subjectID int if subject.Package != nil { p, err := getPkgVersion(ctx, client.Client(), *subject.Package) if err != nil { return nil, Errorf("%v :: %s", funcName, err) } insert.SetPackage(p) + subjectID = p.ID conflictColumns = append(conflictColumns, certifyvex.FieldPackageID) conflictWhere = sql.And( sql.NotNull(certifyvex.FieldPackageID), @@ -82,6 +86,7 @@ func (b *EntBackend) IngestVEXStatement(ctx context.Context, subject model.Packa return nil, Errorf("%v :: %s", funcName, err) } insert.SetArtifactID(artID) + subjectID = artID conflictColumns = append(conflictColumns, certifyvex.FieldArtifactID) conflictWhere = sql.And( sql.IsNull(certifyvex.FieldPackageID), @@ -112,7 +117,7 @@ func (b *EntBackend) IngestVEXStatement(ctx context.Context, subject model.Packa return nil, errors.Wrap(err, "upsert certify vex statement node") } id, err = client.CertifyVex.Query(). - Where(vexStatementInputPredicate(subject, vulnerability, vexStatement)). + Where(vexStatementInputPredicate(subject, subjectID, vulnerability, vulnID, vexStatement)). WithPackage(func(q *ent.PackageVersionQuery) { q.WithName(func(q *ent.PackageNameQuery) { q.WithNamespace(func(q *ent.PackageNamespaceQuery) { @@ -139,19 +144,30 @@ func (b *EntBackend) IngestVEXStatement(ctx context.Context, subject model.Packa } func (b *EntBackend) IngestVEXStatements(ctx context.Context, subjects model.PackageOrArtifactInputs, vulnerabilities []*model.VulnerabilityInputSpec, vexStatements []*model.VexStatementInputSpec) ([]string, error) { - var ids []string + var ids = make([]string, len(vexStatements)) + eg, ctx := errgroup.WithContext(ctx) for i := range vexStatements { + index := i var subject model.PackageOrArtifactInput if len(subjects.Packages) > 0 { - subject = model.PackageOrArtifactInput{Package: subjects.Packages[i]} + subject = model.PackageOrArtifactInput{Package: subjects.Packages[index]} } else { - subject = model.PackageOrArtifactInput{Artifact: subjects.Artifacts[i]} + subject = model.PackageOrArtifactInput{Artifact: subjects.Artifacts[index]} } - statement, err := b.IngestVEXStatement(ctx, subject, *vulnerabilities[i], *vexStatements[i]) - if err != nil { - return nil, gqlerror.Errorf("IngestVEXStatements failed with element #%v with err: %v", i, err) - } - ids = append(ids, statement.ID) + vuln := *vulnerabilities[index] + vexStatement := *vexStatements[index] + concurrently(eg, func() error { + statement, err := b.IngestVEXStatement(ctx, subject, vuln, vexStatement) + if err == nil { + ids[index] = statement.ID + return err + } else { + return gqlerror.Errorf("IngestVEXStatements failed with element #%v with err: %v", i, err) + } + }) + } + if err := eg.Wait(); err != nil { + return nil, err } return ids, nil } @@ -202,27 +218,21 @@ func certifyVexPredicate(filter model.CertifyVEXStatementSpec) predicate.Certify predicates := []predicate.CertifyVex{ optionalPredicate(filter.ID, IDEQ), optionalPredicate(filter.KnownSince, certifyvex.KnownSinceEQ), - optionalPredicate(filter.Statement, certifyvex.StatementEQ), - optionalPredicate(filter.StatusNotes, certifyvex.StatusNotesEQ), - optionalPredicate(filter.Collector, certifyvex.CollectorEQ), - optionalPredicate(filter.Origin, certifyvex.OriginEQ), - } - if filter.Status != nil { - status := filter.Status.String() - predicates = append(predicates, optionalPredicate(&status, certifyvex.StatusEQ)) } if filter.VexJustification != nil { justification := filter.VexJustification.String() predicates = append(predicates, optionalPredicate(&justification, certifyvex.JustificationEQ)) } - - if filter.Subject != nil { - if filter.Subject.Package != nil { - predicates = append(predicates, certifyvex.HasPackageWith(packageVersionQuery(filter.Subject.Package))) - } else if filter.Subject.Artifact != nil { - predicates = append(predicates, certifyvex.HasArtifactWith(artifactQueryPredicates(filter.Subject.Artifact))) - } + if filter.Status != nil { + status := filter.Status.String() + predicates = append(predicates, optionalPredicate(&status, certifyvex.StatusEQ)) } + predicates = append(predicates, + optionalPredicate(filter.Statement, certifyvex.StatementEQ), + optionalPredicate(filter.StatusNotes, certifyvex.StatusNotesEQ), + optionalPredicate(filter.Origin, certifyvex.OriginEQ), + optionalPredicate(filter.Collector, certifyvex.CollectorEQ), + ) if filter.Vulnerability != nil { if filter.Vulnerability.NoVuln != nil && *filter.Vulnerability.NoVuln { @@ -239,23 +249,35 @@ func certifyVexPredicate(filter model.CertifyVEXStatementSpec) predicate.Certify ) } } + + if filter.Subject != nil { + if filter.Subject.Package != nil { + predicates = append(predicates, certifyvex.HasPackageWith(packageVersionQuery(filter.Subject.Package))) + } else if filter.Subject.Artifact != nil { + predicates = append(predicates, certifyvex.HasArtifactWith(artifactQueryPredicates(filter.Subject.Artifact))) + } + } + return certifyvex.And(predicates...) } -func vexStatementInputPredicate(subject model.PackageOrArtifactInput, vulnerability model.VulnerabilityInputSpec, vexStatement model.VexStatementInputSpec) predicate.CertifyVex { +func vexStatementInputPredicate(subject model.PackageOrArtifactInput, subjectID int, vulnerability model.VulnerabilityInputSpec, vulnerabilityID int, vexStatement model.VexStatementInputSpec) predicate.CertifyVex { var sub *model.PackageOrArtifactSpec if subject.Package != nil { sub = &model.PackageOrArtifactSpec{ Package: helper.ConvertPkgInputSpecToPkgSpec(subject.Package), } + sub.Package.ID = ptrfrom.String(nodeID(subjectID)) } else { sub = &model.PackageOrArtifactSpec{ Artifact: helper.ConvertArtInputSpecToArtSpec(subject.Artifact), } + sub.Artifact.ID = ptrfrom.String(nodeID(subjectID)) } return certifyVexPredicate(model.CertifyVEXStatementSpec{ Subject: sub, Vulnerability: &model.VulnerabilitySpec{ + ID: ptrfrom.String(nodeID(vulnerabilityID)), Type: &vulnerability.Type, VulnerabilityID: &vulnerability.VulnerabilityID, }, diff --git a/pkg/assembler/backends/ent/backend/certifyVEXStatement_test.go b/pkg/assembler/backends/ent/backend/certifyVEXStatement_test.go index 83893a0fe5..447901727c 100644 --- a/pkg/assembler/backends/ent/backend/certifyVEXStatement_test.go +++ b/pkg/assembler/backends/ent/backend/certifyVEXStatement_test.go @@ -1063,7 +1063,7 @@ func (s *Suite) TestVEXBulkIngest() { if err != nil { return } - if diff := cmp.Diff(test.ExpVEX, got, ignoreID); diff != "" { + if diff := cmp.Diff(test.ExpVEX, got, IngestPredicatesCmpOpts...); diff != "" { t.Errorf("Unexpected results. (-want +got):\n%s", diff) } }) diff --git a/pkg/assembler/backends/ent/backend/certifyVuln.go b/pkg/assembler/backends/ent/backend/certifyVuln.go index 8c03680e73..8cfd0b2ad3 100644 --- a/pkg/assembler/backends/ent/backend/certifyVuln.go +++ b/pkg/assembler/backends/ent/backend/certifyVuln.go @@ -30,6 +30,7 @@ import ( "github.com/guacsec/guac/pkg/assembler/backends/ent/vulnerabilitytype" "github.com/guacsec/guac/pkg/assembler/graphql/model" "github.com/vektah/gqlparser/v2/gqlerror" + "golang.org/x/sync/errgroup" ) func (b *EntBackend) IngestCertifyVuln(ctx context.Context, pkg model.PkgInputSpec, spec model.VulnerabilityInputSpec, certifyVuln model.ScanMetadataInput) (*model.CertifyVuln, error) { @@ -90,13 +91,25 @@ func (b *EntBackend) IngestCertifyVuln(ctx context.Context, pkg model.PkgInputSp } func (b *EntBackend) IngestCertifyVulns(ctx context.Context, pkgs []*model.PkgInputSpec, vulnerabilities []*model.VulnerabilityInputSpec, certifyVulns []*model.ScanMetadataInput) ([]*model.CertifyVuln, error) { - var modelCertifyVulns []*model.CertifyVuln - for i, certifyVuln := range certifyVulns { - modelCertifyVuln, err := b.IngestCertifyVuln(ctx, *pkgs[i], *vulnerabilities[i], *certifyVuln) - if err != nil { - return nil, gqlerror.Errorf("IngestVulnerability failed with err: %v", err) - } - modelCertifyVulns = append(modelCertifyVulns, modelCertifyVuln) + var modelCertifyVulns = make([]*model.CertifyVuln, len(vulnerabilities)) + eg, ctx := errgroup.WithContext(ctx) + for i := range certifyVulns { + index := i + pkg := *pkgs[index] + vuln := *vulnerabilities[index] + certifyVuln := *certifyVulns[index] + concurrently(eg, func() error { + modelCertifyVuln, err := b.IngestCertifyVuln(ctx, pkg, vuln, certifyVuln) + if err == nil { + modelCertifyVulns[index] = modelCertifyVuln + return err + } else { + return gqlerror.Errorf("IngestCertifyVulns failed with err: %v", err) + } + }) + } + if err := eg.Wait(); err != nil { + return nil, err } return modelCertifyVulns, nil } diff --git a/pkg/assembler/backends/ent/backend/certifyVuln_test.go b/pkg/assembler/backends/ent/backend/certifyVuln_test.go index c4398e8446..d493047a13 100644 --- a/pkg/assembler/backends/ent/backend/certifyVuln_test.go +++ b/pkg/assembler/backends/ent/backend/certifyVuln_test.go @@ -1023,9 +1023,6 @@ func (s *Suite) TestIngestCertifyVulns() { }, }, } - ignoreID := cmp.FilterPath(func(p cmp.Path) bool { - return strings.Compare(".ID", p[len(p)-1].String()) == 0 - }, cmp.Ignore()) ctx := context.Background() for _, test := range tests { s.Run(test.Name, func() { @@ -1072,7 +1069,7 @@ func (s *Suite) TestIngestCertifyVulns() { if err != nil { return } - if diff := cmp.Diff(test.ExpVuln, got, ignoreID); diff != "" { + if diff := cmp.Diff(test.ExpVuln, got, IngestPredicatesCmpOpts...); diff != "" { t.Errorf("Unexpected results. (-want +got):\n%s", diff) } }) diff --git a/pkg/assembler/backends/ent/backend/concurrently.go b/pkg/assembler/backends/ent/backend/concurrently.go new file mode 100644 index 0000000000..8c88e3f9e6 --- /dev/null +++ b/pkg/assembler/backends/ent/backend/concurrently.go @@ -0,0 +1,79 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package backend + +import ( + "context" + "os" + "strconv" + + "github.com/guacsec/guac/pkg/logging" + "golang.org/x/sync/errgroup" +) + +var concurrent chan struct{} +var concurrentRead chan struct{} + +const MaxConcurrentBulkIngestionString string = "MAX_CONCURRENT_BULK_INGESTION" +const defaultMaxConcurrentBulkIngestion int = 50 +const MaxConcurrentReadString string = "MAX_CONCURRENT_READ" +const defaultMaxConcurrentRead int = 40 + +func init() { + logger := logging.FromContext(context.Background()) + concurrentSize := defaultMaxConcurrentBulkIngestion + maxConcurrentBulkIngestionEnv, found := os.LookupEnv(MaxConcurrentBulkIngestionString) + if found { + maxConcurrentBulkIngestion, err := strconv.Atoi(maxConcurrentBulkIngestionEnv) + if err != nil { + logger.Warnf("failed to convert %v value %v to integer. Default value %v will be applied", MaxConcurrentBulkIngestionString, maxConcurrentBulkIngestionEnv, defaultMaxConcurrentBulkIngestion) + concurrentSize = defaultMaxConcurrentBulkIngestion + } else { + concurrentSize = maxConcurrentBulkIngestion + } + } + concurrent = make(chan struct{}, concurrentSize) + + concurrentReadSize := defaultMaxConcurrentRead + maxConcurrentReadEnv, found := os.LookupEnv(MaxConcurrentReadString) + if found { + maxConcurrentBulkIngestion, err := strconv.Atoi(maxConcurrentReadEnv) + if err != nil { + logger.Warnf("failed to convert %v value %v to integer. Default value applied is %v\n", MaxConcurrentReadString, maxConcurrentReadEnv, concurrentReadSize) + } else { + concurrentReadSize = maxConcurrentBulkIngestion + } + } + concurrentRead = make(chan struct{}, concurrentReadSize) +} + +func concurrently(eg *errgroup.Group, fn func() error) { + eg.Go(func() error { + concurrent <- struct{}{} + err := fn() + <-concurrent + return err + }) +} + +func concurrentlyRead(eg *errgroup.Group, fn func() error) { + eg.Go(func() error { + concurrentRead <- struct{}{} + err := fn() + <-concurrentRead + return err + }) +} diff --git a/pkg/assembler/backends/ent/backend/dependency.go b/pkg/assembler/backends/ent/backend/dependency.go index 27c5d0b871..a613a7587f 100644 --- a/pkg/assembler/backends/ent/backend/dependency.go +++ b/pkg/assembler/backends/ent/backend/dependency.go @@ -23,6 +23,7 @@ import ( "github.com/guacsec/guac/pkg/assembler/backends/ent/dependency" "github.com/guacsec/guac/pkg/assembler/graphql/model" "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) func (b *EntBackend) IsDependency(ctx context.Context, spec *model.IsDependencySpec) ([]*model.IsDependency, error) { @@ -74,13 +75,24 @@ func (b *EntBackend) IsDependency(ctx context.Context, spec *model.IsDependencyS func (b *EntBackend) IngestDependencies(ctx context.Context, pkgs []*model.PkgInputSpec, depPkgs []*model.PkgInputSpec, depPkgMatchType model.MatchFlags, dependencies []*model.IsDependencyInputSpec) ([]*model.IsDependency, error) { // TODO: This looks like a good candidate for using BulkCreate() - var modelIsDependencies []*model.IsDependency + var modelIsDependencies = make([]*model.IsDependency, len(dependencies)) + eg, ctx := errgroup.WithContext(ctx) for i := range dependencies { - isDependency, err := b.IngestDependency(ctx, *pkgs[i], *depPkgs[i], depPkgMatchType, *dependencies[i]) - if err != nil { - return nil, Errorf("IngestDependency failed with err: %v", err) - } - modelIsDependencies = append(modelIsDependencies, isDependency) + index := i + pkg := *pkgs[index] + depPkg := *depPkgs[index] + dpmt := depPkgMatchType + dep := *dependencies[index] + concurrently(eg, func() error { + p, err := b.IngestDependency(ctx, pkg, depPkg, dpmt, dep) + if err == nil { + modelIsDependencies[index] = &model.IsDependency{ID: p.ID} + } + return err + }) + } + if err := eg.Wait(); err != nil { + return nil, err } return modelIsDependencies, nil } diff --git a/pkg/assembler/backends/ent/backend/dependency_test.go b/pkg/assembler/backends/ent/backend/dependency_test.go index 232a60ecdf..f3b346e780 100644 --- a/pkg/assembler/backends/ent/backend/dependency_test.go +++ b/pkg/assembler/backends/ent/backend/dependency_test.go @@ -726,9 +726,6 @@ func (s *Suite) TestIngestDependencies() { }, }, } - ignoreID := cmp.FilterPath(func(p cmp.Path) bool { - return strings.Compare(".ID", p[len(p)-1].String()) == 0 - }, cmp.Ignore()) ctx := s.Ctx for _, test := range tests { s.Run(test.Name, func() { @@ -760,7 +757,7 @@ func (s *Suite) TestIngestDependencies() { if err != nil { return } - if diff := cmp.Diff(test.ExpID, got, ignoreID); diff != "" { + if diff := cmp.Diff(test.ExpID, got, IngestPredicatesCmpOpts...); diff != "" { t.Errorf("Unexpected results. (-want +got):\n%s", diff) } }) diff --git a/pkg/assembler/backends/ent/backend/hasMetadata.go b/pkg/assembler/backends/ent/backend/hasMetadata.go index f9363c9013..db7305dc32 100644 --- a/pkg/assembler/backends/ent/backend/hasMetadata.go +++ b/pkg/assembler/backends/ent/backend/hasMetadata.go @@ -21,18 +21,19 @@ import ( "fmt" "entgo.io/ent/dialect/sql" + "github.com/guacsec/guac/internal/testing/ptrfrom" "github.com/guacsec/guac/pkg/assembler/backends/ent" "github.com/guacsec/guac/pkg/assembler/backends/ent/hasmetadata" "github.com/guacsec/guac/pkg/assembler/backends/ent/predicate" - "github.com/guacsec/guac/pkg/assembler/backends/helper" "github.com/guacsec/guac/pkg/assembler/graphql/model" "github.com/pkg/errors" "github.com/vektah/gqlparser/v2/gqlerror" + "golang.org/x/sync/errgroup" ) func (b *EntBackend) HasMetadata(ctx context.Context, filter *model.HasMetadataSpec) ([]*model.HasMetadata, error) { records, err := b.client.HasMetadata.Query(). - Where(hasMetadataPredicate(filter)). + Where(hasMetadataPredicate(filter, nil)). Limit(MaxPageSize). WithSource(withSourceNameTreeQuery()). WithArtifact(). @@ -59,8 +60,11 @@ func (b *EntBackend) IngestHasMetadata(ctx context.Context, subject model.Packag } func (b *EntBackend) IngestBulkHasMetadata(ctx context.Context, subjects model.PackageSourceOrArtifactInputs, pkgMatchType *model.MatchFlags, hasMetadataList []*model.HasMetadataInputSpec) ([]string, error) { - var results []string + var results = make([]string, len(hasMetadataList)) + eg, ctx := errgroup.WithContext(ctx) for i := range hasMetadataList { + index := i + hmSpec := *hasMetadataList[i] var subject model.PackageSourceOrArtifactInput if len(subjects.Packages) > 0 { subject = model.PackageSourceOrArtifactInput{Package: subjects.Packages[i]} @@ -69,16 +73,23 @@ func (b *EntBackend) IngestBulkHasMetadata(ctx context.Context, subjects model.P } else { subject = model.PackageSourceOrArtifactInput{Source: subjects.Sources[i]} } - hm, err := b.IngestHasMetadata(ctx, subject, pkgMatchType, *hasMetadataList[i]) - if err != nil { - return nil, gqlerror.Errorf("IngestBulkHasMetadata failed with element #%v with err: %v", i, err) - } - results = append(results, hm.ID) + concurrently(eg, func() error { + hm, err := b.IngestHasMetadata(ctx, subject, pkgMatchType, hmSpec) + if err == nil { + results[index] = hm.ID + return err + } else { + return gqlerror.Errorf("IngestBulkHasMetadata failed with element #%v %+v with err: %v", i, *subject.Package, err) + } + }) + } + if err := eg.Wait(); err != nil { + return nil, err } return results, nil } -func hasMetadataPredicate(filter *model.HasMetadataSpec) predicate.HasMetadata { +func hasMetadataPredicate(filter *model.HasMetadataSpec, pkgMatchType *model.MatchFlags) predicate.HasMetadata { predicates := []predicate.HasMetadata{ optionalPredicate(filter.ID, IDEQ), optionalPredicate(filter.Since, hasmetadata.TimestampGTE), @@ -92,14 +103,59 @@ func hasMetadataPredicate(filter *model.HasMetadataSpec) predicate.HasMetadata { if filter.Subject != nil { switch { case filter.Subject.Artifact != nil: - predicates = append(predicates, hasmetadata.HasArtifactWith(artifactQueryPredicates(filter.Subject.Artifact))) + if filter.Subject.Artifact.ID != nil { + predicates = append(predicates, + sql.FieldEQ(hasmetadata.FieldArtifactID, filter.Subject.Artifact.ID), + ) + } else { + predicates = append(predicates, + hasmetadata.HasArtifactWith(artifactQueryPredicates(filter.Subject.Artifact)), + ) + } + predicates = append(predicates, + hasmetadata.SourceIDIsNil(), + hasmetadata.PackageNameIDIsNil(), + hasmetadata.PackageVersionIDIsNil(), + ) case filter.Subject.Package != nil: - predicates = append(predicates, hasmetadata.Or( - hasmetadata.HasAllVersionsWith(packageNameQuery(pkgNameQueryFromPkgSpec(filter.Subject.Package))), - hasmetadata.HasPackageVersionWith(packageVersionQuery(filter.Subject.Package)), - )) + if filter.Subject.Package.ID != nil { + if (pkgMatchType != nil && pkgMatchType.Pkg == model.PkgMatchTypeSpecificVersion) || + filter.Subject.Package.Version != nil { + predicates = append(predicates, + sql.FieldEQ(hasmetadata.FieldPackageVersionID, filter.Subject.Package.ID), + hasmetadata.PackageNameIDIsNil(), + ) + } else { + predicates = append(predicates, + hasmetadata.PackageVersionIDIsNil(), + sql.FieldEQ(hasmetadata.FieldPackageNameID, filter.Subject.Package.ID), + ) + } + } else { + predicates = append(predicates, hasmetadata.Or( + hasmetadata.HasAllVersionsWith(packageNameQuery(pkgNameQueryFromPkgSpec(filter.Subject.Package))), + hasmetadata.HasPackageVersionWith(packageVersionQuery(filter.Subject.Package)), + )) + } + predicates = append(predicates, + hasmetadata.SourceIDIsNil(), + hasmetadata.ArtifactIDIsNil(), + ) case filter.Subject.Source != nil: - predicates = append(predicates, hasmetadata.HasSourceWith(sourceQuery(filter.Subject.Source))) + if filter.Subject.Source.ID != nil { + predicates = append(predicates, + sql.FieldEQ(hasmetadata.FieldSourceID, filter.Subject.Source.ID), + ) + } else { + predicates = append(predicates, + hasmetadata.HasSourceWith(sourceQuery(filter.Subject.Source)), + ) + } + predicates = append(predicates, + hasmetadata.PackageNameIDIsNil(), + hasmetadata.PackageVersionIDIsNil(), + hasmetadata.ArtifactIDIsNil(), + ) } } return hasmetadata.And(predicates...) @@ -115,7 +171,6 @@ func upsertHasMetadata(ctx context.Context, client *ent.Tx, subject model.Packag SetCollector(spec.Collector) conflictColumns := []string{ - hasmetadata.FieldTimestamp, hasmetadata.FieldKey, hasmetadata.FieldValue, hasmetadata.FieldJustification, @@ -124,13 +179,14 @@ func upsertHasMetadata(ctx context.Context, client *ent.Tx, subject model.Packag } var conflictWhere *sql.Predicate + var subjectSpec *model.PackageSourceOrArtifactSpec switch { case subject.Artifact != nil: - art, err := client.Artifact.Query().Where(artifactQueryInputPredicates(*subject.Artifact)).Only(ctx) + art, err := client.Artifact.Query().Where(artifactQueryInputPredicates(*subject.Artifact)).OnlyID(ctx) if err != nil { return nil, fmt.Errorf("failed to retrieve subject artifact :: %s", err) } - insert.SetArtifact(art) + insert.SetArtifactID(art) conflictColumns = append(conflictColumns, hasmetadata.FieldArtifactID) conflictWhere = sql.And( sql.NotNull(hasmetadata.FieldArtifactID), @@ -138,14 +194,17 @@ func upsertHasMetadata(ctx context.Context, client *ent.Tx, subject model.Packag sql.IsNull(hasmetadata.FieldPackageVersionID), sql.IsNull(hasmetadata.FieldSourceID), ) + subjectSpec = &model.PackageSourceOrArtifactSpec{ + Artifact: &model.ArtifactSpec{ID: ptrfrom.String(nodeID(art))}, + } case subject.Package != nil: if pkgMatchType.Pkg == model.PkgMatchTypeSpecificVersion { - pv, err := getPkgVersion(ctx, client.Client(), *subject.Package) + pv, err := getPkgVersionID(ctx, client.Client(), *subject.Package) if err != nil { return nil, fmt.Errorf("failed to retrieve subject package version :: %s", err) } - insert.SetPackageVersion(pv) + insert.SetPackageVersionID(pv) conflictColumns = append(conflictColumns, hasmetadata.FieldPackageVersionID) conflictWhere = sql.And( sql.IsNull(hasmetadata.FieldArtifactID), @@ -153,12 +212,15 @@ func upsertHasMetadata(ctx context.Context, client *ent.Tx, subject model.Packag sql.IsNull(hasmetadata.FieldPackageNameID), sql.IsNull(hasmetadata.FieldSourceID), ) + subjectSpec = &model.PackageSourceOrArtifactSpec{ + Package: &model.PkgSpec{ID: ptrfrom.String(nodeID(pv))}, + } } else { - pn, err := getPkgName(ctx, client.Client(), *subject.Package) + pn, err := getPkgNameID(ctx, client.Client(), *subject.Package) if err != nil { return nil, fmt.Errorf("failed to retrieve subject package name :: %s", err) } - insert.SetAllVersions(pn) + insert.SetAllVersionsID(pn) conflictColumns = append(conflictColumns, hasmetadata.FieldPackageNameID) conflictWhere = sql.And( sql.IsNull(hasmetadata.FieldArtifactID), @@ -166,6 +228,10 @@ func upsertHasMetadata(ctx context.Context, client *ent.Tx, subject model.Packag sql.NotNull(hasmetadata.FieldPackageNameID), sql.IsNull(hasmetadata.FieldSourceID), ) + //subject.Package.Version = nil + subjectSpec = &model.PackageSourceOrArtifactSpec{ + Package: &model.PkgSpec{ID: ptrfrom.String(nodeID(pn))}, + } } case subject.Source != nil: @@ -181,6 +247,9 @@ func upsertHasMetadata(ctx context.Context, client *ent.Tx, subject model.Packag sql.IsNull(hasmetadata.FieldPackageNameID), sql.NotNull(hasmetadata.FieldSourceID), ) + subjectSpec = &model.PackageSourceOrArtifactSpec{ + Source: &model.SourceSpec{ID: ptrfrom.String(nodeID(srcID))}, + } } id, err := insert.OnConflict( @@ -194,7 +263,7 @@ func upsertHasMetadata(ctx context.Context, client *ent.Tx, subject model.Packag return nil, errors.Wrap(err, "upsert HasMetadata node") } id, err = client.HasMetadata.Query(). - Where(hasMetadataInputPredicate(subject, pkgMatchType, spec)). + Where(hasMetadataInputPredicate(subjectSpec, pkgMatchType, spec)). OnlyID(ctx) if err != nil { return nil, errors.Wrap(err, "get HasMetadata") @@ -233,31 +302,14 @@ func toModelHasMetadata(v *ent.HasMetadata) *model.HasMetadata { } } -func hasMetadataInputPredicate(subject model.PackageSourceOrArtifactInput, pkgMatchType *model.MatchFlags, filter model.HasMetadataInputSpec) predicate.HasMetadata { - var subjectSpec *model.PackageSourceOrArtifactSpec - if subject.Package != nil { - if pkgMatchType != nil || pkgMatchType.Pkg == model.PkgMatchTypeAllVersions { - subject.Package.Version = nil - } - subjectSpec = &model.PackageSourceOrArtifactSpec{ - Package: helper.ConvertPkgInputSpecToPkgSpec(subject.Package), - } - } else if subject.Artifact != nil { - subjectSpec = &model.PackageSourceOrArtifactSpec{ - Artifact: helper.ConvertArtInputSpecToArtSpec(subject.Artifact), - } - } else { - subjectSpec = &model.PackageSourceOrArtifactSpec{ - Source: helper.ConvertSrcInputSpecToSrcSpec(subject.Source), - } - } +func hasMetadataInputPredicate(subjectSpec *model.PackageSourceOrArtifactSpec, pkgMatchType *model.MatchFlags, filter model.HasMetadataInputSpec) predicate.HasMetadata { return hasMetadataPredicate(&model.HasMetadataSpec{ Subject: subjectSpec, - Since: &filter.Timestamp, Key: &filter.Key, Value: &filter.Value, Justification: &filter.Justification, Origin: &filter.Origin, Collector: &filter.Collector, - }) + }, + pkgMatchType) } diff --git a/pkg/assembler/backends/ent/backend/hasMetadata_test.go b/pkg/assembler/backends/ent/backend/hasMetadata_test.go index be284c7d81..c0c793c6cc 100644 --- a/pkg/assembler/backends/ent/backend/hasMetadata_test.go +++ b/pkg/assembler/backends/ent/backend/hasMetadata_test.go @@ -203,6 +203,43 @@ func (s *Suite) TestHasMetadata() { }, }, }, + { + Name: "Ingest same twice with version", + InPkg: []*model.PkgInputSpec{p2}, + Calls: []call{ + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p2, + }, + Match: &model.MatchFlags{ + Pkg: model.PkgMatchTypeSpecificVersion, + }, + HM: &model.HasMetadataInputSpec{ + Justification: "test justification", + }, + }, + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p2, + }, + Match: &model.MatchFlags{ + Pkg: model.PkgMatchTypeSpecificVersion, + }, + HM: &model.HasMetadataInputSpec{ + Justification: "test justification", + }, + }, + }, + Query: &model.HasMetadataSpec{ + Justification: ptrfrom.String("test justification"), + }, + ExpHM: []*model.HasMetadata{ + { + Subject: p2out, + Justification: "test justification", + }, + }, + }, { Name: "Ingest two different keys", InPkg: []*model.PkgInputSpec{p1}, diff --git a/pkg/assembler/backends/ent/backend/helpers_test.go b/pkg/assembler/backends/ent/backend/helpers_test.go index 34e4f1eb19..fa8130a4d9 100644 --- a/pkg/assembler/backends/ent/backend/helpers_test.go +++ b/pkg/assembler/backends/ent/backend/helpers_test.go @@ -22,6 +22,9 @@ import ( "strings" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/helpers" ) func ptr[T any](s T) *T { @@ -39,3 +42,46 @@ var ignoreEmptySlices = cmp.FilterValues(func(x, y interface{}) bool { } return false }, cmp.Ignore()) + +var IngestPredicatesCmpOpts = []cmp.Option{ + ignoreID, + cmpopts.EquateEmpty(), + cmpopts.SortSlices(isDependencyLess), + cmpopts.SortSlices(packageLess), + cmpopts.SortSlices(certifyVulnLess), + cmpopts.SortSlices(certifyVexLess), + cmpopts.SortSlices(vulnEqualLess), + cmpopts.SortSlices(vulnerabilityLess), +} + +func isDependencyLess(e1, e2 *model.IsDependency) bool { + return packageLess(e1.Package, e2.Package) +} + +func packageLess(e1, e2 *model.Package) bool { + purl1 := helpers.PkgToPurl(e1.Type, e1.Namespaces[0].Namespace, e1.Namespaces[0].Names[0].Name, e1.Namespaces[0].Names[0].Versions[0].Version, e1.Namespaces[0].Names[0].Versions[0].Subpath, nil) + purl2 := helpers.PkgToPurl(e2.Type, e2.Namespaces[0].Namespace, e2.Namespaces[0].Names[0].Name, e2.Namespaces[0].Names[0].Versions[0].Version, e2.Namespaces[0].Names[0].Versions[0].Subpath, nil) + return purl1 < purl2 +} + +func certifyVulnLess(e1, e2 *model.CertifyVuln) bool { + return packageLess(e1.Package, e2.Package) +} + +func certifyVexLess(e1, e2 *model.CertifyVEXStatement) bool { + return e1.Vulnerability.VulnerabilityIDs[0].VulnerabilityID < e2.Vulnerability.VulnerabilityIDs[0].VulnerabilityID +} + +func vulnEqualLess(e1, e2 *model.VulnEqual) bool { + return vulnerabilityLess(e1.Vulnerabilities[0], e2.Vulnerabilities[0]) && vulnerabilityLess(e1.Vulnerabilities[1], e2.Vulnerabilities[1]) +} + +func vulnerabilityLess(e1, e2 *model.Vulnerability) bool { + if e1.Type != e2.Type { + return e1.Type < e2.Type + } else if strings.ToLower(e1.Type) == "novuln" || len(e1.Type) == 0 { + return false + } else { + return e1.VulnerabilityIDs[0].VulnerabilityID < e2.VulnerabilityIDs[0].VulnerabilityID + } +} diff --git a/pkg/assembler/backends/ent/backend/migrations.go b/pkg/assembler/backends/ent/backend/migrations.go index 11bea823c4..b0be9b132a 100644 --- a/pkg/assembler/backends/ent/backend/migrations.go +++ b/pkg/assembler/backends/ent/backend/migrations.go @@ -57,6 +57,7 @@ func SetupBackend(ctx context.Context, options *BackendOptions) (*ent.Client, er client := ent.NewClient(ent.Driver(dialectsql.OpenDB(driver, db))) if options.AutoMigrate { + logger.Infof("ent migrations started") // Run db migrations err = client.Schema.Create( ctx, diff --git a/pkg/assembler/backends/ent/backend/neighbors.go b/pkg/assembler/backends/ent/backend/neighbors.go index b5380a28cf..a891aa1544 100644 --- a/pkg/assembler/backends/ent/backend/neighbors.go +++ b/pkg/assembler/backends/ent/backend/neighbors.go @@ -20,14 +20,15 @@ import ( "log" "strconv" + "github.com/guacsec/guac/pkg/assembler/backends/ent" + "github.com/guacsec/guac/pkg/assembler/backends/ent/dependency" "github.com/guacsec/guac/pkg/assembler/backends/ent/packagename" "github.com/guacsec/guac/pkg/assembler/backends/ent/packagenamespace" "github.com/guacsec/guac/pkg/assembler/backends/ent/packagetype" "github.com/guacsec/guac/pkg/assembler/backends/ent/packageversion" "github.com/guacsec/guac/pkg/assembler/backends/ent/sourcetype" - - "github.com/guacsec/guac/pkg/assembler/backends/ent" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/vektah/gqlparser/v2/gqlerror" ) func (b *EntBackend) Neighbors(ctx context.Context, node string, usingOnly []model.Edge) ([]model.Node, error) { @@ -107,6 +108,17 @@ func (b *EntBackend) Node(ctx context.Context, node string) (model.Node, error) return toModelBuilder(v), nil case *ent.VulnerabilityType: return toModelVulnerability(v), nil + case *ent.Dependency: + isDep, err := b.client.Dependency.Query(). + Where(dependency.ID(v.ID)). + WithPackage(withPackageVersionTree()). + WithDependentPackageName(withPackageNameTree()). + WithDependentPackageVersion(withPackageVersionTree()). + Only(ctx) + if err != nil { + return nil, err + } + return toModelIsDependencyWithBackrefs(isDep), nil default: log.Printf("Unknown node type: %T", v) } @@ -125,3 +137,97 @@ func (b *EntBackend) Nodes(ctx context.Context, nodes []string) ([]model.Node, e } return rv, nil } + +func (b *EntBackend) bfsFromVulnerablePackage(ctx context.Context, pkg int) ([][]model.Node, error) { + queue := make([]int, 0) // the queue of nodes in bfs + type dfsNode struct { + expanded bool // true once all node neighbors are added to queue + parent int + } + nodeMap := map[int]dfsNode{} + + nodeMap[pkg] = dfsNode{} + queue = append(queue, pkg) + + var now int + var productsFound []int + for len(queue) > 0 { + now = queue[0] + queue = queue[1:] + nowNode := nodeMap[now] + + isDependencies, err := b.client.Dependency.Query(). + Select(dependency.FieldID, dependency.FieldPackageID). + Where(dependency.DependentPackageVersionID(now)). + All(ctx) + if err != nil { + return nil, gqlerror.Errorf("bfsThroughIsDependency :: %s", err) + } + foundDependentPkg := false + for _, isDependency := range isDependencies { + foundDependentPkg = true + next := isDependency.PackageID + dfsN, seen := nodeMap[next] + if !seen { + dfsNIsDependency := dfsNode{ + parent: now, + } + nodeMap[isDependency.ID] = dfsNIsDependency + dfsN = dfsNode{ + parent: isDependency.ID, + } + nodeMap[next] = dfsN + } + if !dfsN.expanded { + queue = append(queue, next) + } + } + // if none of the dependencies found has 'depPkg' as dependency package, + // then it means 'depPkg' is a top level package (i.e. "product") + // to be 100% the 'HasSBOM' check should/could be added + if !foundDependentPkg { + productsFound = append(productsFound, now) + } + + nowNode.expanded = true + nodeMap[now] = nowNode + } + + result := [][]model.Node{} + for i := range productsFound { + reversedPath := []int{} + step := productsFound[i] + for step != pkg { + reversedPath = append(reversedPath, step) + step = nodeMap[step].parent + } + reversedPath = append(reversedPath, step) + + // reverse path + path := make([]int, len(reversedPath)) + for i, x := range reversedPath { + path[len(reversedPath)-i-1] = x + } + + nodes, err := b.buildModelNodes(ctx, path) + if err != nil { + return nil, err + } + result = append(result, nodes) + } + return result, nil +} + +func (b *EntBackend) buildModelNodes(ctx context.Context, nodeIDs []int) ([]model.Node, error) { + out := make([]model.Node, len(nodeIDs)) + + for i, nodeID := range nodeIDs { + var err error + out[i], err = b.Node(ctx, strconv.Itoa(nodeID)) + if err != nil { + return nil, gqlerror.Errorf("Internal data error: got invalid node id %d :: %s", nodeID, err) + } + } + + return out, nil +} diff --git a/pkg/assembler/backends/ent/backend/package.go b/pkg/assembler/backends/ent/backend/package.go index bdceb73f70..4875747a3e 100644 --- a/pkg/assembler/backends/ent/backend/package.go +++ b/pkg/assembler/backends/ent/backend/package.go @@ -34,6 +34,7 @@ import ( "github.com/guacsec/guac/pkg/assembler/backends/helper" "github.com/guacsec/guac/pkg/assembler/graphql/model" "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) func (b *EntBackend) Packages(ctx context.Context, pkgSpec *model.PkgSpec) ([]*model.Package, error) { @@ -103,12 +104,20 @@ func (b *EntBackend) Packages(ctx context.Context, pkgSpec *model.PkgSpec) ([]*m func (b *EntBackend) IngestPackages(ctx context.Context, pkgs []*model.PkgInputSpec) ([]*model.Package, error) { // FIXME: (ivanvanderbyl) This will be suboptimal because we can't batch insert relations with upserts. See Readme. models := make([]*model.Package, len(pkgs)) - for i, pkg := range pkgs { - p, err := b.IngestPackage(ctx, *pkg) - if err != nil { - return nil, err - } - models[i] = p + eg, ctx := errgroup.WithContext(ctx) + for i := range pkgs { + index := i + pkg := pkgs[index] + concurrently(eg, func() error { + p, err := b.IngestPackage(ctx, *pkg) + if err == nil { + models[index] = p + } + return err + }) + } + if err := eg.Wait(); err != nil { + return nil, err } return models, nil } @@ -467,18 +476,25 @@ func backReferencePackageNamespace(pns *ent.PackageNamespace) *ent.PackageType { // and should allow using the db index. func getPkgName(ctx context.Context, client *ent.Client, pkgin model.PkgInputSpec) (*ent.PackageName, error) { - return client.PackageName.Query().Where(packageNameInputQuery(pkgin)).Only(ctx) + return getPkgNameQuery(ctx, client, pkgin).Only(ctx) +} + +func getPkgNameID(ctx context.Context, client *ent.Client, pkgin model.PkgInputSpec) (int, error) { + return getPkgNameQuery(ctx, client, pkgin).OnlyID(ctx) +} + +func getPkgNameQuery(ctx context.Context, client *ent.Client, pkgin model.PkgInputSpec) *ent.PackageNameQuery { + return client.PackageName.Query().Where(packageNameInputQuery(pkgin)) } func getPkgVersion(ctx context.Context, client *ent.Client, pkgin model.PkgInputSpec) (*ent.PackageVersion, error) { - return client.PackageVersion.Query().Where(packageVersionInputQuery(pkgin)).Only(ctx) - // return client.PackageType.Query(). - // Where(packagetype.Type(pkgin.Type)). - // QueryNamespaces().Where(packagenamespace.NamespaceEQ(valueOrDefault(pkgin.Namespace, ""))). - // QueryNames().Where(packagename.NameEQ(pkgin.Name)). - // QueryVersions(). - // Where( - // packageVersionInputQuery(pkgin), - // ). - // Only(ctx) + return getPkgVersionQuery(ctx, client, pkgin).Only(ctx) +} + +func getPkgVersionID(ctx context.Context, client *ent.Client, pkgin model.PkgInputSpec) (int, error) { + return getPkgVersionQuery(ctx, client, pkgin).OnlyID(ctx) +} + +func getPkgVersionQuery(ctx context.Context, client *ent.Client, pkgin model.PkgInputSpec) *ent.PackageVersionQuery { + return client.PackageVersion.Query().Where(packageVersionInputQuery(pkgin)) } diff --git a/pkg/assembler/backends/ent/backend/package_test.go b/pkg/assembler/backends/ent/backend/package_test.go index 86d7494384..9ed49e8fb6 100644 --- a/pkg/assembler/backends/ent/backend/package_test.go +++ b/pkg/assembler/backends/ent/backend/package_test.go @@ -227,7 +227,7 @@ func (s *Suite) Test_IngestPackages() { s.T().Errorf("demoClient.IngestPackages() error = %v, wantErr %v", err, tt.wantErr) return } - if diff := cmp.Diff(tt.want, got, ignoreID); diff != "" { + if diff := cmp.Diff(tt.want, got, IngestPredicatesCmpOpts...); diff != "" { s.T().Errorf("Unexpected results. (-want +got):\n%s", diff) } }) diff --git a/pkg/assembler/backends/ent/backend/pkgequal_test.go b/pkg/assembler/backends/ent/backend/pkgequal_test.go index 6202d1ad61..9094243d7c 100644 --- a/pkg/assembler/backends/ent/backend/pkgequal_test.go +++ b/pkg/assembler/backends/ent/backend/pkgequal_test.go @@ -742,7 +742,7 @@ func (s *Suite) TestIngestPkgEquals() { if err != nil { return } - if diff := cmp.Diff(test.ExpHE, got, ignoreID); diff != "" { + if diff := cmp.Diff(test.ExpHE, got, IngestPredicatesCmpOpts...); diff != "" { t.Errorf("Unexpected results. (-want +got):\n%s", diff) } }) diff --git a/pkg/assembler/backends/ent/backend/search.go b/pkg/assembler/backends/ent/backend/search.go index 1243da27ad..ddbdd41255 100644 --- a/pkg/assembler/backends/ent/backend/search.go +++ b/pkg/assembler/backends/ent/backend/search.go @@ -17,14 +17,27 @@ package backend import ( "context" + "fmt" + "slices" + "strconv" + "github.com/guacsec/guac/internal/testing/ptrfrom" "github.com/guacsec/guac/pkg/assembler/backends/ent" "github.com/guacsec/guac/pkg/assembler/backends/ent/artifact" + "github.com/guacsec/guac/pkg/assembler/backends/ent/certifyvex" + "github.com/guacsec/guac/pkg/assembler/backends/ent/certifyvuln" + "github.com/guacsec/guac/pkg/assembler/backends/ent/hasmetadata" "github.com/guacsec/guac/pkg/assembler/backends/ent/packagename" "github.com/guacsec/guac/pkg/assembler/backends/ent/packageversion" "github.com/guacsec/guac/pkg/assembler/backends/ent/sourcename" "github.com/guacsec/guac/pkg/assembler/backends/ent/sourcenamespace" + "github.com/guacsec/guac/pkg/assembler/backends/ent/vulnerabilityid" + "github.com/guacsec/guac/pkg/assembler/backends/ent/vulnerabilitytype" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/helpers" + "github.com/vektah/gqlparser/v2/gqlerror" + "golang.org/x/exp/maps" + "golang.org/x/sync/errgroup" ) // FindSoftware takes in a searchText string and looks for software @@ -107,3 +120,262 @@ func (b *EntBackend) FindSoftware(ctx context.Context, searchText string) ([]mod return results, nil } + +func (b *EntBackend) FindTopLevelPackagesRelatedToVulnerability(ctx context.Context, vulnerabilityID string) ([][]model.Node, error) { + // TODO use directly the query because the EntBackend.HasSBOM is limited to MaxPageSize + hasSBOMs, err := b.HasSBOM(ctx, &model.HasSBOMSpec{}) + if err != nil { + return nil, gqlerror.Errorf("FindTopLevelPackagesRelatedToVulnerability failed with err: %v", err) + } + + result := [][]model.Node{} + productIDsCheckedVulnerable := make(map[string]bool, len(hasSBOMs)) + for _, hasSBOM := range hasSBOMs { + switch v := hasSBOM.Subject.(type) { + case *model.Artifact: + productIDsCheckedVulnerable[v.ID] = false + case *model.Package: + productIDsCheckedVulnerable[v.Namespaces[0].Names[0].Versions[0].ID] = false + } + } + + if len(productIDsCheckedVulnerable) != 0 { + vexStatements, err := b.CertifyVEXStatement(ctx, &model.CertifyVEXStatementSpec{ + Vulnerability: &model.VulnerabilitySpec{ + VulnerabilityID: &vulnerabilityID, + }, + }) + if err != nil { + return nil, gqlerror.Errorf("FindTopLevelPackagesRelatedToVulnerability failed with err: %v", err) + } + packagesAlreadyInvestigated := make([]int, 0) + for _, vexStatement := range vexStatements { + subject := vexStatement.Subject + var pkgVulnerable *model.Package + switch v := subject.(type) { + case *model.Package: + pkgVulnerable = v + case *model.Artifact: + continue + } + pkg, err := strconv.Atoi(pkgVulnerable.Namespaces[0].Names[0].Versions[0].ID) + if err != nil { + return nil, err + } + paths, err := b.bfsFromVulnerablePackage(ctx, pkg) + if err != nil { + return nil, err + } + for i := range paths { + paths[i] = append([]model.Node{vexStatement}, paths[i]...) + } + result = append(result, paths...) + if len(paths) > 0 { + packagesAlreadyInvestigated = append(packagesAlreadyInvestigated, pkg) + } + } + // if no VEX Statements have been found or no path from any VEX statement to product has been found + // then let's check also for CertifyVuln + if len(vexStatements) == 0 || slices.Contains(maps.Values(productIDsCheckedVulnerable), false) { + vulnStatements, err := b.CertifyVuln(ctx, &model.CertifyVulnSpec{ + Vulnerability: &model.VulnerabilitySpec{ + VulnerabilityID: &vulnerabilityID, + }, + }) + if err != nil { + return nil, gqlerror.Errorf("FindTopLevelPackagesRelatedToVulnerability failed with err: %v", err) + } + for _, vuln := range vulnStatements { + pkg, err := strconv.Atoi(vuln.Package.Namespaces[0].Names[0].Versions[0].ID) + if err != nil { + return nil, err + } + if !slices.Contains(packagesAlreadyInvestigated, pkg) { + products, err := b.bfsFromVulnerablePackage(ctx, pkg) + if err != nil { + return nil, err + } + for i := range products { + products[i] = append([]model.Node{vuln}, products[i]...) + } + result = append(result, products...) + } + } + } + + } + return result, nil +} + +// FindVulnerability returns all vulnerabilities related to a package +func (b *EntBackend) FindVulnerability(ctx context.Context, purl string) ([]model.CertifyVulnOrCertifyVEXStatement, error) { + + pkgInput, err := helpers.PurlToPkg(purl) + if err != nil { + return nil, gqlerror.Errorf("failed to parse PURL: %v", err) + } + + pkgQualifierFilter := []*model.PackageQualifierSpec{} + for _, qualifier := range pkgInput.Qualifiers { + pkgQualifierFilter = append(pkgQualifierFilter, &model.PackageQualifierSpec{ + Key: qualifier.Key, + Value: &qualifier.Value, + }) + } + + pkgFilter := &model.PkgSpec{ + Type: &pkgInput.Type, + Namespace: pkgInput.Namespace, + Name: &pkgInput.Name, + Version: pkgInput.Version, + Subpath: pkgInput.Subpath, + Qualifiers: pkgQualifierFilter, + } + + vulnerabilities, err := b.findVulnerabilities(ctx, pkgFilter, purl) + if err != nil { + return nil, err + } + return *vulnerabilities, err +} + +// FindVulnerabilityCPE returns all vulnerabilities related to the package identified by the CPE +func (b *EntBackend) FindVulnerabilityCPE(ctx context.Context, cpe string) ([]model.CertifyVulnOrCertifyVEXStatement, error) { + + packagesFound := map[string]*model.Package{} + metadatas, err := b.HasMetadata(ctx, &model.HasMetadataSpec{Key: ptrfrom.String("cpe"), Value: &cpe}) + if err != nil { + return nil, gqlerror.Errorf("error querying for HasMetadata: %v", err) + } + // if multiple times the same key-value metadata has been attached to the same package, + // it means the referenced package is just only the same one. + for i := range metadatas { + pkg, ok := metadatas[i].Subject.(*model.Package) + if ok { + id := pkg.Namespaces[0].Names[0].Versions[0].ID + if _, found := packagesFound[id]; !found { + packagesFound[id] = pkg + } + } + } + if len(maps.Values(packagesFound)) != 1 { + return nil, gqlerror.Errorf("failed to locate a single package based on the provided CPE") + } + + pkg := maps.Values(packagesFound)[0] + pkgQualifierFilter := []*model.PackageQualifierSpec{} + for _, qualifier := range pkg.Namespaces[0].Names[0].Versions[0].Qualifiers { + pkgQualifierFilter = append(pkgQualifierFilter, &model.PackageQualifierSpec{ + Key: qualifier.Key, + Value: &qualifier.Value, + }) + } + pkgFilter := &model.PkgSpec{ + ID: &pkg.Namespaces[0].Names[0].Versions[0].ID, + } + pkgID, err := strconv.Atoi(*pkgFilter.ID) + if err != nil { + return nil, gqlerror.Errorf("error converting pkg.ID: %v", pkg.ID) + } + purlMetadata, err := b.client.HasMetadata.Query().Select(hasmetadata.FieldValue). + Where( + hasmetadata.PackageVersionID(pkgID), + hasmetadata.KeyEQ("topLevelPackage"), + ). + Only(ctx) + if err != nil { + return nil, gqlerror.Errorf("error querying for HasMetadata: %v", err) + } + vulnerabilities, err := b.findVulnerabilities(ctx, pkgFilter, purlMetadata.Value) + if err != nil { + return nil, err + } + return *vulnerabilities, nil +} + +func (b *EntBackend) findVulnerabilities(ctx context.Context, pkgFilter *model.PkgSpec, purl string) (*[]model.CertifyVulnOrCertifyVEXStatement, error) { + // check the provided input + _, err := b.client.PackageVersion.Query(). + Where(packageVersionQuery(pkgFilter)). + Only(ctx) + if err != nil { + return nil, gqlerror.Errorf("error querying for package: %v", err) + } + vulnerabilities := make(chan model.CertifyVulnOrCertifyVEXStatement) + eg, ctx := errgroup.WithContext(ctx) + concurrentlyRead(eg, func() error { + certifyVexes, err := b.client.CertifyVex.Query(). + Where( + certifyvex.HasPackageWith( + packageversion.HasHasMetadataWith( + hasmetadata.KeyEQ("topLevelPackage"), + hasmetadata.ValueEQ(purl), + hasmetadata.PackageVersionIDNotNil(), + hasmetadata.PackageNameIDIsNil(), + hasmetadata.SourceIDIsNil(), + hasmetadata.ArtifactIDIsNil()), + ), + certifyvex.HasVulnerabilityWith( + vulnerabilityid.HasTypeWith( + vulnerabilitytype.TypeNEQ(NoVuln), + ), + ), + ). + WithVulnerability(func(q *ent.VulnerabilityIDQuery) { + q.WithType() + }). + WithPackage(withPackageVersionTree()). + All(ctx) + if err != nil { + return err + } + for i := range certifyVexes { + index := i + vulnerabilities <- toModelCertifyVEXStatement(certifyVexes[index]) + } + return nil + }) + concurrentlyRead(eg, func() error { + certifyVulns, err := b.client.CertifyVuln.Query(). + Where( + certifyvuln.HasPackageWith( + packageversion.HasHasMetadataWith( + hasmetadata.KeyEQ("topLevelPackage"), + hasmetadata.ValueEQ(purl), + hasmetadata.PackageVersionIDNotNil(), + hasmetadata.PackageNameIDIsNil(), + hasmetadata.SourceIDIsNil(), + hasmetadata.ArtifactIDIsNil()), + ), + certifyvuln.HasVulnerabilityWith( + vulnerabilityid.HasTypeWith( + vulnerabilitytype.TypeNEQ(NoVuln), + ), + ), + ). + WithVulnerability(func(q *ent.VulnerabilityIDQuery) { + q.WithType() + }). + WithPackage(withPackageVersionTree()). + All(ctx) + if err != nil { + return err + } + for i := range certifyVulns { + index := i + vulnerabilities <- toModelCertifyVulnerability(certifyVulns[index]) + } + return nil + }) + go func() { + if err := eg.Wait(); err != nil { + fmt.Printf("WARN: error in a concurrentlyRead :: %s", err) + } + close(vulnerabilities) + }() + var result []model.CertifyVulnOrCertifyVEXStatement + for vulnerability := range vulnerabilities { + result = append(result, vulnerability) + } + return &result, nil +} diff --git a/pkg/assembler/backends/ent/backend/search_test.go b/pkg/assembler/backends/ent/backend/search_test.go index cd4f518514..b14870c010 100644 --- a/pkg/assembler/backends/ent/backend/search_test.go +++ b/pkg/assembler/backends/ent/backend/search_test.go @@ -18,10 +18,42 @@ package backend import ( + "time" + "github.com/google/go-cmp/cmp" + "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/testdata" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/helpers" ) +var p5 = &model.PkgInputSpec{ + Type: p4.Type, + Namespace: p4.Namespace, + Name: p4.Name, + Version: ptrfrom.String("3.0.4"), +} + +var p5out = &model.Package{ + Type: p4out.Type, + Namespaces: []*model.PackageNamespace{{ + Namespace: p4out.Namespaces[0].Namespace, + Names: []*model.PackageName{{ + Name: p4out.Namespaces[0].Names[0].Name, + Versions: []*model.PackageVersion{{ + Version: p4out.Namespaces[0].Names[0].Versions[0].Version, + Qualifiers: []*model.PackageQualifier{}, + }}, + }}, + }}, +} + +var p6 = &model.PkgInputSpec{ + Type: p2.Type, + Name: p2.Name, + Version: ptrfrom.String("3.0.4"), +} + func (s *Suite) Test_FindSoftware() { b, err := GetBackend(s.Client) s.NoError(err) @@ -76,3 +108,996 @@ func (s *Suite) Test_FindSoftware() { s.T().Errorf("Unexpected results. (-want +got):\n%s", diff) } } + +func (s *Suite) Test_FindTopLevelPackagesRelatedToVulnerability() { + type IsDependency struct { + P1 model.PkgInputSpec + P2 model.PkgInputSpec + MF model.MatchFlags + ID model.IsDependencyInputSpec + } + type HasSBOM struct { + Sub model.PackageOrArtifactInput + HS model.HasSBOMInputSpec + } + type CertifyVEXStatement struct { + Sub model.PackageOrArtifactInput + Vuln model.VulnerabilityInputSpec + In model.VexStatementInputSpec + } + type CertifyVuln struct { + Pkg *model.PkgInputSpec + Vuln *model.VulnerabilityInputSpec + CertifyVuln *model.ScanMetadataInput + } + tests := []struct { + name string + InPkg []*model.PkgInputSpec + InIsDependency []IsDependency + InHasSBOM []HasSBOM + InVulnerability []*model.VulnerabilityInputSpec + InCertifyVexStatement []CertifyVEXStatement + InCertifyVuln []CertifyVuln + query string + want [][]model.Node + wantErr bool + }{ + { + name: "Happy Path - VEX", + InPkg: []*model.PkgInputSpec{p1, p4, p5}, + // p1 is a dependency ONLY of p4 + InIsDependency: []IsDependency{ + { + P1: *p4, + P2: *p1, + MF: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + ID: model.IsDependencyInputSpec{ + Justification: "test justification", + }, + }, + }, + // both p4 and p5 are top level packages, i.e. with an SBOM + InHasSBOM: []HasSBOM{ + { + Sub: model.PackageOrArtifactInput{ + Package: p4, + }, + HS: model.HasSBOMInputSpec{ + URI: "test uri", + }, + }, + { + Sub: model.PackageOrArtifactInput{ + Package: p5, + }, + HS: model.HasSBOMInputSpec{ + URI: "test uri", + }, + }, + }, + InVulnerability: []*model.VulnerabilityInputSpec{{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }}, + // vulnerability relates to p1 + InCertifyVexStatement: []CertifyVEXStatement{{ + Sub: model.PackageOrArtifactInput{ + Package: p1, + }, + Vuln: model.VulnerabilityInputSpec{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }, + In: model.VexStatementInputSpec{ + VexJustification: "test justification", + KnownSince: time.Unix(1e9, 0), + }, + }}, + query: "cve-2019-13110", + want: [][]model.Node{{ + &model.CertifyVEXStatement{ + Subject: p1out, + Vulnerability: &model.Vulnerability{ + Type: "cve", + VulnerabilityIDs: []*model.VulnerabilityID{ + {VulnerabilityID: "cve-2019-13110"}, + }, + }, + VexJustification: "test justification", + KnownSince: time.Unix(1e9, 0), + }, + p1out, + &model.IsDependency{ + Package: p4out, + DependencyPackage: p1out, + DependencyType: model.DependencyTypeUnknown, + Justification: "test justification", + }, + p5out, + }}, + wantErr: false, + }, + { + name: "Happy Path - Vuln", + InPkg: []*model.PkgInputSpec{p1, p4, p5}, + // p1 is a dependency ONLY of p4 + InIsDependency: []IsDependency{ + { + P1: *p4, + P2: *p1, + MF: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + ID: model.IsDependencyInputSpec{ + Justification: "test justification", + }, + }, + }, + // both p4 and p5 are top level packages, i.e. with an SBOM + InHasSBOM: []HasSBOM{ + { + Sub: model.PackageOrArtifactInput{ + Package: p4, + }, + HS: model.HasSBOMInputSpec{ + URI: "test uri", + }, + }, + { + Sub: model.PackageOrArtifactInput{ + Package: p5, + }, + HS: model.HasSBOMInputSpec{ + URI: "test uri", + }, + }, + }, + InVulnerability: []*model.VulnerabilityInputSpec{{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }}, + // vulnerability relates to p1 + InCertifyVuln: []CertifyVuln{{ + Pkg: p1, + Vuln: &model.VulnerabilityInputSpec{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }, + CertifyVuln: &model.ScanMetadataInput{ + Collector: "test collector", + Origin: "test origin", + ScannerVersion: "v1.0.0", + ScannerURI: "test scanner uri", + DbVersion: "2023.01.01", + DbURI: "test db uri", + TimeScanned: testdata.T1, + }, + }}, + query: "cve-2019-13110", + want: [][]model.Node{{ + &model.CertifyVuln{ + Package: p1out, + Vulnerability: &model.Vulnerability{ + Type: "cve", + VulnerabilityIDs: []*model.VulnerabilityID{ + {VulnerabilityID: "cve-2019-13110"}, + }, + }, + Metadata: &model.ScanMetadata{ + Collector: "test collector", + Origin: "test origin", + ScannerVersion: "v1.0.0", + ScannerURI: "test scanner uri", + DbVersion: "2023.01.01", + DbURI: "test db uri", + TimeScanned: testdata.T1, + }, + }, + p1out, + &model.IsDependency{ + Package: p4out, + DependencyPackage: p1out, + DependencyType: model.DependencyTypeUnknown, + Justification: "test justification", + }, + p5out, + }}, + wantErr: false, + }, + { + name: "No package with SBOM found", + InPkg: []*model.PkgInputSpec{p1, p4, p5}, + // p1 is a dependency ONLY of p4 + InIsDependency: []IsDependency{ + { + P1: *p4, + P2: *p1, + MF: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + ID: model.IsDependencyInputSpec{ + Justification: "test justification", + }, + }, + }, + InHasSBOM: []HasSBOM{}, + InVulnerability: []*model.VulnerabilityInputSpec{{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }}, + // vulnerability relates to p1 + InCertifyVuln: []CertifyVuln{{ + Pkg: p1, + Vuln: &model.VulnerabilityInputSpec{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }, + CertifyVuln: &model.ScanMetadataInput{ + Collector: "test collector", + Origin: "test origin", + ScannerVersion: "v1.0.0", + ScannerURI: "test scanner uri", + DbVersion: "2023.01.01", + DbURI: "test db uri", + TimeScanned: testdata.T1, + }, + }}, + query: "cve-2019-13110", + want: [][]model.Node{}, + wantErr: false, + }, + } + ctx := s.Ctx + for _, tt := range tests { + s.Run(tt.name, func() { + t := s.T() + b, err := GetBackend(s.Client) + if err != nil { + t.Fatalf("Could not instantiate testing backend: %v", err) + } + + _, err = b.IngestPackages(ctx, tt.InPkg) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + + for _, isDependency := range tt.InIsDependency { + _, err := b.IngestDependency(ctx, isDependency.P1, isDependency.P2, isDependency.MF, isDependency.ID) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + for _, hasSBOM := range tt.InHasSBOM { + _, err := b.IngestHasSbom(ctx, hasSBOM.Sub, hasSBOM.HS) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + _, err = b.IngestVulnerabilities(ctx, tt.InVulnerability) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + + for _, vexStatement := range tt.InCertifyVexStatement { + _, err := b.IngestVEXStatement(ctx, vexStatement.Sub, vexStatement.Vuln, vexStatement.In) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + for _, vuln := range tt.InCertifyVuln { + _, err := b.IngestCertifyVuln(ctx, *vuln.Pkg, *vuln.Vuln, *vuln.CertifyVuln) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + got, err := b.FindTopLevelPackagesRelatedToVulnerability(ctx, tt.query) + if (err != nil) != tt.wantErr { + t.Errorf("FindTopLevelPackagesRelatedToVulnerability error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(tt.want, got, ignoreID); diff != "" { + t.Errorf("Unexpected results. (-want +got):\n%s", diff) + } + }) + } +} + +func (s *Suite) Test_FindVulnerability() { + type IsDependency struct { + P1 model.PkgInputSpec + P2 model.PkgInputSpec + MF model.MatchFlags + ID model.IsDependencyInputSpec + } + type HasMetadata struct { + Sub model.PackageSourceOrArtifactInput + PMT *model.MatchFlags + HM model.HasMetadataInputSpec + } + type HasSBOM struct { + Sub model.PackageOrArtifactInput + HS model.HasSBOMInputSpec + } + type CertifyVEXStatement struct { + Sub model.PackageOrArtifactInput + Vuln model.VulnerabilityInputSpec + In model.VexStatementInputSpec + } + type CertifyVuln struct { + Pkg *model.PkgInputSpec + Vuln *model.VulnerabilityInputSpec + CertifyVuln *model.ScanMetadataInput + } + tests := []struct { + name string + InPkg []*model.PkgInputSpec + InIsDependency []IsDependency + InHasMetadata []HasMetadata + InHasSBOM []HasSBOM + InVulnerability []*model.VulnerabilityInputSpec + InCertifyVexStatement []CertifyVEXStatement + InCertifyVuln []CertifyVuln + query string + want []model.CertifyVulnOrCertifyVEXStatement + wantErr bool + }{ + { + name: "Happy Path - VEX", + InPkg: []*model.PkgInputSpec{p1, p6, p5}, + // p1 is a dependency ONLY of p6 + InIsDependency: []IsDependency{ + { + P1: *p6, + P2: *p1, + MF: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + ID: model.IsDependencyInputSpec{ + Justification: "test justification", + }, + }, + }, + InHasMetadata: []HasMetadata{ + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p1, + }, + PMT: &model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + HM: model.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: helpers.PkgToPurl(p6.Type, "", p6.Name, *p6.Version, "", []string{}), + }, + }, + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p6, + }, + PMT: &model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + HM: model.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: helpers.PkgToPurl(p6.Type, "", p6.Name, *p6.Version, "", []string{}), + }, + }, + }, + // both p6 and p5 are top level packages, i.e. with an SBOM + InHasSBOM: []HasSBOM{ + { + Sub: model.PackageOrArtifactInput{ + Package: p6, + }, + HS: model.HasSBOMInputSpec{ + URI: "test uri", + }, + }, + { + Sub: model.PackageOrArtifactInput{ + Package: p5, + }, + HS: model.HasSBOMInputSpec{ + URI: "test uri", + }, + }, + }, + InVulnerability: []*model.VulnerabilityInputSpec{{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }}, + // vulnerability relates to p1 + InCertifyVexStatement: []CertifyVEXStatement{{ + Sub: model.PackageOrArtifactInput{ + Package: p1, + }, + Vuln: model.VulnerabilityInputSpec{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }, + In: model.VexStatementInputSpec{ + VexJustification: "test justification", + KnownSince: time.Unix(1e9, 0), + }, + }}, + query: helpers.PkgToPurl(p6.Type, "", p6.Name, *p6.Version, "", []string{}), + want: []model.CertifyVulnOrCertifyVEXStatement{ + &model.CertifyVEXStatement{ + Subject: p1out, + Vulnerability: &model.Vulnerability{ + Type: "cve", + VulnerabilityIDs: []*model.VulnerabilityID{ + {VulnerabilityID: "cve-2019-13110"}, + }, + }, + VexJustification: "test justification", + KnownSince: time.Unix(1e9, 0), + }, + }, + wantErr: false, + }, + { + name: "Happy Path - Vuln", + InPkg: []*model.PkgInputSpec{p1, p6, p5}, + // p1 is a dependency ONLY of p6 + InIsDependency: []IsDependency{ + { + P1: *p6, + P2: *p1, + MF: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + ID: model.IsDependencyInputSpec{ + Justification: "test justification", + }, + }, + }, + InHasMetadata: []HasMetadata{ + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p1, + }, + PMT: &model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + HM: model.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: helpers.PkgToPurl(p6.Type, "", p6.Name, *p6.Version, "", []string{}), + }, + }, + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p6, + }, + PMT: &model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + HM: model.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: helpers.PkgToPurl(p6.Type, "", p6.Name, *p6.Version, "", []string{}), + }, + }, + }, + // both p6 and p5 are top level packages, i.e. with an SBOM + InHasSBOM: []HasSBOM{ + { + Sub: model.PackageOrArtifactInput{ + Package: p6, + }, + HS: model.HasSBOMInputSpec{ + URI: "test uri", + }, + }, + { + Sub: model.PackageOrArtifactInput{ + Package: p5, + }, + HS: model.HasSBOMInputSpec{ + URI: "test uri", + }, + }, + }, + InVulnerability: []*model.VulnerabilityInputSpec{{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }}, + // vulnerability relates to p1 + InCertifyVuln: []CertifyVuln{{ + Pkg: p1, + Vuln: &model.VulnerabilityInputSpec{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }, + CertifyVuln: &model.ScanMetadataInput{ + Collector: "test collector", + Origin: "test origin", + ScannerVersion: "v1.0.0", + ScannerURI: "test scanner uri", + DbVersion: "2023.01.01", + DbURI: "test db uri", + TimeScanned: testdata.T1, + }, + }}, + query: helpers.PkgToPurl(p6.Type, "", p6.Name, *p6.Version, "", []string{}), + want: []model.CertifyVulnOrCertifyVEXStatement{ + &model.CertifyVuln{ + Package: p1out, + Vulnerability: &model.Vulnerability{ + Type: "cve", + VulnerabilityIDs: []*model.VulnerabilityID{ + {VulnerabilityID: "cve-2019-13110"}, + }, + }, + Metadata: &model.ScanMetadata{ + Collector: "test collector", + Origin: "test origin", + ScannerVersion: "v1.0.0", + ScannerURI: "test scanner uri", + DbVersion: "2023.01.01", + DbURI: "test db uri", + TimeScanned: testdata.T1, + }, + }, + }, + wantErr: false, + }, + { + name: "No package with SBOM found", + query: helpers.PkgToPurl(p6.Type, "", p6.Name, *p6.Version, "", []string{}), + wantErr: true, + }, + /* InHasMetadata: []HasMetadata{ + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p2, + }, + PMT: &model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + HM: model.HasMetadataInputSpec{ + Key: "cpe", + Value: "cpe:2.3:a:log4j-core:log4j_core:2.8.1:*:*:*:*:*:*:*", + }, + }, + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p6, + }, + PMT: &model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + HM: model.HasMetadataInputSpec{ + Key: "cpe", + Value: "cpe:2.3:a:grep:grep:3.6-1:*:*:*:*:*:*:*", + }, + }, + },*/ + } + ctx := s.Ctx + for _, tt := range tests { + s.Run(tt.name, func() { + t := s.T() + b, err := GetBackend(s.Client) + if err != nil { + t.Fatalf("Could not instantiate testing backend: %v", err) + } + + _, err = b.IngestPackages(ctx, tt.InPkg) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + + for _, isDependency := range tt.InIsDependency { + _, err := b.IngestDependency(ctx, isDependency.P1, isDependency.P2, isDependency.MF, isDependency.ID) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + for _, hasMetadata := range tt.InHasMetadata { + _, err := b.IngestHasMetadata(ctx, hasMetadata.Sub, hasMetadata.PMT, hasMetadata.HM) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + for _, hasSBOM := range tt.InHasSBOM { + _, err := b.IngestHasSbom(ctx, hasSBOM.Sub, hasSBOM.HS) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + _, err = b.IngestVulnerabilities(ctx, tt.InVulnerability) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + + for _, vexStatement := range tt.InCertifyVexStatement { + _, err := b.IngestVEXStatement(ctx, vexStatement.Sub, vexStatement.Vuln, vexStatement.In) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + for _, vuln := range tt.InCertifyVuln { + _, err := b.IngestCertifyVuln(ctx, *vuln.Pkg, *vuln.Vuln, *vuln.CertifyVuln) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + got, err := b.FindVulnerability(ctx, tt.query) + if (err != nil) != tt.wantErr { + t.Errorf("FindTopLevelPackagesRelatedToVulnerability error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(tt.want, got, ignoreID); diff != "" { + t.Errorf("Unexpected results. (-want +got):\n%s", diff) + } + }) + } +} + +func (s *Suite) Test_FindVulnerabilityCPE() { + type IsDependency struct { + P1 model.PkgInputSpec + P2 model.PkgInputSpec + MF model.MatchFlags + ID model.IsDependencyInputSpec + } + type HasMetadata struct { + Sub model.PackageSourceOrArtifactInput + PMT *model.MatchFlags + HM model.HasMetadataInputSpec + } + type HasSBOM struct { + Sub model.PackageOrArtifactInput + HS model.HasSBOMInputSpec + } + type CertifyVEXStatement struct { + Sub model.PackageOrArtifactInput + Vuln model.VulnerabilityInputSpec + In model.VexStatementInputSpec + } + type CertifyVuln struct { + Pkg *model.PkgInputSpec + Vuln *model.VulnerabilityInputSpec + CertifyVuln *model.ScanMetadataInput + } + tests := []struct { + name string + InPkg []*model.PkgInputSpec + InIsDependency []IsDependency + InHasMetadata []HasMetadata + InHasSBOM []HasSBOM + InVulnerability []*model.VulnerabilityInputSpec + InCertifyVexStatement []CertifyVEXStatement + InCertifyVuln []CertifyVuln + query string + want []model.CertifyVulnOrCertifyVEXStatement + wantErr bool + }{ + { + name: "Happy Path - VEX", + InPkg: []*model.PkgInputSpec{p1, p6, p5}, + // p1 is a dependency ONLY of p6 + InIsDependency: []IsDependency{ + { + P1: *p6, + P2: *p1, + MF: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + ID: model.IsDependencyInputSpec{ + Justification: "test justification", + }, + }, + }, + InHasMetadata: []HasMetadata{ + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p1, + }, + PMT: &model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + HM: model.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: helpers.PkgToPurl(p6.Type, "", p6.Name, *p6.Version, "", []string{}), + }, + }, + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p6, + }, + PMT: &model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + HM: model.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: helpers.PkgToPurl(p6.Type, "", p6.Name, *p6.Version, "", []string{}), + }, + }, + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p6, + }, + PMT: &model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + HM: model.HasMetadataInputSpec{ + Key: "cpe", + Value: "cpe:/o:redhat:enterprise_linux:7::server", + }, + }, + }, + // both p6 and p5 are top level packages, i.e. with an SBOM + InHasSBOM: []HasSBOM{ + { + Sub: model.PackageOrArtifactInput{ + Package: p6, + }, + HS: model.HasSBOMInputSpec{ + URI: "test uri", + }, + }, + { + Sub: model.PackageOrArtifactInput{ + Package: p5, + }, + HS: model.HasSBOMInputSpec{ + URI: "test uri", + }, + }, + }, + InVulnerability: []*model.VulnerabilityInputSpec{{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }}, + // vulnerability relates to p1 + InCertifyVexStatement: []CertifyVEXStatement{{ + Sub: model.PackageOrArtifactInput{ + Package: p1, + }, + Vuln: model.VulnerabilityInputSpec{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }, + In: model.VexStatementInputSpec{ + VexJustification: "test justification", + KnownSince: time.Unix(1e9, 0), + }, + }}, + query: "cpe:/o:redhat:enterprise_linux:7::server", + want: []model.CertifyVulnOrCertifyVEXStatement{ + &model.CertifyVEXStatement{ + Subject: p1out, + Vulnerability: &model.Vulnerability{ + Type: "cve", + VulnerabilityIDs: []*model.VulnerabilityID{ + {VulnerabilityID: "cve-2019-13110"}, + }, + }, + VexJustification: "test justification", + KnownSince: time.Unix(1e9, 0), + }, + }, + wantErr: false, + }, + { + name: "Happy Path - Vuln", + InPkg: []*model.PkgInputSpec{p1, p6, p5}, + // p1 is a dependency ONLY of p6 + InIsDependency: []IsDependency{ + { + P1: *p6, + P2: *p1, + MF: model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + ID: model.IsDependencyInputSpec{ + Justification: "test justification", + }, + }, + }, + InHasMetadata: []HasMetadata{ + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p1, + }, + PMT: &model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + HM: model.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: helpers.PkgToPurl(p6.Type, "", p6.Name, *p6.Version, "", []string{}), + }, + }, + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p6, + }, + PMT: &model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + HM: model.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: helpers.PkgToPurl(p6.Type, "", p6.Name, *p6.Version, "", []string{}), + }, + }, + { + Sub: model.PackageSourceOrArtifactInput{ + Package: p6, + }, + PMT: &model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}, + HM: model.HasMetadataInputSpec{ + Key: "cpe", + Value: "cpe:/o:redhat:enterprise_linux:7::server", + }, + }, + }, + // both p6 and p5 are top level packages, i.e. with an SBOM + InHasSBOM: []HasSBOM{ + { + Sub: model.PackageOrArtifactInput{ + Package: p6, + }, + HS: model.HasSBOMInputSpec{ + URI: "test uri", + }, + }, + { + Sub: model.PackageOrArtifactInput{ + Package: p5, + }, + HS: model.HasSBOMInputSpec{ + URI: "test uri", + }, + }, + }, + InVulnerability: []*model.VulnerabilityInputSpec{{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }}, + // vulnerability relates to p1 + InCertifyVuln: []CertifyVuln{{ + Pkg: p1, + Vuln: &model.VulnerabilityInputSpec{ + Type: "cve", + VulnerabilityID: "CVE-2019-13110", + }, + CertifyVuln: &model.ScanMetadataInput{ + Collector: "test collector", + Origin: "test origin", + ScannerVersion: "v1.0.0", + ScannerURI: "test scanner uri", + DbVersion: "2023.01.01", + DbURI: "test db uri", + TimeScanned: testdata.T1, + }, + }}, + query: "cpe:/o:redhat:enterprise_linux:7::server", + want: []model.CertifyVulnOrCertifyVEXStatement{ + &model.CertifyVuln{ + Package: p1out, + Vulnerability: &model.Vulnerability{ + Type: "cve", + VulnerabilityIDs: []*model.VulnerabilityID{ + {VulnerabilityID: "cve-2019-13110"}, + }, + }, + Metadata: &model.ScanMetadata{ + Collector: "test collector", + Origin: "test origin", + ScannerVersion: "v1.0.0", + ScannerURI: "test scanner uri", + DbVersion: "2023.01.01", + DbURI: "test db uri", + TimeScanned: testdata.T1, + }, + }, + }, + wantErr: false, + }, + { + name: "No package with SBOM found", + query: "cpe:/o:redhat:enterprise_linux:7::server", + wantErr: true, + }, + } + ctx := s.Ctx + for _, tt := range tests { + s.Run(tt.name, func() { + t := s.T() + b, err := GetBackend(s.Client) + if err != nil { + t.Fatalf("Could not instantiate testing backend: %v", err) + } + + _, err = b.IngestPackages(ctx, tt.InPkg) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + + for _, isDependency := range tt.InIsDependency { + _, err := b.IngestDependency(ctx, isDependency.P1, isDependency.P2, isDependency.MF, isDependency.ID) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + for _, hasMetadata := range tt.InHasMetadata { + _, err := b.IngestHasMetadata(ctx, hasMetadata.Sub, hasMetadata.PMT, hasMetadata.HM) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + for _, hasSBOM := range tt.InHasSBOM { + _, err := b.IngestHasSbom(ctx, hasSBOM.Sub, hasSBOM.HS) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + _, err = b.IngestVulnerabilities(ctx, tt.InVulnerability) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + + for _, vexStatement := range tt.InCertifyVexStatement { + _, err := b.IngestVEXStatement(ctx, vexStatement.Sub, vexStatement.Vuln, vexStatement.In) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + for _, vuln := range tt.InCertifyVuln { + _, err := b.IngestCertifyVuln(ctx, *vuln.Pkg, *vuln.Vuln, *vuln.CertifyVuln) + if err != nil { + t.Fatalf("did not get expected ingest error, %v", err) + } + if err != nil { + return + } + } + + got, err := b.FindVulnerabilityCPE(ctx, tt.query) + if (err != nil) != tt.wantErr { + t.Errorf("FindTopLevelPackagesRelatedToVulnerability error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(tt.want, got, ignoreID); diff != "" { + t.Errorf("Unexpected results. (-want +got):\n%s", diff) + } + }) + } +} diff --git a/pkg/assembler/backends/ent/backend/vulnEqual_test.go b/pkg/assembler/backends/ent/backend/vulnEqual_test.go index 4550ac8b41..d5360aafb2 100644 --- a/pkg/assembler/backends/ent/backend/vulnEqual_test.go +++ b/pkg/assembler/backends/ent/backend/vulnEqual_test.go @@ -608,23 +608,20 @@ func (s *Suite) TestIngestVulnEquals() { ExpVulnEqual: []*model.VulnEqual{ &model.VulnEqual{ Vulnerabilities: []*model.Vulnerability{ - &model.Vulnerability{ - Type: "osv", - VulnerabilityIDs: []*model.VulnerabilityID{o2out}, - }, &model.Vulnerability{ Type: "ghsa", VulnerabilityIDs: []*model.VulnerabilityID{g2out}, }, + &model.Vulnerability{ + Type: "osv", + VulnerabilityIDs: []*model.VulnerabilityID{o2out}, + }, }, Justification: "test justification", }, }, }, } - ignoreID := cmp.FilterPath(func(p cmp.Path) bool { - return strings.Compare(".ID", p[len(p)-1].String()) == 0 - }, cmp.Ignore()) ctx := context.Background() for _, test := range tests { s.Run(test.Name, func() { @@ -653,7 +650,7 @@ func (s *Suite) TestIngestVulnEquals() { if err != nil { return } - if diff := cmp.Diff(test.ExpVulnEqual, got, ignoreID); diff != "" { + if diff := cmp.Diff(test.ExpVulnEqual, got, IngestPredicatesCmpOpts...); diff != "" { t.Errorf("Unexpected results. (-want +got):\n%s", diff) } }) diff --git a/pkg/assembler/backends/ent/backend/vulnerability.go b/pkg/assembler/backends/ent/backend/vulnerability.go index 0e7338adb8..c552baa3bf 100644 --- a/pkg/assembler/backends/ent/backend/vulnerability.go +++ b/pkg/assembler/backends/ent/backend/vulnerability.go @@ -26,18 +26,29 @@ import ( "github.com/guacsec/guac/pkg/assembler/backends/ent/vulnerabilitytype" "github.com/guacsec/guac/pkg/assembler/graphql/model" "github.com/vektah/gqlparser/v2/gqlerror" + "golang.org/x/sync/errgroup" ) const NoVuln = "novuln" func (b *EntBackend) IngestVulnerabilities(ctx context.Context, vulns []*model.VulnerabilityInputSpec) ([]*model.Vulnerability, error) { - var modelVulnerabilities []*model.Vulnerability - for _, vuln := range vulns { - modelVuln, err := b.IngestVulnerability(ctx, *vuln) - if err != nil { - return nil, gqlerror.Errorf("IngestVulnerability failed with err: %v", err) - } - modelVulnerabilities = append(modelVulnerabilities, modelVuln) + var modelVulnerabilities = make([]*model.Vulnerability, len(vulns)) + eg, ctx := errgroup.WithContext(ctx) + for i := range vulns { + index := i + vuln := vulns[index] + concurrently(eg, func() error { + modelVuln, err := b.IngestVulnerability(ctx, *vuln) + if err == nil { + modelVulnerabilities[index] = modelVuln + return err + } else { + return gqlerror.Errorf("IngestVulnerability failed with err: %v", err) + } + }) + } + if err := eg.Wait(); err != nil { + return nil, err } return modelVulnerabilities, nil } diff --git a/pkg/assembler/backends/ent/backend/vulnerability_test.go b/pkg/assembler/backends/ent/backend/vulnerability_test.go index 07eff26cfb..f9267d4544 100644 --- a/pkg/assembler/backends/ent/backend/vulnerability_test.go +++ b/pkg/assembler/backends/ent/backend/vulnerability_test.go @@ -352,9 +352,6 @@ func (s *Suite) TestIngestVulnerabilities() { ingests: []*model.VulnerabilityInputSpec{c1, c2, c3}, exp: []*model.Vulnerability{{}, {}, {}}, }} - ignoreID := cmp.FilterPath(func(p cmp.Path) bool { - return strings.Compare(".ID", p[len(p)-1].String()) == 0 - }, cmp.Ignore()) for _, test := range tests { s.Run(test.name, func() { ctx := s.Ctx @@ -368,7 +365,7 @@ func (s *Suite) TestIngestVulnerabilities() { t.Fatalf("ingest error: %v", err) return } - if diff := cmp.Diff(test.exp, got, ignoreID); diff != "" { + if diff := cmp.Diff(test.exp, got, IngestPredicatesCmpOpts...); diff != "" { t.Errorf("Unexpected results. (-want +got):\n%s", diff) } }) diff --git a/pkg/assembler/backends/ent/client.go b/pkg/assembler/backends/ent/client.go index ecbff61a5b..64d8d225a9 100644 --- a/pkg/assembler/backends/ent/client.go +++ b/pkg/assembler/backends/ent/client.go @@ -3714,6 +3714,22 @@ func (c *PackageVersionClient) QueryEqualPackages(pv *PackageVersion) *PkgEqualQ return query } +// QueryHasMetadata queries the has_metadata edge of a PackageVersion. +func (c *PackageVersionClient) QueryHasMetadata(pv *PackageVersion) *HasMetadataQuery { + query := (&HasMetadataClient{config: c.config}).Query() + query.path = func(context.Context) (fromV *sql.Selector, _ error) { + id := pv.ID + step := sqlgraph.NewStep( + sqlgraph.From(packageversion.Table, packageversion.FieldID, id), + sqlgraph.To(hasmetadata.Table, hasmetadata.FieldID), + sqlgraph.Edge(sqlgraph.O2M, true, packageversion.HasMetadataTable, packageversion.HasMetadataColumn), + ) + fromV = sqlgraph.Neighbors(pv.driver.Dialect(), step) + return fromV, nil + } + return query +} + // Hooks returns the client hooks. func (c *PackageVersionClient) Hooks() []Hook { return c.hooks.PackageVersion diff --git a/pkg/assembler/backends/ent/gql_collection.go b/pkg/assembler/backends/ent/gql_collection.go index 997c329e25..6a53c8f96e 100644 --- a/pkg/assembler/backends/ent/gql_collection.go +++ b/pkg/assembler/backends/ent/gql_collection.go @@ -2264,6 +2264,18 @@ func (pv *PackageVersionQuery) collectField(ctx context.Context, opCtx *graphql. pv.WithNamedEqualPackages(alias, func(wq *PkgEqualQuery) { *wq = *query }) + case "hasMetadata": + var ( + alias = field.Alias + path = append(path, alias) + query = (&HasMetadataClient{config: pv.config}).Query() + ) + if err := query.collectField(ctx, opCtx, field, path, satisfies...); err != nil { + return err + } + pv.WithNamedHasMetadata(alias, func(wq *HasMetadataQuery) { + *wq = *query + }) case "nameID": if _, ok := fieldSeen[packageversion.FieldNameID]; !ok { selectedFields = append(selectedFields, packageversion.FieldNameID) diff --git a/pkg/assembler/backends/ent/gql_edge.go b/pkg/assembler/backends/ent/gql_edge.go index 3d1a431f76..4b4bf58641 100644 --- a/pkg/assembler/backends/ent/gql_edge.go +++ b/pkg/assembler/backends/ent/gql_edge.go @@ -464,6 +464,18 @@ func (pv *PackageVersion) EqualPackages(ctx context.Context) (result []*PkgEqual return result, err } +func (pv *PackageVersion) HasMetadata(ctx context.Context) (result []*HasMetadata, err error) { + if fc := graphql.GetFieldContext(ctx); fc != nil && fc.Field.Alias != "" { + result, err = pv.NamedHasMetadata(graphql.GetFieldContext(ctx).Field.Alias) + } else { + result, err = pv.Edges.HasMetadataOrErr() + } + if IsNotLoaded(err) { + result, err = pv.QueryHasMetadata().All(ctx) + } + return result, err +} + func (pe *PkgEqual) Packages(ctx context.Context) (result []*PackageVersion, err error) { if fc := graphql.GetFieldContext(ctx); fc != nil && fc.Field.Alias != "" { result, err = pe.NamedPackages(graphql.GetFieldContext(ctx).Field.Alias) diff --git a/pkg/assembler/backends/ent/migrate/schema.go b/pkg/assembler/backends/ent/migrate/schema.go index 25e25e8a21..5c0425d2aa 100644 --- a/pkg/assembler/backends/ent/migrate/schema.go +++ b/pkg/assembler/backends/ent/migrate/schema.go @@ -417,6 +417,16 @@ var ( Where: "dependent_package_name_id IS NULL AND dependent_package_version_id IS NOT NULL", }, }, + { + Name: "dependency_dependent_package_name_id_dependent_package_version_id_package_id", + Unique: false, + Columns: []*schema.Column{DependenciesColumns[7], DependenciesColumns[8], DependenciesColumns[6]}, + }, + { + Name: "dependency_dependent_package_version_id", + Unique: false, + Columns: []*schema.Column{DependenciesColumns[8]}, + }, }, } // HasMetadataColumns holds the columns for the "has_metadata" table. @@ -466,37 +476,47 @@ var ( }, Indexes: []*schema.Index{ { - Name: "hasmetadata_timestamp_key_value_justification_origin_collector_source_id", + Name: "hasmetadata_key_value_justification_origin_collector_source_id", Unique: true, - Columns: []*schema.Column{HasMetadataColumns[1], HasMetadataColumns[2], HasMetadataColumns[3], HasMetadataColumns[4], HasMetadataColumns[5], HasMetadataColumns[6], HasMetadataColumns[7]}, + Columns: []*schema.Column{HasMetadataColumns[2], HasMetadataColumns[3], HasMetadataColumns[4], HasMetadataColumns[5], HasMetadataColumns[6], HasMetadataColumns[7]}, Annotation: &entsql.IndexAnnotation{ Where: "source_id IS NOT NULL AND package_version_id IS NULL AND package_name_id IS NULL AND artifact_id IS NULL", }, }, { - Name: "hasmetadata_timestamp_key_value_justification_origin_collector_package_version_id", + Name: "hasmetadata_key_value_justification_origin_collector_package_version_id", Unique: true, - Columns: []*schema.Column{HasMetadataColumns[1], HasMetadataColumns[2], HasMetadataColumns[3], HasMetadataColumns[4], HasMetadataColumns[5], HasMetadataColumns[6], HasMetadataColumns[8]}, + Columns: []*schema.Column{HasMetadataColumns[2], HasMetadataColumns[3], HasMetadataColumns[4], HasMetadataColumns[5], HasMetadataColumns[6], HasMetadataColumns[8]}, Annotation: &entsql.IndexAnnotation{ Where: "source_id IS NULL AND package_version_id IS NOT NULL AND package_name_id IS NULL AND artifact_id IS NULL", }, }, { - Name: "hasmetadata_timestamp_key_value_justification_origin_collector_package_name_id", + Name: "hasmetadata_key_value_justification_origin_collector_package_name_id", Unique: true, - Columns: []*schema.Column{HasMetadataColumns[1], HasMetadataColumns[2], HasMetadataColumns[3], HasMetadataColumns[4], HasMetadataColumns[5], HasMetadataColumns[6], HasMetadataColumns[9]}, + Columns: []*schema.Column{HasMetadataColumns[2], HasMetadataColumns[3], HasMetadataColumns[4], HasMetadataColumns[5], HasMetadataColumns[6], HasMetadataColumns[9]}, Annotation: &entsql.IndexAnnotation{ Where: "source_id IS NULL AND package_version_id IS NULL AND package_name_id IS NOT NULL AND artifact_id IS NULL", }, }, { - Name: "hasmetadata_timestamp_key_value_justification_origin_collector_artifact_id", + Name: "hasmetadata_key_value_justification_origin_collector_artifact_id", Unique: true, - Columns: []*schema.Column{HasMetadataColumns[1], HasMetadataColumns[2], HasMetadataColumns[3], HasMetadataColumns[4], HasMetadataColumns[5], HasMetadataColumns[6], HasMetadataColumns[10]}, + Columns: []*schema.Column{HasMetadataColumns[2], HasMetadataColumns[3], HasMetadataColumns[4], HasMetadataColumns[5], HasMetadataColumns[6], HasMetadataColumns[10]}, Annotation: &entsql.IndexAnnotation{ Where: "source_id IS NULL AND package_version_id IS NULL AND package_name_id IS NULL AND artifact_id IS NOT NULL", }, }, + { + Name: "hasmetadata_key_value_package_name_id_package_version_id", + Unique: false, + Columns: []*schema.Column{HasMetadataColumns[2], HasMetadataColumns[3], HasMetadataColumns[9], HasMetadataColumns[8]}, + }, + { + Name: "hasmetadata_key_value", + Unique: false, + Columns: []*schema.Column{HasMetadataColumns[2], HasMetadataColumns[3]}, + }, }, } // HasSourceAtsColumns holds the columns for the "has_source_ats" table. @@ -804,6 +824,16 @@ var ( }, }, }, + { + Name: "packageversion_version_subpath_qualifiers_name_id", + Unique: true, + Columns: []*schema.Column{PackageVersionsColumns[1], PackageVersionsColumns[2], PackageVersionsColumns[3], PackageVersionsColumns[5]}, + }, + { + Name: "packageversion_name_id", + Unique: false, + Columns: []*schema.Column{PackageVersionsColumns[5]}, + }, }, } // PkgEqualsColumns holds the columns for the "pkg_equals" table. diff --git a/pkg/assembler/backends/ent/mutation.go b/pkg/assembler/backends/ent/mutation.go index d1e0be0d57..66f786e1fa 100644 --- a/pkg/assembler/backends/ent/mutation.go +++ b/pkg/assembler/backends/ent/mutation.go @@ -13527,6 +13527,9 @@ type PackageVersionMutation struct { equal_packages map[int]struct{} removedequal_packages map[int]struct{} clearedequal_packages bool + has_metadata map[int]struct{} + removedhas_metadata map[int]struct{} + clearedhas_metadata bool done bool oldValue func(context.Context) (*PackageVersion, error) predicates []predicate.PackageVersion @@ -14028,6 +14031,60 @@ func (m *PackageVersionMutation) ResetEqualPackages() { m.removedequal_packages = nil } +// AddHasMetadatumIDs adds the "has_metadata" edge to the HasMetadata entity by ids. +func (m *PackageVersionMutation) AddHasMetadatumIDs(ids ...int) { + if m.has_metadata == nil { + m.has_metadata = make(map[int]struct{}) + } + for i := range ids { + m.has_metadata[ids[i]] = struct{}{} + } +} + +// ClearHasMetadata clears the "has_metadata" edge to the HasMetadata entity. +func (m *PackageVersionMutation) ClearHasMetadata() { + m.clearedhas_metadata = true +} + +// HasMetadataCleared reports if the "has_metadata" edge to the HasMetadata entity was cleared. +func (m *PackageVersionMutation) HasMetadataCleared() bool { + return m.clearedhas_metadata +} + +// RemoveHasMetadatumIDs removes the "has_metadata" edge to the HasMetadata entity by IDs. +func (m *PackageVersionMutation) RemoveHasMetadatumIDs(ids ...int) { + if m.removedhas_metadata == nil { + m.removedhas_metadata = make(map[int]struct{}) + } + for i := range ids { + delete(m.has_metadata, ids[i]) + m.removedhas_metadata[ids[i]] = struct{}{} + } +} + +// RemovedHasMetadata returns the removed IDs of the "has_metadata" edge to the HasMetadata entity. +func (m *PackageVersionMutation) RemovedHasMetadataIDs() (ids []int) { + for id := range m.removedhas_metadata { + ids = append(ids, id) + } + return +} + +// HasMetadataIDs returns the "has_metadata" edge IDs in the mutation. +func (m *PackageVersionMutation) HasMetadataIDs() (ids []int) { + for id := range m.has_metadata { + ids = append(ids, id) + } + return +} + +// ResetHasMetadata resets all changes to the "has_metadata" edge. +func (m *PackageVersionMutation) ResetHasMetadata() { + m.has_metadata = nil + m.clearedhas_metadata = false + m.removedhas_metadata = nil +} + // Where appends a list predicates to the PackageVersionMutation builder. func (m *PackageVersionMutation) Where(ps ...predicate.PackageVersion) { m.predicates = append(m.predicates, ps...) @@ -14241,7 +14298,7 @@ func (m *PackageVersionMutation) ResetField(name string) error { // AddedEdges returns all edge names that were set/added in this mutation. func (m *PackageVersionMutation) AddedEdges() []string { - edges := make([]string, 0, 4) + edges := make([]string, 0, 5) if m.name != nil { edges = append(edges, packageversion.EdgeName) } @@ -14254,6 +14311,9 @@ func (m *PackageVersionMutation) AddedEdges() []string { if m.equal_packages != nil { edges = append(edges, packageversion.EdgeEqualPackages) } + if m.has_metadata != nil { + edges = append(edges, packageversion.EdgeHasMetadata) + } return edges } @@ -14283,13 +14343,19 @@ func (m *PackageVersionMutation) AddedIDs(name string) []ent.Value { ids = append(ids, id) } return ids + case packageversion.EdgeHasMetadata: + ids := make([]ent.Value, 0, len(m.has_metadata)) + for id := range m.has_metadata { + ids = append(ids, id) + } + return ids } return nil } // RemovedEdges returns all edge names that were removed in this mutation. func (m *PackageVersionMutation) RemovedEdges() []string { - edges := make([]string, 0, 4) + edges := make([]string, 0, 5) if m.removedoccurrences != nil { edges = append(edges, packageversion.EdgeOccurrences) } @@ -14299,6 +14365,9 @@ func (m *PackageVersionMutation) RemovedEdges() []string { if m.removedequal_packages != nil { edges = append(edges, packageversion.EdgeEqualPackages) } + if m.removedhas_metadata != nil { + edges = append(edges, packageversion.EdgeHasMetadata) + } return edges } @@ -14324,13 +14393,19 @@ func (m *PackageVersionMutation) RemovedIDs(name string) []ent.Value { ids = append(ids, id) } return ids + case packageversion.EdgeHasMetadata: + ids := make([]ent.Value, 0, len(m.removedhas_metadata)) + for id := range m.removedhas_metadata { + ids = append(ids, id) + } + return ids } return nil } // ClearedEdges returns all edge names that were cleared in this mutation. func (m *PackageVersionMutation) ClearedEdges() []string { - edges := make([]string, 0, 4) + edges := make([]string, 0, 5) if m.clearedname { edges = append(edges, packageversion.EdgeName) } @@ -14343,6 +14418,9 @@ func (m *PackageVersionMutation) ClearedEdges() []string { if m.clearedequal_packages { edges = append(edges, packageversion.EdgeEqualPackages) } + if m.clearedhas_metadata { + edges = append(edges, packageversion.EdgeHasMetadata) + } return edges } @@ -14358,6 +14436,8 @@ func (m *PackageVersionMutation) EdgeCleared(name string) bool { return m.clearedsbom case packageversion.EdgeEqualPackages: return m.clearedequal_packages + case packageversion.EdgeHasMetadata: + return m.clearedhas_metadata } return false } @@ -14389,6 +14469,9 @@ func (m *PackageVersionMutation) ResetEdge(name string) error { case packageversion.EdgeEqualPackages: m.ResetEqualPackages() return nil + case packageversion.EdgeHasMetadata: + m.ResetHasMetadata() + return nil } return fmt.Errorf("unknown PackageVersion edge %s", name) } diff --git a/pkg/assembler/backends/ent/packageversion.go b/pkg/assembler/backends/ent/packageversion.go index 01249a05f0..e786129770 100644 --- a/pkg/assembler/backends/ent/packageversion.go +++ b/pkg/assembler/backends/ent/packageversion.go @@ -45,15 +45,18 @@ type PackageVersionEdges struct { Sbom []*BillOfMaterials `json:"sbom,omitempty"` // EqualPackages holds the value of the equal_packages edge. EqualPackages []*PkgEqual `json:"equal_packages,omitempty"` + // HasMetadata holds the value of the has_metadata edge. + HasMetadata []*HasMetadata `json:"has_metadata,omitempty"` // loadedTypes holds the information for reporting if a // type was loaded (or requested) in eager-loading or not. - loadedTypes [4]bool + loadedTypes [5]bool // totalCount holds the count of the edges above. - totalCount [4]map[string]int + totalCount [5]map[string]int namedOccurrences map[string][]*Occurrence namedSbom map[string][]*BillOfMaterials namedEqualPackages map[string][]*PkgEqual + namedHasMetadata map[string][]*HasMetadata } // NameOrErr returns the Name value or an error if the edge @@ -96,6 +99,15 @@ func (e PackageVersionEdges) EqualPackagesOrErr() ([]*PkgEqual, error) { return nil, &NotLoadedError{edge: "equal_packages"} } +// HasMetadataOrErr returns the HasMetadata value or an error if the edge +// was not loaded in eager-loading. +func (e PackageVersionEdges) HasMetadataOrErr() ([]*HasMetadata, error) { + if e.loadedTypes[4] { + return e.HasMetadata, nil + } + return nil, &NotLoadedError{edge: "has_metadata"} +} + // scanValues returns the types for scanning values from sql.Rows. func (*PackageVersion) scanValues(columns []string) ([]any, error) { values := make([]any, len(columns)) @@ -193,6 +205,11 @@ func (pv *PackageVersion) QueryEqualPackages() *PkgEqualQuery { return NewPackageVersionClient(pv.config).QueryEqualPackages(pv) } +// QueryHasMetadata queries the "has_metadata" edge of the PackageVersion entity. +func (pv *PackageVersion) QueryHasMetadata() *HasMetadataQuery { + return NewPackageVersionClient(pv.config).QueryHasMetadata(pv) +} + // Update returns a builder for updating this PackageVersion. // Note that you need to call PackageVersion.Unwrap() before calling this method if this PackageVersion // was returned from a transaction, and the transaction was committed or rolled back. @@ -306,5 +323,29 @@ func (pv *PackageVersion) appendNamedEqualPackages(name string, edges ...*PkgEqu } } +// NamedHasMetadata returns the HasMetadata named value or an error if the edge was not +// loaded in eager-loading with this name. +func (pv *PackageVersion) NamedHasMetadata(name string) ([]*HasMetadata, error) { + if pv.Edges.namedHasMetadata == nil { + return nil, &NotLoadedError{edge: name} + } + nodes, ok := pv.Edges.namedHasMetadata[name] + if !ok { + return nil, &NotLoadedError{edge: name} + } + return nodes, nil +} + +func (pv *PackageVersion) appendNamedHasMetadata(name string, edges ...*HasMetadata) { + if pv.Edges.namedHasMetadata == nil { + pv.Edges.namedHasMetadata = make(map[string][]*HasMetadata) + } + if len(edges) == 0 { + pv.Edges.namedHasMetadata[name] = []*HasMetadata{} + } else { + pv.Edges.namedHasMetadata[name] = append(pv.Edges.namedHasMetadata[name], edges...) + } +} + // PackageVersions is a parsable slice of PackageVersion. type PackageVersions []*PackageVersion diff --git a/pkg/assembler/backends/ent/packageversion/packageversion.go b/pkg/assembler/backends/ent/packageversion/packageversion.go index 2496aa2c30..8e1d9488e9 100644 --- a/pkg/assembler/backends/ent/packageversion/packageversion.go +++ b/pkg/assembler/backends/ent/packageversion/packageversion.go @@ -30,6 +30,8 @@ const ( EdgeSbom = "sbom" // EdgeEqualPackages holds the string denoting the equal_packages edge name in mutations. EdgeEqualPackages = "equal_packages" + // EdgeHasMetadata holds the string denoting the has_metadata edge name in mutations. + EdgeHasMetadata = "has_metadata" // Table holds the table name of the packageversion in the database. Table = "package_versions" // NameTable is the table that holds the name relation/edge. @@ -58,6 +60,13 @@ const ( // EqualPackagesInverseTable is the table name for the PkgEqual entity. // It exists in this package in order to avoid circular dependency with the "pkgequal" package. EqualPackagesInverseTable = "pkg_equals" + // HasMetadataTable is the table that holds the has_metadata relation/edge. + HasMetadataTable = "has_metadata" + // HasMetadataInverseTable is the table name for the HasMetadata entity. + // It exists in this package in order to avoid circular dependency with the "hasmetadata" package. + HasMetadataInverseTable = "has_metadata" + // HasMetadataColumn is the table column denoting the has_metadata relation/edge. + HasMetadataColumn = "package_version_id" ) // Columns holds all SQL columns for packageversion fields. @@ -169,6 +178,20 @@ func ByEqualPackages(term sql.OrderTerm, terms ...sql.OrderTerm) OrderOption { sqlgraph.OrderByNeighborTerms(s, newEqualPackagesStep(), append([]sql.OrderTerm{term}, terms...)...) } } + +// ByHasMetadataCount orders the results by has_metadata count. +func ByHasMetadataCount(opts ...sql.OrderTermOption) OrderOption { + return func(s *sql.Selector) { + sqlgraph.OrderByNeighborsCount(s, newHasMetadataStep(), opts...) + } +} + +// ByHasMetadata orders the results by has_metadata terms. +func ByHasMetadata(term sql.OrderTerm, terms ...sql.OrderTerm) OrderOption { + return func(s *sql.Selector) { + sqlgraph.OrderByNeighborTerms(s, newHasMetadataStep(), append([]sql.OrderTerm{term}, terms...)...) + } +} func newNameStep() *sqlgraph.Step { return sqlgraph.NewStep( sqlgraph.From(Table, FieldID), @@ -197,3 +220,10 @@ func newEqualPackagesStep() *sqlgraph.Step { sqlgraph.Edge(sqlgraph.M2M, true, EqualPackagesTable, EqualPackagesPrimaryKey...), ) } +func newHasMetadataStep() *sqlgraph.Step { + return sqlgraph.NewStep( + sqlgraph.From(Table, FieldID), + sqlgraph.To(HasMetadataInverseTable, FieldID), + sqlgraph.Edge(sqlgraph.O2M, true, HasMetadataTable, HasMetadataColumn), + ) +} diff --git a/pkg/assembler/backends/ent/packageversion/where.go b/pkg/assembler/backends/ent/packageversion/where.go index e51a41d167..8894385a1f 100644 --- a/pkg/assembler/backends/ent/packageversion/where.go +++ b/pkg/assembler/backends/ent/packageversion/where.go @@ -390,6 +390,29 @@ func HasEqualPackagesWith(preds ...predicate.PkgEqual) predicate.PackageVersion }) } +// HasHasMetadata applies the HasEdge predicate on the "has_metadata" edge. +func HasHasMetadata() predicate.PackageVersion { + return predicate.PackageVersion(func(s *sql.Selector) { + step := sqlgraph.NewStep( + sqlgraph.From(Table, FieldID), + sqlgraph.Edge(sqlgraph.O2M, true, HasMetadataTable, HasMetadataColumn), + ) + sqlgraph.HasNeighbors(s, step) + }) +} + +// HasHasMetadataWith applies the HasEdge predicate on the "has_metadata" edge with a given conditions (other predicates). +func HasHasMetadataWith(preds ...predicate.HasMetadata) predicate.PackageVersion { + return predicate.PackageVersion(func(s *sql.Selector) { + step := newHasMetadataStep() + sqlgraph.HasNeighborsWith(s, step, func(s *sql.Selector) { + for _, p := range preds { + p(s) + } + }) + }) +} + // And groups predicates with the AND operator between them. func And(predicates ...predicate.PackageVersion) predicate.PackageVersion { return predicate.PackageVersion(sql.AndPredicates(predicates...)) diff --git a/pkg/assembler/backends/ent/packageversion_create.go b/pkg/assembler/backends/ent/packageversion_create.go index 0fc5c93fd3..5950f3a283 100644 --- a/pkg/assembler/backends/ent/packageversion_create.go +++ b/pkg/assembler/backends/ent/packageversion_create.go @@ -11,6 +11,7 @@ import ( "entgo.io/ent/dialect/sql/sqlgraph" "entgo.io/ent/schema/field" "github.com/guacsec/guac/pkg/assembler/backends/ent/billofmaterials" + "github.com/guacsec/guac/pkg/assembler/backends/ent/hasmetadata" "github.com/guacsec/guac/pkg/assembler/backends/ent/occurrence" "github.com/guacsec/guac/pkg/assembler/backends/ent/packagename" "github.com/guacsec/guac/pkg/assembler/backends/ent/packageversion" @@ -122,6 +123,21 @@ func (pvc *PackageVersionCreate) AddEqualPackages(p ...*PkgEqual) *PackageVersio return pvc.AddEqualPackageIDs(ids...) } +// AddHasMetadatumIDs adds the "has_metadata" edge to the HasMetadata entity by IDs. +func (pvc *PackageVersionCreate) AddHasMetadatumIDs(ids ...int) *PackageVersionCreate { + pvc.mutation.AddHasMetadatumIDs(ids...) + return pvc +} + +// AddHasMetadata adds the "has_metadata" edges to the HasMetadata entity. +func (pvc *PackageVersionCreate) AddHasMetadata(h ...*HasMetadata) *PackageVersionCreate { + ids := make([]int, len(h)) + for i := range h { + ids[i] = h[i].ID + } + return pvc.AddHasMetadatumIDs(ids...) +} + // Mutation returns the PackageVersionMutation object of the builder. func (pvc *PackageVersionCreate) Mutation() *PackageVersionMutation { return pvc.mutation @@ -292,6 +308,22 @@ func (pvc *PackageVersionCreate) createSpec() (*PackageVersion, *sqlgraph.Create } _spec.Edges = append(_spec.Edges, edge) } + if nodes := pvc.mutation.HasMetadataIDs(); len(nodes) > 0 { + edge := &sqlgraph.EdgeSpec{ + Rel: sqlgraph.O2M, + Inverse: true, + Table: packageversion.HasMetadataTable, + Columns: []string{packageversion.HasMetadataColumn}, + Bidi: false, + Target: &sqlgraph.EdgeTarget{ + IDSpec: sqlgraph.NewFieldSpec(hasmetadata.FieldID, field.TypeInt), + }, + } + for _, k := range nodes { + edge.Target.Nodes = append(edge.Target.Nodes, k) + } + _spec.Edges = append(_spec.Edges, edge) + } return _node, _spec } diff --git a/pkg/assembler/backends/ent/packageversion_query.go b/pkg/assembler/backends/ent/packageversion_query.go index 02e3bf7950..a8848dacb2 100644 --- a/pkg/assembler/backends/ent/packageversion_query.go +++ b/pkg/assembler/backends/ent/packageversion_query.go @@ -12,6 +12,7 @@ import ( "entgo.io/ent/dialect/sql/sqlgraph" "entgo.io/ent/schema/field" "github.com/guacsec/guac/pkg/assembler/backends/ent/billofmaterials" + "github.com/guacsec/guac/pkg/assembler/backends/ent/hasmetadata" "github.com/guacsec/guac/pkg/assembler/backends/ent/occurrence" "github.com/guacsec/guac/pkg/assembler/backends/ent/packagename" "github.com/guacsec/guac/pkg/assembler/backends/ent/packageversion" @@ -30,11 +31,13 @@ type PackageVersionQuery struct { withOccurrences *OccurrenceQuery withSbom *BillOfMaterialsQuery withEqualPackages *PkgEqualQuery + withHasMetadata *HasMetadataQuery modifiers []func(*sql.Selector) loadTotal []func(context.Context, []*PackageVersion) error withNamedOccurrences map[string]*OccurrenceQuery withNamedSbom map[string]*BillOfMaterialsQuery withNamedEqualPackages map[string]*PkgEqualQuery + withNamedHasMetadata map[string]*HasMetadataQuery // intermediate query (i.e. traversal path). sql *sql.Selector path func(context.Context) (*sql.Selector, error) @@ -159,6 +162,28 @@ func (pvq *PackageVersionQuery) QueryEqualPackages() *PkgEqualQuery { return query } +// QueryHasMetadata chains the current query on the "has_metadata" edge. +func (pvq *PackageVersionQuery) QueryHasMetadata() *HasMetadataQuery { + query := (&HasMetadataClient{config: pvq.config}).Query() + query.path = func(ctx context.Context) (fromU *sql.Selector, err error) { + if err := pvq.prepareQuery(ctx); err != nil { + return nil, err + } + selector := pvq.sqlQuery(ctx) + if err := selector.Err(); err != nil { + return nil, err + } + step := sqlgraph.NewStep( + sqlgraph.From(packageversion.Table, packageversion.FieldID, selector), + sqlgraph.To(hasmetadata.Table, hasmetadata.FieldID), + sqlgraph.Edge(sqlgraph.O2M, true, packageversion.HasMetadataTable, packageversion.HasMetadataColumn), + ) + fromU = sqlgraph.SetNeighbors(pvq.driver.Dialect(), step) + return fromU, nil + } + return query +} + // First returns the first PackageVersion entity from the query. // Returns a *NotFoundError when no PackageVersion was found. func (pvq *PackageVersionQuery) First(ctx context.Context) (*PackageVersion, error) { @@ -355,6 +380,7 @@ func (pvq *PackageVersionQuery) Clone() *PackageVersionQuery { withOccurrences: pvq.withOccurrences.Clone(), withSbom: pvq.withSbom.Clone(), withEqualPackages: pvq.withEqualPackages.Clone(), + withHasMetadata: pvq.withHasMetadata.Clone(), // clone intermediate query. sql: pvq.sql.Clone(), path: pvq.path, @@ -405,6 +431,17 @@ func (pvq *PackageVersionQuery) WithEqualPackages(opts ...func(*PkgEqualQuery)) return pvq } +// WithHasMetadata tells the query-builder to eager-load the nodes that are connected to +// the "has_metadata" edge. The optional arguments are used to configure the query builder of the edge. +func (pvq *PackageVersionQuery) WithHasMetadata(opts ...func(*HasMetadataQuery)) *PackageVersionQuery { + query := (&HasMetadataClient{config: pvq.config}).Query() + for _, opt := range opts { + opt(query) + } + pvq.withHasMetadata = query + return pvq +} + // GroupBy is used to group vertices by one or more fields/columns. // It is often used with aggregate functions, like: count, max, mean, min, sum. // @@ -483,11 +520,12 @@ func (pvq *PackageVersionQuery) sqlAll(ctx context.Context, hooks ...queryHook) var ( nodes = []*PackageVersion{} _spec = pvq.querySpec() - loadedTypes = [4]bool{ + loadedTypes = [5]bool{ pvq.withName != nil, pvq.withOccurrences != nil, pvq.withSbom != nil, pvq.withEqualPackages != nil, + pvq.withHasMetadata != nil, } ) _spec.ScanValues = func(columns []string) ([]any, error) { @@ -538,6 +576,13 @@ func (pvq *PackageVersionQuery) sqlAll(ctx context.Context, hooks ...queryHook) return nil, err } } + if query := pvq.withHasMetadata; query != nil { + if err := pvq.loadHasMetadata(ctx, query, nodes, + func(n *PackageVersion) { n.Edges.HasMetadata = []*HasMetadata{} }, + func(n *PackageVersion, e *HasMetadata) { n.Edges.HasMetadata = append(n.Edges.HasMetadata, e) }); err != nil { + return nil, err + } + } for name, query := range pvq.withNamedOccurrences { if err := pvq.loadOccurrences(ctx, query, nodes, func(n *PackageVersion) { n.appendNamedOccurrences(name) }, @@ -559,6 +604,13 @@ func (pvq *PackageVersionQuery) sqlAll(ctx context.Context, hooks ...queryHook) return nil, err } } + for name, query := range pvq.withNamedHasMetadata { + if err := pvq.loadHasMetadata(ctx, query, nodes, + func(n *PackageVersion) { n.appendNamedHasMetadata(name) }, + func(n *PackageVersion, e *HasMetadata) { n.appendNamedHasMetadata(name, e) }); err != nil { + return nil, err + } + } for i := range pvq.loadTotal { if err := pvq.loadTotal[i](ctx, nodes); err != nil { return nil, err @@ -723,6 +775,39 @@ func (pvq *PackageVersionQuery) loadEqualPackages(ctx context.Context, query *Pk } return nil } +func (pvq *PackageVersionQuery) loadHasMetadata(ctx context.Context, query *HasMetadataQuery, nodes []*PackageVersion, init func(*PackageVersion), assign func(*PackageVersion, *HasMetadata)) error { + fks := make([]driver.Value, 0, len(nodes)) + nodeids := make(map[int]*PackageVersion) + for i := range nodes { + fks = append(fks, nodes[i].ID) + nodeids[nodes[i].ID] = nodes[i] + if init != nil { + init(nodes[i]) + } + } + if len(query.ctx.Fields) > 0 { + query.ctx.AppendFieldOnce(hasmetadata.FieldPackageVersionID) + } + query.Where(predicate.HasMetadata(func(s *sql.Selector) { + s.Where(sql.InValues(s.C(packageversion.HasMetadataColumn), fks...)) + })) + neighbors, err := query.All(ctx) + if err != nil { + return err + } + for _, n := range neighbors { + fk := n.PackageVersionID + if fk == nil { + return fmt.Errorf(`foreign-key "package_version_id" is nil for node %v`, n.ID) + } + node, ok := nodeids[*fk] + if !ok { + return fmt.Errorf(`unexpected referenced foreign-key "package_version_id" returned %v for node %v`, *fk, n.ID) + } + assign(node, n) + } + return nil +} func (pvq *PackageVersionQuery) sqlCount(ctx context.Context) (int, error) { _spec := pvq.querySpec() @@ -853,6 +938,20 @@ func (pvq *PackageVersionQuery) WithNamedEqualPackages(name string, opts ...func return pvq } +// WithNamedHasMetadata tells the query-builder to eager-load the nodes that are connected to the "has_metadata" +// edge with the given name. The optional arguments are used to configure the query builder of the edge. +func (pvq *PackageVersionQuery) WithNamedHasMetadata(name string, opts ...func(*HasMetadataQuery)) *PackageVersionQuery { + query := (&HasMetadataClient{config: pvq.config}).Query() + for _, opt := range opts { + opt(query) + } + if pvq.withNamedHasMetadata == nil { + pvq.withNamedHasMetadata = make(map[string]*HasMetadataQuery) + } + pvq.withNamedHasMetadata[name] = query + return pvq +} + // PackageVersionGroupBy is the group-by builder for PackageVersion entities. type PackageVersionGroupBy struct { selector diff --git a/pkg/assembler/backends/ent/packageversion_update.go b/pkg/assembler/backends/ent/packageversion_update.go index 22179d2129..9f1949a53d 100644 --- a/pkg/assembler/backends/ent/packageversion_update.go +++ b/pkg/assembler/backends/ent/packageversion_update.go @@ -12,6 +12,7 @@ import ( "entgo.io/ent/dialect/sql/sqljson" "entgo.io/ent/schema/field" "github.com/guacsec/guac/pkg/assembler/backends/ent/billofmaterials" + "github.com/guacsec/guac/pkg/assembler/backends/ent/hasmetadata" "github.com/guacsec/guac/pkg/assembler/backends/ent/occurrence" "github.com/guacsec/guac/pkg/assembler/backends/ent/packagename" "github.com/guacsec/guac/pkg/assembler/backends/ent/packageversion" @@ -141,6 +142,21 @@ func (pvu *PackageVersionUpdate) AddEqualPackages(p ...*PkgEqual) *PackageVersio return pvu.AddEqualPackageIDs(ids...) } +// AddHasMetadatumIDs adds the "has_metadata" edge to the HasMetadata entity by IDs. +func (pvu *PackageVersionUpdate) AddHasMetadatumIDs(ids ...int) *PackageVersionUpdate { + pvu.mutation.AddHasMetadatumIDs(ids...) + return pvu +} + +// AddHasMetadata adds the "has_metadata" edges to the HasMetadata entity. +func (pvu *PackageVersionUpdate) AddHasMetadata(h ...*HasMetadata) *PackageVersionUpdate { + ids := make([]int, len(h)) + for i := range h { + ids[i] = h[i].ID + } + return pvu.AddHasMetadatumIDs(ids...) +} + // Mutation returns the PackageVersionMutation object of the builder. func (pvu *PackageVersionUpdate) Mutation() *PackageVersionMutation { return pvu.mutation @@ -215,6 +231,27 @@ func (pvu *PackageVersionUpdate) RemoveEqualPackages(p ...*PkgEqual) *PackageVer return pvu.RemoveEqualPackageIDs(ids...) } +// ClearHasMetadata clears all "has_metadata" edges to the HasMetadata entity. +func (pvu *PackageVersionUpdate) ClearHasMetadata() *PackageVersionUpdate { + pvu.mutation.ClearHasMetadata() + return pvu +} + +// RemoveHasMetadatumIDs removes the "has_metadata" edge to HasMetadata entities by IDs. +func (pvu *PackageVersionUpdate) RemoveHasMetadatumIDs(ids ...int) *PackageVersionUpdate { + pvu.mutation.RemoveHasMetadatumIDs(ids...) + return pvu +} + +// RemoveHasMetadata removes "has_metadata" edges to HasMetadata entities. +func (pvu *PackageVersionUpdate) RemoveHasMetadata(h ...*HasMetadata) *PackageVersionUpdate { + ids := make([]int, len(h)) + for i := range h { + ids[i] = h[i].ID + } + return pvu.RemoveHasMetadatumIDs(ids...) +} + // Save executes the query and returns the number of nodes affected by the update operation. func (pvu *PackageVersionUpdate) Save(ctx context.Context) (int, error) { return withHooks(ctx, pvu.sqlSave, pvu.mutation, pvu.hooks) @@ -446,6 +483,51 @@ func (pvu *PackageVersionUpdate) sqlSave(ctx context.Context) (n int, err error) } _spec.Edges.Add = append(_spec.Edges.Add, edge) } + if pvu.mutation.HasMetadataCleared() { + edge := &sqlgraph.EdgeSpec{ + Rel: sqlgraph.O2M, + Inverse: true, + Table: packageversion.HasMetadataTable, + Columns: []string{packageversion.HasMetadataColumn}, + Bidi: false, + Target: &sqlgraph.EdgeTarget{ + IDSpec: sqlgraph.NewFieldSpec(hasmetadata.FieldID, field.TypeInt), + }, + } + _spec.Edges.Clear = append(_spec.Edges.Clear, edge) + } + if nodes := pvu.mutation.RemovedHasMetadataIDs(); len(nodes) > 0 && !pvu.mutation.HasMetadataCleared() { + edge := &sqlgraph.EdgeSpec{ + Rel: sqlgraph.O2M, + Inverse: true, + Table: packageversion.HasMetadataTable, + Columns: []string{packageversion.HasMetadataColumn}, + Bidi: false, + Target: &sqlgraph.EdgeTarget{ + IDSpec: sqlgraph.NewFieldSpec(hasmetadata.FieldID, field.TypeInt), + }, + } + for _, k := range nodes { + edge.Target.Nodes = append(edge.Target.Nodes, k) + } + _spec.Edges.Clear = append(_spec.Edges.Clear, edge) + } + if nodes := pvu.mutation.HasMetadataIDs(); len(nodes) > 0 { + edge := &sqlgraph.EdgeSpec{ + Rel: sqlgraph.O2M, + Inverse: true, + Table: packageversion.HasMetadataTable, + Columns: []string{packageversion.HasMetadataColumn}, + Bidi: false, + Target: &sqlgraph.EdgeTarget{ + IDSpec: sqlgraph.NewFieldSpec(hasmetadata.FieldID, field.TypeInt), + }, + } + for _, k := range nodes { + edge.Target.Nodes = append(edge.Target.Nodes, k) + } + _spec.Edges.Add = append(_spec.Edges.Add, edge) + } if n, err = sqlgraph.UpdateNodes(ctx, pvu.driver, _spec); err != nil { if _, ok := err.(*sqlgraph.NotFoundError); ok { err = &NotFoundError{packageversion.Label} @@ -574,6 +656,21 @@ func (pvuo *PackageVersionUpdateOne) AddEqualPackages(p ...*PkgEqual) *PackageVe return pvuo.AddEqualPackageIDs(ids...) } +// AddHasMetadatumIDs adds the "has_metadata" edge to the HasMetadata entity by IDs. +func (pvuo *PackageVersionUpdateOne) AddHasMetadatumIDs(ids ...int) *PackageVersionUpdateOne { + pvuo.mutation.AddHasMetadatumIDs(ids...) + return pvuo +} + +// AddHasMetadata adds the "has_metadata" edges to the HasMetadata entity. +func (pvuo *PackageVersionUpdateOne) AddHasMetadata(h ...*HasMetadata) *PackageVersionUpdateOne { + ids := make([]int, len(h)) + for i := range h { + ids[i] = h[i].ID + } + return pvuo.AddHasMetadatumIDs(ids...) +} + // Mutation returns the PackageVersionMutation object of the builder. func (pvuo *PackageVersionUpdateOne) Mutation() *PackageVersionMutation { return pvuo.mutation @@ -648,6 +745,27 @@ func (pvuo *PackageVersionUpdateOne) RemoveEqualPackages(p ...*PkgEqual) *Packag return pvuo.RemoveEqualPackageIDs(ids...) } +// ClearHasMetadata clears all "has_metadata" edges to the HasMetadata entity. +func (pvuo *PackageVersionUpdateOne) ClearHasMetadata() *PackageVersionUpdateOne { + pvuo.mutation.ClearHasMetadata() + return pvuo +} + +// RemoveHasMetadatumIDs removes the "has_metadata" edge to HasMetadata entities by IDs. +func (pvuo *PackageVersionUpdateOne) RemoveHasMetadatumIDs(ids ...int) *PackageVersionUpdateOne { + pvuo.mutation.RemoveHasMetadatumIDs(ids...) + return pvuo +} + +// RemoveHasMetadata removes "has_metadata" edges to HasMetadata entities. +func (pvuo *PackageVersionUpdateOne) RemoveHasMetadata(h ...*HasMetadata) *PackageVersionUpdateOne { + ids := make([]int, len(h)) + for i := range h { + ids[i] = h[i].ID + } + return pvuo.RemoveHasMetadatumIDs(ids...) +} + // Where appends a list predicates to the PackageVersionUpdate builder. func (pvuo *PackageVersionUpdateOne) Where(ps ...predicate.PackageVersion) *PackageVersionUpdateOne { pvuo.mutation.Where(ps...) @@ -909,6 +1027,51 @@ func (pvuo *PackageVersionUpdateOne) sqlSave(ctx context.Context) (_node *Packag } _spec.Edges.Add = append(_spec.Edges.Add, edge) } + if pvuo.mutation.HasMetadataCleared() { + edge := &sqlgraph.EdgeSpec{ + Rel: sqlgraph.O2M, + Inverse: true, + Table: packageversion.HasMetadataTable, + Columns: []string{packageversion.HasMetadataColumn}, + Bidi: false, + Target: &sqlgraph.EdgeTarget{ + IDSpec: sqlgraph.NewFieldSpec(hasmetadata.FieldID, field.TypeInt), + }, + } + _spec.Edges.Clear = append(_spec.Edges.Clear, edge) + } + if nodes := pvuo.mutation.RemovedHasMetadataIDs(); len(nodes) > 0 && !pvuo.mutation.HasMetadataCleared() { + edge := &sqlgraph.EdgeSpec{ + Rel: sqlgraph.O2M, + Inverse: true, + Table: packageversion.HasMetadataTable, + Columns: []string{packageversion.HasMetadataColumn}, + Bidi: false, + Target: &sqlgraph.EdgeTarget{ + IDSpec: sqlgraph.NewFieldSpec(hasmetadata.FieldID, field.TypeInt), + }, + } + for _, k := range nodes { + edge.Target.Nodes = append(edge.Target.Nodes, k) + } + _spec.Edges.Clear = append(_spec.Edges.Clear, edge) + } + if nodes := pvuo.mutation.HasMetadataIDs(); len(nodes) > 0 { + edge := &sqlgraph.EdgeSpec{ + Rel: sqlgraph.O2M, + Inverse: true, + Table: packageversion.HasMetadataTable, + Columns: []string{packageversion.HasMetadataColumn}, + Bidi: false, + Target: &sqlgraph.EdgeTarget{ + IDSpec: sqlgraph.NewFieldSpec(hasmetadata.FieldID, field.TypeInt), + }, + } + for _, k := range nodes { + edge.Target.Nodes = append(edge.Target.Nodes, k) + } + _spec.Edges.Add = append(_spec.Edges.Add, edge) + } _node = &PackageVersion{config: pvuo.config} _spec.Assign = _node.assignValues _spec.ScanValues = _node.scanValues diff --git a/pkg/assembler/backends/ent/schema/dependency.go b/pkg/assembler/backends/ent/schema/dependency.go index 977c0f635f..b3ae8974a4 100644 --- a/pkg/assembler/backends/ent/schema/dependency.go +++ b/pkg/assembler/backends/ent/schema/dependency.go @@ -78,5 +78,9 @@ func (Dependency) Indexes() []ent.Index { Edges("package", "dependent_package_version"). Unique(). Annotations(entsql.IndexWhere("dependent_package_name_id IS NULL AND dependent_package_version_id IS NOT NULL")).StorageKey("dep_package_version"), + // sustain parser_csaf_red_hat.go in finding the correlation + index.Fields("dependent_package_name_id", "dependent_package_version_id", "package_id"), + // sustain bfsFromVulnerablePackage + index.Fields("dependent_package_version_id"), } } diff --git a/pkg/assembler/backends/ent/schema/hasmetadata.go b/pkg/assembler/backends/ent/schema/hasmetadata.go index 47c3c18f63..a0c13d326d 100644 --- a/pkg/assembler/backends/ent/schema/hasmetadata.go +++ b/pkg/assembler/backends/ent/schema/hasmetadata.go @@ -56,9 +56,11 @@ func (HasMetadata) Edges() []ent.Edge { func (HasMetadata) Indexes() []ent.Index { return []ent.Index{ - index.Fields("timestamp", "key", "value", "justification", "origin", "collector", "source_id").Unique().Annotations(entsql.IndexWhere("source_id IS NOT NULL AND package_version_id IS NULL AND package_name_id IS NULL AND artifact_id IS NULL")), - index.Fields("timestamp", "key", "value", "justification", "origin", "collector", "package_version_id").Unique().Annotations(entsql.IndexWhere("source_id IS NULL AND package_version_id IS NOT NULL AND package_name_id IS NULL AND artifact_id IS NULL")), - index.Fields("timestamp", "key", "value", "justification", "origin", "collector", "package_name_id").Unique().Annotations(entsql.IndexWhere("source_id IS NULL AND package_version_id IS NULL AND package_name_id IS NOT NULL AND artifact_id IS NULL")), - index.Fields("timestamp", "key", "value", "justification", "origin", "collector", "artifact_id").Unique().Annotations(entsql.IndexWhere("source_id IS NULL AND package_version_id IS NULL AND package_name_id IS NULL AND artifact_id IS NOT NULL")), + index.Fields("key", "value", "justification", "origin", "collector", "source_id").Unique().Annotations(entsql.IndexWhere("source_id IS NOT NULL AND package_version_id IS NULL AND package_name_id IS NULL AND artifact_id IS NULL")), + index.Fields("key", "value", "justification", "origin", "collector", "package_version_id").Unique().Annotations(entsql.IndexWhere("source_id IS NULL AND package_version_id IS NOT NULL AND package_name_id IS NULL AND artifact_id IS NULL")), + index.Fields("key", "value", "justification", "origin", "collector", "package_name_id").Unique().Annotations(entsql.IndexWhere("source_id IS NULL AND package_version_id IS NULL AND package_name_id IS NOT NULL AND artifact_id IS NULL")), + index.Fields("key", "value", "justification", "origin", "collector", "artifact_id").Unique().Annotations(entsql.IndexWhere("source_id IS NULL AND package_version_id IS NULL AND package_name_id IS NULL AND artifact_id IS NOT NULL")), + index.Fields("key", "value", "package_name_id", "package_version_id"), + index.Fields("key", "value"), } } diff --git a/pkg/assembler/backends/ent/schema/packageversion.go b/pkg/assembler/backends/ent/schema/packageversion.go index 6405d79f95..08ab25eb2c 100644 --- a/pkg/assembler/backends/ent/schema/packageversion.go +++ b/pkg/assembler/backends/ent/schema/packageversion.go @@ -51,6 +51,7 @@ func (PackageVersion) Edges() []ent.Edge { // edge.To("equal_packages", PackageVersion.Type).Through("equals", PkgEqual.Type), edge.From("equal_packages", PkgEqual.Type).Ref("packages"), // edge.From("pkg_equal_dependant", PkgEqual.Type).Ref("dependant_package"), + edge.From("has_metadata", HasMetadata.Type).Ref("package_version"), } } @@ -61,5 +62,7 @@ func (PackageVersion) Indexes() []ent.Index { index.Fields("qualifiers").Annotations( entsql.IndexTypes(map[string]string{dialect.Postgres: "GIN"}), ), + index.Fields("version", "subpath", "qualifiers").Edges("name").Unique(), + index.Fields("name_id"), } } diff --git a/pkg/assembler/backends/ent/testutils/suite.go b/pkg/assembler/backends/ent/testutils/suite.go index 4b55d4b2f2..06348eecf9 100644 --- a/pkg/assembler/backends/ent/testutils/suite.go +++ b/pkg/assembler/backends/ent/testutils/suite.go @@ -41,6 +41,10 @@ func init() { } txdb.Register("txdb", "postgres", db) + err := os.Setenv("MAX_CONCURRENT_BULK_INGESTION", "1") + if err != nil { + log.Fatal(err) + } } type Suite struct { diff --git a/pkg/assembler/clients/helpers/bulk.go b/pkg/assembler/clients/helpers/bulk.go index b111e8efd5..60d1802241 100644 --- a/pkg/assembler/clients/helpers/bulk.go +++ b/pkg/assembler/clients/helpers/bulk.go @@ -181,10 +181,10 @@ func GetBulkAssembler(ctx context.Context, gqlclient graphql.Client) func([]asse logger.Errorf("ingestPkgEquals failed with error: %v", err) } - logger.Infof("assembling CertifyLegal : %v", len(p.CertifyLegal)) - if err := ingestCertifyLegals(ctx, gqlclient, p.CertifyLegal); err != nil { - logger.Errorf("ingestCertifyLegals failed with error: %v", err) - } + //logger.Infof("assembling CertifyLegal : %v", len(p.CertifyLegal)) + //if err := ingestCertifyLegals(ctx, gqlclient, p.CertifyLegal); err != nil { + // logger.Errorf("ingestCertifyLegals failed with error: %v", err) + //} } return nil } diff --git a/pkg/ingestor/parser/csaf/parser_csaf_red_hat.go b/pkg/ingestor/parser/csaf/parser_csaf_red_hat.go index 208eb3d3f5..dcd9ba746b 100644 --- a/pkg/ingestor/parser/csaf/parser_csaf_red_hat.go +++ b/pkg/ingestor/parser/csaf/parser_csaf_red_hat.go @@ -78,28 +78,35 @@ func (c *csafParserRedHat) findPkgSpec(ctx context.Context, product_id string) ( for _, pkgWithMetadata := range pkgsWithMetadata.HasMetadata { // check the ones whose value starts with the CPE found in the VEX if strings.HasPrefix(pkgWithMetadata.Value, *cpe) { - depPkg := &generated.PkgSpec{Name: pref} - filterIsDependency := &generated.IsDependencySpec{ - DependencyPackage: depPkg, - } - switch subject := pkgWithMetadata.Subject.(type) { + switch productPkg := pkgWithMetadata.Subject.(type) { case *generated.AllHasMetadataSubjectPackage: - filterIsDependency.Package = &generated.PkgSpec{ - Type: &subject.Type, - Namespace: &subject.Namespaces[0].Namespace, - Name: &subject.Namespaces[0].Names[0].Name, - Version: &subject.Namespaces[0].Names[0].Versions[0].Version, + toPkg, err := helpers.PurlToPkg(helpers.AllPkgTreeToPurl(productPkg.AllPkgTree, true)) + if err != nil { + logger.Warnf("Failed to handle the HasMetadata response %+v\n", productPkg) + continue + } + filterProductDependenciesHasMetadata := &generated.HasMetadataSpec{ + Key: ptrfrom.String("topLevelPackage"), + Value: ptrfrom.String(helpers.PkgInputSpecToPurl(toPkg)), + Subject: &generated.PackageSourceOrArtifactSpec{ + Package: &generated.PkgSpec{Name: pref}, + }, } - isDependency, err := generated.IsDependency(ctx, gqlclient, *filterIsDependency) + dependenciesHasMetadataResponse, err := generated.HasMetadata(ctx, gqlclient, *filterProductDependenciesHasMetadata) if err != nil { return nil, err } - for _, isDep := range isDependency.IsDependency { - toPkg, err := helpers.PurlToPkg(helpers.AllPkgTreeToPurl(isDep.DependencyPackage.AllPkgTree, true)) - if err != nil { - logger.Warnf("Failed to handle the IsDependency response %+v\n", isDep) - } else { - purlsMap[helpers.PkgInputSpecToPurl(toPkg)] = struct{}{} + for _, dependencyHasMetadata := range dependenciesHasMetadataResponse.HasMetadata { + switch vulnerableDependencyPkg := dependencyHasMetadata.Subject.(type) { + case *generated.AllHasMetadataSubjectPackage: + vulnerablePkgInputSpec, err := helpers.PurlToPkg(helpers.AllPkgTreeToPurl(vulnerableDependencyPkg.AllPkgTree, true)) + if err != nil { + logger.Warnf("Failed to handle the IsDependency response %+v\n", dependencyHasMetadata) + } else { + purlsMap[helpers.PkgInputSpecToPurl(vulnerablePkgInputSpec)] = struct{}{} + } + default: + continue } } case *generated.AllHasMetadataSubjectSource: diff --git a/pkg/ingestor/parser/spdx/parse_spdx.go b/pkg/ingestor/parser/spdx/parse_spdx.go index 9f4a3ac3ab..0748fc8d65 100644 --- a/pkg/ingestor/parser/spdx/parse_spdx.go +++ b/pkg/ingestor/parser/spdx/parse_spdx.go @@ -362,18 +362,19 @@ func (s *spdxParser) GetPredicates(ctx context.Context) *assembler.IngestPredica for _, pkg := range s.spdxDoc.Packages { pkgInputSpecs := s.getPackageElement(string(pkg.PackageSPDXIdentifier)) - for _, extRef := range pkg.PackageExternalReferences { - if extRef.Category == spdx_common.CategorySecurity { - locator := extRef.Locator - metadataInputSpec := &model.HasMetadataInputSpec{ - Key: "cpe", - Value: locator, - Timestamp: time.Now().UTC(), - Justification: "spdx cpe external reference", - Origin: "GUAC SPDX", - Collector: "GUAC", - } - for i := range pkgInputSpecs { + for i := range pkgInputSpecs { + // add all CPEs found for each package with HasMetadata nodes + for _, extRef := range pkg.PackageExternalReferences { + if extRef.Category == spdx_common.CategorySecurity { + locator := extRef.Locator + metadataInputSpec := &model.HasMetadataInputSpec{ + Key: "cpe", + Value: locator, + Timestamp: time.Now().UTC(), + Justification: "spdx cpe external reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + } hasMetadata := assembler.HasMetadataIngest{ Pkg: pkgInputSpecs[i], PkgMatchFlag: model.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, @@ -382,6 +383,22 @@ func (s *spdxParser) GetPredicates(ctx context.Context) *assembler.IngestPredica preds.HasMetadata = append(preds.HasMetadata, hasMetadata) } } + // add top level package reference to each package with a HasMetadata node + for _, topLevelPkg := range topLevel { + hasMetadata := assembler.HasMetadataIngest{ + Pkg: pkgInputSpecs[i], + PkgMatchFlag: model.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &model.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: asmhelpers.PkgInputSpecToPurl(topLevelPkg), + Timestamp: time.Now().UTC(), + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + } + preds.HasMetadata = append(preds.HasMetadata, hasMetadata) + } } } diff --git a/pkg/ingestor/parser/spdx/parse_spdx_test.go b/pkg/ingestor/parser/spdx/parse_spdx_test.go index 89d718e411..7ede32f30a 100644 --- a/pkg/ingestor/parser/spdx/parse_spdx_test.go +++ b/pkg/ingestor/parser/spdx/parse_spdx_test.go @@ -72,6 +72,8 @@ func Test_spdxParser(t *testing.T) { additionalOpts: []cmp.Option{ cmpopts.IgnoreFields(assembler.HasSBOMIngest{}, "HasSBOM"), + cmpopts.IgnoreFields(generated.HasMetadataInputSpec{}, + "Timestamp"), }, doc: &processor.Document{ Blob: []byte(` @@ -153,6 +155,20 @@ func Test_spdxParser(t *testing.T) { }, }, }, + + HasMetadata: []assembler.HasMetadataIngest{ + { + Pkg: pUrlToPkgDiscardError("pkg:oci/image@sha256:a743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10?mediaType=application%2Fvnd.oci.image.manifest.v1%2Bjson"), + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:guac/spdx/sbom-sha256%253Aa743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, + }, }, wantErr: false, }, @@ -161,6 +177,8 @@ func Test_spdxParser(t *testing.T) { additionalOpts: []cmp.Option{ cmpopts.IgnoreFields(assembler.HasSBOMIngest{}, "HasSBOM"), + cmpopts.IgnoreFields(generated.HasMetadataInputSpec{}, + "Timestamp"), }, doc: &processor.Document{ Blob: []byte(` @@ -202,6 +220,19 @@ func Test_spdxParser(t *testing.T) { HasSBOM: []assembler.HasSBOMIngest{ {Pkg: pUrlToPkgDiscardError("pkg:oci/image@sha256:a743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10?mediaType=application%2Fvnd.oci.image.manifest.v1%2Bjson")}, }, + HasMetadata: []assembler.HasMetadataIngest{ + { + Pkg: pUrlToPkgDiscardError("pkg:oci/image@sha256:a743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10?mediaType=application%2Fvnd.oci.image.manifest.v1%2Bjson"), + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:oci/image@sha256:a743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10?mediatype=application%2Fvnd.oci.image.manifest.v1+json", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, + }, }, wantErr: false, }, @@ -210,6 +241,8 @@ func Test_spdxParser(t *testing.T) { additionalOpts: []cmp.Option{ cmpopts.IgnoreFields(assembler.HasSBOMIngest{}, "HasSBOM"), + cmpopts.IgnoreFields(generated.HasMetadataInputSpec{}, + "Timestamp"), }, doc: &processor.Document{ Blob: []byte(` @@ -268,6 +301,52 @@ func Test_spdxParser(t *testing.T) { {Pkg: pUrlToPkgDiscardError("pkg:oci/image@sha256:a743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10?mediaType=application%2Fvnd.oci.image.manifest.v1%2Bjson")}, {Pkg: pUrlToPkgDiscardError("pkg:oci/image@sha256:abc123?mediaType=application%2Fvnd.oci.image.manifest.v1%2Bjson")}, }, + HasMetadata: []assembler.HasMetadataIngest{ + { + Pkg: pUrlToPkgDiscardError("pkg:oci/image@sha256:a743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10?mediaType=application%2Fvnd.oci.image.manifest.v1%2Bjson"), + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:oci/image@sha256:a743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10?mediatype=application%2Fvnd.oci.image.manifest.v1+json", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, + { + Pkg: pUrlToPkgDiscardError("pkg:oci/image@sha256:a743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10?mediaType=application%2Fvnd.oci.image.manifest.v1%2Bjson"), + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:oci/image@sha256:abc123?mediatype=application%2Fvnd.oci.image.manifest.v1+json", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, + { + Pkg: pUrlToPkgDiscardError("pkg:oci/image@sha256:abc123?mediaType=application%2Fvnd.oci.image.manifest.v1%2Bjson"), + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:oci/image@sha256:a743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10?mediatype=application%2Fvnd.oci.image.manifest.v1+json", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, + { + Pkg: pUrlToPkgDiscardError("pkg:oci/image@sha256:abc123?mediaType=application%2Fvnd.oci.image.manifest.v1%2Bjson"), + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:oci/image@sha256:abc123?mediatype=application%2Fvnd.oci.image.manifest.v1+json", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, + }, }, wantErr: false, }, @@ -276,6 +355,8 @@ func Test_spdxParser(t *testing.T) { additionalOpts: []cmp.Option{ cmpopts.IgnoreFields(assembler.HasSBOMIngest{}, "HasSBOM"), + cmpopts.IgnoreFields(generated.HasMetadataInputSpec{}, + "Timestamp"), }, doc: &processor.Document{ Blob: []byte(` @@ -317,6 +398,19 @@ func Test_spdxParser(t *testing.T) { HasSBOM: []assembler.HasSBOMIngest{ {Pkg: pUrlToPkgDiscardError("pkg:oci/image@sha256:a743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10?mediaType=application%2Fvnd.oci.image.manifest.v1%2Bjson")}, }, + HasMetadata: []assembler.HasMetadataIngest{ + { + Pkg: pUrlToPkgDiscardError("pkg:oci/image@sha256:a743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10?mediaType=application%2Fvnd.oci.image.manifest.v1%2Bjson"), + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:oci/image@sha256:a743268cd3c56f921f3fb706cc0425c8ab78119fd433e38bb7c5dcd5635b0d10?mediatype=application%2Fvnd.oci.image.manifest.v1+json", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, + }, }, wantErr: false, }, @@ -361,6 +455,8 @@ func Test_spdxParser(t *testing.T) { additionalOpts: []cmp.Option{ cmpopts.IgnoreFields(assembler.HasSBOMIngest{}, "HasSBOM"), + cmpopts.IgnoreFields(generated.HasMetadataInputSpec{}, + "Timestamp"), }, doc: &processor.Document{ Blob: []byte(` @@ -435,6 +531,30 @@ func Test_spdxParser(t *testing.T) { HasSBOM: []assembler.HasSBOMIngest{ {Pkg: pUrlToPkgDiscardError("pkg:oci/redhat/ubi9-container@sha256:4227a4b5013999a412196237c62e40d778d09cdc751720a66ff3701fbe5a4a9d?repository_url=registry.redhat.io/ubi9&tag=9.1.0-1750")}, }, + HasMetadata: []assembler.HasMetadataIngest{ + { + Pkg: pUrlToPkgDiscardError("pkg:oci/redhat/ubi9-container@sha256:4227a4b5013999a412196237c62e40d778d09cdc751720a66ff3701fbe5a4a9d?repository_url=registry.redhat.io/ubi9&tag=9.1.0-1750"), + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:oci/ubi9-container@sha256:4227a4b5013999a412196237c62e40d778d09cdc751720a66ff3701fbe5a4a9d?tag=9.1.0-1750&repository_url=registry.redhat.", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, + { + Pkg: pUrlToPkgDiscardError("pkg:rpm/redhat/python3-libcomps@0.1.18-1.el9?arch=x86_64"), + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:oci/ubi9-container@sha256:4227a4b5013999a412196237c62e40d778d09cdc751720a66ff3701fbe5a4a9d?tag=9.1.0-1750&repository_url=registry.redhat.", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, + }, }, wantErr: false, }, @@ -804,6 +924,8 @@ func Test_spdxParser(t *testing.T) { additionalOpts: []cmp.Option{ cmpopts.IgnoreFields(assembler.IngestPredicates{}, "HasSBOM", "IsDependency", "IsOccurrence"), + cmpopts.IgnoreFields(generated.HasMetadataInputSpec{}, + "Timestamp"), }, doc: &processor.Document{ Blob: []byte(` @@ -878,6 +1000,19 @@ func Test_spdxParser(t *testing.T) { }, }, }, + HasMetadata: []assembler.HasMetadataIngest{ + { + Pkg: pUrlToPkgDiscardError("pkg:guac/pkg/mypackage@3.2.0-r22"), + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:guac/spdx/testsbom", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, + }, }, wantErr: false, }, @@ -886,6 +1021,8 @@ func Test_spdxParser(t *testing.T) { additionalOpts: []cmp.Option{ cmpopts.IgnoreFields(assembler.IngestPredicates{}, "HasSBOM", "IsDependency", "IsOccurrence"), + cmpopts.IgnoreFields(generated.HasMetadataInputSpec{}, + "Timestamp"), }, doc: &processor.Document{ Blob: []byte(` @@ -957,6 +1094,19 @@ func Test_spdxParser(t *testing.T) { }, }, }, + HasMetadata: []assembler.HasMetadataIngest{ + { + Pkg: pUrlToPkgDiscardError("pkg:guac/pkg/mypackage@3.2.0-r22"), + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:guac/spdx/testsbom", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, + }, }, wantErr: false, }, @@ -965,6 +1115,8 @@ func Test_spdxParser(t *testing.T) { additionalOpts: []cmp.Option{ cmpopts.IgnoreFields(assembler.IngestPredicates{}, "HasSBOM", "IsDependency", "IsOccurrence"), + cmpopts.IgnoreFields(generated.HasMetadataInputSpec{}, + "Timestamp"), }, doc: &processor.Document{ Blob: []byte(` @@ -1030,6 +1182,19 @@ func Test_spdxParser(t *testing.T) { }, }, }, + HasMetadata: []assembler.HasMetadataIngest{ + { + Pkg: pUrlToPkgDiscardError("pkg:guac/pkg/mypackage@3.2.0-r22"), + PkgMatchFlag: generated.MatchFlags{Pkg: generated.PkgMatchTypeSpecificVersion}, + HasMetadata: &generated.HasMetadataInputSpec{ + Key: "topLevelPackage", + Value: "pkg:guac/spdx/testsbom", + Justification: "spdx top level package reference", + Origin: "GUAC SPDX", + Collector: "GUAC", + }, + }, + }, }, wantErr: false, },