From 44ff51fef73084154383bcaa1e0c548585640e5b Mon Sep 17 00:00:00 2001 From: Christopher Phillips <32073428+spiffcs@users.noreply.github.com> Date: Sun, 17 Nov 2024 18:26:17 -0500 Subject: [PATCH 1/2] feat: add skeleton goModSource cataloger Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com> --- syft/pkg/cataloger/golang/cataloger.go | 13 +++++-- syft/pkg/cataloger/golang/parse_go_mod.go | 44 +++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/syft/pkg/cataloger/golang/cataloger.go b/syft/pkg/cataloger/golang/cataloger.go index c7d483faa09..07843629d9d 100644 --- a/syft/pkg/cataloger/golang/cataloger.go +++ b/syft/pkg/cataloger/golang/cataloger.go @@ -14,8 +14,9 @@ import ( var versionCandidateGroups = regexp.MustCompile(`(?P\d+(\.\d+)?(\.\d+)?)(?P\w*)`) const ( - modFileCatalogerName = "go-module-file-cataloger" - binaryCatalogerName = "go-module-binary-cataloger" + modFileCatalogerName = "go-module-file-cataloger" + binaryCatalogerName = "go-module-binary-cataloger" + sourceFileCatalogerName = "go-module-source-file-cataloger" ) // NewGoModuleFileCataloger returns a new cataloger object that searches within go.mod files. @@ -33,3 +34,11 @@ func NewGoModuleBinaryCataloger(opts CatalogerConfig) pkg.Cataloger { ). WithProcessors(stdlibProcessor) } + +// NewGoModuleSourceFileCataloger returns a new cataloger object that uses the go.mod file +// to extract the module name and then searches all direct and transitive dependencies +// for the given module source tree +func NewGoModuleSourceFileCataloger(opts CatalogerConfig) pkg.Cataloger { + return generic.NewCataloger(modFileCatalogerName). + WithParserByGlobs(newGoModSourceCataloger(opts).parseGoModFile, "**/go.mod") +} diff --git a/syft/pkg/cataloger/golang/parse_go_mod.go b/syft/pkg/cataloger/golang/parse_go_mod.go index cfb373e9048..919102c6458 100644 --- a/syft/pkg/cataloger/golang/parse_go_mod.go +++ b/syft/pkg/cataloger/golang/parse_go_mod.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "fmt" + "golang.org/x/tools/go/packages" "io" "sort" "strings" @@ -19,16 +20,59 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/generic" ) +var searchSuffix = "/..." + type goModCataloger struct { licenseResolver goLicenseResolver } +type goModSourceCataloger struct{} + func newGoModCataloger(opts CatalogerConfig) *goModCataloger { return &goModCataloger{ licenseResolver: newGoLicenseResolver(modFileCatalogerName, opts), } } +func newGoModSourceCataloger(opts CatalogerConfig) *goModSourceCataloger { + return &goModSourceCataloger{} +} + +func (c *goModSourceCataloger) parseGoModFile(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + contents, err := io.ReadAll(reader) + if err != nil { + return nil, nil, fmt.Errorf("failed to read module file: %w", err) + } + + file, err := modfile.Parse("go.mod", contents, nil) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse module file: %w", err) + } + + // extract the module name and add the search suffix + mainModuleName := file.Module.Mod.Path + mainModuleName = fmt.Sprintf("%s/%s", mainModuleName, searchSuffix) + + cfg := &packages.Config{ + Context: ctx, + Mode: packages.NeedImports | packages.NeedDeps | packages.NeedFiles | packages.NeedName | packages.NeedModule, + Tests: true, + } + + rootPkgs, err := packages.Load(cfg, mainModuleName) + if err != nil { + return nil, nil, fmt.Errorf("failed to load packages for %s: %w", mainModuleName, err) + } + + discovered := make([]pkg.Package, 0) + for _, rootPkg := range rootPkgs { + discovered = append(discovered, pkg.Package{ + Name: rootPkg.Name, + }) + } + return discovered, nil, nil +} + // parseGoModFile takes a go.mod and lists all packages discovered. // //nolint:funlen From 7262452efb89d8cee9a9d3cdd04e574eeb27a339 Mon Sep 17 00:00:00 2001 From: Christopher Phillips <32073428+spiffcs@users.noreply.github.com> Date: Sun, 17 Nov 2024 19:45:08 -0500 Subject: [PATCH 2/2] feat: add new go module source cataloger Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com> --- go.mod | 1 + internal/task/package_tasks.go | 5 ++ syft/pkg/cataloger/golang/cataloger.go | 2 +- syft/pkg/cataloger/golang/parse_go_mod.go | 61 ++++++++++++++++++++--- syft/pkg/golang.go | 19 +++++++ 5 files changed, 81 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 07ade518b87..e5c8a7cae8a 100644 --- a/go.mod +++ b/go.mod @@ -92,6 +92,7 @@ require ( github.com/adrg/xdg v0.5.3 github.com/magiconair/properties v1.8.7 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 + golang.org/x/tools v0.23.0 ) require ( diff --git a/internal/task/package_tasks.go b/internal/task/package_tasks.go index 431c77590f4..dd9b28fa652 100644 --- a/internal/task/package_tasks.go +++ b/internal/task/package_tasks.go @@ -84,6 +84,11 @@ func DefaultPackageTaskFactories() PackageTaskFactories { }, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, Go, Golang, "gomod", ), + newPackageTaskFactory( + func(cfg CatalogingFactoryConfig) pkg.Cataloger { + return golang.NewGoModuleSourceFileCataloger(cfg.PackagesConfig.Golang) + }, + pkgcataloging.InstalledTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, Go, Golang, "gosource"), newSimplePackageTaskFactory(java.NewGradleLockfileCataloger, pkgcataloging.DeclaredTag, pkgcataloging.DirectoryTag, pkgcataloging.LanguageTag, Java, "gradle"), newPackageTaskFactory( func(cfg CatalogingFactoryConfig) pkg.Cataloger { diff --git a/syft/pkg/cataloger/golang/cataloger.go b/syft/pkg/cataloger/golang/cataloger.go index 07843629d9d..14c1e051c63 100644 --- a/syft/pkg/cataloger/golang/cataloger.go +++ b/syft/pkg/cataloger/golang/cataloger.go @@ -39,6 +39,6 @@ func NewGoModuleBinaryCataloger(opts CatalogerConfig) pkg.Cataloger { // to extract the module name and then searches all direct and transitive dependencies // for the given module source tree func NewGoModuleSourceFileCataloger(opts CatalogerConfig) pkg.Cataloger { - return generic.NewCataloger(modFileCatalogerName). + return generic.NewCataloger(sourceFileCatalogerName). WithParserByGlobs(newGoModSourceCataloger(opts).parseGoModFile, "**/go.mod") } diff --git a/syft/pkg/cataloger/golang/parse_go_mod.go b/syft/pkg/cataloger/golang/parse_go_mod.go index 919102c6458..f33662ca787 100644 --- a/syft/pkg/cataloger/golang/parse_go_mod.go +++ b/syft/pkg/cataloger/golang/parse_go_mod.go @@ -51,7 +51,7 @@ func (c *goModSourceCataloger) parseGoModFile(ctx context.Context, resolver file // extract the module name and add the search suffix mainModuleName := file.Module.Mod.Path - mainModuleName = fmt.Sprintf("%s/%s", mainModuleName, searchSuffix) + mainModuleName = fmt.Sprintf("%s%s", mainModuleName, searchSuffix) cfg := &packages.Config{ Context: ctx, @@ -64,13 +64,62 @@ func (c *goModSourceCataloger) parseGoModFile(ctx context.Context, resolver file return nil, nil, fmt.Errorf("failed to load packages for %s: %w", mainModuleName, err) } - discovered := make([]pkg.Package, 0) - for _, rootPkg := range rootPkgs { - discovered = append(discovered, pkg.Package{ - Name: rootPkg.Name, + syftPackages := make([]pkg.Package, 0) + pkgErrorOccurred := false + otherErrorOccurred := false + packages.Visit(rootPkgs, func(p *packages.Package) bool { + if len(p.Errors) > 0 { + pkgErrorOccurred = true + return false + } + if p.Module == nil { + otherErrorOccurred = true + return false + } + + if !isValid(p) { + return false + } + + syftPackages = append(syftPackages, pkg.Package{ + Name: p.Name, + Version: p.Module.Version, + // Licenses (TODO) + // Locations: file.NewLocationSet(reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), + PURL: packageURL(p.PkgPath, p.Module.Version), + Language: pkg.Go, + Type: pkg.GoModulePkg, + Metadata: pkg.GolangModuleEntryMetadata{ + Path: p.Module.Path, + Version: p.Module.Version, + Replace: p.Module.Replace, + Time: p.Module.Time, + Main: p.Module.Main, + Indirect: p.Module.Indirect, + Dir: p.Module.Dir, + GoMod: p.Module.GoMod, + GoVersion: p.Module.GoVersion, + }, }) + return true + }, nil) + if pkgErrorOccurred { + // TODO: log error as warning for packages that could not be analyzed + } + if otherErrorOccurred { + // TODO: log errors for direct/transitive dependency loading + } + return syftPackages, nil, nil +} + +func isValid(p *packages.Package) bool { + if p.Name == "" { + return false + } + if p.Module.Version == "" { + return false } - return discovered, nil, nil + return true } // parseGoModFile takes a go.mod and lists all packages discovered. diff --git a/syft/pkg/golang.go b/syft/pkg/golang.go index 4e0269c4491..54dafa2520e 100644 --- a/syft/pkg/golang.go +++ b/syft/pkg/golang.go @@ -1,5 +1,10 @@ package pkg +import ( + "golang.org/x/tools/go/packages" + "time" +) + // GolangBinaryBuildinfoEntry represents all captured data for a Golang binary type GolangBinaryBuildinfoEntry struct { BuildSettings KeyValues `json:"goBuildSettings,omitempty" cyclonedx:"goBuildSettings"` @@ -15,3 +20,17 @@ type GolangBinaryBuildinfoEntry struct { type GolangModuleEntry struct { H1Digest string `json:"h1Digest,omitempty" cyclonedx:"h1Digest"` } + +// GolangModuleEntryMetadata represetns all captured data from the golang.org/x/tools/go/packages package +// when scanning a golang source directory for direct and indirect dependencies +type GolangModuleEntryMetadata struct { + Path string // module path + Version string // module version + Replace *packages.Module // replaced by this module + Time *time.Time // time version was created + Main bool // is this the main module? + Indirect bool // is this module only an indirect dependency of main module? + Dir string // directory holding files for this module, if any + GoMod string // path to go.mod file used when loading this module, if any + GoVersion string // go version used in module +}