From fef951c29bdaea4424d4393435abde6935d7c798 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 28 Oct 2021 09:53:32 -0400 Subject: [PATCH] return relationships from tasks Signed-off-by: Alex Goodman --- cmd/packages.go | 5 +- cmd/power_user.go | 60 +++++++++++++++---- cmd/power_user_tasks.go | 52 ++++++++-------- syft/artifact/relationship.go | 8 +-- syft/lib.go | 14 +++-- syft/pkg/cataloger/apkdb/parse_apk_db.go | 10 ++-- syft/pkg/cataloger/catalog.go | 12 ++-- syft/pkg/cataloger/cataloger.go | 3 +- .../pkg/cataloger/common/generic_cataloger.go | 24 ++++---- .../common/generic_cataloger_test.go | 7 ++- syft/pkg/cataloger/common/parser.go | 3 +- syft/pkg/cataloger/deb/cataloger.go | 20 ++++--- syft/pkg/cataloger/deb/parse_dpkg_status.go | 9 +-- .../{bin_cataloger.go => binary_cataloger.go} | 20 ++++--- syft/pkg/cataloger/golang/parse_go_bin.go | 9 ++- syft/pkg/cataloger/golang/parse_go_mod.go | 9 +-- syft/pkg/cataloger/java/archive_parser.go | 40 +++++++------ .../javascript/parse_package_json.go | 13 ++-- .../javascript/parse_package_lock.go | 9 +-- .../cataloger/javascript/parse_yarn_lock.go | 9 +-- .../pkg/cataloger/python/package_cataloger.go | 9 +-- .../cataloger/python/parse_pipfile_lock.go | 7 ++- .../pkg/cataloger/python/parse_poetry_lock.go | 9 +-- .../cataloger/python/parse_requirements.go | 7 ++- syft/pkg/cataloger/python/parse_setup.go | 5 +- syft/pkg/cataloger/rpmdb/cataloger.go | 15 +++-- syft/pkg/cataloger/ruby/parse_gemfile_lock.go | 7 ++- syft/pkg/cataloger/ruby/parse_gemspec.go | 7 ++- syft/pkg/cataloger/rust/parse_cargo_lock.go | 9 +-- syft/pkg/package.go | 2 +- syft/sbom/sbom.go | 6 +- 31 files changed, 251 insertions(+), 168 deletions(-) rename syft/pkg/cataloger/golang/{bin_cataloger.go => binary_cataloger.go} (71%) diff --git a/cmd/packages.go b/cmd/packages.go index cceb095c5..e7f3f9c8a 100644 --- a/cmd/packages.go +++ b/cmd/packages.go @@ -255,7 +255,7 @@ func packagesExecWorker(userInput string) <-chan error { } defer cleanup() - catalog, d, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt) + catalog, relationships, d, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt) if err != nil { errs <- fmt.Errorf("failed to catalog input: %w", err) return @@ -273,7 +273,8 @@ func packagesExecWorker(userInput string) <-chan error { PackageCatalog: catalog, Distro: d, }, - Source: src.Metadata, + Relationships: relationships, + Source: src.Metadata, } bus.Publish(partybus.Event{ diff --git a/cmd/power_user.go b/cmd/power_user.go index 607ca48cd..a65e54415 100644 --- a/cmd/power_user.go +++ b/cmd/power_user.go @@ -4,6 +4,8 @@ import ( "fmt" "sync" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/sbom" "github.com/anchore/stereoscope" @@ -88,7 +90,6 @@ func powerUserExec(_ *cobra.Command, args []string) error { ui.Select(isVerbose(), appConfig.Quiet, reporter)..., ) } - func powerUserExecWorker(userInput string) <-chan error { errs := make(chan error) go func() { @@ -109,28 +110,61 @@ func powerUserExecWorker(userInput string) <-chan error { } defer cleanup() - analysisResults := sbom.SBOM{ + s := sbom.SBOM{ Source: src.Metadata, } - wg := &sync.WaitGroup{} + var results []<-chan artifact.Relationship for _, task := range tasks { - wg.Add(1) - go func(task powerUserTask) { - defer wg.Done() - if err = task(&analysisResults.Artifacts, src); err != nil { - errs <- err - return - } - }(task) + c := make(chan artifact.Relationship) + results = append(results, c) + + go runTask(task, &s.Artifacts, src, c, errs) } - wg.Wait() + for relationship := range mergeResults(results...) { + s.Relationships = append(s.Relationships, relationship) + } bus.Publish(partybus.Event{ Type: event.PresenterReady, - Value: poweruser.NewJSONPresenter(analysisResults, *appConfig), + Value: poweruser.NewJSONPresenter(s, *appConfig), }) }() return errs } + +func runTask(t powerUserTask, a *sbom.Artifacts, src *source.Source, c chan<- artifact.Relationship, errs chan<- error) { + defer close(c) + + relationships, err := t(a, src) + if err != nil { + errs <- err + return + } + + for _, relationship := range relationships { + c <- relationship + } +} + +func mergeResults(cs ...<-chan artifact.Relationship) <-chan artifact.Relationship { + var wg sync.WaitGroup + var results = make(chan artifact.Relationship) + + wg.Add(len(cs)) + for _, c := range cs { + go func(c <-chan artifact.Relationship) { + for n := range c { + results <- n + } + wg.Done() + }(c) + } + + go func() { + wg.Wait() + close(results) + }() + return results +} diff --git a/cmd/power_user_tasks.go b/cmd/power_user_tasks.go index ab493ddd5..365a8fdd5 100644 --- a/cmd/power_user_tasks.go +++ b/cmd/power_user_tasks.go @@ -4,6 +4,8 @@ import ( "crypto" "fmt" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft" @@ -11,7 +13,7 @@ import ( "github.com/anchore/syft/syft/source" ) -type powerUserTask func(*sbom.Artifacts, *source.Source) error +type powerUserTask func(*sbom.Artifacts, *source.Source) ([]artifact.Relationship, error) func powerUserTasks() ([]powerUserTask, error) { var tasks []powerUserTask @@ -43,16 +45,16 @@ func catalogPackagesTask() (powerUserTask, error) { return nil, nil } - task := func(results *sbom.Artifacts, src *source.Source) error { - packageCatalog, theDistro, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt) + task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { + packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt) if err != nil { - return err + return nil, err } results.PackageCatalog = packageCatalog results.Distro = theDistro - return nil + return relationships, nil } return task, nil @@ -65,18 +67,18 @@ func catalogFileMetadataTask() (powerUserTask, error) { metadataCataloger := file.NewMetadataCataloger() - task := func(results *sbom.Artifacts, src *source.Source) error { + task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(appConfig.FileMetadata.Cataloger.ScopeOpt) if err != nil { - return err + return nil, err } result, err := metadataCataloger.Catalog(resolver) if err != nil { - return err + return nil, err } results.FileMetadata = result - return nil + return nil, nil } return task, nil @@ -111,18 +113,18 @@ func catalogFileDigestsTask() (powerUserTask, error) { return nil, err } - task := func(results *sbom.Artifacts, src *source.Source) error { + task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(appConfig.FileMetadata.Cataloger.ScopeOpt) if err != nil { - return err + return nil, err } result, err := digestsCataloger.Catalog(resolver) if err != nil { - return err + return nil, err } results.FileDigests = result - return nil + return nil, nil } return task, nil @@ -143,18 +145,18 @@ func catalogSecretsTask() (powerUserTask, error) { return nil, err } - task := func(results *sbom.Artifacts, src *source.Source) error { + task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(appConfig.Secrets.Cataloger.ScopeOpt) if err != nil { - return err + return nil, err } result, err := secretsCataloger.Catalog(resolver) if err != nil { - return err + return nil, err } results.Secrets = result - return nil + return nil, nil } return task, nil @@ -171,18 +173,18 @@ func catalogFileClassificationsTask() (powerUserTask, error) { return nil, err } - task := func(results *sbom.Artifacts, src *source.Source) error { + task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(appConfig.FileClassification.Cataloger.ScopeOpt) if err != nil { - return err + return nil, err } result, err := classifierCataloger.Catalog(resolver) if err != nil { - return err + return nil, err } results.FileClassifications = result - return nil + return nil, nil } return task, nil @@ -198,18 +200,18 @@ func catalogContentsTask() (powerUserTask, error) { return nil, err } - task := func(results *sbom.Artifacts, src *source.Source) error { + task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(appConfig.FileContents.Cataloger.ScopeOpt) if err != nil { - return err + return nil, err } result, err := contentsCataloger.Catalog(resolver) if err != nil { - return err + return nil, err } results.FileContents = result - return nil + return nil, nil } return task, nil diff --git a/syft/artifact/relationship.go b/syft/artifact/relationship.go index adc867b57..be81a311d 100644 --- a/syft/artifact/relationship.go +++ b/syft/artifact/relationship.go @@ -8,8 +8,8 @@ const ( type RelationshipType string type Relationship struct { - From ID - To ID - Type RelationshipType - Data interface{} + From ID `json:"from"` + To ID `json:"to"` + Type RelationshipType `json:"type"` + Data interface{} `json:"data,omitempty"` } diff --git a/syft/lib.go b/syft/lib.go index ccf85cd6b..376315335 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -19,6 +19,8 @@ package syft import ( "fmt" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/distro" @@ -32,10 +34,10 @@ import ( // CatalogPackages takes an inventory of packages from the given image from a particular perspective // (e.g. squashed source, all-layers source). Returns the discovered set of packages, the identified Linux // distribution, and the source object used to wrap the data source. -func CatalogPackages(src *source.Source, scope source.Scope) (*pkg.Catalog, *distro.Distro, error) { +func CatalogPackages(src *source.Source, scope source.Scope) (*pkg.Catalog, []artifact.Relationship, *distro.Distro, error) { resolver, err := src.FileResolver(scope) if err != nil { - return nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err) + return nil, nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err) } // find the distro @@ -59,15 +61,15 @@ func CatalogPackages(src *source.Source, scope source.Scope) (*pkg.Catalog, *dis log.Info("cataloging directory") catalogers = cataloger.DirectoryCatalogers() default: - return nil, nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", src.Metadata.Scheme) + return nil, nil, nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", src.Metadata.Scheme) } - catalog, err := cataloger.Catalog(resolver, theDistro, catalogers...) + catalog, relationships, err := cataloger.Catalog(resolver, theDistro, catalogers...) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - return catalog, theDistro, nil + return catalog, relationships, theDistro, nil } // SetLogger sets the logger object used for all syft logging calls. diff --git a/syft/pkg/cataloger/apkdb/parse_apk_db.go b/syft/pkg/cataloger/apkdb/parse_apk_db.go index e79942c87..835822f5d 100644 --- a/syft/pkg/cataloger/apkdb/parse_apk_db.go +++ b/syft/pkg/cataloger/apkdb/parse_apk_db.go @@ -8,6 +8,8 @@ import ( "strconv" "strings" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/internal/log" @@ -21,7 +23,7 @@ var _ common.ParserFn = parseApkDB // parseApkDb parses individual packages from a given Alpine DB file. For more information on specific fields // see https://wiki.alpinelinux.org/wiki/Apk_spec . -func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { // larger capacity for the scanner. const maxScannerCapacity = 1024 * 1024 // a new larger buffer for the scanner @@ -47,7 +49,7 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) { for scanner.Scan() { metadata, err := parseApkDBEntry(strings.NewReader(scanner.Text())) if err != nil { - return nil, err + return nil, nil, err } if metadata != nil { packages = append(packages, pkg.Package{ @@ -62,10 +64,10 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) { } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("failed to parse APK DB file: %w", err) + return nil, nil, fmt.Errorf("failed to parse APK DB file: %w", err) } - return packages, nil + return packages, nil, nil } // nolint:funlen diff --git a/syft/pkg/cataloger/catalog.go b/syft/pkg/cataloger/catalog.go index b76e73a45..49afdc930 100644 --- a/syft/pkg/cataloger/catalog.go +++ b/syft/pkg/cataloger/catalog.go @@ -3,6 +3,7 @@ package cataloger import ( "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/pkg" @@ -38,8 +39,9 @@ func newMonitor() (*progress.Manual, *progress.Manual) { // In order to efficiently retrieve contents from a underlying container image the content fetch requests are // done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single // request. -func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers ...Cataloger) (*pkg.Catalog, error) { +func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers ...Cataloger) (*pkg.Catalog, []artifact.Relationship, error) { catalog := pkg.NewCatalog() + var allRelationships []artifact.Relationship filesProcessed, packagesDiscovered := newMonitor() @@ -47,7 +49,7 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers var errs error for _, theCataloger := range catalogers { // find packages from the underlying raw data - packages, err := theCataloger.Catalog(resolver) + packages, relationships, err := theCataloger.Catalog(resolver) if err != nil { errs = multierror.Append(errs, err) continue @@ -68,14 +70,16 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers // add to catalog catalog.Add(p) } + + allRelationships = append(allRelationships, relationships...) } if errs != nil { - return nil, errs + return nil, nil, errs } filesProcessed.SetCompleted() packagesDiscovered.SetCompleted() - return catalog, nil + return catalog, allRelationships, nil } diff --git a/syft/pkg/cataloger/cataloger.go b/syft/pkg/cataloger/cataloger.go index a92a2dfbe..6e9fbcb52 100644 --- a/syft/pkg/cataloger/cataloger.go +++ b/syft/pkg/cataloger/cataloger.go @@ -6,6 +6,7 @@ catalogers defined in child packages as well as the interface definition to impl package cataloger import ( + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/apkdb" "github.com/anchore/syft/syft/pkg/cataloger/deb" @@ -26,7 +27,7 @@ type Cataloger interface { // Name returns a string that uniquely describes a cataloger Name() string // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. - Catalog(resolver source.FileResolver) ([]pkg.Package, error) + Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) } // ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages. diff --git a/syft/pkg/cataloger/common/generic_cataloger.go b/syft/pkg/cataloger/common/generic_cataloger.go index 84ef0f865..9035bda7a 100644 --- a/syft/pkg/cataloger/common/generic_cataloger.go +++ b/syft/pkg/cataloger/common/generic_cataloger.go @@ -6,6 +6,8 @@ package common import ( "fmt" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" @@ -35,18 +37,18 @@ func (c *GenericCataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. -func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { +func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { var packages []pkg.Package - parserByLocation := c.selectFiles(resolver) + var relationships []artifact.Relationship - for location, parser := range parserByLocation { + for location, parser := range c.selectFiles(resolver) { contentReader, err := resolver.FileContentsByLocation(location) if err != nil { // TODO: fail or log? - return nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err) + return nil, nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err) } - entries, err := parser(location.RealPath, contentReader) + discoveredPackages, discoveredRelationships, err := parser(location.RealPath, contentReader) internal.CloseAndLogError(contentReader, location.VirtualPath) if err != nil { // TODO: should we fail? or only log? @@ -54,14 +56,16 @@ func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, continue } - for _, entry := range entries { - entry.FoundBy = c.upstreamCataloger - entry.Locations = []source.Location{location} + for _, p := range discoveredPackages { + p.FoundBy = c.upstreamCataloger + p.Locations = append(p.Locations, location) - packages = append(packages, entry) + packages = append(packages, p) } + + relationships = append(relationships, discoveredRelationships...) } - return packages, nil + return packages, relationships, nil } // SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging diff --git a/syft/pkg/cataloger/common/generic_cataloger_test.go b/syft/pkg/cataloger/common/generic_cataloger_test.go index a30a3d5f7..36a0cb61a 100644 --- a/syft/pkg/cataloger/common/generic_cataloger_test.go +++ b/syft/pkg/cataloger/common/generic_cataloger_test.go @@ -8,11 +8,12 @@ import ( "github.com/stretchr/testify/assert" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) -func parser(_ string, reader io.Reader) ([]pkg.Package, error) { +func parser(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { contents, err := ioutil.ReadAll(reader) if err != nil { panic(err) @@ -21,7 +22,7 @@ func parser(_ string, reader io.Reader) ([]pkg.Package, error) { { Name: string(contents), }, - }, nil + }, nil, nil } func TestGenericCataloger(t *testing.T) { @@ -47,7 +48,7 @@ func TestGenericCataloger(t *testing.T) { } } - actualPkgs, err := cataloger.Catalog(resolver) + actualPkgs, _, err := cataloger.Catalog(resolver) assert.NoError(t, err) assert.Len(t, actualPkgs, len(expectedPkgs)) diff --git a/syft/pkg/cataloger/common/parser.go b/syft/pkg/cataloger/common/parser.go index 15ca17d11..b2094f276 100644 --- a/syft/pkg/cataloger/common/parser.go +++ b/syft/pkg/cataloger/common/parser.go @@ -3,8 +3,9 @@ package common import ( "io" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" ) // ParserFn standardizes a function signature for parser functions that accept the virtual file path (not usable for file reads) and contents and return any discovered packages from that file -type ParserFn func(string, io.Reader) ([]pkg.Package, error) +type ParserFn func(string, io.Reader) ([]pkg.Package, []artifact.Relationship, error) diff --git a/syft/pkg/cataloger/deb/cataloger.go b/syft/pkg/cataloger/deb/cataloger.go index 20e510160..7235891b4 100644 --- a/syft/pkg/cataloger/deb/cataloger.go +++ b/syft/pkg/cataloger/deb/cataloger.go @@ -13,6 +13,7 @@ import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) @@ -36,24 +37,24 @@ func (c *Cataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files. -func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { +func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { dbFileMatches, err := resolver.FilesByGlob(pkg.DpkgDBGlob) if err != nil { - return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err) + return nil, nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err) } - var results []pkg.Package - var pkgs []pkg.Package + var allPackages []pkg.Package + var allRelationships []artifact.Relationship for _, dbLocation := range dbFileMatches { dbContents, err := resolver.FileContentsByLocation(dbLocation) if err != nil { - return nil, err + return nil, nil, err } - pkgs, err = parseDpkgStatus(dbContents) + pkgs, relationships, err := parseDpkgStatus(dbContents) internal.CloseAndLogError(dbContents, dbLocation.VirtualPath) if err != nil { - return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err) + return nil, nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err) } for i := range pkgs { @@ -70,9 +71,10 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) addLicenses(resolver, dbLocation, p) } - results = append(results, pkgs...) + allPackages = append(allPackages, pkgs...) + allRelationships = append(allRelationships, relationships...) } - return results, nil + return allPackages, allRelationships, nil } func addLicenses(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) { diff --git a/syft/pkg/cataloger/deb/parse_dpkg_status.go b/syft/pkg/cataloger/deb/parse_dpkg_status.go index ea25584dd..fe5581824 100644 --- a/syft/pkg/cataloger/deb/parse_dpkg_status.go +++ b/syft/pkg/cataloger/deb/parse_dpkg_status.go @@ -11,6 +11,7 @@ import ( "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/mitchellh/mapstructure" ) @@ -21,9 +22,9 @@ var ( ) // parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed. -func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) { +func parseDpkgStatus(reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { buffedReader := bufio.NewReader(reader) - var packages = make([]pkg.Package, 0) + var packages []pkg.Package continueProcessing := true for continueProcessing { @@ -32,7 +33,7 @@ func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) { if errors.Is(err, errEndOfPackages) { continueProcessing = false } else { - return nil, err + return nil, nil, err } } @@ -47,7 +48,7 @@ func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) { } } - return packages, nil + return packages, nil, nil } // parseDpkgStatusEntry returns an individual Dpkg entry, or returns errEndOfPackages if there are no more packages to parse from the reader. diff --git a/syft/pkg/cataloger/golang/bin_cataloger.go b/syft/pkg/cataloger/golang/binary_cataloger.go similarity index 71% rename from syft/pkg/cataloger/golang/bin_cataloger.go rename to syft/pkg/cataloger/golang/binary_cataloger.go index cd1a91e5f..20b19796c 100644 --- a/syft/pkg/cataloger/golang/bin_cataloger.go +++ b/syft/pkg/cataloger/golang/binary_cataloger.go @@ -6,7 +6,10 @@ package golang import ( "fmt" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) @@ -35,27 +38,30 @@ func (c *Cataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation. -func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { - pkgs := make([]pkg.Package, 0) +func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { + var pkgs []pkg.Package + var relationships []artifact.Relationship + fileMatches, err := resolver.FilesByMIMEType(mimeTypes...) if err != nil { - return pkgs, fmt.Errorf("failed to find bin by mime types: %w", err) + return pkgs, nil, fmt.Errorf("failed to find bin by mime types: %w", err) } for _, location := range fileMatches { r, err := resolver.FileContentsByLocation(location) if err != nil { - return pkgs, fmt.Errorf("failed to resolve file contents by location: %w", err) + return pkgs, nil, fmt.Errorf("failed to resolve file contents by location: %w", err) } - goPkgs, err := parseGoBin(location, r) + goPkgs, goRelationships, err := parseGoBin(location, r) if err != nil { log.Warnf("could not parse possible go binary: %+v", err) } - r.Close() + internal.CloseAndLogError(r, location.RealPath) pkgs = append(pkgs, goPkgs...) + relationships = append(relationships, goRelationships...) } - return pkgs, nil + return pkgs, relationships, nil } diff --git a/syft/pkg/cataloger/golang/parse_go_bin.go b/syft/pkg/cataloger/golang/parse_go_bin.go index 311bc9681..fade0d102 100644 --- a/syft/pkg/cataloger/golang/parse_go_bin.go +++ b/syft/pkg/cataloger/golang/parse_go_bin.go @@ -5,6 +5,7 @@ import ( "io" "strings" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) @@ -14,18 +15,16 @@ const ( replaceIdentifier = "=>" ) -func parseGoBin(location source.Location, reader io.ReadCloser) ([]pkg.Package, error) { +func parseGoBin(location source.Location, reader io.ReadCloser) ([]pkg.Package, []artifact.Relationship, error) { // Identify if bin was compiled by go x, err := openExe(reader) if err != nil { - return nil, err + return nil, nil, err } goVersion, mod := findVers(x) - pkgs := buildGoPkgInfo(location, mod, goVersion) - - return pkgs, nil + return buildGoPkgInfo(location, mod, goVersion), nil, nil } func buildGoPkgInfo(location source.Location, mod, goVersion string) []pkg.Package { diff --git a/syft/pkg/cataloger/golang/parse_go_mod.go b/syft/pkg/cataloger/golang/parse_go_mod.go index c8facdc25..c7cc49e5a 100644 --- a/syft/pkg/cataloger/golang/parse_go_mod.go +++ b/syft/pkg/cataloger/golang/parse_go_mod.go @@ -6,22 +6,23 @@ import ( "io/ioutil" "sort" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "golang.org/x/mod/modfile" ) // parseGoMod takes a go.mod and lists all packages discovered. -func parseGoMod(path string, reader io.Reader) ([]pkg.Package, error) { +func parseGoMod(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { packages := make(map[string]pkg.Package) contents, err := ioutil.ReadAll(reader) if err != nil { - return nil, fmt.Errorf("failed to read go module: %w", err) + return nil, nil, fmt.Errorf("failed to read go module: %w", err) } file, err := modfile.Parse(path, contents, nil) if err != nil { - return nil, fmt.Errorf("failed to parse go module: %w", err) + return nil, nil, fmt.Errorf("failed to parse go module: %w", err) } for _, m := range file.Require { @@ -59,5 +60,5 @@ func parseGoMod(path string, reader io.Reader) ([]pkg.Package, error) { return pkgsSlice[i].Name < pkgsSlice[j].Name }) - return pkgsSlice, nil + return pkgsSlice, nil, nil } diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index ca1c369a3..7e188dce9 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -9,6 +9,7 @@ import ( "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -34,12 +35,12 @@ type archiveParser struct { } // parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives. -func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, error) { +func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true) // note: even on error, we should always run cleanup functions defer cleanupFn() if err != nil { - return nil, err + return nil, nil, err } return parser.parse() } @@ -80,29 +81,31 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo } // parse the loaded archive and return all packages found. -func (j *archiveParser) parse() ([]pkg.Package, error) { - var pkgs = make([]pkg.Package, 0) +func (j *archiveParser) parse() ([]pkg.Package, []artifact.Relationship, error) { + var pkgs []pkg.Package + var relationships []artifact.Relationship // find the parent package from the java manifest parentPkg, err := j.discoverMainPackage() if err != nil { - return nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err) + return nil, nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err) } // find aux packages from pom.properties/pom.xml and potentially modify the existing parentPkg auxPkgs, err := j.discoverPkgsFromAllMavenFiles(parentPkg) if err != nil { - return nil, err + return nil, nil, err } pkgs = append(pkgs, auxPkgs...) if j.detectNested { // find nested java archive packages - nestedPkgs, err := j.discoverPkgsFromNestedArchives(parentPkg) + nestedPkgs, nestedRelationships, err := j.discoverPkgsFromNestedArchives(parentPkg) if err != nil { - return nil, err + return nil, nil, err } pkgs = append(pkgs, nestedPkgs...) + relationships = append(relationships, nestedRelationships...) } // lastly, add the parent package to the list (assuming the parent exists) @@ -110,7 +113,7 @@ func (j *archiveParser) parse() ([]pkg.Package, error) { pkgs = append([]pkg.Package{*parentPkg}, pkgs...) } - return pkgs, nil + return pkgs, relationships, nil } // discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages. @@ -189,31 +192,32 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(parentPkg *pkg.Package) ([ // discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and // associating each discovered package to the given parent package. -func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, error) { - var pkgs = make([]pkg.Package, 0) +func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) { + var pkgs []pkg.Package + var relationships []artifact.Relationship // search and parse pom.properties files & fetch the contents openers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...) if err != nil { - return nil, fmt.Errorf("unable to extract files from zip: %w", err) + return nil, nil, fmt.Errorf("unable to extract files from zip: %w", err) } // discover nested artifacts for archivePath, archiveOpener := range openers { archiveReadCloser, err := archiveOpener.Open() if err != nil { - return nil, fmt.Errorf("unable to open archived file from tempdir: %w", err) + return nil, nil, fmt.Errorf("unable to open archived file from tempdir: %w", err) } nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath) - nestedPkgs, err := parseJavaArchive(nestedPath, archiveReadCloser) + nestedPkgs, nestedRelationships, err := parseJavaArchive(nestedPath, archiveReadCloser) if err != nil { if closeErr := archiveReadCloser.Close(); closeErr != nil { log.Warnf("unable to close archived file from tempdir: %+v", closeErr) } - return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err) + return nil, nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err) } if err = archiveReadCloser.Close(); err != nil { - return nil, fmt.Errorf("unable to close archived file from tempdir: %w", err) + return nil, nil, fmt.Errorf("unable to close archived file from tempdir: %w", err) } // attach the parent package to all discovered packages that are not already associated with a java archive @@ -226,9 +230,11 @@ func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ( } pkgs = append(pkgs, p) } + + relationships = append(relationships, nestedRelationships...) } - return pkgs, nil + return pkgs, relationships, nil } func pomPropertiesByParentPath(archivePath string, extractPaths []string, virtualPath string) (map[string]pkg.PomProperties, error) { diff --git a/syft/pkg/cataloger/javascript/parse_package_json.go b/syft/pkg/cataloger/javascript/parse_package_json.go index 1a5be5e44..ad570a740 100644 --- a/syft/pkg/cataloger/javascript/parse_package_json.go +++ b/syft/pkg/cataloger/javascript/parse_package_json.go @@ -13,6 +13,7 @@ import ( "github.com/mitchellh/mapstructure" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -162,8 +163,8 @@ func licensesFromJSON(p PackageJSON) ([]string, error) { } // parsePackageJSON parses a package.json and returns the discovered JavaScript packages. -func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { - packages := make([]pkg.Package, 0) +func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { + var packages []pkg.Package dec := json.NewDecoder(reader) for { @@ -171,17 +172,17 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { if err := dec.Decode(&p); err == io.EOF { break } else if err != nil { - return nil, fmt.Errorf("failed to parse package.json file: %w", err) + return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err) } if !p.hasNameAndVersionValues() { log.Debug("encountered package.json file without a name and/or version field, ignoring this file") - return nil, nil + return nil, nil, nil } licenses, err := licensesFromJSON(p) if err != nil { - return nil, fmt.Errorf("failed to parse package.json file: %w", err) + return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err) } packages = append(packages, pkg.Package{ @@ -200,7 +201,7 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { }) } - return packages, nil + return packages, nil, nil } func (p PackageJSON) hasNameAndVersionValues() bool { diff --git a/syft/pkg/cataloger/javascript/parse_package_lock.go b/syft/pkg/cataloger/javascript/parse_package_lock.go index 3a1289230..880674963 100644 --- a/syft/pkg/cataloger/javascript/parse_package_lock.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock.go @@ -5,6 +5,7 @@ import ( "fmt" "io" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -28,11 +29,11 @@ type Dependency struct { } // parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages. -func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, error) { +func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { // in the case we find package-lock.json files in the node_modules directories, skip those // as the whole purpose of the lock file is for the specific dependencies of the root project if pathContainsNodeModulesDirectory(path) { - return nil, nil + return nil, nil, nil } var packages []pkg.Package @@ -43,7 +44,7 @@ func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, error) { if err := dec.Decode(&lock); err == io.EOF { break } else if err != nil { - return nil, fmt.Errorf("failed to parse package-lock.json file: %w", err) + return nil, nil, fmt.Errorf("failed to parse package-lock.json file: %w", err) } for name, pkgMeta := range lock.Dependencies { packages = append(packages, pkg.Package{ @@ -55,5 +56,5 @@ func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, error) { } } - return packages, nil + return packages, nil, nil } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index bd527f9b7..71e9dd3da 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -7,6 +7,7 @@ import ( "regexp" "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -34,11 +35,11 @@ const ( noVersion = "" ) -func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, error) { +func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { // in the case we find yarn.lock files in the node_modules directories, skip those // as the whole purpose of the lock file is for the specific dependencies of the project if pathContainsNodeModulesDirectory(path) { - return nil, nil + return nil, nil, nil } var packages []pkg.Package @@ -79,10 +80,10 @@ func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, error) { } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("failed to parse yarn.lock file: %w", err) + return nil, nil, fmt.Errorf("failed to parse yarn.lock file: %w", err) } - return packages, nil + return packages, nil, nil } func findPackageName(line string) string { diff --git a/syft/pkg/cataloger/python/package_cataloger.go b/syft/pkg/cataloger/python/package_cataloger.go index b49512125..d4749c874 100644 --- a/syft/pkg/cataloger/python/package_cataloger.go +++ b/syft/pkg/cataloger/python/package_cataloger.go @@ -7,6 +7,7 @@ import ( "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" @@ -31,13 +32,13 @@ func (c *PackageCataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations. -func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { +func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { var fileMatches []source.Location for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob, eggFileMetadataGlob} { matches, err := resolver.FilesByGlob(glob) if err != nil { - return nil, fmt.Errorf("failed to find files by glob: %s", glob) + return nil, nil, fmt.Errorf("failed to find files by glob: %s", glob) } fileMatches = append(fileMatches, matches...) } @@ -46,13 +47,13 @@ func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, for _, location := range fileMatches { p, err := c.catalogEggOrWheel(resolver, location) if err != nil { - return nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.RealPath, err) + return nil, nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.RealPath, err) } if p != nil { pkgs = append(pkgs, *p) } } - return pkgs, nil + return pkgs, nil, nil } // catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents. diff --git a/syft/pkg/cataloger/python/parse_pipfile_lock.go b/syft/pkg/cataloger/python/parse_pipfile_lock.go index 3a03218a1..3d34bbd0f 100644 --- a/syft/pkg/cataloger/python/parse_pipfile_lock.go +++ b/syft/pkg/cataloger/python/parse_pipfile_lock.go @@ -6,6 +6,7 @@ import ( "io" "strings" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -37,7 +38,7 @@ type Dependency struct { var _ common.ParserFn = parsePipfileLock // parsePipfileLock is a parser function for Pipfile.lock contents, returning "Default" python packages discovered. -func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, error) { +func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { packages := make([]pkg.Package, 0) dec := json.NewDecoder(reader) @@ -46,7 +47,7 @@ func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, error) { if err := dec.Decode(&lock); err == io.EOF { break } else if err != nil { - return nil, fmt.Errorf("failed to parse Pipfile.lock file: %w", err) + return nil, nil, fmt.Errorf("failed to parse Pipfile.lock file: %w", err) } for name, pkgMeta := range lock.Default { version := strings.TrimPrefix(pkgMeta.Version, "==") @@ -59,5 +60,5 @@ func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, error) { } } - return packages, nil + return packages, nil, nil } diff --git a/syft/pkg/cataloger/python/parse_poetry_lock.go b/syft/pkg/cataloger/python/parse_poetry_lock.go index b6981a1b0..70ab2f471 100644 --- a/syft/pkg/cataloger/python/parse_poetry_lock.go +++ b/syft/pkg/cataloger/python/parse_poetry_lock.go @@ -4,6 +4,7 @@ import ( "fmt" "io" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" "github.com/pelletier/go-toml" @@ -13,17 +14,17 @@ import ( var _ common.ParserFn = parsePoetryLock // parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered. -func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, error) { +func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { tree, err := toml.LoadReader(reader) if err != nil { - return nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err) + return nil, nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err) } metadata := PoetryMetadata{} err = tree.Unmarshal(&metadata) if err != nil { - return nil, fmt.Errorf("unable to parse poetry.lock: %v", err) + return nil, nil, fmt.Errorf("unable to parse poetry.lock: %v", err) } - return metadata.Pkgs(), nil + return metadata.Pkgs(), nil, nil } diff --git a/syft/pkg/cataloger/python/parse_requirements.go b/syft/pkg/cataloger/python/parse_requirements.go index b206224dd..9a7d403e6 100644 --- a/syft/pkg/cataloger/python/parse_requirements.go +++ b/syft/pkg/cataloger/python/parse_requirements.go @@ -6,6 +6,7 @@ import ( "io" "strings" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -15,7 +16,7 @@ var _ common.ParserFn = parseRequirementsTxt // parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a // specific version. -func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { packages := make([]pkg.Package, 0) scanner := bufio.NewScanner(reader) @@ -55,10 +56,10 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) { } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("failed to parse python requirements file: %w", err) + return nil, nil, fmt.Errorf("failed to parse python requirements file: %w", err) } - return packages, nil + return packages, nil, nil } // removeTrailingComment takes a requirements.txt line and strips off comment strings. diff --git a/syft/pkg/cataloger/python/parse_setup.go b/syft/pkg/cataloger/python/parse_setup.go index e1b0c39ce..d21bbabba 100644 --- a/syft/pkg/cataloger/python/parse_setup.go +++ b/syft/pkg/cataloger/python/parse_setup.go @@ -6,6 +6,7 @@ import ( "regexp" "strings" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -19,7 +20,7 @@ var _ common.ParserFn = parseSetup // " mypy2 == v0.770", ' mypy3== v0.770', --> match(name=mypy2 version=v0.770), match(name=mypy3, version=v0.770) var pinnedDependency = regexp.MustCompile(`['"]\W?(\w+\W?==\W?[\w\.]*)`) -func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseSetup(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { packages := make([]pkg.Package, 0) scanner := bufio.NewScanner(reader) @@ -46,5 +47,5 @@ func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) { } } - return packages, nil + return packages, nil, nil } diff --git a/syft/pkg/cataloger/rpmdb/cataloger.go b/syft/pkg/cataloger/rpmdb/cataloger.go index 642c494e9..4571e67e4 100644 --- a/syft/pkg/cataloger/rpmdb/cataloger.go +++ b/syft/pkg/cataloger/rpmdb/cataloger.go @@ -8,6 +8,7 @@ import ( "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) @@ -27,24 +28,26 @@ func (c *Cataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation. -func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { +func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { fileMatches, err := resolver.FilesByGlob(pkg.RpmDBGlob) if err != nil { - return nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err) + return nil, nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err) } var pkgs []pkg.Package for _, location := range fileMatches { dbContentReader, err := resolver.FileContentsByLocation(location) if err != nil { - return nil, err + return nil, nil, err } - pkgs, err = parseRpmDB(resolver, location, dbContentReader) + discoveredPkgs, err := parseRpmDB(resolver, location, dbContentReader) internal.CloseAndLogError(dbContentReader, location.VirtualPath) if err != nil { - return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err) + return nil, nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err) } + + pkgs = append(pkgs, discoveredPkgs...) } - return pkgs, nil + return pkgs, nil, nil } diff --git a/syft/pkg/cataloger/ruby/parse_gemfile_lock.go b/syft/pkg/cataloger/ruby/parse_gemfile_lock.go index b27b3ba66..2d2f62c42 100644 --- a/syft/pkg/cataloger/ruby/parse_gemfile_lock.go +++ b/syft/pkg/cataloger/ruby/parse_gemfile_lock.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -16,7 +17,7 @@ var _ common.ParserFn = parseGemFileLockEntries var sectionsOfInterest = internal.NewStringSetFromSlice([]string{"GEM"}) // parseGemFileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered. -func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { pkgs := make([]pkg.Package, 0) scanner := bufio.NewScanner(reader) @@ -49,9 +50,9 @@ func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, error) } } if err := scanner.Err(); err != nil { - return nil, err + return nil, nil, err } - return pkgs, nil + return pkgs, nil, nil } func isDependencyLine(line string) bool { diff --git a/syft/pkg/cataloger/ruby/parse_gemspec.go b/syft/pkg/cataloger/ruby/parse_gemspec.go index f743a2f5c..afab064e7 100644 --- a/syft/pkg/cataloger/ruby/parse_gemspec.go +++ b/syft/pkg/cataloger/ruby/parse_gemspec.go @@ -12,6 +12,7 @@ import ( "github.com/mitchellh/mapstructure" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -60,7 +61,7 @@ func processList(s string) []string { return results } -func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { var pkgs []pkg.Package var fields = make(map[string]interface{}) scanner := bufio.NewScanner(reader) @@ -93,7 +94,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) { if fields["name"] != "" && fields["version"] != "" { var metadata pkg.GemMetadata if err := mapstructure.Decode(fields, &metadata); err != nil { - return nil, fmt.Errorf("unable to decode gem metadata: %w", err) + return nil, nil, fmt.Errorf("unable to decode gem metadata: %w", err) } pkgs = append(pkgs, pkg.Package{ @@ -107,7 +108,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) { }) } - return pkgs, nil + return pkgs, nil, nil } // renderUtf8 takes any string escaped string sub-sections from the ruby string and replaces those sections with the UTF8 runes. diff --git a/syft/pkg/cataloger/rust/parse_cargo_lock.go b/syft/pkg/cataloger/rust/parse_cargo_lock.go index ccb7dafd8..ab5918978 100644 --- a/syft/pkg/cataloger/rust/parse_cargo_lock.go +++ b/syft/pkg/cataloger/rust/parse_cargo_lock.go @@ -4,6 +4,7 @@ import ( "fmt" "io" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" "github.com/pelletier/go-toml" @@ -13,17 +14,17 @@ import ( var _ common.ParserFn = parseCargoLock // parseCargoLock is a parser function for Cargo.lock contents, returning all rust cargo crates discovered. -func parseCargoLock(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseCargoLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { tree, err := toml.LoadReader(reader) if err != nil { - return nil, fmt.Errorf("unable to load Cargo.lock for parsing: %v", err) + return nil, nil, fmt.Errorf("unable to load Cargo.lock for parsing: %v", err) } metadata := CargoMetadata{} err = tree.Unmarshal(&metadata) if err != nil { - return nil, fmt.Errorf("unable to parse Cargo.lock: %v", err) + return nil, nil, fmt.Errorf("unable to parse Cargo.lock: %v", err) } - return metadata.Pkgs(), nil + return metadata.Pkgs(), nil, nil } diff --git a/syft/pkg/package.go b/syft/pkg/package.go index 8bd92b260..937aeb60b 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -15,7 +15,7 @@ import ( // Package represents an application or library that has been bundled into a distributable format. // TODO: if we ignore FoundBy for ID generation should we merge the field to show it was found in two places? type Package struct { - ID artifact.ID `hash:"ignore"` // uniquely identifies a package, set by the cataloger + ID artifact.ID `hash:"ignore"` // uniquely identifies a package Name string // the package name Version string // the version of the package FoundBy string // the specific cataloger that discovered this package diff --git a/syft/sbom/sbom.go b/syft/sbom/sbom.go index 411bcc28c..82e97dd2a 100644 --- a/syft/sbom/sbom.go +++ b/syft/sbom/sbom.go @@ -1,6 +1,7 @@ package sbom import ( + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -8,8 +9,9 @@ import ( ) type SBOM struct { - Artifacts Artifacts - Source source.Metadata + Artifacts Artifacts + Relationships []artifact.Relationship + Source source.Metadata } type Artifacts struct {