Replace core SBOM-creation API with builder pattern (#1383)

* remove existing cataloging API

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add file cataloging config

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add package cataloging config

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add configs for cross-cutting concerns

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename CLI option configs to not require import aliases later

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update all nested structs for the Catalog struct

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update Catalog cli options

- add new cataloger selection options (selection and default)
- remove the excludeBinaryOverlapByOwnership
- deprecate "catalogers" flag
- add new javascript configuration

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* migrate relationship capabilities to separate internal package

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor golang cataloger to use configuration options when creating packages

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* create internal object to facilitate reading from and writing to an SBOM

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* create a command-like object (task) to facilitate partial SBOM creation

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add cataloger selection capability

- be able to parse string expressions into a set of resolved actions against sets
- be able to use expressions to select/add/remove tasks to/from the final set of tasks to run

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add package, file, and environment related tasks

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update existing file catalogers to use nested UI elements

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add CreateSBOMConfig that drives the SBOM creation process

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* capture SBOM creation info as a struct

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add CreateSBOM() function

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix tests

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update docs with SBOM selection help + breaking changes

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix multiple override default inputs

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix deprecation flag printing to stdout

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor cataloger selection description to separate object

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* keep expression errors and show specific suggestions only

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address additional review feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* address more review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* addressed additional PR review feedback

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* fix file selection references

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove guess language data generation option

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add tests for coordinatesForSelection

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename relationship attributes

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add descriptions to relationships config fields

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* improve documentation around configuration options

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add explicit errors around legacy config entries

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

syft/pkg/cataloger/binary/cataloger.go

@@ -13,7 +13,7 @@ import (
 const catalogerName = "binary-cataloger"
 
 type CatalogerConfig struct {
-	Classifiers []Classifier
+	Classifiers []Classifier `yaml:"classifiers" json:"classifiers" mapstructure:"classifiers"`
 }
 
 func DefaultCatalogerConfig() CatalogerConfig {

syft/pkg/cataloger/binary/classifier.go

@@ -23,25 +23,25 @@ import (
 // Classifier is a generic package classifier that can be used to match a package definition
 // to a file that meets the given content criteria of the EvidenceMatcher.
 type Classifier struct {
-	Class string
+	Class string `json:"class"`
 
 	// FileGlob is a selector to narrow down file inspection using the **/glob* syntax
-	FileGlob string
+	FileGlob string `json:"fileGlob"`
 
 	// EvidenceMatcher is what will be used to match against the file in the source
 	// location. If the matcher returns a package, the file will be considered a candidate.
-	EvidenceMatcher EvidenceMatcher
+	EvidenceMatcher EvidenceMatcher `json:"-"`
 
 	// Information below is used to specify the Package information when returned
 
 	// Package is the name to use for the package
-	Package string
+	Package string `json:"package"`
 
 	// PURL is the Package URL to use when generating a package
-	PURL packageurl.PackageURL
+	PURL packageurl.PackageURL `json:"purl"`
 
 	// CPEs are the specific CPEs we want to include for this binary with updated version information
-	CPEs []cpe.CPE
+	CPEs []cpe.CPE `json:"cpes"`
 }
 
 // EvidenceMatcher is a function called to catalog Packages that match some sort of evidence

syft/pkg/cataloger/catalog.go

@@ -1,229 +0,0 @@
-package cataloger
-
-import (
-	"fmt"
-	"math"
-	"runtime/debug"
-	"sync"
-
-	"github.com/dustin/go-humanize"
-	"github.com/hashicorp/go-multierror"
-	"github.com/wagoodman/go-progress"
-
-	"github.com/anchore/syft/internal/bus"
-	"github.com/anchore/syft/internal/log"
-	"github.com/anchore/syft/syft/artifact"
-	"github.com/anchore/syft/syft/event/monitor"
-	"github.com/anchore/syft/syft/file"
-	"github.com/anchore/syft/syft/linux"
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/pkg/cataloger/common/cpe"
-)
-
-// Monitor provides progress-related data for observing the progress of a Catalog() call (published on the event bus).
-type Monitor struct {
-	FilesProcessed     progress.Monitorable // the number of files selected and contents analyzed from all registered catalogers
-	PackagesDiscovered progress.Monitorable // the number of packages discovered from all registered catalogers
-}
-
-// catalogResult provides the result of running a single cataloger against source
-type catalogResult struct {
-	Packages      []pkg.Package
-	Relationships []artifact.Relationship
-	// Discovered may sometimes be more than len(packages)
-	Discovered int64
-	Error      error
-}
-
-func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerResult *catalogResult, err error) {
-	// handle individual cataloger panics
-	defer func() {
-		if e := recover(); e != nil {
-			err = fmt.Errorf("%v at:\n%s", e, string(debug.Stack()))
-		}
-	}()
-
-	catalogerResult = new(catalogResult)
-
-	// find packages from the underlying raw data
-	log.WithFields("cataloger", cataloger.Name()).Trace("cataloging started")
-	packages, relationships, err := cataloger.Catalog(resolver)
-	if err != nil {
-		log.WithFields("cataloger", cataloger.Name()).Warn("error while cataloging")
-		return catalogerResult, err
-	}
-
-	catalogedPackages := len(packages)
-
-	log.WithFields("cataloger", cataloger.Name()).Debugf("discovered %d packages", catalogedPackages)
-	catalogerResult.Discovered = int64(catalogedPackages)
-
-	for _, p := range packages {
-		// generate CPEs (note: this is excluded from package ID, so is safe to mutate)
-		// we might have binary classified CPE already with the package so we want to append here
-
-		dictionaryCPE, ok := cpe.DictionaryFind(p)
-		if ok {
-			log.Debugf("used CPE dictionary to find CPE for %s package %q: %s", p.Type, p.Name, dictionaryCPE.BindToFmtString())
-			p.CPEs = append(p.CPEs, dictionaryCPE)
-		} else {
-			p.CPEs = append(p.CPEs, cpe.Generate(p)...)
-		}
-
-		// if we were not able to identify the language we have an opportunity
-		// to try and get this value from the PURL. Worst case we assert that
-		// we could not identify the language at either stage and set UnknownLanguage
-		if p.Language == "" {
-			p.Language = pkg.LanguageFromPURL(p.PURL)
-		}
-
-		// create file-to-package relationships for files owned by the package
-		owningRelationships, err := packageFileOwnershipRelationships(p, resolver)
-		if err != nil {
-			log.WithFields("cataloger", cataloger.Name(), "package", p.Name, "error", err).Warnf("unable to create any package-file relationships")
-		} else {
-			catalogerResult.Relationships = append(catalogerResult.Relationships, owningRelationships...)
-		}
-		catalogerResult.Packages = append(catalogerResult.Packages, p)
-	}
-
-	catalogerResult.Relationships = append(catalogerResult.Relationships, relationships...)
-	log.WithFields("cataloger", cataloger.Name()).Trace("cataloging complete")
-	return catalogerResult, err
-}
-
-// Catalog a given source (container image or filesystem) with the given catalogers, returning all discovered packages.
-// In order to efficiently retrieve contents from a underlying container image the content fetch requests are
-// done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single
-// request.
-//
-//nolint:funlen
-func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, catalogers ...pkg.Cataloger) (*pkg.Collection, []artifact.Relationship, error) {
-	catalog := pkg.NewCollection()
-	var allRelationships []artifact.Relationship
-
-	prog := monitorPackageCatalogingTask()
-
-	// perform analysis, accumulating errors for each failed analysis
-	var errs error
-
-	nCatalogers := len(catalogers)
-
-	// we do not need more parallelism than there are `catalogers`.
-	parallelism = int(math.Min(float64(nCatalogers), math.Max(1.0, float64(parallelism))))
-	log.WithFields("parallelism", parallelism, "catalogers", nCatalogers).Debug("cataloging packages")
-
-	jobs := make(chan pkg.Cataloger, nCatalogers)
-	results := make(chan *catalogResult, nCatalogers)
-
-	waitGroup := sync.WaitGroup{}
-
-	var totalPackagesDiscovered int64
-
-	for i := 0; i < parallelism; i++ {
-		waitGroup.Add(1)
-
-		go func() {
-			defer waitGroup.Done()
-
-			// wait for / get the next cataloger job available.
-			for cataloger := range jobs {
-				result, err := runCataloger(cataloger, resolver)
-
-				// ensure we set the error to be aggregated
-				result.Error = err
-
-				prog.Add(result.Discovered)
-				totalPackagesDiscovered += result.Discovered
-				count := humanize.Comma(totalPackagesDiscovered)
-				prog.AtomicStage.Set(fmt.Sprintf("%s packages", count))
-
-				results <- result
-			}
-		}()
-	}
-
-	// Enqueue the jobs
-	for _, cataloger := range catalogers {
-		jobs <- cataloger
-	}
-	close(jobs)
-
-	// Wait for the jobs to finish
-	waitGroup.Wait()
-	close(results)
-
-	// collect the results
-	for result := range results {
-		if result.Error != nil {
-			errs = multierror.Append(errs, result.Error)
-		}
-		for _, p := range result.Packages {
-			catalog.Add(p)
-		}
-		allRelationships = append(allRelationships, result.Relationships...)
-	}
-
-	allRelationships = append(allRelationships, pkg.NewRelationships(catalog)...)
-
-	if errs != nil {
-		prog.SetError(errs)
-	} else {
-		prog.SetCompleted()
-	}
-
-	return catalog, allRelationships, errs
-}
-
-func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver) ([]artifact.Relationship, error) {
-	fileOwner, ok := p.Metadata.(pkg.FileOwner)
-	if !ok {
-		return nil, nil
-	}
-
-	locations := map[artifact.ID]file.Location{}
-
-	for _, path := range fileOwner.OwnedFiles() {
-		pathRefs, err := resolver.FilesByPath(path)
-		if err != nil {
-			return nil, fmt.Errorf("unable to find path for path=%q: %w", path, err)
-		}
-
-		if len(pathRefs) == 0 {
-			// ideally we want to warn users about missing files from a package, however, it is very common for
-			// container image authors to delete files that are not needed in order to keep image sizes small. Adding
-			// a warning here would be needlessly noisy (even for popular base images).
-			continue
-		}
-
-		for _, ref := range pathRefs {
-			if oldRef, ok := locations[ref.Coordinates.ID()]; ok {
-				log.Debugf("found path duplicate of %s", oldRef.RealPath)
-			}
-			locations[ref.Coordinates.ID()] = ref
-		}
-	}
-
-	var relationships []artifact.Relationship
-	for _, location := range locations {
-		relationships = append(relationships, artifact.Relationship{
-			From: p,
-			To:   location.Coordinates,
-			Type: artifact.ContainsRelationship,
-		})
-	}
-	return relationships, nil
-}
-
-func monitorPackageCatalogingTask() *monitor.CatalogerTaskProgress {
-	info := monitor.GenericTask{
-		Title: monitor.Title{
-			Default:      "Catalog packages",
-			WhileRunning: "Cataloging packages",
-			OnSuccess:    "Cataloged packages",
-		},
-		HideOnSuccess: false,
-	}
-
-	return bus.StartCatalogerTask(info, -1, "")
-}

syft/pkg/cataloger/catalog_test.go

@@ -1,67 +0,0 @@
-package cataloger
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/anchore/syft/syft/artifact"
-	"github.com/anchore/syft/syft/file"
-	"github.com/anchore/syft/syft/linux"
-	"github.com/anchore/syft/syft/pkg"
-)
-
-func Test_CatalogPanicHandling(t *testing.T) {
-	catalog, relationships, err := Catalog(
-		file.NewMockResolverForPaths(),
-		&linux.Release{},
-		1,
-		panickingCataloger{},
-		returningCataloger{},
-	)
-
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "catalog_test.go")
-	require.Len(t, catalog.Sorted(), 2)
-	require.Len(t, relationships, 1)
-}
-
-type panickingCataloger struct{}
-
-func (p panickingCataloger) Name() string {
-	return "panicking-cataloger"
-}
-
-func (p panickingCataloger) Catalog(_ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
-	panic("something bad happened")
-}
-
-var _ pkg.Cataloger = (*panickingCataloger)(nil)
-
-type returningCataloger struct{}
-
-func (p returningCataloger) Name() string {
-	return "returning-cataloger"
-}
-
-func (p returningCataloger) Catalog(_ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
-	pkg1 := pkg.Package{
-		Name:    "package-1",
-		Version: "1.0",
-	}
-	pkg1.SetID()
-	pkg2 := pkg.Package{
-		Name:    "package-2",
-		Version: "2.0",
-	}
-	pkg2.SetID()
-	return []pkg.Package{pkg1, pkg2}, []artifact.Relationship{
-		{
-			From: pkg1,
-			To:   pkg2,
-			Type: artifact.DependencyOfRelationship,
-		},
-	}, nil
-}
-
-var _ pkg.Cataloger = (*returningCataloger)(nil)

syft/pkg/cataloger/cataloger.go

@@ -1,212 +0,0 @@
-/*
-Package cataloger provides the ability to process files from a container image or file system and discover packages
-(gems, wheels, jars, rpms, debs, etc). Specifically, this package contains both a catalog function to utilize all
-catalogers defined in child packages as well as the interface definition to implement a cataloger.
-*/
-package cataloger
-
-import (
-	"strings"
-
-	"github.com/anchore/syft/internal/log"
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/pkg/cataloger/alpine"
-	"github.com/anchore/syft/syft/pkg/cataloger/arch"
-	"github.com/anchore/syft/syft/pkg/cataloger/binary"
-	"github.com/anchore/syft/syft/pkg/cataloger/cpp"
-	"github.com/anchore/syft/syft/pkg/cataloger/dart"
-	"github.com/anchore/syft/syft/pkg/cataloger/debian"
-	"github.com/anchore/syft/syft/pkg/cataloger/dotnet"
-	"github.com/anchore/syft/syft/pkg/cataloger/elixir"
-	"github.com/anchore/syft/syft/pkg/cataloger/erlang"
-	"github.com/anchore/syft/syft/pkg/cataloger/gentoo"
-	"github.com/anchore/syft/syft/pkg/cataloger/githubactions"
-	"github.com/anchore/syft/syft/pkg/cataloger/golang"
-	"github.com/anchore/syft/syft/pkg/cataloger/haskell"
-	"github.com/anchore/syft/syft/pkg/cataloger/java"
-	"github.com/anchore/syft/syft/pkg/cataloger/javascript"
-	"github.com/anchore/syft/syft/pkg/cataloger/kernel"
-	"github.com/anchore/syft/syft/pkg/cataloger/nix"
-	"github.com/anchore/syft/syft/pkg/cataloger/php"
-	"github.com/anchore/syft/syft/pkg/cataloger/python"
-	"github.com/anchore/syft/syft/pkg/cataloger/r"
-	"github.com/anchore/syft/syft/pkg/cataloger/redhat"
-	"github.com/anchore/syft/syft/pkg/cataloger/ruby"
-	"github.com/anchore/syft/syft/pkg/cataloger/rust"
-	"github.com/anchore/syft/syft/pkg/cataloger/sbom"
-	"github.com/anchore/syft/syft/pkg/cataloger/swift"
-)
-
-const AllCatalogersPattern = "all"
-
-// ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
-func ImageCatalogers(cfg Config) []pkg.Cataloger {
-	return filterCatalogers([]pkg.Cataloger{
-		arch.NewDBCataloger(),
-		alpine.NewDBCataloger(),
-		binary.NewCataloger(cfg.Binary),
-		cpp.NewConanInfoCataloger(),
-		debian.NewDBCataloger(),
-		dotnet.NewDotnetPortableExecutableCataloger(),
-		golang.NewGoModuleBinaryCataloger(cfg.Golang),
-		java.NewArchiveCataloger(cfg.JavaConfig()),
-		java.NewNativeImageCataloger(),
-		javascript.NewPackageCataloger(),
-		nix.NewStoreCataloger(),
-		php.NewComposerInstalledCataloger(),
-		gentoo.NewPortageCataloger(),
-		python.NewInstalledPackageCataloger(),
-		r.NewPackageCataloger(),
-		redhat.NewDBCataloger(),
-		ruby.NewInstalledGemSpecCataloger(),
-		sbom.NewCataloger(),
-	}, cfg.Catalogers)
-}
-
-// DirectoryCatalogers returns a slice of locally implemented catalogers that are fit for detecting packages from index files (and select installations)
-func DirectoryCatalogers(cfg Config) []pkg.Cataloger {
-	return filterCatalogers([]pkg.Cataloger{
-		arch.NewDBCataloger(),
-		alpine.NewDBCataloger(),
-		binary.NewCataloger(cfg.Binary),
-		cpp.NewConanCataloger(),
-		dart.NewPubspecLockCataloger(),
-		debian.NewDBCataloger(),
-		dotnet.NewDotnetDepsCataloger(),
-		dotnet.NewDotnetPortableExecutableCataloger(),
-		elixir.NewMixLockCataloger(),
-		erlang.NewRebarLockCataloger(),
-		githubactions.NewActionUsageCataloger(),
-		githubactions.NewWorkflowUsageCataloger(),
-		golang.NewGoModuleFileCataloger(cfg.Golang),
-		golang.NewGoModuleBinaryCataloger(cfg.Golang),
-		haskell.NewHackageCataloger(),
-		java.NewArchiveCataloger(cfg.JavaConfig()),
-		java.NewGradleLockfileCataloger(),
-		java.NewPomCataloger(cfg.JavaConfig()),
-		java.NewNativeImageCataloger(),
-		javascript.NewLockCataloger(cfg.Javascript),
-		nix.NewStoreCataloger(),
-		php.NewComposerLockCataloger(),
-		gentoo.NewPortageCataloger(),
-		python.NewPackageCataloger(cfg.Python),
-		python.NewInstalledPackageCataloger(),
-		redhat.NewArchiveCataloger(),
-		redhat.NewDBCataloger(),
-		ruby.NewGemFileLockCataloger(),
-		ruby.NewGemSpecCataloger(),
-		rust.NewCargoLockCataloger(),
-		sbom.NewCataloger(),
-		swift.NewCocoapodsCataloger(),
-		swift.NewSwiftPackageManagerCataloger(),
-	}, cfg.Catalogers)
-}
-
-// AllCatalogers returns all implemented catalogers
-func AllCatalogers(cfg Config) []pkg.Cataloger {
-	return filterCatalogers([]pkg.Cataloger{
-		arch.NewDBCataloger(),
-		alpine.NewDBCataloger(),
-		binary.NewCataloger(cfg.Binary),
-		cpp.NewConanCataloger(),
-		dart.NewPubspecLockCataloger(),
-		debian.NewDBCataloger(),
-		dotnet.NewDotnetDepsCataloger(),
-		dotnet.NewDotnetPortableExecutableCataloger(),
-		elixir.NewMixLockCataloger(),
-		erlang.NewRebarLockCataloger(),
-		githubactions.NewActionUsageCataloger(),
-		githubactions.NewWorkflowUsageCataloger(),
-		golang.NewGoModuleFileCataloger(cfg.Golang),
-		golang.NewGoModuleBinaryCataloger(cfg.Golang),
-		haskell.NewHackageCataloger(),
-		java.NewArchiveCataloger(cfg.JavaConfig()),
-		java.NewGradleLockfileCataloger(),
-		java.NewPomCataloger(cfg.JavaConfig()),
-		java.NewNativeImageCataloger(),
-		javascript.NewLockCataloger(cfg.Javascript),
-		javascript.NewPackageCataloger(),
-		kernel.NewLinuxKernelCataloger(cfg.LinuxKernel),
-		nix.NewStoreCataloger(),
-		php.NewComposerInstalledCataloger(),
-		php.NewComposerLockCataloger(),
-		gentoo.NewPortageCataloger(),
-		python.NewPackageCataloger(cfg.Python),
-		python.NewInstalledPackageCataloger(),
-		r.NewPackageCataloger(),
-		redhat.NewArchiveCataloger(),
-		redhat.NewDBCataloger(),
-		ruby.NewGemFileLockCataloger(),
-		ruby.NewGemSpecCataloger(),
-		ruby.NewInstalledGemSpecCataloger(),
-		rust.NewAuditBinaryCataloger(),
-		rust.NewCargoLockCataloger(),
-		sbom.NewCataloger(),
-		swift.NewCocoapodsCataloger(),
-		swift.NewSwiftPackageManagerCataloger(),
-	}, cfg.Catalogers)
-}
-
-func RequestedAllCatalogers(cfg Config) bool {
-	for _, enableCatalogerPattern := range cfg.Catalogers {
-		if enableCatalogerPattern == AllCatalogersPattern {
-			return true
-		}
-	}
-	return false
-}
-
-func filterCatalogers(catalogers []pkg.Cataloger, enabledCatalogerPatterns []string) []pkg.Cataloger {
-	// if cataloger is not set, all applicable catalogers are enabled by default
-	if len(enabledCatalogerPatterns) == 0 {
-		return catalogers
-	}
-	for _, enableCatalogerPattern := range enabledCatalogerPatterns {
-		if enableCatalogerPattern == AllCatalogersPattern {
-			return catalogers
-		}
-	}
-	var keepCatalogers []pkg.Cataloger
-	for _, cataloger := range catalogers {
-		if contains(enabledCatalogerPatterns, cataloger.Name()) {
-			keepCatalogers = append(keepCatalogers, cataloger)
-			continue
-		}
-		log.Infof("skipping cataloger %q", cataloger.Name())
-	}
-	return keepCatalogers
-}
-
-func contains(enabledPartial []string, catalogerName string) bool {
-	catalogerName = strings.TrimSuffix(catalogerName, "-cataloger")
-	for _, partial := range enabledPartial {
-		partial = strings.TrimSuffix(partial, "-cataloger")
-		if partial == "" {
-			continue
-		}
-		if hasFullWord(partial, catalogerName) {
-			return true
-		}
-	}
-	return false
-}
-
-func hasFullWord(targetPhrase, candidate string) bool {
-	if targetPhrase == "cataloger" || targetPhrase == "" {
-		return false
-	}
-	start := strings.Index(candidate, targetPhrase)
-	if start == -1 {
-		return false
-	}
-
-	if start > 0 && candidate[start-1] != '-' {
-		return false
-	}
-
-	end := start + len(targetPhrase)
-	if end < len(candidate) && candidate[end] != '-' {
-		return false
-	}
-	return true
-}

syft/pkg/cataloger/cataloger_test.go

@@ -1,342 +0,0 @@
-package cataloger
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/anchore/syft/syft/artifact"
-	"github.com/anchore/syft/syft/file"
-	"github.com/anchore/syft/syft/pkg"
-)
-
-var _ pkg.Cataloger = (*dummy)(nil)
-
-type dummy struct {
-	name string
-}
-
-func (d dummy) Name() string {
-	return d.name
-}
-
-func (d dummy) Catalog(_ file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
-	panic("not implemented")
-}
-
-func Test_filterCatalogers(t *testing.T) {
-	largeCatalogerList := []string{
-		"alpm-db-cataloger",
-		"apkdb-cataloger",
-		"binary-cataloger",
-		"conan-cataloger",
-		"dartlang-lock-cataloger",
-		"dpkg-db-cataloger",
-		"dotnet-deps-cataloger",
-		"elixir-mix-lock-cataloger",
-		"erlang-rebar-lock-cataloger",
-		"go-module-file-cataloger",
-		"go-module-binary-cataloger",
-		"haskell-cataloger",
-		"graalvm-native-image-cataloger",
-		"java-cataloger",
-		"java-pom-cataloger",
-		"javascript-package-cataloger",
-		"javascript-lock-cataloger",
-		"php-composer-installed-cataloger",
-		"php-composer-lock-cataloger",
-		"portage-cataloger",
-		"python-package-cataloger",
-		"python-installed-package-cataloger",
-		"rpm-db-cataloger",
-		"rpm-archive-cataloger",
-		"ruby-gemfile-cataloger",
-		"ruby-installed-gemspec-cataloger",
-		"rust-cargo-lock-cataloger",
-		"cargo-auditable-binary-cataloger",
-		"sbom-cataloger",
-		"cocoapods-cataloger",
-	}
-	tests := []struct {
-		name       string
-		patterns   []string
-		catalogers []string
-		want       []string
-	}{
-		{
-			name:     "no filtering",
-			patterns: nil,
-			catalogers: []string{
-				"ruby-installed-gemspec-cataloger",
-				"python-installed-package-cataloger",
-				"php-composer-installed-cataloger",
-				"javascript-package-cataloger",
-				"dpkg-db-cataloger",
-				"rpm-db-cataloger",
-				"java-cataloger",
-				"apkdb-cataloger",
-				"go-module-binary-cataloger",
-			},
-			want: []string{
-				"ruby-installed-gemspec-cataloger",
-				"python-installed-package-cataloger",
-				"php-composer-installed-cataloger",
-				"javascript-package-cataloger",
-				"dpkg-db-cataloger",
-				"rpm-db-cataloger",
-				"java-cataloger",
-				"apkdb-cataloger",
-				"go-module-binary-cataloger",
-			},
-		},
-		{
-			name: "exact name match",
-			patterns: []string{
-				"rpm-db-cataloger",
-				"javascript-package-cataloger",
-			},
-			catalogers: []string{
-				"ruby-installed-gemspec-cataloger",
-				"python-installed-package-cataloger",
-				"php-composer-installed-cataloger",
-				"javascript-package-cataloger",
-				"dpkg-db-cataloger",
-				"rpm-db-cataloger",
-				"java-cataloger",
-				"apkdb-cataloger",
-				"go-module-binary-cataloger",
-			},
-			want: []string{
-				"javascript-package-cataloger",
-				"rpm-db-cataloger",
-			},
-		},
-		{
-			name: "partial name match",
-			patterns: []string{
-				"ruby",
-				"installed",
-			},
-			catalogers: []string{
-				"ruby-installed-gemspec-cataloger",
-				"ruby-gemfile-cataloger",
-				"python-installed-package-cataloger",
-				"php-composer-installed-cataloger",
-				"javascript-package-cataloger",
-				"dpkg-db-cataloger",
-				"rpm-db-cataloger",
-				"java-cataloger",
-				"apkdb-cataloger",
-				"go-module-binary-cataloger",
-			},
-			want: []string{
-				"php-composer-installed-cataloger",
-				"python-installed-package-cataloger",
-				"ruby-installed-gemspec-cataloger",
-				"ruby-gemfile-cataloger",
-			},
-		},
-		{
-			name: "ignore 'cataloger' keyword",
-			patterns: []string{
-				"cataloger",
-			},
-			catalogers: []string{
-				"ruby-installed-gemspec-cataloger",
-				"ruby-gemfile-cataloger",
-				"python-installed-package-cataloger",
-				"php-composer-installed-cataloger",
-				"javascript-package-cataloger",
-				"dpkg-db-cataloger",
-				"rpm-db-cataloger",
-				"java-cataloger",
-				"apkdb-cataloger",
-				"go-module-binary-cataloger",
-			},
-			want: []string{},
-		},
-		{
-			name: "only some patterns match",
-			patterns: []string{
-				"cataloger",
-				"go-module",
-			},
-			catalogers: []string{
-				"ruby-installed-gemspec-cataloger",
-				"ruby-gemfile-cataloger",
-				"python-installed-package-cataloger",
-				"php-composer-installed-cataloger",
-				"javascript-package-cataloger",
-				"dpkg-db-cataloger",
-				"rpm-db-cataloger",
-				"java-cataloger",
-				"apkdb-cataloger",
-				"go-module-binary-cataloger",
-			},
-			want: []string{
-				"go-module-binary-cataloger",
-			},
-		},
-		{
-			name: "don't cross match ecosystems with matching prefix",
-			patterns: []string{
-				"java-cataloger",
-			},
-			catalogers: []string{
-				"javascript-package-cataloger",
-				"java-cataloger",
-			},
-			want: []string{
-				"java-cataloger",
-			},
-		},
-		{
-			name: "don't cross match ecosystems with short, common name",
-			patterns: []string{
-				"go",
-			},
-			catalogers: largeCatalogerList,
-			want: []string{
-				"go-module-file-cataloger",
-				"go-module-binary-cataloger",
-				//"rust-cargo-lock-cataloger",  // with naive "contains" matching
-				//"cargo-auditable-binary-cataloger",  // with naive "contains" matching
-			},
-		},
-		{
-			name: "ignore partial matches",
-			patterns: []string{
-				"mod",
-			},
-			catalogers: largeCatalogerList,
-			want:       []string{
-				// important! these are NOT found
-				//"go-module-file-cataloger",
-				//"go-module-binary-cataloger",
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			var catalogers []pkg.Cataloger
-			for _, n := range tt.catalogers {
-				catalogers = append(catalogers, dummy{name: n})
-			}
-			got := filterCatalogers(catalogers, tt.patterns)
-			var gotNames []string
-			for _, g := range got {
-				gotNames = append(gotNames, g.Name())
-			}
-			assert.ElementsMatch(t, tt.want, gotNames)
-		})
-	}
-}
-
-func Test_contains(t *testing.T) {
-	tests := []struct {
-		name              string
-		enabledCatalogers []string
-		catalogerName     string
-		want              bool
-	}{
-		{
-			name: "keep exact match",
-			enabledCatalogers: []string{
-				"php-composer-installed-cataloger",
-			},
-			catalogerName: "php-composer-installed-cataloger",
-			want:          true,
-		},
-		{
-			name: "match substring",
-			enabledCatalogers: []string{
-				"python",
-			},
-			catalogerName: "python-installed-package-cataloger",
-			want:          true,
-		},
-		{
-			name: "dont match on 'cataloger'",
-			enabledCatalogers: []string{
-				"cataloger",
-			},
-			catalogerName: "python-installed-package-cataloger",
-			want:          false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			assert.Equal(t, tt.want, contains(tt.enabledCatalogers, tt.catalogerName))
-		})
-	}
-}
-
-func Test_hasFullWord(t *testing.T) {
-
-	tests := []struct {
-		name         string
-		targetPhrase string
-		candidate    string
-		want         bool
-	}{
-		{
-			name:         "exact match",
-			targetPhrase: "php-composer-installed-cataloger",
-			candidate:    "php-composer-installed-cataloger",
-			want:         true,
-		},
-		{
-			name:         "partial, full word match",
-			targetPhrase: "composer",
-			candidate:    "php-composer-installed-cataloger",
-			want:         true,
-		},
-		{
-			name:         "partial, full, multi-word match",
-			targetPhrase: "php-composer",
-			candidate:    "php-composer-installed-cataloger",
-			want:         true,
-		},
-		{
-			name:         "prefix match",
-			targetPhrase: "php",
-			candidate:    "php-composer-installed-cataloger",
-			want:         true,
-		},
-		{
-			name:         "postfix match with -cataloger suffix",
-			targetPhrase: "installed",
-			candidate:    "php-composer-installed-cataloger",
-			want:         true,
-		},
-		{
-			name:         "postfix match",
-			targetPhrase: "installed",
-			candidate:    "php-composer-installed",
-			want:         true,
-		},
-		{
-			name:         "ignore cataloger keyword",
-			targetPhrase: "cataloger",
-			candidate:    "php-composer-installed-cataloger",
-			want:         false,
-		},
-		{
-			name:         "ignore partial match",
-			targetPhrase: "hp",
-			candidate:    "php-composer-installed-cataloger",
-			want:         false,
-		},
-		{
-			name:         "ignore empty string",
-			targetPhrase: "",
-			candidate:    "php-composer-installed-cataloger",
-			want:         false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			assert.Equalf(t, tt.want, hasFullWord(tt.targetPhrase, tt.candidate), "hasFullWord(%v, %v)", tt.targetPhrase, tt.candidate)
-		})
-	}
-}

syft/pkg/cataloger/config.go

@@ -1,53 +0,0 @@
-package cataloger
-
-import (
-	"github.com/anchore/syft/syft/cataloging"
-	"github.com/anchore/syft/syft/pkg/cataloger/binary"
-	"github.com/anchore/syft/syft/pkg/cataloger/golang"
-	"github.com/anchore/syft/syft/pkg/cataloger/java"
-	"github.com/anchore/syft/syft/pkg/cataloger/javascript"
-	"github.com/anchore/syft/syft/pkg/cataloger/kernel"
-	"github.com/anchore/syft/syft/pkg/cataloger/python"
-)
-
-// TODO: these field naming vs helper function naming schemes are inconsistent.
-type Config struct {
-	Search                          SearchConfig
-	Golang                          golang.CatalogerConfig
-	LinuxKernel                     kernel.LinuxKernelCatalogerConfig
-	Python                          python.CatalogerConfig
-	Java                            java.ArchiveCatalogerConfig
-	Javascript                      javascript.CatalogerConfig
-	Binary                          binary.CatalogerConfig
-	Catalogers                      []string
-	Parallelism                     int
-	ExcludeBinaryOverlapByOwnership bool
-}
-
-func DefaultConfig() Config {
-	return Config{
-		Search:                          DefaultSearchConfig(),
-		Parallelism:                     1,
-		LinuxKernel:                     kernel.DefaultLinuxCatalogerConfig(),
-		Python:                          python.DefaultCatalogerConfig(),
-		Java:                            java.DefaultArchiveCatalogerConfig(),
-		Javascript:                      javascript.DefaultCatalogerConfig(),
-		Binary:                          binary.DefaultCatalogerConfig(),
-		ExcludeBinaryOverlapByOwnership: true,
-	}
-}
-
-// JavaConfig merges relevant config values from Config to return a java.Config struct.
-// Values like IncludeUnindexedArchives and IncludeIndexedArchives are used across catalogers
-// and are not specific to Java requiring this merge.
-func (c Config) JavaConfig() java.ArchiveCatalogerConfig {
-	return java.ArchiveCatalogerConfig{
-		ArchiveSearchConfig: cataloging.ArchiveSearchConfig{
-			IncludeUnindexedArchives: c.Search.IncludeUnindexedArchives,
-			IncludeIndexedArchives:   c.Search.IncludeIndexedArchives,
-		},
-		UseNetwork:              c.Java.UseNetwork,
-		MavenBaseURL:            c.Java.MavenBaseURL,
-		MaxParentRecursiveDepth: c.Java.MaxParentRecursiveDepth,
-	}
-}

syft/pkg/cataloger/dotnet/parse_dotnet_deps.go

@@ -6,6 +6,7 @@ import (
 	"sort"
 
 	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/internal/relationship"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
@@ -122,7 +123,7 @@ func parseDotnetDeps(_ file.Resolver, _ *generic.Environment, reader file.Locati
 	// sort the relationships for deterministic output
 	// TODO: ideally this would be replaced with artifact.SortRelationships when one exists and is type agnostic.
 	// this will only consider package-to-package relationships.
-	pkg.SortRelationships(relationships)
+	relationship.Sort(relationships)
 
 	return pkgs, relationships, nil
 }

syft/pkg/cataloger/golang/cataloger.go

@@ -18,13 +18,18 @@ import (
 
 var versionCandidateGroups = regexp.MustCompile(`(?P<version>\d+(\.\d+)?(\.\d+)?)(?P<candidate>\w*)`)
 
+const (
+	modFileCatalogerName = "go-module-file-cataloger"
+	binaryCatalogerName  = "go-module-binary-cataloger"
+)
+
 // NewGoModuleFileCataloger returns a new cataloger object that searches within go.mod files.
 func NewGoModuleFileCataloger(opts CatalogerConfig) pkg.Cataloger {
 	c := goModCataloger{
-		licenses: newGoLicenses(opts),
+		licenses: newGoLicenses(modFileCatalogerName, opts),
 	}
 	return &progressingCataloger{
-		cataloger: generic.NewCataloger("go-module-file-cataloger").
+		cataloger: generic.NewCataloger(modFileCatalogerName).
 			WithParserByGlobs(c.parseGoModFile, "**/go.mod"),
 	}
 }
@@ -32,10 +37,10 @@ func NewGoModuleFileCataloger(opts CatalogerConfig) pkg.Cataloger {
 // NewGoModuleBinaryCataloger returns a new cataloger object that searches within binaries built by the go compiler.
 func NewGoModuleBinaryCataloger(opts CatalogerConfig) pkg.Cataloger {
 	c := goBinaryCataloger{
-		licenses: newGoLicenses(opts),
+		licenses: newGoLicenses(binaryCatalogerName, opts),
 	}
 	return &progressingCataloger{
-		cataloger: generic.NewCataloger("go-module-binary-cataloger").
+		cataloger: generic.NewCataloger(binaryCatalogerName).
 			WithParserByMimeTypes(c.parseGoBinary, internal.ExecutableMIMETypeSet.List()...),
 	}
 }

syft/pkg/cataloger/golang/licenses.go

@@ -30,13 +30,15 @@ import (
 )
 
 type goLicenses struct {
+	catalogerName         string
 	opts                  CatalogerConfig
 	localModCacheResolver file.WritableResolver
 	lowerLicenseFileNames *strset.Set
 }
 
-func newGoLicenses(opts CatalogerConfig) goLicenses {
+func newGoLicenses(catalogerName string, opts CatalogerConfig) goLicenses {
 	return goLicenses{
+		catalogerName:         catalogerName,
 		opts:                  opts,
 		localModCacheResolver: modCacheResolver(opts.LocalModCacheDir),
 		lowerLicenseFileNames: strset.New(lowercaseLicenseFiles()...),
@@ -125,6 +127,7 @@ func (c *goLicenses) getLicensesFromRemote(moduleName, moduleVersion string) ([]
 			OnSuccess:    "Downloaded go mod",
 		},
 		HideOnSuccess: true,
+		ParentID:      c.catalogerName,
 	}, -1, "")
 
 	fsys, err := getModule(prog, proxies, moduleName, moduleVersion)

syft/pkg/cataloger/golang/licenses_test.go

@@ -67,6 +67,7 @@ func Test_LocalLicenseSearch(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			l := newGoLicenses(
+				"",
 				CatalogerConfig{
 					SearchLocalModCacheLicenses: true,
 					LocalModCacheDir:            path.Join(wd, "test-fixtures", "licenses", "pkg", "mod"),
@@ -154,11 +155,14 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			l := newGoLicenses(CatalogerConfig{
-				SearchRemoteLicenses: true,
-				Proxies:              []string{server.URL},
-				LocalModCacheDir:     modDir,
-			})
+			l := newGoLicenses(
+				"",
+				CatalogerConfig{
+					SearchRemoteLicenses: true,
+					Proxies:              []string{server.URL},
+					LocalModCacheDir:     modDir,
+				},
+			)
 
 			licenses, err := l.getLicenses(fileresolver.Empty{}, test.name, test.version)
 			require.NoError(t, err)

syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go

@@ -14,6 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/internal/relationship"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/linux"
@@ -346,8 +347,8 @@ func (p *CatalogTester) assertPkgs(t *testing.T, pkgs []pkg.Package, relationshi
 		opts = append(opts, cmp.Reporter(&r))
 
 		// order should not matter
-		pkg.SortRelationships(p.expectedRelationships)
-		pkg.SortRelationships(relationships)
+		relationship.Sort(p.expectedRelationships)
+		relationship.Sort(relationships)
 
 		if diff := cmp.Diff(p.expectedRelationships, relationships, opts...); diff != "" {
 			t.Log("Specific Differences:\n" + r.String())

syft/pkg/cataloger/package_exclusions.go

@@ -1,51 +0,0 @@
-package cataloger
-
-import (
-	"slices"
-
-	"github.com/anchore/syft/syft/artifact"
-	"github.com/anchore/syft/syft/pkg"
-)
-
-var (
-	osCatalogerTypes = []pkg.Type{
-		pkg.AlpmPkg,
-		pkg.ApkPkg,
-		pkg.DebPkg,
-		pkg.NixPkg,
-		pkg.PortagePkg,
-		pkg.RpmPkg,
-	}
-	binaryCatalogerTypes = []pkg.Type{
-		pkg.BinaryPkg,
-	}
-)
-
-// ExcludeBinaryByFileOwnershipOverlap will remove packages from a collection given the following properties are true
-// 1) the relationship between packages is OwnershipByFileOverlap
-// 2) the parent is an "os" package
-// 3) the child is a synthetic package generated by the binary cataloger
-// 4) the package names are identical
-// This was implemented as a way to help resolve: https://github.com/anchore/syft/issues/931
-func ExcludeBinaryByFileOwnershipOverlap(r artifact.Relationship, c *pkg.Collection) bool {
-	if artifact.OwnershipByFileOverlapRelationship != r.Type {
-		return false
-	}
-
-	parent := c.Package(r.From.ID())
-	if parent == nil {
-		return false
-	}
-
-	parentInExclusion := slices.Contains(osCatalogerTypes, parent.Type)
-	if !parentInExclusion {
-		return false
-	}
-
-	child := c.Package(r.To.ID())
-	if child == nil {
-		return false
-	}
-
-	return slices.Contains(binaryCatalogerTypes, child.Type)
-}

syft/pkg/cataloger/package_exclusions_test.go

@@ -1,76 +0,0 @@
-package cataloger
-
-import (
-	"testing"
-
-	"github.com/anchore/syft/syft/artifact"
-	"github.com/anchore/syft/syft/pkg"
-)
-
-func TestExclude(t *testing.T) {
-	packageA := pkg.Package{Name: "package-a", Type: pkg.ApkPkg}
-	packageB := pkg.Package{Name: "package-a", Type: pkg.PythonPkg}
-	packageC := pkg.Package{Name: "package-a", Type: pkg.BinaryPkg}
-	packageD := pkg.Package{Name: "package-d", Type: pkg.BinaryPkg}
-	for _, p := range []*pkg.Package{&packageA, &packageB, &packageC, &packageD} {
-		p := p
-		p.SetID()
-	}
-
-	tests := []struct {
-		name          string
-		relationship  artifact.Relationship
-		packages      *pkg.Collection
-		shouldExclude bool
-	}{
-		{
-			name: "no exclusions from os -> python",
-			relationship: artifact.Relationship{
-				Type: artifact.OwnershipByFileOverlapRelationship,
-				From: packageA,
-				To:   packageB,
-			},
-			packages:      pkg.NewCollection(packageA, packageB),
-			shouldExclude: false,
-		},
-		{
-			name: "exclusions from os -> binary",
-			relationship: artifact.Relationship{
-				Type: artifact.OwnershipByFileOverlapRelationship,
-				From: packageA,
-				To:   packageC,
-			},
-			packages:      pkg.NewCollection(packageA, packageC),
-			shouldExclude: true,
-		},
-		{
-			name: "no exclusions from python -> binary",
-			relationship: artifact.Relationship{
-				Type: artifact.OwnershipByFileOverlapRelationship,
-				From: packageB,
-				To:   packageC,
-			},
-			packages:      pkg.NewCollection(packageB, packageC),
-			shouldExclude: false,
-		},
-		{
-			name: "no exclusions for different package names",
-			relationship: artifact.Relationship{
-				Type: artifact.OwnershipByFileOverlapRelationship,
-				From: packageA,
-				To:   packageD,
-			},
-			packages:      pkg.NewCollection(packageA, packageD),
-			shouldExclude: false,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			if !ExcludeBinaryByFileOwnershipOverlap(test.relationship, test.packages) && test.shouldExclude {
-				t.Errorf("expected to exclude relationship %+v", test.relationship)
-			}
-		})
-
-	}
-}

syft/pkg/cataloger/search_config.go

@@ -1,19 +0,0 @@
-package cataloger
-
-import (
-	"github.com/anchore/syft/syft/source"
-)
-
-type SearchConfig struct {
-	IncludeIndexedArchives   bool
-	IncludeUnindexedArchives bool
-	Scope                    source.Scope
-}
-
-func DefaultSearchConfig() SearchConfig {
-	return SearchConfig{
-		IncludeIndexedArchives:   true,
-		IncludeUnindexedArchives: false,
-		Scope:                    source.SquashedScope,
-	}
-}