diff --git a/cmd/packages.go b/cmd/packages.go index ee49c2087..a11c078b1 100644 --- a/cmd/packages.go +++ b/cmd/packages.go @@ -253,7 +253,7 @@ func packagesExecWorker(userInput string) <-chan error { } defer cleanup() - catalog, d, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt) + catalog, relationships, d, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt) if err != nil { errs <- fmt.Errorf("failed to catalog input: %w", err) return @@ -264,7 +264,8 @@ func packagesExecWorker(userInput string) <-chan error { PackageCatalog: catalog, Distro: d, }, - Source: src.Metadata, + Relationships: relationships, + Source: src.Metadata, } if appConfig.Anchore.Host != "" { diff --git a/cmd/power_user.go b/cmd/power_user.go index 607ca48cd..a65e54415 100644 --- a/cmd/power_user.go +++ b/cmd/power_user.go @@ -4,6 +4,8 @@ import ( "fmt" "sync" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/sbom" "github.com/anchore/stereoscope" @@ -88,7 +90,6 @@ func powerUserExec(_ *cobra.Command, args []string) error { ui.Select(isVerbose(), appConfig.Quiet, reporter)..., ) } - func powerUserExecWorker(userInput string) <-chan error { errs := make(chan error) go func() { @@ -109,28 +110,61 @@ func powerUserExecWorker(userInput string) <-chan error { } defer cleanup() - analysisResults := sbom.SBOM{ + s := sbom.SBOM{ Source: src.Metadata, } - wg := &sync.WaitGroup{} + var results []<-chan artifact.Relationship for _, task := range tasks { - wg.Add(1) - go func(task powerUserTask) { - defer wg.Done() - if err = task(&analysisResults.Artifacts, src); err != nil { - errs <- err - return - } - }(task) + c := make(chan artifact.Relationship) + results = append(results, c) + + go runTask(task, &s.Artifacts, src, c, errs) } - wg.Wait() + for relationship := range mergeResults(results...) { + s.Relationships = append(s.Relationships, relationship) + } bus.Publish(partybus.Event{ Type: event.PresenterReady, - Value: poweruser.NewJSONPresenter(analysisResults, *appConfig), + Value: poweruser.NewJSONPresenter(s, *appConfig), }) }() return errs } + +func runTask(t powerUserTask, a *sbom.Artifacts, src *source.Source, c chan<- artifact.Relationship, errs chan<- error) { + defer close(c) + + relationships, err := t(a, src) + if err != nil { + errs <- err + return + } + + for _, relationship := range relationships { + c <- relationship + } +} + +func mergeResults(cs ...<-chan artifact.Relationship) <-chan artifact.Relationship { + var wg sync.WaitGroup + var results = make(chan artifact.Relationship) + + wg.Add(len(cs)) + for _, c := range cs { + go func(c <-chan artifact.Relationship) { + for n := range c { + results <- n + } + wg.Done() + }(c) + } + + go func() { + wg.Wait() + close(results) + }() + return results +} diff --git a/cmd/power_user_tasks.go b/cmd/power_user_tasks.go index ab493ddd5..365a8fdd5 100644 --- a/cmd/power_user_tasks.go +++ b/cmd/power_user_tasks.go @@ -4,6 +4,8 @@ import ( "crypto" "fmt" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft" @@ -11,7 +13,7 @@ import ( "github.com/anchore/syft/syft/source" ) -type powerUserTask func(*sbom.Artifacts, *source.Source) error +type powerUserTask func(*sbom.Artifacts, *source.Source) ([]artifact.Relationship, error) func powerUserTasks() ([]powerUserTask, error) { var tasks []powerUserTask @@ -43,16 +45,16 @@ func catalogPackagesTask() (powerUserTask, error) { return nil, nil } - task := func(results *sbom.Artifacts, src *source.Source) error { - packageCatalog, theDistro, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt) + task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { + packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt) if err != nil { - return err + return nil, err } results.PackageCatalog = packageCatalog results.Distro = theDistro - return nil + return relationships, nil } return task, nil @@ -65,18 +67,18 @@ func catalogFileMetadataTask() (powerUserTask, error) { metadataCataloger := file.NewMetadataCataloger() - task := func(results *sbom.Artifacts, src *source.Source) error { + task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(appConfig.FileMetadata.Cataloger.ScopeOpt) if err != nil { - return err + return nil, err } result, err := metadataCataloger.Catalog(resolver) if err != nil { - return err + return nil, err } results.FileMetadata = result - return nil + return nil, nil } return task, nil @@ -111,18 +113,18 @@ func catalogFileDigestsTask() (powerUserTask, error) { return nil, err } - task := func(results *sbom.Artifacts, src *source.Source) error { + task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(appConfig.FileMetadata.Cataloger.ScopeOpt) if err != nil { - return err + return nil, err } result, err := digestsCataloger.Catalog(resolver) if err != nil { - return err + return nil, err } results.FileDigests = result - return nil + return nil, nil } return task, nil @@ -143,18 +145,18 @@ func catalogSecretsTask() (powerUserTask, error) { return nil, err } - task := func(results *sbom.Artifacts, src *source.Source) error { + task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(appConfig.Secrets.Cataloger.ScopeOpt) if err != nil { - return err + return nil, err } result, err := secretsCataloger.Catalog(resolver) if err != nil { - return err + return nil, err } results.Secrets = result - return nil + return nil, nil } return task, nil @@ -171,18 +173,18 @@ func catalogFileClassificationsTask() (powerUserTask, error) { return nil, err } - task := func(results *sbom.Artifacts, src *source.Source) error { + task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(appConfig.FileClassification.Cataloger.ScopeOpt) if err != nil { - return err + return nil, err } result, err := classifierCataloger.Catalog(resolver) if err != nil { - return err + return nil, err } results.FileClassifications = result - return nil + return nil, nil } return task, nil @@ -198,18 +200,18 @@ func catalogContentsTask() (powerUserTask, error) { return nil, err } - task := func(results *sbom.Artifacts, src *source.Source) error { + task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { resolver, err := src.FileResolver(appConfig.FileContents.Cataloger.ScopeOpt) if err != nil { - return err + return nil, err } result, err := contentsCataloger.Catalog(resolver) if err != nil { - return err + return nil, err } results.FileContents = result - return nil + return nil, nil } return task, nil diff --git a/go.mod b/go.mod index fba115bf1..ea7804280 100644 --- a/go.mod +++ b/go.mod @@ -24,8 +24,9 @@ require ( github.com/gookit/color v1.2.7 github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/go-version v1.2.0 + github.com/jinzhu/copier v0.3.2 github.com/mitchellh/go-homedir v1.1.0 - github.com/mitchellh/hashstructure v1.1.0 + github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/mitchellh/mapstructure v1.3.1 github.com/olekukonko/tablewriter v0.0.4 github.com/pelletier/go-toml v1.8.1 diff --git a/go.sum b/go.sum index d2a6395ed..e373a0c9c 100644 --- a/go.sum +++ b/go.sum @@ -454,6 +454,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= +github.com/jinzhu/copier v0.3.2 h1:QdBOCbaouLDYaIPFfi1bKv5F5tPpeTwXe4sD0jqtz5w= +github.com/jinzhu/copier v0.3.2/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro= github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -544,8 +546,8 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0= -github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA= +github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= diff --git a/internal/formats/common/spdxhelpers/description.go b/internal/formats/common/spdxhelpers/description.go index 4ea9f906c..8bad4797a 100644 --- a/internal/formats/common/spdxhelpers/description.go +++ b/internal/formats/common/spdxhelpers/description.go @@ -2,7 +2,7 @@ package spdxhelpers import "github.com/anchore/syft/syft/pkg" -func Description(p *pkg.Package) string { +func Description(p pkg.Package) string { if hasMetadata(p) { switch metadata := p.Metadata.(type) { case pkg.ApkMetadata: @@ -14,10 +14,6 @@ func Description(p *pkg.Package) string { return "" } -func packageExists(p *pkg.Package) bool { - return p != nil -} - -func hasMetadata(p *pkg.Package) bool { - return packageExists(p) && p.Metadata != nil +func hasMetadata(p pkg.Package) bool { + return p.Metadata != nil } diff --git a/internal/formats/common/spdxhelpers/description_test.go b/internal/formats/common/spdxhelpers/description_test.go index 77bc424ed..e62018622 100644 --- a/internal/formats/common/spdxhelpers/description_test.go +++ b/internal/formats/common/spdxhelpers/description_test.go @@ -50,7 +50,7 @@ func Test_Description(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, Description(&test.input)) + assert.Equal(t, test.expected, Description(test.input)) }) } } diff --git a/internal/formats/common/spdxhelpers/download_location.go b/internal/formats/common/spdxhelpers/download_location.go index d99f6a7be..7c22aa870 100644 --- a/internal/formats/common/spdxhelpers/download_location.go +++ b/internal/formats/common/spdxhelpers/download_location.go @@ -2,7 +2,7 @@ package spdxhelpers import "github.com/anchore/syft/syft/pkg" -func DownloadLocation(p *pkg.Package) string { +func DownloadLocation(p pkg.Package) string { // 3.7: Package Download Location // Cardinality: mandatory, one // NONE if there is no download location whatsoever. diff --git a/internal/formats/common/spdxhelpers/download_location_test.go b/internal/formats/common/spdxhelpers/download_location_test.go index 3636c7c77..a20bef6d1 100644 --- a/internal/formats/common/spdxhelpers/download_location_test.go +++ b/internal/formats/common/spdxhelpers/download_location_test.go @@ -48,7 +48,7 @@ func Test_DownloadLocation(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, DownloadLocation(&test.input)) + assert.Equal(t, test.expected, DownloadLocation(test.input)) }) } } diff --git a/internal/formats/common/spdxhelpers/external_refs.go b/internal/formats/common/spdxhelpers/external_refs.go index 8aa863fd5..3a536da8a 100644 --- a/internal/formats/common/spdxhelpers/external_refs.go +++ b/internal/formats/common/spdxhelpers/external_refs.go @@ -6,13 +6,9 @@ import ( "github.com/anchore/syft/syft/pkg" ) -func ExternalRefs(p *pkg.Package) (externalRefs []model.ExternalRef) { +func ExternalRefs(p pkg.Package) (externalRefs []model.ExternalRef) { externalRefs = make([]model.ExternalRef, 0) - if !packageExists(p) { - return externalRefs - } - for _, c := range p.CPEs { externalRefs = append(externalRefs, model.ExternalRef{ ReferenceCategory: model.SecurityReferenceCategory, diff --git a/internal/formats/common/spdxhelpers/external_refs_test.go b/internal/formats/common/spdxhelpers/external_refs_test.go index 8f0860577..a3c2b3372 100644 --- a/internal/formats/common/spdxhelpers/external_refs_test.go +++ b/internal/formats/common/spdxhelpers/external_refs_test.go @@ -39,7 +39,7 @@ func Test_ExternalRefs(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - assert.ElementsMatch(t, test.expected, ExternalRefs(&test.input)) + assert.ElementsMatch(t, test.expected, ExternalRefs(test.input)) }) } } diff --git a/internal/formats/common/spdxhelpers/files.go b/internal/formats/common/spdxhelpers/files.go index 247acf277..c0d2be4f2 100644 --- a/internal/formats/common/spdxhelpers/files.go +++ b/internal/formats/common/spdxhelpers/files.go @@ -9,7 +9,7 @@ import ( "github.com/anchore/syft/syft/pkg" ) -func Files(packageSpdxID string, p *pkg.Package) (files []model.File, fileIDs []string, relationships []model.Relationship) { +func Files(packageSpdxID string, p pkg.Package) (files []model.File, fileIDs []string, relationships []model.Relationship) { files = make([]model.File, 0) fileIDs = make([]string, 0) relationships = make([]model.Relationship, 0) diff --git a/internal/formats/common/spdxhelpers/homepage.go b/internal/formats/common/spdxhelpers/homepage.go index 936e89eee..b790ba614 100644 --- a/internal/formats/common/spdxhelpers/homepage.go +++ b/internal/formats/common/spdxhelpers/homepage.go @@ -2,7 +2,7 @@ package spdxhelpers import "github.com/anchore/syft/syft/pkg" -func Homepage(p *pkg.Package) string { +func Homepage(p pkg.Package) string { if hasMetadata(p) { switch metadata := p.Metadata.(type) { case pkg.GemMetadata: diff --git a/internal/formats/common/spdxhelpers/homepage_test.go b/internal/formats/common/spdxhelpers/homepage_test.go index 781873f7a..371f45bfb 100644 --- a/internal/formats/common/spdxhelpers/homepage_test.go +++ b/internal/formats/common/spdxhelpers/homepage_test.go @@ -50,7 +50,7 @@ func Test_Homepage(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, Homepage(&test.input)) + assert.Equal(t, test.expected, Homepage(test.input)) }) } } diff --git a/internal/formats/common/spdxhelpers/license.go b/internal/formats/common/spdxhelpers/license.go index f53881c01..80d84bb3d 100644 --- a/internal/formats/common/spdxhelpers/license.go +++ b/internal/formats/common/spdxhelpers/license.go @@ -7,7 +7,7 @@ import ( "github.com/anchore/syft/syft/pkg" ) -func License(p *pkg.Package) string { +func License(p pkg.Package) string { // source: https://spdx.github.io/spdx-spec/3-package-information/#313-concluded-license // The options to populate this field are limited to: // A valid SPDX License Expression as defined in Appendix IV; @@ -17,7 +17,7 @@ func License(p *pkg.Package) string { // (ii) the SPDX file creator has made no attempt to determine this field; or // (iii) the SPDX file creator has intentionally provided no information (no meaning should be implied by doing so). - if !packageExists(p) || len(p.Licenses) == 0 { + if len(p.Licenses) == 0 { return "NONE" } diff --git a/internal/formats/common/spdxhelpers/license_test.go b/internal/formats/common/spdxhelpers/license_test.go index c4762ee18..13a62fe98 100644 --- a/internal/formats/common/spdxhelpers/license_test.go +++ b/internal/formats/common/spdxhelpers/license_test.go @@ -67,7 +67,7 @@ func Test_License(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, License(&test.input)) + assert.Equal(t, test.expected, License(test.input)) }) } } diff --git a/internal/formats/common/spdxhelpers/originator_test.go b/internal/formats/common/spdxhelpers/originator_test.go index 7e3ec04ed..ec443b862 100644 --- a/internal/formats/common/spdxhelpers/originator_test.go +++ b/internal/formats/common/spdxhelpers/originator_test.go @@ -108,7 +108,7 @@ func Test_Originator(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, Originator(&test.input)) + assert.Equal(t, test.expected, Originator(test.input)) }) } } diff --git a/internal/formats/common/spdxhelpers/origintor.go b/internal/formats/common/spdxhelpers/origintor.go index 24f98a54c..852189b1f 100644 --- a/internal/formats/common/spdxhelpers/origintor.go +++ b/internal/formats/common/spdxhelpers/origintor.go @@ -6,7 +6,7 @@ import ( "github.com/anchore/syft/syft/pkg" ) -func Originator(p *pkg.Package) string { +func Originator(p pkg.Package) string { if hasMetadata(p) { switch metadata := p.Metadata.(type) { case pkg.ApkMetadata: diff --git a/internal/formats/common/spdxhelpers/source_info.go b/internal/formats/common/spdxhelpers/source_info.go index 793a86940..d0ae4ed80 100644 --- a/internal/formats/common/spdxhelpers/source_info.go +++ b/internal/formats/common/spdxhelpers/source_info.go @@ -6,11 +6,7 @@ import ( "github.com/anchore/syft/syft/pkg" ) -func SourceInfo(p *pkg.Package) string { - if !packageExists(p) { - return "" - } - +func SourceInfo(p pkg.Package) string { answer := "" switch p.Type { case pkg.RpmPkg: diff --git a/internal/formats/common/spdxhelpers/source_info_test.go b/internal/formats/common/spdxhelpers/source_info_test.go index 2a668152e..e236f5a70 100644 --- a/internal/formats/common/spdxhelpers/source_info_test.go +++ b/internal/formats/common/spdxhelpers/source_info_test.go @@ -139,7 +139,7 @@ func Test_SourceInfo(t *testing.T) { if test.input.Type != "" { pkgTypes = append(pkgTypes, test.input.Type) } - actual := SourceInfo(&test.input) + actual := SourceInfo(test.input) for _, expected := range test.expected { assert.Contains(t, actual, expected) } diff --git a/internal/formats/common/testutils/utils.go b/internal/formats/common/testutils/utils.go index 69b3243ac..d48f9f9ba 100644 --- a/internal/formats/common/testutils/utils.go +++ b/internal/formats/common/testutils/utils.go @@ -133,7 +133,6 @@ func populateImageCatalog(catalog *pkg.Catalog, img *image.Image) { // populate catalog with test data catalog.Add(pkg.Package{ - ID: "package-1-id", Name: "package-1", Version: "1.0.1", Locations: []source.Location{ @@ -154,7 +153,6 @@ func populateImageCatalog(catalog *pkg.Catalog, img *image.Image) { }, }) catalog.Add(pkg.Package{ - ID: "package-2-id", Name: "package-2", Version: "2.0.1", Locations: []source.Location{ @@ -197,7 +195,6 @@ func newDirectoryCatalog() *pkg.Catalog { // populate catalog with test data catalog.Add(pkg.Package{ - ID: "package-1-id", Name: "package-1", Version: "1.0.1", Type: pkg.PythonPkg, @@ -223,7 +220,6 @@ func newDirectoryCatalog() *pkg.Catalog { }, }) catalog.Add(pkg.Package{ - ID: "package-2-id", Name: "package-2", Version: "2.0.1", Type: pkg.DebPkg, diff --git a/internal/formats/cyclonedx12xml/to_format_model.go b/internal/formats/cyclonedx12xml/to_format_model.go index 70744662a..6a6c43703 100644 --- a/internal/formats/cyclonedx12xml/to_format_model.go +++ b/internal/formats/cyclonedx12xml/to_format_model.go @@ -33,7 +33,7 @@ func toFormatModel(s sbom.SBOM) model.Document { return doc } -func toComponent(p *pkg.Package) model.Component { +func toComponent(p pkg.Package) model.Component { return model.Component{ Type: "library", // TODO: this is not accurate Name: p.Name, diff --git a/internal/formats/spdx22tagvalue/to_format_model.go b/internal/formats/spdx22tagvalue/to_format_model.go index 44c728565..4e9fb5d1f 100644 --- a/internal/formats/spdx22tagvalue/to_format_model.go +++ b/internal/formats/spdx22tagvalue/to_format_model.go @@ -256,7 +256,7 @@ func toFormatPackages(catalog *pkg.Catalog) map[spdx.ElementID]*spdx.Package2_2 return results } -func formatSPDXExternalRefs(p *pkg.Package) (refs []*spdx.PackageExternalReference2_2) { +func formatSPDXExternalRefs(p pkg.Package) (refs []*spdx.PackageExternalReference2_2) { for _, ref := range spdxhelpers.ExternalRefs(p) { refs = append(refs, &spdx.PackageExternalReference2_2{ Category: string(ref.ReferenceCategory), diff --git a/internal/formats/syftjson/decoder_test.go b/internal/formats/syftjson/decoder_test.go index bd537c366..ea69ee2c7 100644 --- a/internal/formats/syftjson/decoder_test.go +++ b/internal/formats/syftjson/decoder_test.go @@ -31,11 +31,7 @@ func TestEncodeDecodeCycle(t *testing.T) { continue } - // ids will never be equal - p.ID = "" - actualPackages[idx].ID = "" - - for _, d := range deep.Equal(*p, *actualPackages[idx]) { + for _, d := range deep.Equal(p, actualPackages[idx]) { if strings.Contains(d, ".VirtualPath: ") { // location.Virtual path is not exposed in the json output continue diff --git a/internal/formats/syftjson/test-fixtures/snapshot/TestDirectoryPresenter.golden b/internal/formats/syftjson/test-fixtures/snapshot/TestDirectoryPresenter.golden index 7289b4213..c61896ca0 100644 --- a/internal/formats/syftjson/test-fixtures/snapshot/TestDirectoryPresenter.golden +++ b/internal/formats/syftjson/test-fixtures/snapshot/TestDirectoryPresenter.golden @@ -1,7 +1,7 @@ { "artifacts": [ { - "id": "package-1-id", + "id": "cbf4f3077fc7deee", "name": "package-1", "version": "1.0.1", "type": "python", @@ -36,7 +36,7 @@ } }, { - "id": "package-2-id", + "id": "1a39aadd9705c2b9", "name": "package-2", "version": "2.0.1", "type": "deb", diff --git a/internal/formats/syftjson/test-fixtures/snapshot/TestImagePresenter.golden b/internal/formats/syftjson/test-fixtures/snapshot/TestImagePresenter.golden index 42fce092a..602b9731f 100644 --- a/internal/formats/syftjson/test-fixtures/snapshot/TestImagePresenter.golden +++ b/internal/formats/syftjson/test-fixtures/snapshot/TestImagePresenter.golden @@ -1,7 +1,7 @@ { "artifacts": [ { - "id": "package-1-id", + "id": "d1d433485a31ed07", "name": "package-1", "version": "1.0.1", "type": "python", @@ -32,7 +32,7 @@ } }, { - "id": "package-2-id", + "id": "2db629ca48fa6786", "name": "package-2", "version": "2.0.1", "type": "deb", diff --git a/internal/formats/syftjson/to_format_model.go b/internal/formats/syftjson/to_format_model.go index 1b2a7ca48..cea9a9159 100644 --- a/internal/formats/syftjson/to_format_model.go +++ b/internal/formats/syftjson/to_format_model.go @@ -3,6 +3,8 @@ package syftjson import ( "fmt" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/internal" @@ -23,7 +25,7 @@ func ToFormatModel(s sbom.SBOM, applicationConfig interface{}) model.Document { return model.Document{ Artifacts: toPackageModels(s.Artifacts.PackageCatalog), - ArtifactRelationships: toRelationshipModel(pkg.NewRelationships(s.Artifacts.PackageCatalog)), + ArtifactRelationships: toRelationshipModel(s.Relationships), Source: src, Distro: toDistroModel(s.Artifacts.Distro), Descriptor: model.Descriptor{ @@ -50,7 +52,7 @@ func toPackageModels(catalog *pkg.Catalog) []model.Package { } // toPackageModel crates a new Package from the given pkg.Package. -func toPackageModel(p *pkg.Package) model.Package { +func toPackageModel(p pkg.Package) model.Package { var cpes = make([]string, len(p.CPEs)) for i, c := range p.CPEs { cpes[i] = c.BindToFmtString() @@ -69,7 +71,7 @@ func toPackageModel(p *pkg.Package) model.Package { return model.Package{ PackageBasicData: model.PackageBasicData{ - ID: string(p.ID), + ID: string(p.ID()), Name: p.Name, Version: p.Version, Type: p.Type, @@ -87,14 +89,14 @@ func toPackageModel(p *pkg.Package) model.Package { } } -func toRelationshipModel(relationships []pkg.Relationship) []model.Relationship { +func toRelationshipModel(relationships []artifact.Relationship) []model.Relationship { result := make([]model.Relationship, len(relationships)) for i, r := range relationships { result[i] = model.Relationship{ - Parent: string(r.Parent), - Child: string(r.Child), + Parent: string(r.From.ID()), + Child: string(r.To.ID()), Type: string(r.Type), - Metadata: r.Metadata, + Metadata: r.Data, } } return result diff --git a/internal/formats/syftjson/to_syft_model.go b/internal/formats/syftjson/to_syft_model.go index e2c6ee40c..b49789cac 100644 --- a/internal/formats/syftjson/to_syft_model.go +++ b/internal/formats/syftjson/to_syft_model.go @@ -61,7 +61,6 @@ func toSyftPackage(p model.Package) pkg.Package { } return pkg.Package{ - ID: pkg.ID(p.ID), Name: p.Name, Version: p.Version, FoundBy: p.FoundBy, diff --git a/internal/presenter/poweruser/json_presenter_test.go b/internal/presenter/poweruser/json_presenter_test.go index f4dbda5be..40b0f40c3 100644 --- a/internal/presenter/poweruser/json_presenter_test.go +++ b/internal/presenter/poweruser/json_presenter_test.go @@ -33,7 +33,6 @@ func TestJSONPresenter(t *testing.T) { catalog := pkg.NewCatalog() catalog.Add(pkg.Package{ - ID: "package-1-id", Name: "package-1", Version: "1.0.1", Locations: []source.Location{ @@ -57,7 +56,6 @@ func TestJSONPresenter(t *testing.T) { }, }) catalog.Add(pkg.Package{ - ID: "package-2-id", Name: "package-2", Version: "2.0.1", Locations: []source.Location{ diff --git a/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden b/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden index d71439257..8b0293ab7 100644 --- a/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden +++ b/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden @@ -72,7 +72,7 @@ ], "artifacts": [ { - "id": "package-1-id", + "id": "b84dfe0eb2c5670f", "name": "package-1", "version": "1.0.1", "type": "python", @@ -102,7 +102,7 @@ } }, { - "id": "package-2-id", + "id": "6619226d6979963f", "name": "package-2", "version": "2.0.1", "type": "deb", diff --git a/syft/artifact/id.go b/syft/artifact/id.go new file mode 100644 index 000000000..50498467c --- /dev/null +++ b/syft/artifact/id.go @@ -0,0 +1,26 @@ +package artifact + +import ( + "fmt" + + "github.com/mitchellh/hashstructure/v2" +) + +// ID represents a unique value for each package added to a package catalog. +type ID string + +type Identifiable interface { + ID() ID +} + +func IDFromHash(obj interface{}) (ID, error) { + f, err := hashstructure.Hash(obj, hashstructure.FormatV2, &hashstructure.HashOptions{ + ZeroNil: true, + SlicesAsSets: true, + }) + if err != nil { + return "", fmt.Errorf("could not build ID for object=%+v: %+v", obj, err) + } + + return ID(fmt.Sprintf("%x", f)), nil +} diff --git a/syft/artifact/relationship.go b/syft/artifact/relationship.go new file mode 100644 index 000000000..fc381c712 --- /dev/null +++ b/syft/artifact/relationship.go @@ -0,0 +1,17 @@ +package artifact + +const ( + // OwnershipByFileOverlapRelationship indicates that the parent package claims ownership of a child package since + // the parent metadata indicates overlap with a location that a cataloger found the child package by. This is + // by definition a package-to-package relationship and is created only after all package cataloging has been completed. + OwnershipByFileOverlapRelationship RelationshipType = "ownership-by-file-overlap" +) + +type RelationshipType string + +type Relationship struct { + From Identifiable `json:"from"` + To Identifiable `json:"to"` + Type RelationshipType `json:"type"` + Data interface{} `json:"data,omitempty"` +} diff --git a/syft/lib.go b/syft/lib.go index ccf85cd6b..376315335 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -19,6 +19,8 @@ package syft import ( "fmt" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/distro" @@ -32,10 +34,10 @@ import ( // CatalogPackages takes an inventory of packages from the given image from a particular perspective // (e.g. squashed source, all-layers source). Returns the discovered set of packages, the identified Linux // distribution, and the source object used to wrap the data source. -func CatalogPackages(src *source.Source, scope source.Scope) (*pkg.Catalog, *distro.Distro, error) { +func CatalogPackages(src *source.Source, scope source.Scope) (*pkg.Catalog, []artifact.Relationship, *distro.Distro, error) { resolver, err := src.FileResolver(scope) if err != nil { - return nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err) + return nil, nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err) } // find the distro @@ -59,15 +61,15 @@ func CatalogPackages(src *source.Source, scope source.Scope) (*pkg.Catalog, *dis log.Info("cataloging directory") catalogers = cataloger.DirectoryCatalogers() default: - return nil, nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", src.Metadata.Scheme) + return nil, nil, nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", src.Metadata.Scheme) } - catalog, err := cataloger.Catalog(resolver, theDistro, catalogers...) + catalog, relationships, err := cataloger.Catalog(resolver, theDistro, catalogers...) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - return catalog, theDistro, nil + return catalog, relationships, theDistro, nil } // SetLogger sets the logger object used for all syft logging calls. diff --git a/syft/pkg/catalog.go b/syft/pkg/catalog.go index 66c37c9c3..ce3d4c7ba 100644 --- a/syft/pkg/catalog.go +++ b/syft/pkg/catalog.go @@ -4,6 +4,9 @@ import ( "sort" "sync" + "github.com/anchore/syft/syft/artifact" + "github.com/jinzhu/copier" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" @@ -11,18 +14,18 @@ import ( // Catalog represents a collection of Packages. type Catalog struct { - byID map[ID]*Package - idsByType map[Type][]ID - idsByPath map[string][]ID // note: this is real path or virtual path + byID map[artifact.ID]Package + idsByType map[Type][]artifact.ID + idsByPath map[string][]artifact.ID // note: this is real path or virtual path lock sync.RWMutex } // NewCatalog returns a new empty Catalog func NewCatalog(pkgs ...Package) *Catalog { catalog := Catalog{ - byID: make(map[ID]*Package), - idsByType: make(map[Type][]ID), - idsByPath: make(map[string][]ID), + byID: make(map[artifact.ID]Package), + idsByType: make(map[Type][]artifact.ID), + idsByPath: make(map[string][]artifact.ID), } for _, p := range pkgs { @@ -38,21 +41,26 @@ func (c *Catalog) PackageCount() int { } // Package returns the package with the given ID. -func (c *Catalog) Package(id ID) *Package { +func (c *Catalog) Package(id artifact.ID) *Package { v, exists := c.byID[id] if !exists { return nil } - return v + var p Package + if err := copier.Copy(&p, &v); err != nil { + log.Warnf("unable to copy package id=%q name=%q: %+v", id, v.Name, err) + return nil + } + return &p } // PackagesByPath returns all packages that were discovered from the given path. -func (c *Catalog) PackagesByPath(path string) []*Package { +func (c *Catalog) PackagesByPath(path string) []Package { return c.Packages(c.idsByPath[path]) } // Packages returns all packages for the given ID. -func (c *Catalog) Packages(ids []ID) (result []*Package) { +func (c *Catalog) Packages(ids []artifact.ID) (result []Package) { for _, i := range ids { p, exists := c.byID[i] if exists { @@ -67,68 +75,32 @@ func (c *Catalog) Add(p Package) { c.lock.Lock() defer c.lock.Unlock() - if p.ID == "" { - fingerprint, err := p.Fingerprint() - if err != nil { - log.Warnf("failed to add package to catalog: %w", err) - return - } - - p.ID = ID(fingerprint) - } + // note: since we are capturing the ID, we cannot modify the package being added from this point forward + id := p.ID() // store by package ID - c.byID[p.ID] = &p + c.byID[id] = p // store by package type - c.idsByType[p.Type] = append(c.idsByType[p.Type], p.ID) + c.idsByType[p.Type] = append(c.idsByType[p.Type], id) // store by file location paths observedPaths := internal.NewStringSet() for _, l := range p.Locations { if l.RealPath != "" && !observedPaths.Contains(l.RealPath) { - c.idsByPath[l.RealPath] = append(c.idsByPath[l.RealPath], p.ID) + c.idsByPath[l.RealPath] = append(c.idsByPath[l.RealPath], id) observedPaths.Add(l.RealPath) } if l.VirtualPath != "" && l.RealPath != l.VirtualPath && !observedPaths.Contains(l.VirtualPath) { - c.idsByPath[l.VirtualPath] = append(c.idsByPath[l.VirtualPath], p.ID) + c.idsByPath[l.VirtualPath] = append(c.idsByPath[l.VirtualPath], id) observedPaths.Add(l.VirtualPath) } } } -func (c *Catalog) Remove(id ID) { - c.lock.Lock() - defer c.lock.Unlock() - - _, exists := c.byID[id] - if !exists { - log.Errorf("package ID does not exist in the catalog : id=%+v", id) - return - } - - // Remove all index references to this package ID - for t, ids := range c.idsByType { - c.idsByType[t] = removeID(id, ids) - if len(c.idsByType[t]) == 0 { - delete(c.idsByType, t) - } - } - - for p, ids := range c.idsByPath { - c.idsByPath[p] = removeID(id, ids) - if len(c.idsByPath[p]) == 0 { - delete(c.idsByPath, p) - } - } - - // Remove package - delete(c.byID, id) -} - // Enumerate all packages for the given type(s), enumerating all packages if no type is specified. -func (c *Catalog) Enumerate(types ...Type) <-chan *Package { - channel := make(chan *Package) +func (c *Catalog) Enumerate(types ...Type) <-chan Package { + channel := make(chan Package) go func() { defer close(channel) for ty, ids := range c.idsByType { @@ -146,7 +118,10 @@ func (c *Catalog) Enumerate(types ...Type) <-chan *Package { } } for _, id := range ids { - channel <- c.Package(id) + p := c.Package(id) + if p != nil { + channel <- *p + } } } }() @@ -155,8 +130,7 @@ func (c *Catalog) Enumerate(types ...Type) <-chan *Package { // Sorted enumerates all packages for the given types sorted by package name. Enumerates all packages if no type // is specified. -func (c *Catalog) Sorted(types ...Type) []*Package { - pkgs := make([]*Package, 0) +func (c *Catalog) Sorted(types ...Type) (pkgs []Package) { for p := range c.Enumerate(types...) { pkgs = append(pkgs, p) } @@ -176,12 +150,3 @@ func (c *Catalog) Sorted(types ...Type) []*Package { return pkgs } - -func removeID(id ID, target []ID) (result []ID) { - for _, value := range target { - if value != id { - result = append(result, value) - } - } - return result -} diff --git a/syft/pkg/catalog_test.go b/syft/pkg/catalog_test.go index a11c238ae..14ece77f1 100644 --- a/syft/pkg/catalog_test.go +++ b/syft/pkg/catalog_test.go @@ -10,7 +10,6 @@ import ( var catalogAddAndRemoveTestPkgs = []Package{ { - ID: "my-id", Locations: []source.Location{ { RealPath: "/a/path", @@ -24,7 +23,6 @@ var catalogAddAndRemoveTestPkgs = []Package{ Type: RpmPkg, }, { - ID: "my-other-id", Locations: []source.Location{ { RealPath: "/c/path", @@ -45,6 +43,11 @@ type expectedIndexes struct { } func TestCatalogAddPopulatesIndex(t *testing.T) { + + fixtureID := func(i int) string { + return string(catalogAddAndRemoveTestPkgs[i].ID()) + } + tests := []struct { name string pkgs []Package @@ -55,16 +58,16 @@ func TestCatalogAddPopulatesIndex(t *testing.T) { pkgs: catalogAddAndRemoveTestPkgs, expectedIndexes: expectedIndexes{ byType: map[Type]*strset.Set{ - RpmPkg: strset.New("my-id"), - NpmPkg: strset.New("my-other-id"), + RpmPkg: strset.New(fixtureID(0)), + NpmPkg: strset.New(fixtureID(1)), }, byPath: map[string]*strset.Set{ - "/another/path": strset.New("my-id", "my-other-id"), - "/a/path": strset.New("my-id"), - "/b/path": strset.New("my-id"), - "/bee/path": strset.New("my-id"), - "/c/path": strset.New("my-other-id"), - "/d/path": strset.New("my-other-id"), + "/another/path": strset.New(fixtureID(0), fixtureID(1)), + "/a/path": strset.New(fixtureID(0)), + "/b/path": strset.New(fixtureID(0)), + "/bee/path": strset.New(fixtureID(0)), + "/c/path": strset.New(fixtureID(1)), + "/d/path": strset.New(fixtureID(1)), }, }, }, @@ -80,50 +83,6 @@ func TestCatalogAddPopulatesIndex(t *testing.T) { } } -func TestCatalogRemove(t *testing.T) { - tests := []struct { - name string - pkgs []Package - removeId ID - expectedIndexes expectedIndexes - }{ - { - name: "vanilla-add", - removeId: "my-other-id", - pkgs: catalogAddAndRemoveTestPkgs, - expectedIndexes: expectedIndexes{ - byType: map[Type]*strset.Set{ - RpmPkg: strset.New("my-id"), - }, - byPath: map[string]*strset.Set{ - "/another/path": strset.New("my-id"), - "/a/path": strset.New("my-id"), - "/b/path": strset.New("my-id"), - "/bee/path": strset.New("my-id"), - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - c := NewCatalog(test.pkgs...) - c.Remove(test.removeId) - - assertIndexes(t, c, test.expectedIndexes) - - if c.Package(test.removeId) != nil { - t.Errorf("expected package to be removed, but was found!") - } - - if c.PackageCount() != len(test.pkgs)-1 { - t.Errorf("expected count to be affected but was not") - } - - }) - } -} - func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) { // assert path index if len(c.idsByPath) != len(expectedIndexes.byPath) { @@ -132,7 +91,7 @@ func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) { for path, expectedIds := range expectedIndexes.byPath { actualIds := strset.New() for _, p := range c.PackagesByPath(path) { - actualIds.Add(string(p.ID)) + actualIds.Add(string(p.ID())) } if !expectedIds.IsEqual(actualIds) { @@ -147,7 +106,7 @@ func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) { for ty, expectedIds := range expectedIndexes.byType { actualIds := strset.New() for p := range c.Enumerate(ty) { - actualIds.Add(string(p.ID)) + actualIds.Add(string(p.ID())) } if !expectedIds.IsEqual(actualIds) { @@ -157,39 +116,42 @@ func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) { } func TestCatalog_PathIndexDeduplicatesRealVsVirtualPaths(t *testing.T) { + p1 := Package{ + Locations: []source.Location{ + { + RealPath: "/b/path", + VirtualPath: "/another/path", + }, + { + RealPath: "/b/path", + VirtualPath: "/b/path", + }, + }, + Type: RpmPkg, + Name: "Package-1", + } + + p2 := Package{ + Locations: []source.Location{ + { + RealPath: "/b/path", + VirtualPath: "/b/path", + }, + }, + Type: RpmPkg, + Name: "Package-2", + } tests := []struct { name string pkg Package }{ { name: "multiple locations with shared path", - pkg: Package{ - ID: "my-id", - Locations: []source.Location{ - { - RealPath: "/b/path", - VirtualPath: "/another/path", - }, - { - RealPath: "/b/path", - VirtualPath: "/b/path", - }, - }, - Type: RpmPkg, - }, + pkg: p1, }, { name: "one location with shared path", - pkg: Package{ - ID: "my-id", - Locations: []source.Location{ - { - RealPath: "/b/path", - VirtualPath: "/b/path", - }, - }, - Type: RpmPkg, - }, + pkg: p2, }, } diff --git a/syft/pkg/cataloger/apkdb/parse_apk_db.go b/syft/pkg/cataloger/apkdb/parse_apk_db.go index e79942c87..835822f5d 100644 --- a/syft/pkg/cataloger/apkdb/parse_apk_db.go +++ b/syft/pkg/cataloger/apkdb/parse_apk_db.go @@ -8,6 +8,8 @@ import ( "strconv" "strings" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/internal/log" @@ -21,7 +23,7 @@ var _ common.ParserFn = parseApkDB // parseApkDb parses individual packages from a given Alpine DB file. For more information on specific fields // see https://wiki.alpinelinux.org/wiki/Apk_spec . -func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { // larger capacity for the scanner. const maxScannerCapacity = 1024 * 1024 // a new larger buffer for the scanner @@ -47,7 +49,7 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) { for scanner.Scan() { metadata, err := parseApkDBEntry(strings.NewReader(scanner.Text())) if err != nil { - return nil, err + return nil, nil, err } if metadata != nil { packages = append(packages, pkg.Package{ @@ -62,10 +64,10 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) { } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("failed to parse APK DB file: %w", err) + return nil, nil, fmt.Errorf("failed to parse APK DB file: %w", err) } - return packages, nil + return packages, nil, nil } // nolint:funlen diff --git a/syft/pkg/cataloger/apkdb/parse_apk_db_test.go b/syft/pkg/cataloger/apkdb/parse_apk_db_test.go index dea6bb796..13926274e 100644 --- a/syft/pkg/cataloger/apkdb/parse_apk_db_test.go +++ b/syft/pkg/cataloger/apkdb/parse_apk_db_test.go @@ -775,7 +775,8 @@ func TestMultiplePackages(t *testing.T) { } }() - pkgs, err := parseApkDB(file.Name(), file) + // TODO: no relationships are under test yet + pkgs, _, err := parseApkDB(file.Name(), file) if err != nil { t.Fatal("Unable to read file contents: ", err) } diff --git a/syft/pkg/cataloger/catalog.go b/syft/pkg/cataloger/catalog.go index b76e73a45..93678370b 100644 --- a/syft/pkg/cataloger/catalog.go +++ b/syft/pkg/cataloger/catalog.go @@ -3,6 +3,7 @@ package cataloger import ( "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/pkg" @@ -38,8 +39,9 @@ func newMonitor() (*progress.Manual, *progress.Manual) { // In order to efficiently retrieve contents from a underlying container image the content fetch requests are // done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single // request. -func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers ...Cataloger) (*pkg.Catalog, error) { +func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers ...Cataloger) (*pkg.Catalog, []artifact.Relationship, error) { catalog := pkg.NewCatalog() + var allRelationships []artifact.Relationship filesProcessed, packagesDiscovered := newMonitor() @@ -47,7 +49,7 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers var errs error for _, theCataloger := range catalogers { // find packages from the underlying raw data - packages, err := theCataloger.Catalog(resolver) + packages, relationships, err := theCataloger.Catalog(resolver) if err != nil { errs = multierror.Append(errs, err) continue @@ -68,14 +70,18 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers // add to catalog catalog.Add(p) } + + allRelationships = append(allRelationships, relationships...) } + allRelationships = append(allRelationships, pkg.NewRelationships(catalog)...) + if errs != nil { - return nil, errs + return nil, nil, errs } filesProcessed.SetCompleted() packagesDiscovered.SetCompleted() - return catalog, nil + return catalog, allRelationships, nil } diff --git a/syft/pkg/cataloger/cataloger.go b/syft/pkg/cataloger/cataloger.go index d35e50ec7..096d51d76 100644 --- a/syft/pkg/cataloger/cataloger.go +++ b/syft/pkg/cataloger/cataloger.go @@ -6,6 +6,7 @@ catalogers defined in child packages as well as the interface definition to impl package cataloger import ( + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/apkdb" "github.com/anchore/syft/syft/pkg/cataloger/deb" @@ -27,7 +28,7 @@ type Cataloger interface { // Name returns a string that uniquely describes a cataloger Name() string // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. - Catalog(resolver source.FileResolver) ([]pkg.Package, error) + Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) } // ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages. diff --git a/syft/pkg/cataloger/common/cpe/generate.go b/syft/pkg/cataloger/common/cpe/generate.go index 46febb2df..daa1820c1 100644 --- a/syft/pkg/cataloger/common/cpe/generate.go +++ b/syft/pkg/cataloger/common/cpe/generate.go @@ -83,7 +83,7 @@ func candidateVendors(p pkg.Package) []string { // allow * as a candidate. Note: do NOT allow Java packages to have * vendors. switch p.Language { case pkg.Ruby, pkg.JavaScript: - vendors.addValue("*") + vendors.addValue(wfn.Any) } switch p.MetadataType { diff --git a/syft/pkg/cataloger/common/cpe/generate_test.go b/syft/pkg/cataloger/common/cpe/generate_test.go index 884c2c244..9f3eb7dd0 100644 --- a/syft/pkg/cataloger/common/cpe/generate_test.go +++ b/syft/pkg/cataloger/common/cpe/generate_test.go @@ -637,7 +637,7 @@ func TestCandidateProducts(t *testing.T) { } for _, test := range tests { - t.Run(fmt.Sprintf("%+v %+v", test.p, test.expected), func(t *testing.T) { + t.Run(test.name, func(t *testing.T) { assert.ElementsMatch(t, test.expected, candidateProducts(test.p)) }) } diff --git a/syft/pkg/cataloger/common/generic_cataloger.go b/syft/pkg/cataloger/common/generic_cataloger.go index 84ef0f865..9035bda7a 100644 --- a/syft/pkg/cataloger/common/generic_cataloger.go +++ b/syft/pkg/cataloger/common/generic_cataloger.go @@ -6,6 +6,8 @@ package common import ( "fmt" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" @@ -35,18 +37,18 @@ func (c *GenericCataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. -func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { +func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { var packages []pkg.Package - parserByLocation := c.selectFiles(resolver) + var relationships []artifact.Relationship - for location, parser := range parserByLocation { + for location, parser := range c.selectFiles(resolver) { contentReader, err := resolver.FileContentsByLocation(location) if err != nil { // TODO: fail or log? - return nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err) + return nil, nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err) } - entries, err := parser(location.RealPath, contentReader) + discoveredPackages, discoveredRelationships, err := parser(location.RealPath, contentReader) internal.CloseAndLogError(contentReader, location.VirtualPath) if err != nil { // TODO: should we fail? or only log? @@ -54,14 +56,16 @@ func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, continue } - for _, entry := range entries { - entry.FoundBy = c.upstreamCataloger - entry.Locations = []source.Location{location} + for _, p := range discoveredPackages { + p.FoundBy = c.upstreamCataloger + p.Locations = append(p.Locations, location) - packages = append(packages, entry) + packages = append(packages, p) } + + relationships = append(relationships, discoveredRelationships...) } - return packages, nil + return packages, relationships, nil } // SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging diff --git a/syft/pkg/cataloger/common/generic_cataloger_test.go b/syft/pkg/cataloger/common/generic_cataloger_test.go index a30a3d5f7..36a0cb61a 100644 --- a/syft/pkg/cataloger/common/generic_cataloger_test.go +++ b/syft/pkg/cataloger/common/generic_cataloger_test.go @@ -8,11 +8,12 @@ import ( "github.com/stretchr/testify/assert" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) -func parser(_ string, reader io.Reader) ([]pkg.Package, error) { +func parser(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { contents, err := ioutil.ReadAll(reader) if err != nil { panic(err) @@ -21,7 +22,7 @@ func parser(_ string, reader io.Reader) ([]pkg.Package, error) { { Name: string(contents), }, - }, nil + }, nil, nil } func TestGenericCataloger(t *testing.T) { @@ -47,7 +48,7 @@ func TestGenericCataloger(t *testing.T) { } } - actualPkgs, err := cataloger.Catalog(resolver) + actualPkgs, _, err := cataloger.Catalog(resolver) assert.NoError(t, err) assert.Len(t, actualPkgs, len(expectedPkgs)) diff --git a/syft/pkg/cataloger/common/parser.go b/syft/pkg/cataloger/common/parser.go index 15ca17d11..b2094f276 100644 --- a/syft/pkg/cataloger/common/parser.go +++ b/syft/pkg/cataloger/common/parser.go @@ -3,8 +3,9 @@ package common import ( "io" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" ) // ParserFn standardizes a function signature for parser functions that accept the virtual file path (not usable for file reads) and contents and return any discovered packages from that file -type ParserFn func(string, io.Reader) ([]pkg.Package, error) +type ParserFn func(string, io.Reader) ([]pkg.Package, []artifact.Relationship, error) diff --git a/syft/pkg/cataloger/deb/cataloger.go b/syft/pkg/cataloger/deb/cataloger.go index 20e510160..39c893747 100644 --- a/syft/pkg/cataloger/deb/cataloger.go +++ b/syft/pkg/cataloger/deb/cataloger.go @@ -13,6 +13,7 @@ import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) @@ -36,24 +37,23 @@ func (c *Cataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files. -func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { +func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { dbFileMatches, err := resolver.FilesByGlob(pkg.DpkgDBGlob) if err != nil { - return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err) + return nil, nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err) } - var results []pkg.Package - var pkgs []pkg.Package + var allPackages []pkg.Package for _, dbLocation := range dbFileMatches { dbContents, err := resolver.FileContentsByLocation(dbLocation) if err != nil { - return nil, err + return nil, nil, err } - pkgs, err = parseDpkgStatus(dbContents) + pkgs, err := parseDpkgStatus(dbContents) internal.CloseAndLogError(dbContents, dbLocation.VirtualPath) if err != nil { - return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err) + return nil, nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err) } for i := range pkgs { @@ -70,9 +70,9 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) addLicenses(resolver, dbLocation, p) } - results = append(results, pkgs...) + allPackages = append(allPackages, pkgs...) } - return results, nil + return allPackages, nil, nil } func addLicenses(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) { diff --git a/syft/pkg/cataloger/deb/cataloger_test.go b/syft/pkg/cataloger/deb/cataloger_test.go index 482ef7755..4568c72b9 100644 --- a/syft/pkg/cataloger/deb/cataloger_test.go +++ b/syft/pkg/cataloger/deb/cataloger_test.go @@ -100,7 +100,7 @@ func TestDpkgCataloger(t *testing.T) { t.Errorf("could not get resolver error: %+v", err) } - actual, err := c.Catalog(resolver) + actual, _, err := c.Catalog(resolver) if err != nil { t.Fatalf("failed to catalog: %+v", err) } diff --git a/syft/pkg/cataloger/deb/parse_dpkg_status.go b/syft/pkg/cataloger/deb/parse_dpkg_status.go index ea25584dd..a5338640f 100644 --- a/syft/pkg/cataloger/deb/parse_dpkg_status.go +++ b/syft/pkg/cataloger/deb/parse_dpkg_status.go @@ -23,7 +23,7 @@ var ( // parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed. func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) { buffedReader := bufio.NewReader(reader) - var packages = make([]pkg.Package, 0) + var packages []pkg.Package continueProcessing := true for continueProcessing { diff --git a/syft/pkg/cataloger/golang/bin_cataloger.go b/syft/pkg/cataloger/golang/binary_cataloger.go similarity index 78% rename from syft/pkg/cataloger/golang/bin_cataloger.go rename to syft/pkg/cataloger/golang/binary_cataloger.go index cd1a91e5f..2ebd0e849 100644 --- a/syft/pkg/cataloger/golang/bin_cataloger.go +++ b/syft/pkg/cataloger/golang/binary_cataloger.go @@ -6,7 +6,10 @@ package golang import ( "fmt" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) @@ -35,17 +38,18 @@ func (c *Cataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation. -func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { - pkgs := make([]pkg.Package, 0) +func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { + var pkgs []pkg.Package + fileMatches, err := resolver.FilesByMIMEType(mimeTypes...) if err != nil { - return pkgs, fmt.Errorf("failed to find bin by mime types: %w", err) + return pkgs, nil, fmt.Errorf("failed to find bin by mime types: %w", err) } for _, location := range fileMatches { r, err := resolver.FileContentsByLocation(location) if err != nil { - return pkgs, fmt.Errorf("failed to resolve file contents by location: %w", err) + return pkgs, nil, fmt.Errorf("failed to resolve file contents by location: %w", err) } goPkgs, err := parseGoBin(location, r) @@ -53,9 +57,9 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) log.Warnf("could not parse possible go binary: %+v", err) } - r.Close() + internal.CloseAndLogError(r, location.RealPath) pkgs = append(pkgs, goPkgs...) } - return pkgs, nil + return pkgs, nil, nil } diff --git a/syft/pkg/cataloger/golang/parse_go_bin.go b/syft/pkg/cataloger/golang/parse_go_bin.go index 311bc9681..a44b676a2 100644 --- a/syft/pkg/cataloger/golang/parse_go_bin.go +++ b/syft/pkg/cataloger/golang/parse_go_bin.go @@ -23,9 +23,7 @@ func parseGoBin(location source.Location, reader io.ReadCloser) ([]pkg.Package, goVersion, mod := findVers(x) - pkgs := buildGoPkgInfo(location, mod, goVersion) - - return pkgs, nil + return buildGoPkgInfo(location, mod, goVersion), nil } func buildGoPkgInfo(location source.Location, mod, goVersion string) []pkg.Package { diff --git a/syft/pkg/cataloger/golang/parse_go_mod.go b/syft/pkg/cataloger/golang/parse_go_mod.go index c8facdc25..c7cc49e5a 100644 --- a/syft/pkg/cataloger/golang/parse_go_mod.go +++ b/syft/pkg/cataloger/golang/parse_go_mod.go @@ -6,22 +6,23 @@ import ( "io/ioutil" "sort" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "golang.org/x/mod/modfile" ) // parseGoMod takes a go.mod and lists all packages discovered. -func parseGoMod(path string, reader io.Reader) ([]pkg.Package, error) { +func parseGoMod(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { packages := make(map[string]pkg.Package) contents, err := ioutil.ReadAll(reader) if err != nil { - return nil, fmt.Errorf("failed to read go module: %w", err) + return nil, nil, fmt.Errorf("failed to read go module: %w", err) } file, err := modfile.Parse(path, contents, nil) if err != nil { - return nil, fmt.Errorf("failed to parse go module: %w", err) + return nil, nil, fmt.Errorf("failed to parse go module: %w", err) } for _, m := range file.Require { @@ -59,5 +60,5 @@ func parseGoMod(path string, reader io.Reader) ([]pkg.Package, error) { return pkgsSlice[i].Name < pkgsSlice[j].Name }) - return pkgsSlice, nil + return pkgsSlice, nil, nil } diff --git a/syft/pkg/cataloger/golang/parse_go_mod_test.go b/syft/pkg/cataloger/golang/parse_go_mod_test.go index 292107ef6..54d5b5465 100644 --- a/syft/pkg/cataloger/golang/parse_go_mod_test.go +++ b/syft/pkg/cataloger/golang/parse_go_mod_test.go @@ -70,7 +70,8 @@ func TestParseGoMod(t *testing.T) { t.Fatalf(err.Error()) } - actual, err := parseGoMod(test.fixture, f) + // TODO: no relationships are under test yet + actual, _, err := parseGoMod(test.fixture, f) if err != nil { t.Fatalf(err.Error()) } diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index ca1c369a3..7e188dce9 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -9,6 +9,7 @@ import ( "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -34,12 +35,12 @@ type archiveParser struct { } // parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives. -func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, error) { +func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true) // note: even on error, we should always run cleanup functions defer cleanupFn() if err != nil { - return nil, err + return nil, nil, err } return parser.parse() } @@ -80,29 +81,31 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo } // parse the loaded archive and return all packages found. -func (j *archiveParser) parse() ([]pkg.Package, error) { - var pkgs = make([]pkg.Package, 0) +func (j *archiveParser) parse() ([]pkg.Package, []artifact.Relationship, error) { + var pkgs []pkg.Package + var relationships []artifact.Relationship // find the parent package from the java manifest parentPkg, err := j.discoverMainPackage() if err != nil { - return nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err) + return nil, nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err) } // find aux packages from pom.properties/pom.xml and potentially modify the existing parentPkg auxPkgs, err := j.discoverPkgsFromAllMavenFiles(parentPkg) if err != nil { - return nil, err + return nil, nil, err } pkgs = append(pkgs, auxPkgs...) if j.detectNested { // find nested java archive packages - nestedPkgs, err := j.discoverPkgsFromNestedArchives(parentPkg) + nestedPkgs, nestedRelationships, err := j.discoverPkgsFromNestedArchives(parentPkg) if err != nil { - return nil, err + return nil, nil, err } pkgs = append(pkgs, nestedPkgs...) + relationships = append(relationships, nestedRelationships...) } // lastly, add the parent package to the list (assuming the parent exists) @@ -110,7 +113,7 @@ func (j *archiveParser) parse() ([]pkg.Package, error) { pkgs = append([]pkg.Package{*parentPkg}, pkgs...) } - return pkgs, nil + return pkgs, relationships, nil } // discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages. @@ -189,31 +192,32 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(parentPkg *pkg.Package) ([ // discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and // associating each discovered package to the given parent package. -func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, error) { - var pkgs = make([]pkg.Package, 0) +func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) { + var pkgs []pkg.Package + var relationships []artifact.Relationship // search and parse pom.properties files & fetch the contents openers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...) if err != nil { - return nil, fmt.Errorf("unable to extract files from zip: %w", err) + return nil, nil, fmt.Errorf("unable to extract files from zip: %w", err) } // discover nested artifacts for archivePath, archiveOpener := range openers { archiveReadCloser, err := archiveOpener.Open() if err != nil { - return nil, fmt.Errorf("unable to open archived file from tempdir: %w", err) + return nil, nil, fmt.Errorf("unable to open archived file from tempdir: %w", err) } nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath) - nestedPkgs, err := parseJavaArchive(nestedPath, archiveReadCloser) + nestedPkgs, nestedRelationships, err := parseJavaArchive(nestedPath, archiveReadCloser) if err != nil { if closeErr := archiveReadCloser.Close(); closeErr != nil { log.Warnf("unable to close archived file from tempdir: %+v", closeErr) } - return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err) + return nil, nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err) } if err = archiveReadCloser.Close(); err != nil { - return nil, fmt.Errorf("unable to close archived file from tempdir: %w", err) + return nil, nil, fmt.Errorf("unable to close archived file from tempdir: %w", err) } // attach the parent package to all discovered packages that are not already associated with a java archive @@ -226,9 +230,11 @@ func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ( } pkgs = append(pkgs, p) } + + relationships = append(relationships, nestedRelationships...) } - return pkgs, nil + return pkgs, relationships, nil } func pomPropertiesByParentPath(archivePath string, extractPaths []string, virtualPath string) (map[string]pkg.PomProperties, error) { diff --git a/syft/pkg/cataloger/java/archive_parser_test.go b/syft/pkg/cataloger/java/archive_parser_test.go index ac8c826d3..64e51b8e9 100644 --- a/syft/pkg/cataloger/java/archive_parser_test.go +++ b/syft/pkg/cataloger/java/archive_parser_test.go @@ -242,7 +242,7 @@ func TestParseJar(t *testing.T) { t.Fatalf("should not have filed... %+v", err) } - actual, err := parser.parse() + actual, _, err := parser.parse() if err != nil { t.Fatalf("failed to parse java archive: %+v", err) } @@ -507,7 +507,7 @@ func TestParseNestedJar(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseJavaArchive(fixture.Name(), fixture) + actual, _, err := parseJavaArchive(fixture.Name(), fixture) if err != nil { t.Fatalf("failed to parse java archive: %+v", err) } diff --git a/syft/pkg/cataloger/javascript/parse_package_json.go b/syft/pkg/cataloger/javascript/parse_package_json.go index 1a5be5e44..ad570a740 100644 --- a/syft/pkg/cataloger/javascript/parse_package_json.go +++ b/syft/pkg/cataloger/javascript/parse_package_json.go @@ -13,6 +13,7 @@ import ( "github.com/mitchellh/mapstructure" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -162,8 +163,8 @@ func licensesFromJSON(p PackageJSON) ([]string, error) { } // parsePackageJSON parses a package.json and returns the discovered JavaScript packages. -func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { - packages := make([]pkg.Package, 0) +func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { + var packages []pkg.Package dec := json.NewDecoder(reader) for { @@ -171,17 +172,17 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { if err := dec.Decode(&p); err == io.EOF { break } else if err != nil { - return nil, fmt.Errorf("failed to parse package.json file: %w", err) + return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err) } if !p.hasNameAndVersionValues() { log.Debug("encountered package.json file without a name and/or version field, ignoring this file") - return nil, nil + return nil, nil, nil } licenses, err := licensesFromJSON(p) if err != nil { - return nil, fmt.Errorf("failed to parse package.json file: %w", err) + return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err) } packages = append(packages, pkg.Package{ @@ -200,7 +201,7 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) { }) } - return packages, nil + return packages, nil, nil } func (p PackageJSON) hasNameAndVersionValues() bool { diff --git a/syft/pkg/cataloger/javascript/parse_package_json_test.go b/syft/pkg/cataloger/javascript/parse_package_json_test.go index 83af1f9b1..cce594cb1 100644 --- a/syft/pkg/cataloger/javascript/parse_package_json_test.go +++ b/syft/pkg/cataloger/javascript/parse_package_json_test.go @@ -124,7 +124,7 @@ func TestParsePackageJSON(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parsePackageJSON("", fixture) + actual, _, err := parsePackageJSON("", fixture) if err != nil { t.Fatalf("failed to parse package-lock.json: %+v", err) } @@ -150,7 +150,8 @@ func TestParsePackageJSON_Partial(t *testing.T) { // see https://github.com/anch t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parsePackageJSON("", fixture) + // TODO: no relationships are under test yet + actual, _, err := parsePackageJSON("", fixture) if err != nil { t.Fatalf("failed to parse package-lock.json: %+v", err) } diff --git a/syft/pkg/cataloger/javascript/parse_package_lock.go b/syft/pkg/cataloger/javascript/parse_package_lock.go index 3a1289230..880674963 100644 --- a/syft/pkg/cataloger/javascript/parse_package_lock.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock.go @@ -5,6 +5,7 @@ import ( "fmt" "io" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -28,11 +29,11 @@ type Dependency struct { } // parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages. -func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, error) { +func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { // in the case we find package-lock.json files in the node_modules directories, skip those // as the whole purpose of the lock file is for the specific dependencies of the root project if pathContainsNodeModulesDirectory(path) { - return nil, nil + return nil, nil, nil } var packages []pkg.Package @@ -43,7 +44,7 @@ func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, error) { if err := dec.Decode(&lock); err == io.EOF { break } else if err != nil { - return nil, fmt.Errorf("failed to parse package-lock.json file: %w", err) + return nil, nil, fmt.Errorf("failed to parse package-lock.json file: %w", err) } for name, pkgMeta := range lock.Dependencies { packages = append(packages, pkg.Package{ @@ -55,5 +56,5 @@ func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, error) { } } - return packages, nil + return packages, nil, nil } diff --git a/syft/pkg/cataloger/javascript/parse_package_lock_test.go b/syft/pkg/cataloger/javascript/parse_package_lock_test.go index 1f0b0a086..0f33aaa83 100644 --- a/syft/pkg/cataloger/javascript/parse_package_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock_test.go @@ -109,7 +109,8 @@ func TestParsePackageLock(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parsePackageLock(fixture.Name(), fixture) + // TODO: no relationships are under test yet + actual, _, err := parsePackageLock(fixture.Name(), fixture) if err != nil { t.Fatalf("failed to parse package-lock.json: %+v", err) } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index bd527f9b7..71e9dd3da 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -7,6 +7,7 @@ import ( "regexp" "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -34,11 +35,11 @@ const ( noVersion = "" ) -func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, error) { +func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { // in the case we find yarn.lock files in the node_modules directories, skip those // as the whole purpose of the lock file is for the specific dependencies of the project if pathContainsNodeModulesDirectory(path) { - return nil, nil + return nil, nil, nil } var packages []pkg.Package @@ -79,10 +80,10 @@ func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, error) { } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("failed to parse yarn.lock file: %w", err) + return nil, nil, fmt.Errorf("failed to parse yarn.lock file: %w", err) } - return packages, nil + return packages, nil, nil } func findPackageName(line string) string { diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go index fd4ecd2e3..5ef0242d9 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go @@ -70,7 +70,8 @@ func TestParseYarnLock(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseYarnLock(fixture.Name(), fixture) + // TODO: no relationships are under test yet + actual, _, err := parseYarnLock(fixture.Name(), fixture) if err != nil { t.Fatalf("failed to parse yarn.lock: %+v", err) } diff --git a/syft/pkg/cataloger/php/parse_composer_lock.go b/syft/pkg/cataloger/php/parse_composer_lock.go index e3a0c41c1..b92f4fa64 100644 --- a/syft/pkg/cataloger/php/parse_composer_lock.go +++ b/syft/pkg/cataloger/php/parse_composer_lock.go @@ -5,6 +5,8 @@ import ( "fmt" "io" + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -23,7 +25,7 @@ type Dependency struct { var _ common.ParserFn = parseComposerLock // parseComposerLock is a parser function for Composer.lock contents, returning "Default" php packages discovered. -func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { packages := make([]pkg.Package, 0) dec := json.NewDecoder(reader) @@ -32,7 +34,7 @@ func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, error) { if err := dec.Decode(&lock); err == io.EOF { break } else if err != nil { - return nil, fmt.Errorf("failed to parse composer.lock file: %w", err) + return nil, nil, fmt.Errorf("failed to parse composer.lock file: %w", err) } for _, pkgMeta := range lock.Packages { version := pkgMeta.Version @@ -46,5 +48,5 @@ func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, error) { } } - return packages, nil + return packages, nil, nil } diff --git a/syft/pkg/cataloger/php/parse_composer_lock_test.go b/syft/pkg/cataloger/php/parse_composer_lock_test.go index 92ffbfaef..16b97fc89 100644 --- a/syft/pkg/cataloger/php/parse_composer_lock_test.go +++ b/syft/pkg/cataloger/php/parse_composer_lock_test.go @@ -28,7 +28,8 @@ func TestParseComposerFileLock(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseComposerLock(fixture.Name(), fixture) + // TODO: no relationships are under test yet + actual, _, err := parseComposerLock(fixture.Name(), fixture) if err != nil { t.Fatalf("failed to parse requirements: %+v", err) } diff --git a/syft/pkg/cataloger/python/package_cataloger.go b/syft/pkg/cataloger/python/package_cataloger.go index b49512125..d4749c874 100644 --- a/syft/pkg/cataloger/python/package_cataloger.go +++ b/syft/pkg/cataloger/python/package_cataloger.go @@ -7,6 +7,7 @@ import ( "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" @@ -31,13 +32,13 @@ func (c *PackageCataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations. -func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { +func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { var fileMatches []source.Location for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob, eggFileMetadataGlob} { matches, err := resolver.FilesByGlob(glob) if err != nil { - return nil, fmt.Errorf("failed to find files by glob: %s", glob) + return nil, nil, fmt.Errorf("failed to find files by glob: %s", glob) } fileMatches = append(fileMatches, matches...) } @@ -46,13 +47,13 @@ func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, for _, location := range fileMatches { p, err := c.catalogEggOrWheel(resolver, location) if err != nil { - return nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.RealPath, err) + return nil, nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.RealPath, err) } if p != nil { pkgs = append(pkgs, *p) } } - return pkgs, nil + return pkgs, nil, nil } // catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents. diff --git a/syft/pkg/cataloger/python/package_cataloger_test.go b/syft/pkg/cataloger/python/package_cataloger_test.go index 4aced6a4d..8766c42f6 100644 --- a/syft/pkg/cataloger/python/package_cataloger_test.go +++ b/syft/pkg/cataloger/python/package_cataloger_test.go @@ -144,7 +144,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) { test.expectedPackage.Locations = locations - actual, err := NewPythonPackageCataloger().Catalog(resolver) + actual, _, err := NewPythonPackageCataloger().Catalog(resolver) if err != nil { t.Fatalf("failed to catalog python package: %+v", err) } @@ -173,7 +173,7 @@ func TestIgnorePackage(t *testing.T) { t.Run(test.MetadataFixture, func(t *testing.T) { resolver := source.NewMockResolverForPaths(test.MetadataFixture) - actual, err := NewPythonPackageCataloger().Catalog(resolver) + actual, _, err := NewPythonPackageCataloger().Catalog(resolver) if err != nil { t.Fatalf("failed to catalog python package: %+v", err) } diff --git a/syft/pkg/cataloger/python/parse_pipfile_lock.go b/syft/pkg/cataloger/python/parse_pipfile_lock.go index 3a03218a1..3d34bbd0f 100644 --- a/syft/pkg/cataloger/python/parse_pipfile_lock.go +++ b/syft/pkg/cataloger/python/parse_pipfile_lock.go @@ -6,6 +6,7 @@ import ( "io" "strings" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -37,7 +38,7 @@ type Dependency struct { var _ common.ParserFn = parsePipfileLock // parsePipfileLock is a parser function for Pipfile.lock contents, returning "Default" python packages discovered. -func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, error) { +func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { packages := make([]pkg.Package, 0) dec := json.NewDecoder(reader) @@ -46,7 +47,7 @@ func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, error) { if err := dec.Decode(&lock); err == io.EOF { break } else if err != nil { - return nil, fmt.Errorf("failed to parse Pipfile.lock file: %w", err) + return nil, nil, fmt.Errorf("failed to parse Pipfile.lock file: %w", err) } for name, pkgMeta := range lock.Default { version := strings.TrimPrefix(pkgMeta.Version, "==") @@ -59,5 +60,5 @@ func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, error) { } } - return packages, nil + return packages, nil, nil } diff --git a/syft/pkg/cataloger/python/parse_pipfile_lock_test.go b/syft/pkg/cataloger/python/parse_pipfile_lock_test.go index f65864e0e..2e4fedc9d 100644 --- a/syft/pkg/cataloger/python/parse_pipfile_lock_test.go +++ b/syft/pkg/cataloger/python/parse_pipfile_lock_test.go @@ -39,7 +39,8 @@ func TestParsePipFileLock(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parsePipfileLock(fixture.Name(), fixture) + // TODO: no relationships are under test yet + actual, _, err := parsePipfileLock(fixture.Name(), fixture) if err != nil { t.Fatalf("failed to parse requirements: %+v", err) } diff --git a/syft/pkg/cataloger/python/parse_poetry_lock.go b/syft/pkg/cataloger/python/parse_poetry_lock.go index b6981a1b0..70ab2f471 100644 --- a/syft/pkg/cataloger/python/parse_poetry_lock.go +++ b/syft/pkg/cataloger/python/parse_poetry_lock.go @@ -4,6 +4,7 @@ import ( "fmt" "io" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" "github.com/pelletier/go-toml" @@ -13,17 +14,17 @@ import ( var _ common.ParserFn = parsePoetryLock // parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered. -func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, error) { +func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { tree, err := toml.LoadReader(reader) if err != nil { - return nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err) + return nil, nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err) } metadata := PoetryMetadata{} err = tree.Unmarshal(&metadata) if err != nil { - return nil, fmt.Errorf("unable to parse poetry.lock: %v", err) + return nil, nil, fmt.Errorf("unable to parse poetry.lock: %v", err) } - return metadata.Pkgs(), nil + return metadata.Pkgs(), nil, nil } diff --git a/syft/pkg/cataloger/python/parse_poetry_lock_test.go b/syft/pkg/cataloger/python/parse_poetry_lock_test.go index 80cc6b625..5f30c213f 100644 --- a/syft/pkg/cataloger/python/parse_poetry_lock_test.go +++ b/syft/pkg/cataloger/python/parse_poetry_lock_test.go @@ -45,7 +45,8 @@ func TestParsePoetryLock(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parsePoetryLock(fixture.Name(), fixture) + // TODO: no relationships are under test yet + actual, _, err := parsePoetryLock(fixture.Name(), fixture) if err != nil { t.Error(err) } diff --git a/syft/pkg/cataloger/python/parse_requirements.go b/syft/pkg/cataloger/python/parse_requirements.go index b206224dd..9a7d403e6 100644 --- a/syft/pkg/cataloger/python/parse_requirements.go +++ b/syft/pkg/cataloger/python/parse_requirements.go @@ -6,6 +6,7 @@ import ( "io" "strings" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -15,7 +16,7 @@ var _ common.ParserFn = parseRequirementsTxt // parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a // specific version. -func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { packages := make([]pkg.Package, 0) scanner := bufio.NewScanner(reader) @@ -55,10 +56,10 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) { } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("failed to parse python requirements file: %w", err) + return nil, nil, fmt.Errorf("failed to parse python requirements file: %w", err) } - return packages, nil + return packages, nil, nil } // removeTrailingComment takes a requirements.txt line and strips off comment strings. diff --git a/syft/pkg/cataloger/python/parse_requirements_test.go b/syft/pkg/cataloger/python/parse_requirements_test.go index 8dd66092a..f84913ac0 100644 --- a/syft/pkg/cataloger/python/parse_requirements_test.go +++ b/syft/pkg/cataloger/python/parse_requirements_test.go @@ -50,7 +50,8 @@ func TestParseRequirementsTxt(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseRequirementsTxt(fixture.Name(), fixture) + // TODO: no relationships are under test yet + actual, _, err := parseRequirementsTxt(fixture.Name(), fixture) if err != nil { t.Fatalf("failed to parse requirements: %+v", err) } diff --git a/syft/pkg/cataloger/python/parse_setup.go b/syft/pkg/cataloger/python/parse_setup.go index e1b0c39ce..d21bbabba 100644 --- a/syft/pkg/cataloger/python/parse_setup.go +++ b/syft/pkg/cataloger/python/parse_setup.go @@ -6,6 +6,7 @@ import ( "regexp" "strings" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -19,7 +20,7 @@ var _ common.ParserFn = parseSetup // " mypy2 == v0.770", ' mypy3== v0.770', --> match(name=mypy2 version=v0.770), match(name=mypy3, version=v0.770) var pinnedDependency = regexp.MustCompile(`['"]\W?(\w+\W?==\W?[\w\.]*)`) -func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseSetup(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { packages := make([]pkg.Package, 0) scanner := bufio.NewScanner(reader) @@ -46,5 +47,5 @@ func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) { } } - return packages, nil + return packages, nil, nil } diff --git a/syft/pkg/cataloger/python/parse_setup_test.go b/syft/pkg/cataloger/python/parse_setup_test.go index 3b8fa8edc..c8106157e 100644 --- a/syft/pkg/cataloger/python/parse_setup_test.go +++ b/syft/pkg/cataloger/python/parse_setup_test.go @@ -45,7 +45,7 @@ func TestParseSetup(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseSetup(fixture.Name(), fixture) + actual, _, err := parseSetup(fixture.Name(), fixture) if err != nil { t.Fatalf("failed to parse requirements: %+v", err) } diff --git a/syft/pkg/cataloger/rpmdb/cataloger.go b/syft/pkg/cataloger/rpmdb/cataloger.go index 642c494e9..4571e67e4 100644 --- a/syft/pkg/cataloger/rpmdb/cataloger.go +++ b/syft/pkg/cataloger/rpmdb/cataloger.go @@ -8,6 +8,7 @@ import ( "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) @@ -27,24 +28,26 @@ func (c *Cataloger) Name() string { } // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation. -func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) { +func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) { fileMatches, err := resolver.FilesByGlob(pkg.RpmDBGlob) if err != nil { - return nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err) + return nil, nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err) } var pkgs []pkg.Package for _, location := range fileMatches { dbContentReader, err := resolver.FileContentsByLocation(location) if err != nil { - return nil, err + return nil, nil, err } - pkgs, err = parseRpmDB(resolver, location, dbContentReader) + discoveredPkgs, err := parseRpmDB(resolver, location, dbContentReader) internal.CloseAndLogError(dbContentReader, location.VirtualPath) if err != nil { - return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err) + return nil, nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err) } + + pkgs = append(pkgs, discoveredPkgs...) } - return pkgs, nil + return pkgs, nil, nil } diff --git a/syft/pkg/cataloger/ruby/parse_gemfile_lock.go b/syft/pkg/cataloger/ruby/parse_gemfile_lock.go index b27b3ba66..2d2f62c42 100644 --- a/syft/pkg/cataloger/ruby/parse_gemfile_lock.go +++ b/syft/pkg/cataloger/ruby/parse_gemfile_lock.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -16,7 +17,7 @@ var _ common.ParserFn = parseGemFileLockEntries var sectionsOfInterest = internal.NewStringSetFromSlice([]string{"GEM"}) // parseGemFileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered. -func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { pkgs := make([]pkg.Package, 0) scanner := bufio.NewScanner(reader) @@ -49,9 +50,9 @@ func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, error) } } if err := scanner.Err(); err != nil { - return nil, err + return nil, nil, err } - return pkgs, nil + return pkgs, nil, nil } func isDependencyLine(line string) bool { diff --git a/syft/pkg/cataloger/ruby/parse_gemfile_lock_test.go b/syft/pkg/cataloger/ruby/parse_gemfile_lock_test.go index 4307c34fa..56cb5f277 100644 --- a/syft/pkg/cataloger/ruby/parse_gemfile_lock_test.go +++ b/syft/pkg/cataloger/ruby/parse_gemfile_lock_test.go @@ -68,7 +68,8 @@ func TestParseGemfileLockEntries(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseGemFileLockEntries(fixture.Name(), fixture) + // TODO: no relationships are under test yet + actual, _, err := parseGemFileLockEntries(fixture.Name(), fixture) if err != nil { t.Fatalf("failed to parse gemfile lock: %+v", err) } diff --git a/syft/pkg/cataloger/ruby/parse_gemspec.go b/syft/pkg/cataloger/ruby/parse_gemspec.go index f743a2f5c..afab064e7 100644 --- a/syft/pkg/cataloger/ruby/parse_gemspec.go +++ b/syft/pkg/cataloger/ruby/parse_gemspec.go @@ -12,6 +12,7 @@ import ( "github.com/mitchellh/mapstructure" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" ) @@ -60,7 +61,7 @@ func processList(s string) []string { return results } -func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { var pkgs []pkg.Package var fields = make(map[string]interface{}) scanner := bufio.NewScanner(reader) @@ -93,7 +94,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) { if fields["name"] != "" && fields["version"] != "" { var metadata pkg.GemMetadata if err := mapstructure.Decode(fields, &metadata); err != nil { - return nil, fmt.Errorf("unable to decode gem metadata: %w", err) + return nil, nil, fmt.Errorf("unable to decode gem metadata: %w", err) } pkgs = append(pkgs, pkg.Package{ @@ -107,7 +108,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) { }) } - return pkgs, nil + return pkgs, nil, nil } // renderUtf8 takes any string escaped string sub-sections from the ruby string and replaces those sections with the UTF8 runes. diff --git a/syft/pkg/cataloger/ruby/parse_gemspec_test.go b/syft/pkg/cataloger/ruby/parse_gemspec_test.go index 2a32ae0b3..d98cc4370 100644 --- a/syft/pkg/cataloger/ruby/parse_gemspec_test.go +++ b/syft/pkg/cataloger/ruby/parse_gemspec_test.go @@ -31,7 +31,8 @@ func TestParseGemspec(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseGemSpecEntries(fixture.Name(), fixture) + // TODO: no relationships are under test yet + actual, _, err := parseGemSpecEntries(fixture.Name(), fixture) if err != nil { t.Fatalf("failed to parse gemspec: %+v", err) } diff --git a/syft/pkg/cataloger/rust/parse_cargo_lock.go b/syft/pkg/cataloger/rust/parse_cargo_lock.go index ccb7dafd8..ab5918978 100644 --- a/syft/pkg/cataloger/rust/parse_cargo_lock.go +++ b/syft/pkg/cataloger/rust/parse_cargo_lock.go @@ -4,6 +4,7 @@ import ( "fmt" "io" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" "github.com/pelletier/go-toml" @@ -13,17 +14,17 @@ import ( var _ common.ParserFn = parseCargoLock // parseCargoLock is a parser function for Cargo.lock contents, returning all rust cargo crates discovered. -func parseCargoLock(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseCargoLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) { tree, err := toml.LoadReader(reader) if err != nil { - return nil, fmt.Errorf("unable to load Cargo.lock for parsing: %v", err) + return nil, nil, fmt.Errorf("unable to load Cargo.lock for parsing: %v", err) } metadata := CargoMetadata{} err = tree.Unmarshal(&metadata) if err != nil { - return nil, fmt.Errorf("unable to parse Cargo.lock: %v", err) + return nil, nil, fmt.Errorf("unable to parse Cargo.lock: %v", err) } - return metadata.Pkgs(), nil + return metadata.Pkgs(), nil, nil } diff --git a/syft/pkg/cataloger/rust/parse_cargo_lock_test.go b/syft/pkg/cataloger/rust/parse_cargo_lock_test.go index 334090b8f..c5c88ffa4 100644 --- a/syft/pkg/cataloger/rust/parse_cargo_lock_test.go +++ b/syft/pkg/cataloger/rust/parse_cargo_lock_test.go @@ -177,7 +177,8 @@ func TestParseCargoLock(t *testing.T) { t.Fatalf("failed to open fixture: %+v", err) } - actual, err := parseCargoLock(fixture.Name(), fixture) + // TODO: no relationships are under test yet + actual, _, err := parseCargoLock(fixture.Name(), fixture) if err != nil { t.Error(err) } diff --git a/syft/pkg/cpe.go b/syft/pkg/cpe.go index 52a86adce..c009f4e29 100644 --- a/syft/pkg/cpe.go +++ b/syft/pkg/cpe.go @@ -45,5 +45,8 @@ func MustCPE(cpeStr string) CPE { func normalizeCpeField(field string) string { // keep dashes and forward slashes unescaped + if field == "*" { + return wfn.Any + } return strings.ReplaceAll(wfn.StripSlashes(field), `\/`, "/") } diff --git a/syft/pkg/cpe_test.go b/syft/pkg/cpe_test.go index d06efa34b..cf0a1e1f7 100644 --- a/syft/pkg/cpe_test.go +++ b/syft/pkg/cpe_test.go @@ -1,6 +1,10 @@ package pkg -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func must(c CPE, e error) CPE { if e != nil { @@ -46,3 +50,33 @@ func TestNewCPE(t *testing.T) { }) } } + +func Test_normalizeCpeField(t *testing.T) { + + tests := []struct { + field string + expected string + }{ + { + field: "something", + expected: "something", + }, + { + field: "some\\thing", + expected: `some\thing`, + }, + { + field: "*", + expected: "", + }, + { + field: "", + expected: "", + }, + } + for _, test := range tests { + t.Run(test.field, func(t *testing.T) { + assert.Equal(t, test.expected, normalizeCpeField(test.field)) + }) + } +} diff --git a/syft/pkg/id.go b/syft/pkg/id.go deleted file mode 100644 index 7b3e6b2d7..000000000 --- a/syft/pkg/id.go +++ /dev/null @@ -1,4 +0,0 @@ -package pkg - -// ID represents a unique value for each package added to a package catalog. -type ID string diff --git a/syft/pkg/java_metadata.go b/syft/pkg/java_metadata.go index 641b9294f..ceaf476e9 100644 --- a/syft/pkg/java_metadata.go +++ b/syft/pkg/java_metadata.go @@ -21,7 +21,7 @@ type JavaMetadata struct { Manifest *JavaManifest `mapstructure:"Manifest" json:"manifest,omitempty"` PomProperties *PomProperties `mapstructure:"PomProperties" json:"pomProperties,omitempty"` PomProject *PomProject `mapstructure:"PomProject" json:"pomProject,omitempty"` - Parent *Package `json:"-"` + Parent *Package `hash:"ignore" json:"-"` // note: the parent cannot be included in the minimal definition of uniqueness since this field is not reproducible in an encode-decode cycle (is lossy). } // PomProperties represents the fields of interest extracted from a Java archive's pom.properties file. diff --git a/syft/pkg/package.go b/syft/pkg/package.go index 08c5ebbdb..d91b211cb 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -6,41 +6,39 @@ package pkg import ( "fmt" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/source" - "github.com/mitchellh/hashstructure" ) // Package represents an application or library that has been bundled into a distributable format. // TODO: if we ignore FoundBy for ID generation should we merge the field to show it was found in two places? type Package struct { - ID ID `hash:"ignore"` // uniquely identifies a package, set by the cataloger - Name string // the package name - Version string // the version of the package - FoundBy string // the specific cataloger that discovered this package - Locations []source.Location // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) - // TODO: should we move licenses into metadata? - Licenses []string // licenses discovered with the package metadata - Language Language // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc) - Type Type // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc) - CPEs []CPE // all possible Common Platform Enumerators - PURL string // the Package URL (see https://github.com/package-url/purl-spec) - MetadataType MetadataType // the shape of the additional data in the "metadata" field - Metadata interface{} // additional data found while parsing the package source + Name string // the package name + Version string // the version of the package + FoundBy string // the specific cataloger that discovered this package + Locations []source.Location // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package) + Licenses []string // licenses discovered with the package metadata + Language Language // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc) + Type Type // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc) + CPEs []CPE // all possible Common Platform Enumerators + PURL string // the Package URL (see https://github.com/package-url/purl-spec) + MetadataType MetadataType // the shape of the additional data in the "metadata" field + Metadata interface{} // additional data found while parsing the package source +} + +func (p Package) ID() artifact.ID { + f, err := artifact.IDFromHash(p) + if err != nil { + // TODO: what to do in this case? + log.Warnf("unable to get fingerprint of package=%s@%s: %+v", p.Name, p.Version, err) + return "" + } + + return f } // Stringer to represent a package. func (p Package) String() string { return fmt.Sprintf("Pkg(type=%s, name=%s, version=%s)", p.Type, p.Name, p.Version) } - -func (p Package) Fingerprint() (string, error) { - f, err := hashstructure.Hash(p, &hashstructure.HashOptions{ - ZeroNil: true, - SlicesAsSets: true, - }) - if err != nil { - return "", fmt.Errorf("could not build package fingerprint for: %s version: %s", p.Name, p.Version) - } - - return fmt.Sprint(f), nil -} diff --git a/syft/pkg/package_test.go b/syft/pkg/package_test.go index 4edf88dac..60fe51a95 100644 --- a/syft/pkg/package_test.go +++ b/syft/pkg/package_test.go @@ -9,7 +9,6 @@ import ( func TestFingerprint(t *testing.T) { originalPkg := Package{ - ID: "π", Name: "pi", Version: "3.14", FoundBy: "Archimedes", @@ -190,10 +189,10 @@ func TestFingerprint(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { transformedPkg := test.transform(originalPkg) - originalFingerprint, err := originalPkg.Fingerprint() - assert.NoError(t, err, "expected no error on package fingerprint") - transformedFingerprint, err := transformedPkg.Fingerprint() - assert.NoError(t, err, "expected no error on package fingerprint") + originalFingerprint := originalPkg.ID() + assert.NotEmpty(t, originalFingerprint) + transformedFingerprint := transformedPkg.ID() + assert.NotEmpty(t, transformedFingerprint) if test.expectIdentical { assert.Equal(t, originalFingerprint, transformedFingerprint) diff --git a/syft/pkg/relationship.go b/syft/pkg/relationship.go deleted file mode 100644 index 09271e564..000000000 --- a/syft/pkg/relationship.go +++ /dev/null @@ -1,20 +0,0 @@ -package pkg - -const ( - // OwnershipByFileOverlapRelationship indicates that the parent package owns the child package made evident by the set of provided files - OwnershipByFileOverlapRelationship RelationshipType = "ownership-by-file-overlap" -) - -type RelationshipType string - -type Relationship struct { - Parent ID - Child ID - Type RelationshipType - Metadata interface{} -} - -// TODO: as more relationships are added, this function signature will probably accommodate selection -func NewRelationships(catalog *Catalog) []Relationship { - return ownershipByFilesRelationships(catalog) -} diff --git a/syft/pkg/relationships.go b/syft/pkg/relationships.go new file mode 100644 index 000000000..fe73c1b0f --- /dev/null +++ b/syft/pkg/relationships.go @@ -0,0 +1,8 @@ +package pkg + +import "github.com/anchore/syft/syft/artifact" + +// TODO: as more relationships are added, this function signature will probably accommodate selection +func NewRelationships(catalog *Catalog) []artifact.Relationship { + return RelationshipsByFileOwnership(catalog) +} diff --git a/syft/pkg/ownership_by_files_relationship.go b/syft/pkg/relationships_by_file_ownership.go similarity index 66% rename from syft/pkg/ownership_by_files_relationship.go rename to syft/pkg/relationships_by_file_ownership.go index 3aa2f37b7..5c01c949d 100644 --- a/syft/pkg/ownership_by_files_relationship.go +++ b/syft/pkg/relationships_by_file_ownership.go @@ -2,6 +2,7 @@ package pkg import ( "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" "github.com/bmatcuk/doublestar/v2" "github.com/scylladb/go-set/strset" ) @@ -20,17 +21,19 @@ type ownershipByFilesMetadata struct { Files []string `json:"files"` } -func ownershipByFilesRelationships(catalog *Catalog) []Relationship { +// RelationshipsByFileOwnership creates a package-to-package relationship based on discovering which packages have +// evidence locations that overlap with ownership claim from another package's package manager metadata. +func RelationshipsByFileOwnership(catalog *Catalog) []artifact.Relationship { var relationships = findOwnershipByFilesRelationships(catalog) - var edges []Relationship + var edges []artifact.Relationship for parent, children := range relationships { for child, files := range children { - edges = append(edges, Relationship{ - Parent: parent, - Child: child, - Type: OwnershipByFileOverlapRelationship, - Metadata: ownershipByFilesMetadata{ + edges = append(edges, artifact.Relationship{ + From: catalog.byID[parent], + To: catalog.byID[child], + Type: artifact.OwnershipByFileOverlapRelationship, + Data: ownershipByFilesMetadata{ Files: files.List(), }, }) @@ -42,14 +45,15 @@ func ownershipByFilesRelationships(catalog *Catalog) []Relationship { // findOwnershipByFilesRelationships find overlaps in file ownership with a file that defines another package. Specifically, a .Location.Path of // a package is found to be owned by another (from the owner's .Metadata.Files[]). -func findOwnershipByFilesRelationships(catalog *Catalog) map[ID]map[ID]*strset.Set { - var relationships = make(map[ID]map[ID]*strset.Set) +func findOwnershipByFilesRelationships(catalog *Catalog) map[artifact.ID]map[artifact.ID]*strset.Set { + var relationships = make(map[artifact.ID]map[artifact.ID]*strset.Set) if catalog == nil { return relationships } for _, candidateOwnerPkg := range catalog.Sorted() { + id := candidateOwnerPkg.ID() if candidateOwnerPkg.Metadata == nil { continue } @@ -68,17 +72,18 @@ func findOwnershipByFilesRelationships(catalog *Catalog) map[ID]map[ID]*strset.S // look for package(s) in the catalog that may be owned by this package and mark the relationship for _, subPackage := range catalog.PackagesByPath(ownedFilePath) { - if subPackage.ID == candidateOwnerPkg.ID { + subID := subPackage.ID() + if subID == id { continue } - if _, exists := relationships[candidateOwnerPkg.ID]; !exists { - relationships[candidateOwnerPkg.ID] = make(map[ID]*strset.Set) + if _, exists := relationships[id]; !exists { + relationships[id] = make(map[artifact.ID]*strset.Set) } - if _, exists := relationships[candidateOwnerPkg.ID][subPackage.ID]; !exists { - relationships[candidateOwnerPkg.ID][subPackage.ID] = strset.New() + if _, exists := relationships[id][subID]; !exists { + relationships[id][subID] = strset.New() } - relationships[candidateOwnerPkg.ID][subPackage.ID].Add(ownedFilePath) + relationships[id][subID].Add(ownedFilePath) } } } diff --git a/syft/pkg/ownership_by_files_relationship_test.go b/syft/pkg/relationships_by_file_ownership_test.go similarity index 59% rename from syft/pkg/ownership_by_files_relationship_test.go rename to syft/pkg/relationships_by_file_ownership_test.go index 3e8bf4597..9199b0b13 100644 --- a/syft/pkg/ownership_by_files_relationship_test.go +++ b/syft/pkg/relationships_by_file_ownership_test.go @@ -3,21 +3,21 @@ package pkg import ( "testing" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/source" - "github.com/go-test/deep" + "github.com/stretchr/testify/assert" ) func TestOwnershipByFilesRelationship(t *testing.T) { + tests := []struct { - name string - pkgs []Package - expectedRelations []Relationship + name string + setup func(t testing.TB) ([]Package, []artifact.Relationship) }{ { name: "owns-by-real-path", - pkgs: []Package{ - { - ID: "parent", + setup: func(t testing.TB) ([]Package, []artifact.Relationship) { + parent := Package{ Locations: []source.Location{ { RealPath: "/a/path", @@ -37,9 +37,9 @@ func TestOwnershipByFilesRelationship(t *testing.T) { {Path: "/d/path"}, }, }, - }, - { - ID: "child", + } + + child := Package{ Locations: []source.Location{ { RealPath: "/c/path", @@ -51,26 +51,26 @@ func TestOwnershipByFilesRelationship(t *testing.T) { }, }, Type: NpmPkg, - }, - }, - expectedRelations: []Relationship{ - { - Parent: "parent", - Child: "child", - Type: OwnershipByFileOverlapRelationship, - Metadata: ownershipByFilesMetadata{ + } + + relationship := artifact.Relationship{ + From: parent, + To: child, + Type: artifact.OwnershipByFileOverlapRelationship, + Data: ownershipByFilesMetadata{ Files: []string{ "/d/path", }, }, - }, + } + + return []Package{parent, child}, []artifact.Relationship{relationship} }, }, { name: "owns-by-virtual-path", - pkgs: []Package{ - { - ID: "parent", + setup: func(t testing.TB) ([]Package, []artifact.Relationship) { + parent := Package{ Locations: []source.Location{ { RealPath: "/a/path", @@ -90,9 +90,9 @@ func TestOwnershipByFilesRelationship(t *testing.T) { {Path: "/another/path"}, }, }, - }, - { - ID: "child", + } + + child := Package{ Locations: []source.Location{ { RealPath: "/c/path", @@ -104,26 +104,25 @@ func TestOwnershipByFilesRelationship(t *testing.T) { }, }, Type: NpmPkg, - }, - }, - expectedRelations: []Relationship{ - { - Parent: "parent", - Child: "child", - Type: OwnershipByFileOverlapRelationship, - Metadata: ownershipByFilesMetadata{ + } + + relationship := artifact.Relationship{ + From: parent, + To: child, + Type: artifact.OwnershipByFileOverlapRelationship, + Data: ownershipByFilesMetadata{ Files: []string{ "/another/path", }, }, - }, + } + return []Package{parent, child}, []artifact.Relationship{relationship} }, }, { name: "ignore-empty-path", - pkgs: []Package{ - { - ID: "parent", + setup: func(t testing.TB) ([]Package, []artifact.Relationship) { + parent := Package{ Locations: []source.Location{ { RealPath: "/a/path", @@ -143,9 +142,9 @@ func TestOwnershipByFilesRelationship(t *testing.T) { {Path: ""}, }, }, - }, - { - ID: "child", + } + + child := Package{ Locations: []source.Location{ { RealPath: "/c/path", @@ -157,18 +156,26 @@ func TestOwnershipByFilesRelationship(t *testing.T) { }, }, Type: NpmPkg, - }, + } + + return []Package{parent, child}, nil }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - c := NewCatalog(test.pkgs...) - relationships := ownershipByFilesRelationships(c) + pkgs, expectedRelations := test.setup(t) + c := NewCatalog(pkgs...) + relationships := RelationshipsByFileOwnership(c) - for _, d := range deep.Equal(test.expectedRelations, relationships) { - t.Errorf("diff: %+v", d) + assert.Len(t, relationships, len(expectedRelations)) + for idx, expectedRelationship := range expectedRelations { + actualRelationship := relationships[idx] + assert.Equal(t, expectedRelationship.From.ID(), actualRelationship.From.ID()) + assert.Equal(t, expectedRelationship.To.ID(), actualRelationship.To.ID()) + assert.Equal(t, expectedRelationship.Type, actualRelationship.Type) + assert.Equal(t, expectedRelationship.Data, actualRelationship.Data) } }) } diff --git a/syft/sbom/sbom.go b/syft/sbom/sbom.go index 411bcc28c..82e97dd2a 100644 --- a/syft/sbom/sbom.go +++ b/syft/sbom/sbom.go @@ -1,6 +1,7 @@ package sbom import ( + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/distro" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -8,8 +9,9 @@ import ( ) type SBOM struct { - Artifacts Artifacts - Source source.Metadata + Artifacts Artifacts + Relationships []artifact.Relationship + Source source.Metadata } type Artifacts struct { diff --git a/syft/source/location.go b/syft/source/location.go index f7cb729f3..6d1907bcb 100644 --- a/syft/source/location.go +++ b/syft/source/location.go @@ -3,19 +3,22 @@ package source import ( "fmt" - "github.com/anchore/syft/internal/log" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" ) // Location represents a path relative to a particular filesystem resolved to a specific file.Reference. This struct is used as a key -// in content fetching to uniquely identify a file relative to a request (the VirtualPath). +// in content fetching to uniquely identify a file relative to a request (the VirtualPath). Note that the VirtualPath +// and ref are ignored fields when using github.com/mitchellh/hashstructure. The reason for this is to ensure that +// only the minimally expressible fields of a location are baked into the uniqueness of a Location. Since VirutalPath +// and ref are not captured in JSON output they cannot be included in this minimal definition. type Location struct { RealPath string `json:"path"` // The path where all path ancestors have no hardlinks / symlinks - VirtualPath string `json:"-"` // The path to the file which may or may not have hardlinks / symlinks + VirtualPath string `hash:"ignore" json:"-"` // The path to the file which may or may not have hardlinks / symlinks FileSystemID string `json:"layerID,omitempty"` // An ID representing the filesystem. For container images this is a layer digest, directories or root filesystem this is blank. - ref file.Reference // The file reference relative to the stereoscope.FileCatalog that has more information about this location. + ref file.Reference `hash:"ignore"` // The file reference relative to the stereoscope.FileCatalog that has more information about this location. } // NewLocation creates a new Location representing a path without denoting a filesystem or FileCatalog reference. @@ -70,3 +73,14 @@ func (l Location) String() string { } return fmt.Sprintf("Location<%s>", str) } + +func (l Location) ID() artifact.ID { + f, err := artifact.IDFromHash(l) + if err != nil { + // TODO: what to do in this case? + log.Warnf("unable to get fingerprint of location=%+v: %+v", l, err) + return "" + } + + return f +} diff --git a/syft/test-fixtures/pkgs/project/package-lock.json b/syft/test-fixtures/pkgs/project/package-lock.json deleted file mode 100644 index f18505de8..000000000 --- a/syft/test-fixtures/pkgs/project/package-lock.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "name": "npm-lock", - "version": "1.0.0", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "collapse-white-space": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/collapse-white-space/-/collapse-white-space-2.0.0.tgz", - "integrity": "sha512-eh9krktAIMDL0KHuN7WTBJ/0PMv8KUvfQRBkIlGmW61idRM2DJjgd1qXEPr4wyk2PimZZeNww3RVYo6CMvDGlg==" - }, - "end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, - "requires": { - "once": "^1.4.0" - } - }, - "insert-css": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/insert-css/-/insert-css-2.0.0.tgz", - "integrity": "sha1-610Ql7dUL0x56jBg067gfQU4gPQ=" - }, - "once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "dev": true, - "requires": { - "wrappy": "1" - } - }, - "pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "dev": true, - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "dev": true - } - } -} diff --git a/test/cli/trait_assertions_test.go b/test/cli/trait_assertions_test.go index 3aa43a5d8..d19f14a4c 100644 --- a/test/cli/trait_assertions_test.go +++ b/test/cli/trait_assertions_test.go @@ -2,13 +2,11 @@ package cli import ( "encoding/json" - "fmt" "regexp" "strings" "testing" "github.com/acarl005/stripansi" - "github.com/anchore/syft/syft/source" ) type traitAssertion func(tb testing.TB, stdout, stderr string, rc int) @@ -29,17 +27,17 @@ func assertTableReport(tb testing.TB, stdout, _ string, _ int) { } } -func assertScope(scope source.Scope) traitAssertion { - return func(tb testing.TB, stdout, stderr string, rc int) { - tb.Helper() - // we can only verify source with the json report - assertJsonReport(tb, stdout, stderr, rc) - - if !strings.Contains(stdout, fmt.Sprintf(`"scope": "%s"`, scope.String())) { - tb.Errorf("JSON report did not indicate the %q scope", scope) - } - } -} +//func assertScope(scope source.Scope) traitAssertion { +// return func(tb testing.TB, stdout, stderr string, rc int) { +// tb.Helper() +// // we can only verify source with the json report +// assertJsonReport(tb, stdout, stderr, rc) +// +// if !strings.Contains(stdout, fmt.Sprintf(`"scope": "%s"`, scope.String())) { +// tb.Errorf("JSON report did not indicate the %q scope", scope) +// } +// } +//} func assertLoggingLevel(level string) traitAssertion { // match examples: diff --git a/test/integration/catalog_packages_test.go b/test/integration/catalog_packages_test.go index ecc2baa9b..a4a092ac1 100644 --- a/test/integration/catalog_packages_test.go +++ b/test/integration/catalog_packages_test.go @@ -37,7 +37,7 @@ func BenchmarkImagePackageCatalogers(b *testing.B) { b.Run(c.Name(), func(b *testing.B) { for i := 0; i < b.N; i++ { - pc, err = cataloger.Catalog(resolver, theDistro, c) + pc, _, err = cataloger.Catalog(resolver, theDistro, c) if err != nil { b.Fatalf("failure during benchmark: %+v", err) } @@ -49,7 +49,7 @@ func BenchmarkImagePackageCatalogers(b *testing.B) { } func TestPkgCoverageImage(t *testing.T) { - catalog, _, _ := catalogFixtureImage(t, "image-pkg-coverage") + sbom, _ := catalogFixtureImage(t, "image-pkg-coverage") observedLanguages := internal.NewStringSet() definedLanguages := internal.NewStringSet() @@ -82,7 +82,7 @@ func TestPkgCoverageImage(t *testing.T) { t.Run(c.name, func(t *testing.T) { pkgCount := 0 - for a := range catalog.Enumerate(c.pkgType) { + for a := range sbom.Artifacts.PackageCatalog.Enumerate(c.pkgType) { if a.Language.String() != "" { observedLanguages.Add(a.Language.String()) @@ -110,7 +110,7 @@ func TestPkgCoverageImage(t *testing.T) { if pkgCount != len(c.pkgInfo)+c.duplicates { t.Logf("Discovered packages of type %+v", c.pkgType) - for a := range catalog.Enumerate(c.pkgType) { + for a := range sbom.Artifacts.PackageCatalog.Enumerate(c.pkgType) { t.Log(" ", a) } t.Fatalf("unexpected package count: %d!=%d", pkgCount, len(c.pkgInfo)) @@ -137,7 +137,7 @@ func TestPkgCoverageImage(t *testing.T) { } func TestPkgCoverageDirectory(t *testing.T) { - catalog, _, _ := catalogDirectory(t, "test-fixtures/image-pkg-coverage") + sbom, _ := catalogDirectory(t, "test-fixtures/image-pkg-coverage") observedLanguages := internal.NewStringSet() definedLanguages := internal.NewStringSet() @@ -159,7 +159,7 @@ func TestPkgCoverageDirectory(t *testing.T) { t.Run(test.name, func(t *testing.T) { actualPkgCount := 0 - for actualPkg := range catalog.Enumerate(test.pkgType) { + for actualPkg := range sbom.Artifacts.PackageCatalog.Enumerate(test.pkgType) { observedLanguages.Add(actualPkg.Language.String()) observedPkgs.Add(string(actualPkg.Type)) @@ -184,7 +184,7 @@ func TestPkgCoverageDirectory(t *testing.T) { } if actualPkgCount != len(test.pkgInfo)+test.duplicates { - for actualPkg := range catalog.Enumerate(test.pkgType) { + for actualPkg := range sbom.Artifacts.PackageCatalog.Enumerate(test.pkgType) { t.Log(" ", actualPkg) } t.Fatalf("unexpected package count: %d!=%d", actualPkgCount, len(test.pkgInfo)) diff --git a/test/integration/distro_test.go b/test/integration/distro_test.go index a54c472f3..b5662454c 100644 --- a/test/integration/distro_test.go +++ b/test/integration/distro_test.go @@ -8,14 +8,14 @@ import ( ) func TestDistroImage(t *testing.T) { - _, actualDistro, _ := catalogFixtureImage(t, "image-distro-id") + sbom, _ := catalogFixtureImage(t, "image-distro-id") expected, err := distro.NewDistro(distro.Busybox, "1.31.1", "") if err != nil { t.Fatalf("could not create distro: %+v", err) } - for _, d := range deep.Equal(actualDistro, &expected) { + for _, d := range deep.Equal(sbom.Artifacts.Distro, &expected) { t.Errorf("found distro difference: %+v", d) } diff --git a/syft/encode_decode_test.go b/test/integration/encode_decode_cycle_test.go similarity index 58% rename from syft/encode_decode_test.go rename to test/integration/encode_decode_cycle_test.go index 6adc6ead4..7c520b076 100644 --- a/syft/encode_decode_test.go +++ b/test/integration/encode_decode_cycle_test.go @@ -1,15 +1,14 @@ -package syft +package integration import ( "bytes" "testing" - "github.com/anchore/syft/syft/sbom" + "github.com/anchore/syft/syft" - "github.com/go-test/deep" + "github.com/sergi/go-diff/diffmatchpatch" "github.com/anchore/syft/syft/format" - "github.com/anchore/syft/syft/source" "github.com/stretchr/testify/assert" ) @@ -20,7 +19,6 @@ import ( // to do an object-to-object comparison. For this reason this test focuses on a bytes-to-bytes comparison after an // encode-decode-encode loop which will detect lossy behavior in both directions. func TestEncodeDecodeEncodeCycleComparison(t *testing.T) { - testImage := "image-simple" tests := []struct { format format.Option }{ @@ -29,35 +27,25 @@ func TestEncodeDecodeEncodeCycleComparison(t *testing.T) { }, } for _, test := range tests { - t.Run(testImage, func(t *testing.T) { + t.Run(string(test.format), func(t *testing.T) { - src, err := source.NewFromDirectory("./test-fixtures/pkgs") - if err != nil { - t.Fatalf("cant get dir") - } - originalCatalog, d, err := CatalogPackages(&src, source.SquashedScope) + originalSBOM, _ := catalogFixtureImage(t, "image-pkg-coverage") - originalSBOM := sbom.SBOM{ - Artifacts: sbom.Artifacts{ - PackageCatalog: originalCatalog, - Distro: d, - }, - Source: src.Metadata, - } - - by1, err := Encode(originalSBOM, test.format) + by1, err := syft.Encode(originalSBOM, test.format) assert.NoError(t, err) - newSBOM, newFormat, err := Decode(bytes.NewReader(by1)) + newSBOM, newFormat, err := syft.Decode(bytes.NewReader(by1)) assert.NoError(t, err) assert.Equal(t, test.format, newFormat) - by2, err := Encode(*newSBOM, test.format) + by2, err := syft.Encode(*newSBOM, test.format) assert.NoError(t, err) - for _, diff := range deep.Equal(by1, by2) { - t.Errorf(diff) + + if !assert.True(t, bytes.Equal(by1, by2)) { + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(string(by1), string(by2), true) + t.Errorf("diff: %s", dmp.DiffPrettyText(diffs)) } - assert.True(t, bytes.Equal(by1, by2)) }) } } diff --git a/test/integration/node_packages_test.go b/test/integration/node_packages_test.go index b4d270ae5..8505e5c78 100644 --- a/test/integration/node_packages_test.go +++ b/test/integration/node_packages_test.go @@ -9,11 +9,11 @@ import ( ) func TestNpmPackageLockDirectory(t *testing.T) { - catalog, _, _ := catalogDirectory(t, "test-fixtures/npm-lock") + sbom, _ := catalogDirectory(t, "test-fixtures/npm-lock") foundPackages := internal.NewStringSet() - for actualPkg := range catalog.Enumerate(pkg.NpmPkg) { + for actualPkg := range sbom.Artifacts.PackageCatalog.Enumerate(pkg.NpmPkg) { for _, actualLocation := range actualPkg.Locations { if strings.Contains(actualLocation.RealPath, "node_modules") { t.Errorf("found packages from package-lock.json in node_modules: %s", actualLocation) @@ -30,11 +30,11 @@ func TestNpmPackageLockDirectory(t *testing.T) { } func TestYarnPackageLockDirectory(t *testing.T) { - catalog, _, _ := catalogDirectory(t, "test-fixtures/yarn-lock") + sbom, _ := catalogDirectory(t, "test-fixtures/yarn-lock") foundPackages := internal.NewStringSet() - for actualPkg := range catalog.Enumerate(pkg.NpmPkg) { + for actualPkg := range sbom.Artifacts.PackageCatalog.Enumerate(pkg.NpmPkg) { for _, actualLocation := range actualPkg.Locations { if strings.Contains(actualLocation.RealPath, "node_modules") { t.Errorf("found packages from yarn.lock in node_modules: %s", actualLocation) diff --git a/test/integration/package_ownership_relationship_test.go b/test/integration/package_ownership_relationship_test.go index 42f9206af..2166bd9b7 100644 --- a/test/integration/package_ownership_relationship_test.go +++ b/test/integration/package_ownership_relationship_test.go @@ -7,7 +7,6 @@ import ( "github.com/anchore/syft/internal/formats/syftjson" syftjsonModel "github.com/anchore/syft/internal/formats/syftjson/model" - "github.com/anchore/syft/syft/sbom" ) func TestPackageOwnershipRelationships(t *testing.T) { @@ -23,15 +22,9 @@ func TestPackageOwnershipRelationships(t *testing.T) { for _, test := range tests { t.Run(test.fixture, func(t *testing.T) { - catalog, d, src := catalogFixtureImage(t, test.fixture) + sbom, _ := catalogFixtureImage(t, test.fixture) - p := syftjson.Format().Presenter(sbom.SBOM{ - Artifacts: sbom.Artifacts{ - PackageCatalog: catalog, - Distro: d, - }, - Source: src.Metadata, - }) + p := syftjson.Format().Presenter(sbom) if p == nil { t.Fatal("unable to get presenter") } diff --git a/test/integration/regression_apk_scanner_buffer_size_test.go b/test/integration/regression_apk_scanner_buffer_size_test.go index 5aaa4f24d..19bf9f92f 100644 --- a/test/integration/regression_apk_scanner_buffer_size_test.go +++ b/test/integration/regression_apk_scanner_buffer_size_test.go @@ -9,11 +9,11 @@ import ( func TestRegression212ApkBufferSize(t *testing.T) { // This is a regression test for issue #212 (https://github.com/anchore/syft/issues/212) in which the apk db could // not be processed due to a scanner buffer that was too small - catalog, _, _ := catalogFixtureImage(t, "image-large-apk-data") + sbom, _ := catalogFixtureImage(t, "image-large-apk-data") expectedPkgs := 58 actualPkgs := 0 - for range catalog.Enumerate(pkg.ApkPkg) { + for range sbom.Artifacts.PackageCatalog.Enumerate(pkg.ApkPkg) { actualPkgs += 1 } diff --git a/test/integration/regression_go_bin_scanner_arch_test.go b/test/integration/regression_go_bin_scanner_arch_test.go index 3a76ff335..f5e21727e 100644 --- a/test/integration/regression_go_bin_scanner_arch_test.go +++ b/test/integration/regression_go_bin_scanner_arch_test.go @@ -15,11 +15,11 @@ func TestRegressionGoArchDiscovery(t *testing.T) { ) // This is a regression test to make sure the way we detect go binary packages // stays consistent and reproducible as the tool chain evolves - catalog, _, _ := catalogFixtureImage(t, "image-go-bin-arch-coverage") + sbom, _ := catalogFixtureImage(t, "image-go-bin-arch-coverage") var actualELF, actualWIN, actualMACOS int - for p := range catalog.Enumerate(pkg.GoModulePkg) { + for p := range sbom.Artifacts.PackageCatalog.Enumerate(pkg.GoModulePkg) { for _, l := range p.Locations { switch { case strings.Contains(l.RealPath, "elf"): diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go index bd6b7435c..167c3469d 100644 --- a/test/integration/utils_test.go +++ b/test/integration/utils_test.go @@ -3,14 +3,14 @@ package integration import ( "testing" + "github.com/anchore/syft/syft/sbom" + "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/syft" - "github.com/anchore/syft/syft/distro" - "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) -func catalogFixtureImage(t *testing.T, fixtureImageName string) (*pkg.Catalog, *distro.Distro, *source.Source) { +func catalogFixtureImage(t *testing.T, fixtureImageName string) (sbom.SBOM, *source.Source) { imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName) tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName) @@ -20,25 +20,39 @@ func catalogFixtureImage(t *testing.T, fixtureImageName string) (*pkg.Catalog, * t.Fatalf("unable to get source: %+v", err) } - pkgCatalog, actualDistro, err := syft.CatalogPackages(theSource, source.SquashedScope) + pkgCatalog, relationships, actualDistro, err := syft.CatalogPackages(theSource, source.SquashedScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } - return pkgCatalog, actualDistro, theSource + return sbom.SBOM{ + Artifacts: sbom.Artifacts{ + PackageCatalog: pkgCatalog, + Distro: actualDistro, + }, + Relationships: relationships, + Source: theSource.Metadata, + }, theSource } -func catalogDirectory(t *testing.T, dir string) (*pkg.Catalog, *distro.Distro, *source.Source) { +func catalogDirectory(t *testing.T, dir string) (sbom.SBOM, *source.Source) { theSource, cleanupSource, err := source.New("dir:"+dir, nil) t.Cleanup(cleanupSource) if err != nil { t.Fatalf("unable to get source: %+v", err) } - pkgCatalog, actualDistro, err := syft.CatalogPackages(theSource, source.AllLayersScope) + pkgCatalog, relationships, actualDistro, err := syft.CatalogPackages(theSource, source.AllLayersScope) if err != nil { t.Fatalf("failed to catalog image: %+v", err) } - return pkgCatalog, actualDistro, theSource + return sbom.SBOM{ + Artifacts: sbom.Artifacts{ + PackageCatalog: pkgCatalog, + Distro: actualDistro, + }, + Relationships: relationships, + Source: theSource.Metadata, + }, theSource }