From eb8ebd9ffceaed608e64fd29a2f18ebccc1c3fd5 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Mon, 24 Oct 2022 16:11:20 -0400 Subject: [PATCH] port conan cataloger to new generic cataloger pattern (#1284) Signed-off-by: Alex Goodman Signed-off-by: Alex Goodman --- syft/pkg/cataloger/cataloger.go | 4 +- syft/pkg/cataloger/cpp/cataloger.go | 15 ++-- syft/pkg/cataloger/cpp/package.go | 76 +++++++++++++++++++ syft/pkg/cataloger/cpp/parse_conanfile.go | 28 +++---- .../pkg/cataloger/cpp/parse_conanfile_test.go | 23 ++++-- syft/pkg/cataloger/cpp/parse_conanlock.go | 27 +++---- .../pkg/cataloger/cpp/parse_conanlock_test.go | 18 +++-- syft/pkg/conan_metadata.go | 39 ---------- syft/pkg/conan_metadata_test.go | 27 ------- syft/pkg/url_test.go | 31 +------- 10 files changed, 132 insertions(+), 156 deletions(-) create mode 100644 syft/pkg/cataloger/cpp/package.go delete mode 100644 syft/pkg/conan_metadata_test.go diff --git a/syft/pkg/cataloger/cataloger.go b/syft/pkg/cataloger/cataloger.go index ec2be5ad3..821926d67 100644 --- a/syft/pkg/cataloger/cataloger.go +++ b/syft/pkg/cataloger/cataloger.go @@ -82,7 +82,7 @@ func DirectoryCatalogers(cfg Config) []Cataloger { dart.NewPubspecLockCataloger(), dotnet.NewDotnetDepsCataloger(), swift.NewCocoapodsCataloger(), - cpp.NewConanfileCataloger(), + cpp.NewConanCataloger(), portage.NewPortageCataloger(), haskell.NewHackageCataloger(), }, cfg.Catalogers) @@ -113,7 +113,7 @@ func AllCatalogers(cfg Config) []Cataloger { php.NewPHPComposerInstalledCataloger(), php.NewPHPComposerLockCataloger(), swift.NewCocoapodsCataloger(), - cpp.NewConanfileCataloger(), + cpp.NewConanCataloger(), portage.NewPortageCataloger(), haskell.NewHackageCataloger(), }, cfg.Catalogers) diff --git a/syft/pkg/cataloger/cpp/cataloger.go b/syft/pkg/cataloger/cpp/cataloger.go index 0fb822259..80c6b5b11 100644 --- a/syft/pkg/cataloger/cpp/cataloger.go +++ b/syft/pkg/cataloger/cpp/cataloger.go @@ -1,15 +1,14 @@ package cpp import ( - "github.com/anchore/syft/syft/pkg/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/generic" ) -// NewConanfileCataloger returns a new C++ Conanfile cataloger object. -func NewConanfileCataloger() *common.GenericCataloger { - globParsers := map[string]common.ParserFn{ - "**/conanfile.txt": parseConanfile, - "**/conan.lock": parseConanlock, - } +const catalogerName = "conan-cataloger" - return common.NewGenericCataloger(nil, globParsers, "conan-cataloger") +// NewConanCataloger returns a new C++ conanfile.txt and conan.lock cataloger object. +func NewConanCataloger() *generic.Cataloger { + return generic.NewCataloger(catalogerName). + WithParserByGlobs(parseConanfile, "**/conanfile.txt"). + WithParserByGlobs(parseConanlock, "**/conan.lock") } diff --git a/syft/pkg/cataloger/cpp/package.go b/syft/pkg/cataloger/cpp/package.go new file mode 100644 index 000000000..ba54add77 --- /dev/null +++ b/syft/pkg/cataloger/cpp/package.go @@ -0,0 +1,76 @@ +package cpp + +import ( + "strings" + + "github.com/anchore/packageurl-go" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +func newConanfilePackage(m pkg.ConanMetadata, locations ...source.Location) *pkg.Package { + fields := strings.Split(strings.TrimSpace(m.Ref), "/") + if len(fields) < 2 { + return nil + } + + pkgName, pkgVersion := fields[0], fields[1] + + if pkgName == "" || pkgVersion == "" { + return nil + } + + p := pkg.Package{ + Name: pkgName, + Version: pkgVersion, + Locations: source.NewLocationSet(locations...), + PURL: packageURL(pkgName, pkgVersion), + Language: pkg.CPP, + Type: pkg.ConanPkg, + MetadataType: pkg.ConanMetadataType, + Metadata: m, + } + + p.SetID() + + return &p +} + +func newConanlockPackage(m pkg.ConanLockMetadata, locations ...source.Location) *pkg.Package { + fields := strings.Split(strings.Split(m.Ref, "@")[0], "/") + if len(fields) < 2 { + return nil + } + + pkgName, pkgVersion := fields[0], fields[1] + + if pkgName == "" || pkgVersion == "" { + return nil + } + + p := pkg.Package{ + Name: pkgName, + Version: pkgVersion, + Locations: source.NewLocationSet(locations...), + PURL: packageURL(pkgName, pkgVersion), + Language: pkg.CPP, + Type: pkg.ConanPkg, + MetadataType: pkg.ConanLockMetadataType, + Metadata: m, + } + + p.SetID() + + return &p +} + +func packageURL(name, version string) string { + return packageurl.NewPackageURL( + packageurl.TypeConan, + "", + name, + version, + nil, // TODO: no qualifiers (...yet) + "", + ).ToString() +} diff --git a/syft/pkg/cataloger/cpp/parse_conanfile.go b/syft/pkg/cataloger/cpp/parse_conanfile.go index e3c70a90d..eaa2d1b5b 100644 --- a/syft/pkg/cataloger/cpp/parse_conanfile.go +++ b/syft/pkg/cataloger/cpp/parse_conanfile.go @@ -9,21 +9,21 @@ import ( "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/source" ) -// integrity check -var _ common.ParserFn = parseConanfile +var _ generic.Parser = parseConanfile type Conanfile struct { Requires []string `toml:"requires"` } // parseConanfile is a parser function for conanfile.txt contents, returning all packages discovered. -func parseConanfile(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) { +func parseConanfile(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { r := bufio.NewReader(reader) inRequirements := false - pkgs := []*pkg.Package{} + var pkgs []pkg.Package for { line, err := r.ReadString('\n') switch { @@ -44,19 +44,15 @@ func parseConanfile(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Rela Ref: strings.Trim(line, "\n"), } - pkgName, pkgVersion := m.NameAndVersion() - - if pkgName == "" || pkgVersion == "" || !inRequirements { + if !inRequirements { continue } - pkgs = append(pkgs, &pkg.Package{ - Name: pkgName, - Version: pkgVersion, - Language: pkg.CPP, - Type: pkg.ConanPkg, - MetadataType: pkg.ConanMetadataType, - Metadata: m, - }) + p := newConanfilePackage(m, reader.Location) + if p == nil { + continue + } + + pkgs = append(pkgs, *p) } } diff --git a/syft/pkg/cataloger/cpp/parse_conanfile_test.go b/syft/pkg/cataloger/cpp/parse_conanfile_test.go index 253fc6b0a..a14e122cd 100644 --- a/syft/pkg/cataloger/cpp/parse_conanfile_test.go +++ b/syft/pkg/cataloger/cpp/parse_conanfile_test.go @@ -5,15 +5,18 @@ import ( "testing" "github.com/go-test/deep" + "github.com/stretchr/testify/require" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" ) func TestParseConanfile(t *testing.T) { - expected := []*pkg.Package{ + expected := []pkg.Package{ { Name: "catch2", Version: "2.13.8", + PURL: "pkg:conan/catch2@2.13.8", Language: pkg.CPP, Type: pkg.ConanPkg, MetadataType: pkg.ConanMetadataType, @@ -24,6 +27,7 @@ func TestParseConanfile(t *testing.T) { { Name: "docopt.cpp", Version: "0.6.3", + PURL: "pkg:conan/docopt.cpp@0.6.3", Language: pkg.CPP, Type: pkg.ConanPkg, MetadataType: pkg.ConanMetadataType, @@ -34,6 +38,7 @@ func TestParseConanfile(t *testing.T) { { Name: "fmt", Version: "8.1.1", + PURL: "pkg:conan/fmt@8.1.1", Language: pkg.CPP, Type: pkg.ConanPkg, MetadataType: pkg.ConanMetadataType, @@ -44,6 +49,7 @@ func TestParseConanfile(t *testing.T) { { Name: "spdlog", Version: "1.9.2", + PURL: "pkg:conan/spdlog@1.9.2", Language: pkg.CPP, Type: pkg.ConanPkg, MetadataType: pkg.ConanMetadataType, @@ -54,6 +60,7 @@ func TestParseConanfile(t *testing.T) { { Name: "sdl", Version: "2.0.20", + PURL: "pkg:conan/sdl@2.0.20", Language: pkg.CPP, Type: pkg.ConanPkg, MetadataType: pkg.ConanMetadataType, @@ -64,6 +71,7 @@ func TestParseConanfile(t *testing.T) { { Name: "fltk", Version: "1.3.8", + PURL: "pkg:conan/fltk@1.3.8", Language: pkg.CPP, Type: pkg.ConanPkg, MetadataType: pkg.ConanMetadataType, @@ -74,15 +82,14 @@ func TestParseConanfile(t *testing.T) { } fixture, err := os.Open("test-fixtures/conanfile.txt") - if err != nil { - t.Fatalf("failed to open fixture: %+v", err) - } + require.NoError(t, err) // TODO: no relationships are under test yet - actual, _, err := parseConanfile(fixture.Name(), fixture) - if err != nil { - t.Error(err) - } + actual, _, err := parseConanfile(nil, nil, source.LocationReadCloser{ + Location: source.NewLocation(fixture.Name()), + ReadCloser: fixture, + }) + require.NoError(t, err) differences := deep.Equal(expected, actual) if differences != nil { diff --git a/syft/pkg/cataloger/cpp/parse_conanlock.go b/syft/pkg/cataloger/cpp/parse_conanlock.go index b49bf5f11..e7310c22f 100644 --- a/syft/pkg/cataloger/cpp/parse_conanlock.go +++ b/syft/pkg/cataloger/cpp/parse_conanlock.go @@ -2,16 +2,15 @@ package cpp import ( "encoding/json" - "io" "strings" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" - "github.com/anchore/syft/syft/pkg/cataloger/common" + "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/source" ) -// integrity check -var _ common.ParserFn = parseConanlock +var _ generic.Parser = parseConanlock type conanLock struct { GraphLock struct { @@ -31,8 +30,8 @@ type conanLock struct { } // parseConanlock is a parser function for conan.lock contents, returning all packages discovered. -func parseConanlock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) { - pkgs := []*pkg.Package{} +func parseConanlock(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + var pkgs []pkg.Package var cl conanLock if err := json.NewDecoder(reader).Decode(&cl); err != nil { return nil, nil, err @@ -45,19 +44,11 @@ func parseConanlock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Rela Context: node.Context, } - pkgName, pkgVersion := metadata.NameAndVersion() - if pkgName == "" || pkgVersion == "" { - continue - } + p := newConanlockPackage(metadata, reader.Location) - pkgs = append(pkgs, &pkg.Package{ - Name: pkgName, - Version: pkgVersion, - Language: pkg.CPP, - Type: pkg.ConanPkg, - MetadataType: pkg.ConanLockMetadataType, - Metadata: metadata, - }) + if p != nil { + pkgs = append(pkgs, *p) + } } return pkgs, nil, nil diff --git a/syft/pkg/cataloger/cpp/parse_conanlock_test.go b/syft/pkg/cataloger/cpp/parse_conanlock_test.go index ffb2b7c84..9a5c69556 100644 --- a/syft/pkg/cataloger/cpp/parse_conanlock_test.go +++ b/syft/pkg/cataloger/cpp/parse_conanlock_test.go @@ -5,15 +5,18 @@ import ( "testing" "github.com/go-test/deep" + "github.com/stretchr/testify/require" "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" ) func TestParseConanlock(t *testing.T) { - expected := []*pkg.Package{ + expected := []pkg.Package{ { Name: "zlib", Version: "1.2.12", + PURL: "pkg:conan/zlib@1.2.12", Language: pkg.CPP, Type: pkg.ConanPkg, MetadataType: pkg.ConanLockMetadataType, @@ -30,15 +33,14 @@ func TestParseConanlock(t *testing.T) { } fixture, err := os.Open("test-fixtures/conan.lock") - if err != nil { - t.Fatalf("failed to open fixture: %+v", err) - } + require.NoError(t, err) // TODO: no relationships are under test yet - actual, _, err := parseConanlock(fixture.Name(), fixture) - if err != nil { - t.Error(err) - } + actual, _, err := parseConanlock(nil, nil, source.LocationReadCloser{ + Location: source.NewLocation(fixture.Name()), + ReadCloser: fixture, + }) + require.NoError(t, err) differences := deep.Equal(expected, actual) if differences != nil { diff --git a/syft/pkg/conan_metadata.go b/syft/pkg/conan_metadata.go index 1e287ba73..a138c279c 100644 --- a/syft/pkg/conan_metadata.go +++ b/syft/pkg/conan_metadata.go @@ -1,44 +1,5 @@ package pkg -import ( - "strings" - - "github.com/anchore/packageurl-go" - "github.com/anchore/syft/syft/linux" -) - type ConanMetadata struct { Ref string `mapstructure:"ref" json:"ref"` } - -func (m ConanMetadata) PackageURL(_ *linux.Release) string { - var qualifiers packageurl.Qualifiers - - name, version := m.NameAndVersion() - - return packageurl.NewPackageURL( - packageurl.TypeConan, - "", - name, - version, - qualifiers, - "", - ).ToString() -} - -// NameAndVersion tries to return the name and version of a cpp package -// given the ref format: pkg/version -// it returns empty strings if ref is empty or parsing is unsuccessful -func (m ConanMetadata) NameAndVersion() (name, version string) { - if len(m.Ref) < 1 { - return name, version - } - - splits := strings.Split(strings.TrimSpace(m.Ref), "/") - - if len(splits) < 2 { - return name, version - } - - return splits[0], splits[1] -} diff --git a/syft/pkg/conan_metadata_test.go b/syft/pkg/conan_metadata_test.go deleted file mode 100644 index 7bf13cf41..000000000 --- a/syft/pkg/conan_metadata_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package pkg - -import "testing" - -func TestConanMetadata_PackageURL(t *testing.T) { - tests := []struct { - name string - m ConanMetadata - want string - }{ - { - name: "happy path", - m: ConanMetadata{ - Ref: "catch2/2.13.8", - }, - want: "pkg:conan/catch2@2.13.8", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if got := test.m.PackageURL(nil); got != test.want { - t.Errorf("ConanMetadata.PackageURL() = %v, want %v", got, test.want) - } - }) - } -} diff --git a/syft/pkg/url_test.go b/syft/pkg/url_test.go index 93de269b5..0604e739e 100644 --- a/syft/pkg/url_test.go +++ b/syft/pkg/url_test.go @@ -196,36 +196,6 @@ func TestPackageURL(t *testing.T) { }, expected: "pkg:cocoapods/GlossButtonNode@3.1.2", }, - { - name: "conan", - pkg: Package{ - Name: "catch2", - Version: "2.13.8", - Type: ConanPkg, - Language: CPP, - MetadataType: ConanMetadataType, - Metadata: ConanMetadata{ - Ref: "catch2/2.13.8", - }, - }, - expected: "pkg:conan/catch2@2.13.8", - }, - // note both Ref should parse the same for conan ecosystem - { - name: "conan lock", - pkg: Package{ - Name: "catch2", - Version: "2.13.8", - Type: ConanPkg, - Language: CPP, - MetadataType: ConanLockMetadataType, - Metadata: ConanLockMetadata{ - Ref: "catch2/2.13.8", - }, - }, - expected: "pkg:conan/catch2@2.13.8", - }, - { name: "hackage", pkg: Package{ @@ -254,6 +224,7 @@ func TestPackageURL(t *testing.T) { expectedTypes.Remove(string(PortagePkg)) expectedTypes.Remove(string(AlpmPkg)) expectedTypes.Remove(string(ApkPkg)) + expectedTypes.Remove(string(ConanPkg)) for _, test := range tests { t.Run(test.name, func(t *testing.T) {