Update portage cataloger to new generic cataloger (#1316)

* port portage (ha) cataloger to new generic cataloger pattern

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update JSON schema to account for removing portage fields

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

internal/constants.go

@@ -6,5 +6,5 @@ const (
 
 	// JSONSchemaVersion is the current schema version output by the JSON encoder
 	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
-	JSONSchemaVersion = "4.1.0"
+	JSONSchemaVersion = "5.0.0"
 )

syft/formats/syftjson/format.go

@@ -4,7 +4,7 @@ import (
 	"github.com/anchore/syft/syft/sbom"
 )
 
-const ID sbom.FormatID = "syft-4-json"
+const ID sbom.FormatID = "syft-5-json"
 
 func Format() sbom.Format {
 	return sbom.NewFormat(

syft/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden

@@ -89,7 +89,7 @@
   }
  },
  "schema": {
-  "version": "4.1.0",
+  "version": "5.0.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-4.1.0.json"
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-5.0.0.json"
  }
 }

syft/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden

@@ -185,7 +185,7 @@
   }
  },
  "schema": {
-  "version": "4.1.0",
+  "version": "5.0.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-4.1.0.json"
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-5.0.0.json"
  }
 }

syft/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden

@@ -112,7 +112,7 @@
   }
  },
  "schema": {
-  "version": "4.1.0",
+  "version": "5.0.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-4.1.0.json"
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-5.0.0.json"
  }
 }

syft/pkg/cataloger/portage/cataloger.go

@@ -18,71 +18,63 @@ import (
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 	"github.com/anchore/syft/syft/source"
 )
 
 var (
 	cpvRe                = regexp.MustCompile(`/([^/]*/[\w+][\w+-]*)-((\d+)((\.\d+)*)([a-z]?)((_(pre|p|beta|alpha|rc)\d*)*)(-r\d+)?)/CONTENTS$`)
+	_     generic.Parser = parsePortageContents
 )
 
-type Cataloger struct{}
+func NewPortageCataloger() *generic.Cataloger {
-
+	return generic.NewCataloger("portage-cataloger").
-// NewPortageCataloger returns a new Portage package cataloger object.
+		WithParserByGlobs(parsePortageContents, "**/var/db/pkg/*/*/CONTENTS")
-func NewPortageCataloger() *Cataloger {
-	return &Cataloger{}
 }
 
-// Name returns a string that uniquely describes a cataloger
+func parsePortageContents(resolver source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
-func (c *Cataloger) Name() string {
+	cpvMatch := cpvRe.FindStringSubmatch(reader.Location.RealPath)
-	return "portage-cataloger"
-}
-
-// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing portage support files.
-func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
-	dbFileMatches, err := resolver.FilesByGlob(pkg.PortageDBGlob)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to find portage files by glob: %w", err)
-	}
-	var allPackages []pkg.Package
-	for _, dbLocation := range dbFileMatches {
-		cpvMatch := cpvRe.FindStringSubmatch(dbLocation.RealPath)
 	if cpvMatch == nil {
-			return nil, nil, fmt.Errorf("failed to match package and version in %s", dbLocation.RealPath)
+		return nil, nil, fmt.Errorf("failed to match package and version in %s", reader.Location.RealPath)
-		}
-		entry := pkg.PortageMetadata{
-			// ensure the default value for a collection is never nil since this may be shown as JSON
-			Files:   make([]pkg.PortageFileRecord, 0),
-			Package: cpvMatch[1],
-			Version: cpvMatch[2],
 	}
 
-		err = addFiles(resolver, dbLocation, &entry)
+	name, version := cpvMatch[1], cpvMatch[2]
-		if err != nil {
+	if name == "" || version == "" {
-			return nil, nil, err
+		log.WithFields("path", reader.Location.RealPath).Warnf("failed to parse portage name and version")
+		return nil, nil, nil
 	}
 
-		addSize(resolver, dbLocation, &entry)
-
 	p := pkg.Package{
-			Name:         entry.Package,
+		Name:         name,
-			Version:      entry.Version,
+		Version:      version,
+		PURL:         packageURL(name, version),
+		Locations:    source.NewLocationSet(),
 		Type:         pkg.PortagePkg,
 		MetadataType: pkg.PortageMetadataType,
-			Metadata:     entry,
+		Metadata: pkg.PortageMetadata{
+			// ensure the default value for a collection is never nil since this may be shown as JSON
+			Files: make([]pkg.PortageFileRecord, 0),
+		},
 	}
-		addLicenses(resolver, dbLocation, &p)
+	addLicenses(resolver, reader.Location, &p)
-		p.FoundBy = c.Name()
+	addSize(resolver, reader.Location, &p)
-		p.Locations.Add(dbLocation)
+	addFiles(resolver, reader.Location, &p)
+
 	p.SetID()
-		allPackages = append(allPackages, p)
+
-	}
+	return []pkg.Package{p}, nil, nil
-	return allPackages, nil, nil
 }
 
-func addFiles(resolver source.FileResolver, dbLocation source.Location, entry *pkg.PortageMetadata) error {
+func addFiles(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) {
 	contentsReader, err := resolver.FileContentsByLocation(dbLocation)
 	if err != nil {
-		return err
+		log.WithFields("path", dbLocation.RealPath).Warnf("failed to fetch portage contents (package=%s): %+v", p.Name, err)
+		return
+	}
+
+	entry, ok := p.Metadata.(pkg.PortageMetadata)
+	if !ok {
+		return
 	}
 
 	scanner := bufio.NewScanner(contentsReader)
@@ -101,7 +93,9 @@ func addFiles(resolver source.FileResolver, dbLocation source.Location, entry *p
 			entry.Files = append(entry.Files, record)
 		}
 	}
-	return nil
+
+	p.Metadata = entry
+	p.Locations.Add(dbLocation)
 }
 
 func addLicenses(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) {
@@ -109,9 +103,16 @@ func addLicenses(resolver source.FileResolver, dbLocation source.Location, p *pk
 
 	location := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "LICENSE"))
 
-	if location != nil {
+	if location == nil {
+		return
+	}
+
 	licenseReader, err := resolver.FileContentsByLocation(*location)
-		if err == nil {
+	if err != nil {
+		log.WithFields("path", dbLocation.RealPath).Warnf("failed to fetch portage LICENSE: %+v", err)
+		return
+	}
+
 	findings := internal.NewStringSet()
 	scanner := bufio.NewScanner(licenseReader)
 	scanner.Split(bufio.ScanWords)
@@ -121,23 +122,32 @@ func addLicenses(resolver source.FileResolver, dbLocation source.Location, p *pk
 			findings.Add(token)
 		}
 	}
-			p.Licenses = findings.ToSlice()
+	licenses := findings.ToSlice()
-
+	sort.Strings(licenses)
-			sort.Strings(p.Licenses)
+	p.Licenses = licenses
-		}
+	p.Locations.Add(*location)
-	}
 }
 
-func addSize(resolver source.FileResolver, dbLocation source.Location, entry *pkg.PortageMetadata) {
+func addSize(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) {
 	parentPath := filepath.Dir(dbLocation.RealPath)
 
 	location := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "SIZE"))
 
-	if location != nil {
+	if location == nil {
+		return
+	}
+
+	entry, ok := p.Metadata.(pkg.PortageMetadata)
+	if !ok {
+		return
+	}
+
 	sizeReader, err := resolver.FileContentsByLocation(*location)
 	if err != nil {
-			log.Warnf("failed to fetch portage SIZE (package=%s): %+v", entry.Package, err)
+		log.WithFields("name", p.Name).Warnf("failed to fetch portage SIZE: %+v", err)
-		} else {
+		return
+	}
+
 	scanner := bufio.NewScanner(sizeReader)
 	for scanner.Scan() {
 		line := strings.Trim(scanner.Text(), "\n")
@@ -146,6 +156,7 @@ func addSize(resolver source.FileResolver, dbLocation source.Location, entry *pk
 			entry.InstalledSize = size
 		}
 	}
-		}
+
-	}
+	p.Metadata = entry
+	p.Locations.Add(*location)
 }

syft/pkg/cataloger/portage/cataloger_test.go

@@ -3,32 +3,30 @@ package portage
 import (
 	"testing"
 
-	"github.com/go-test/deep"
+	"github.com/anchore/syft/syft/artifact"
-
-	"github.com/anchore/stereoscope/pkg/imagetest"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
 	"github.com/anchore/syft/syft/source"
 )
 
 func TestPortageCataloger(t *testing.T) {
-	tests := []struct {
+
-		name     string
+	expectedPkgs := []pkg.Package{
-		expected []pkg.Package
-	}{
-		{
-			name: "go-case",
-			expected: []pkg.Package{
 		{
 			Name:    "app-containers/skopeo",
 			Version: "1.5.1",
 			FoundBy: "portage-cataloger",
+			PURL:    "pkg:ebuild/app-containers/skopeo@1.5.1",
+			Locations: source.NewLocationSet(
+				source.NewLocation("var/db/pkg/app-containers/skopeo-1.5.1/CONTENTS"),
+				source.NewLocation("var/db/pkg/app-containers/skopeo-1.5.1/LICENSE"),
+				source.NewLocation("var/db/pkg/app-containers/skopeo-1.5.1/SIZE"),
+			),
 			Licenses:     []string{"Apache-2.0", "BSD", "BSD-2", "CC-BY-SA-4.0", "ISC", "MIT"},
 			Type:         pkg.PortagePkg,
 			MetadataType: pkg.PortageMetadataType,
 			Metadata: pkg.PortageMetadata{
-						Package:       "app-containers/skopeo",
-						Version:       "1.5.1",
 				InstalledSize: 27937835,
 				Files: []pkg.PortageFileRecord{
 					{
@@ -62,45 +60,14 @@ func TestPortageCataloger(t *testing.T) {
 				},
 			},
 		},
-			},
-		},
 	}
 
-	for _, test := range tests {
+	// TODO: relationships are not under test yet
-		t.Run(test.name, func(t *testing.T) {
+	var expectedRelationships []artifact.Relationship
 
-			img := imagetest.GetFixtureImage(t, "docker-archive", "image-portage")
+	pkgtest.NewCatalogTester().
-
+		FromDirectory(t, "test-fixtures/image-portage").
-			s, err := source.NewFromImage(img, "")
+		Expects(expectedPkgs, expectedRelationships).
-			if err != nil {
+		TestCataloger(t, NewPortageCataloger())
-				t.Fatal(err)
-			}
-
-			c := NewPortageCataloger()
-
-			resolver, err := s.FileResolver(source.SquashedScope)
-			if err != nil {
-				t.Errorf("could not get resolver error: %+v", err)
-			}
-
-			actual, _, err := c.Catalog(resolver)
-			if err != nil {
-				t.Fatalf("failed to catalog: %+v", err)
-			}
-
-			if len(actual) != len(test.expected) {
-				for _, a := range actual {
-					t.Logf("   %+v", a)
-				}
-				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
-			}
-
-			// test remaining fields...
-			for _, d := range deep.Equal(actual, test.expected) {
-				t.Errorf("diff: %+v", d)
-			}
-
-		})
-	}
 
 }

syft/pkg/cataloger/portage/purl.go

@@ -0,0 +1,18 @@
+package portage
+
+import (
+	"github.com/anchore/packageurl-go"
+)
+
+func packageURL(name, version string) string {
+	var qualifiers packageurl.Qualifiers
+
+	return packageurl.NewPackageURL(
+		"ebuild", // currently this is the proposed type for portage packages at https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst
+		"",
+		name,
+		version,
+		qualifiers,
+		"",
+	).ToString()
+}

syft/pkg/cataloger/portage/purl_test.go

@@ -0,0 +1,28 @@
+package portage
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_packageURL(t *testing.T) {
+
+	tests := []struct {
+		name    string
+		version string
+		want    string
+	}{
+		{
+			"app-admin/eselect",
+			"1.4.15",
+			"pkg:ebuild/app-admin/eselect@1.4.15",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("%s@%s", tt.name, tt.version), func(t *testing.T) {
+			assert.Equal(t, tt.want, packageURL(tt.name, tt.version))
+		})
+	}
+}

syft/pkg/cataloger/portage/test-fixtures/image-portage/Dockerfile

@@ -1,2 +0,0 @@
-FROM scratch
-COPY . .

syft/pkg/portage_metadata.go

@@ -4,12 +4,8 @@ import (
 	"github.com/anchore/syft/syft/file"
 )
 
-const PortageDBGlob = "**/var/db/pkg/*/*/CONTENTS"
-
 // PortageMetadata represents all captured data for a Package package DB entry.
 type PortageMetadata struct {
-	Package       string              `mapstructure:"Package" json:"package"`
-	Version       string              `mapstructure:"Version" json:"version"`
 	InstalledSize int                 `mapstructure:"InstalledSize" json:"installedSize" cyclonedx:"installedSize"`
 	Files         []PortageFileRecord `json:"files"`
 }