feat: Generic Binary Cataloger (#1336)

syft/pkg/cataloger/binary/cataloger.go

@@ -0,0 +1,78 @@
+package binary
+
+import (
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+const catalogerName = "binary-cataloger"
+
+func NewCataloger() *Cataloger {
+	return &Cataloger{}
+}
+
+// Cataloger is the cataloger responsible for surfacing evidence of a very limited set of binary files,
+// which have been identified by the classifiers. The Cataloger is _NOT_ a place to catalog any and every
+// binary, but rather the specific set that has been curated to be important, predominantly related to toolchain-
+// related runtimes like Python, Go, Java, or Node. Some exceptions can be made for widely-used binaries such
+// as busybox.
+type Cataloger struct{}
+
+// Name returns a string that uniquely describes the Cataloger
+func (c Cataloger) Name() string {
+	return catalogerName
+}
+
+// Catalog is given an object to resolve file references and content, this function returns any discovered Packages
+// after analyzing the catalog source.
+func (c Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
+	var packages []pkg.Package
+	var relationships []artifact.Relationship
+
+	for _, classifier := range defaultClassifiers {
+		locations, err := resolver.FilesByGlob(classifier.FileGlob)
+		if err != nil {
+			return nil, nil, err
+		}
+		for _, location := range locations {
+			reader, err := resolver.FileContentsByLocation(location)
+			if err != nil {
+				return nil, nil, err
+			}
+			locationReader := source.NewLocationReadCloser(location, reader)
+			newPkgs, err := classifier.EvidenceMatcher(classifier, locationReader)
+			if err != nil {
+				return nil, nil, err
+			}
+		newPackages:
+			for i := range newPkgs {
+				newPkg := &newPkgs[i]
+				for j := range packages {
+					p := &packages[j]
+					// consolidate identical packages found in different locations,
+					// but continue to track each location
+					if packagesMatch(p, newPkg) {
+						p.Locations.Add(newPkg.Locations.ToSlice()...)
+						continue newPackages
+					}
+				}
+				packages = append(packages, *newPkg)
+			}
+		}
+	}
+
+	return packages, relationships, nil
+}
+
+// packagesMatch returns true if the binary packages "match" based on basic criteria
+func packagesMatch(p1 *pkg.Package, p2 *pkg.Package) bool {
+	if p1.Name != p2.Name ||
+		p1.Version != p2.Version ||
+		p1.Language != p2.Language ||
+		p1.Type != p2.Type {
+		return false
+	}
+
+	return true
+}

syft/pkg/cataloger/binary/cataloger_test.go

@@ -0,0 +1,201 @@
+package binary
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
+	tests := []struct {
+		name       string
+		fixtureDir string
+		expected   pkg.Package
+	}{
+		{
+			name:       "positive-libpython3.7.so",
+			fixtureDir: "test-fixtures/classifiers/positive",
+			expected: pkg.Package{
+				Name:      "python",
+				Version:   "3.7.4a-vZ9",
+				Locations: singleLocation("libpython3.7.so"),
+				Metadata: pkg.BinaryMetadata{
+					Classifier: "python-binary-lib",
+				},
+			},
+		},
+		{
+			name:       "positive-python3.6",
+			fixtureDir: "test-fixtures/classifiers/positive",
+			expected: pkg.Package{
+				Name:      "python",
+				Version:   "3.6.3a-vZ9",
+				Locations: singleLocation("python3.6"),
+				Metadata: pkg.BinaryMetadata{
+					Classifier: "python-binary",
+				},
+			},
+		},
+		{
+			name:       "positive-patchlevel.h",
+			fixtureDir: "test-fixtures/classifiers/positive",
+			expected: pkg.Package{
+				Name:      "python",
+				Version:   "3.9-aZ5",
+				Locations: singleLocation("patchlevel.h"),
+				Metadata: pkg.BinaryMetadata{
+					Classifier: "cpython-source",
+				},
+			},
+		},
+		{
+			name:       "positive-go",
+			fixtureDir: "test-fixtures/classifiers/positive",
+			expected: pkg.Package{
+				Name:      "go",
+				Version:   "1.14",
+				Locations: singleLocation("go"),
+				Metadata: pkg.BinaryMetadata{
+					Classifier: "go-binary",
+				},
+			},
+		},
+		{
+			name:       "positive-go-hint",
+			fixtureDir: "test-fixtures/classifiers/positive",
+			expected: pkg.Package{
+				Name:      "go",
+				Version:   "1.15",
+				Locations: singleLocation("VERSION"),
+				Metadata: pkg.BinaryMetadata{
+					Classifier: "go-binary-hint",
+				},
+			},
+		},
+		{
+			name:       "positive-busybox",
+			fixtureDir: "test-fixtures/classifiers/positive",
+			expected: pkg.Package{
+				Name:      "busybox",
+				Version:   "3.33.3",
+				Locations: singleLocation("["), // note: busybox is a link to [
+				Metadata: pkg.BinaryMetadata{
+					Classifier:  "busybox-binary",
+					VirtualPath: "busybox",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			c := NewCataloger()
+
+			src, err := source.NewFromDirectory(test.fixtureDir)
+			require.NoError(t, err)
+
+			resolver, err := src.FileResolver(source.SquashedScope)
+			require.NoError(t, err)
+
+			packages, _, err := c.Catalog(resolver)
+			require.NoError(t, err)
+
+			ok := false
+			for _, p := range packages {
+				if test.expected.Locations.ToSlice()[0].RealPath == p.Locations.ToSlice()[0].RealPath {
+					ok = true
+					assertPackagesAreEqual(t, test.expected, p)
+				}
+			}
+
+			if !ok {
+				t.Fatalf("could not find test location=%q", test.expected.Locations.ToSlice()[0].RealPath)
+			}
+
+		})
+	}
+}
+
+func TestClassifierCataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T) {
+	tests := []struct {
+		name         string
+		fixtureImage string
+		expected     pkg.Package
+	}{
+		{
+			name:         "busybox-regression",
+			fixtureImage: "image-busybox",
+			expected: pkg.Package{
+				Name:      "busybox",
+				Version:   "1.35.0",
+				Locations: singleLocation("/bin/["),
+				Metadata: pkg.BinaryMetadata{
+					Classifier:  "busybox-binary",
+					VirtualPath: "/bin/busybox",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			c := NewCataloger()
+
+			img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureImage)
+			src, err := source.NewFromImage(img, "test-img")
+			require.NoError(t, err)
+
+			resolver, err := src.FileResolver(source.SquashedScope)
+			require.NoError(t, err)
+
+			packages, _, err := c.Catalog(resolver)
+			require.NoError(t, err)
+
+			ok := false
+			for _, p := range packages {
+				if test.expected.Locations.ToSlice()[0].RealPath == p.Locations.ToSlice()[0].RealPath {
+					ok = true
+					assertPackagesAreEqual(t, test.expected, p)
+				}
+			}
+
+			if !ok {
+				t.Fatalf("could not find test location=%q", test.expected.Locations.ToSlice()[0].RealPath)
+			}
+
+		})
+	}
+}
+
+func TestClassifierCataloger_DefaultClassifiers_NegativeCases(t *testing.T) {
+	c := NewCataloger()
+
+	src, err := source.NewFromDirectory("test-fixtures/classifiers/negative")
+	assert.NoError(t, err)
+
+	resolver, err := src.FileResolver(source.SquashedScope)
+	assert.NoError(t, err)
+
+	actualResults, _, err := c.Catalog(resolver)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(actualResults))
+}
+
+func singleLocation(s string) source.LocationSet {
+	return source.NewLocationSet(source.NewLocation(s))
+}
+
+func assertPackagesAreEqual(t *testing.T, expected pkg.Package, p pkg.Package) {
+	meta1 := expected.Metadata.(pkg.BinaryMetadata)
+	meta2 := p.Metadata.(pkg.BinaryMetadata)
+	if expected.Name != p.Name ||
+		expected.Version != p.Version ||
+		meta1.Classifier != meta2.Classifier {
+		assert.Failf(t, "packages not equal", "%v != %v", expected, p)
+	}
+}

syft/pkg/cataloger/binary/classifier.go

@@ -0,0 +1,180 @@
+package binary
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"reflect"
+	"regexp"
+	"text/template"
+
+	"github.com/anchore/packageurl-go"
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/internal/unionreader"
+	"github.com/anchore/syft/syft/source"
+)
+
+var emptyPURL = packageurl.PackageURL{}
+
+// classifier is a generic package classifier that can be used to match a package definition
+// to a file that meets the given content criteria of the evidenceMatcher.
+type classifier struct {
+	Class string
+
+	// FileGlob is a selector to narrow down file inspection using the **/glob* syntax
+	FileGlob string
+
+	// EvidenceMatcher is what will be used to match against the file in the source
+	// location. If the matcher returns a package, the file will be considered a candidate.
+	EvidenceMatcher evidenceMatcher
+
+	// Information below is used to specify the Package information when returned
+
+	// Package is the name to use for the package
+	Package string
+
+	// Language is the language to classify this package as
+	Language pkg.Language
+
+	// Type is the package type to use for the package
+	Type pkg.Type
+
+	// PURL is the Package URL to use when generating a package
+	PURL packageurl.PackageURL
+
+	// CPEs are the specific CPEs we want to include for this binary with updated version information
+	CPEs []pkg.CPE
+}
+
+// evidenceMatcher is a function called to catalog Packages that match some sort of evidence
+type evidenceMatcher func(classifier classifier, reader source.LocationReadCloser) ([]pkg.Package, error)
+
+func fileNameTemplateVersionMatcher(fileNamePattern string, contentTemplate string) evidenceMatcher {
+	pat := regexp.MustCompile(fileNamePattern)
+	return func(classifier classifier, reader source.LocationReadCloser) ([]pkg.Package, error) {
+		if !pat.MatchString(reader.RealPath) {
+			return nil, nil
+		}
+
+		filepathNamedGroupValues := internal.MatchNamedCaptureGroups(pat, reader.RealPath)
+
+		tmpl, err := template.New("").Parse(contentTemplate)
+		if err != nil {
+			return nil, fmt.Errorf("unable to parse classifier template=%q : %w", contentTemplate, err)
+		}
+
+		patternBuf := &bytes.Buffer{}
+		err = tmpl.Execute(patternBuf, filepathNamedGroupValues)
+		if err != nil {
+			return nil, fmt.Errorf("unable to render template: %w", err)
+		}
+
+		tmplPattern, err := regexp.Compile(patternBuf.String())
+		if err != nil {
+			return nil, fmt.Errorf("unable to compile rendered regex=%q: %w", patternBuf.String(), err)
+		}
+
+		contents, err := getContents(reader)
+		if err != nil {
+			return nil, fmt.Errorf("unable to get read contents for file: %w", err)
+		}
+
+		matchMetadata := internal.MatchNamedCaptureGroups(tmplPattern, string(contents))
+
+		version, ok := matchMetadata["version"]
+		if ok {
+			return singlePackage(classifier, reader, version), nil
+		}
+
+		return nil, nil
+	}
+}
+
+func fileContentsVersionMatcher(pattern string) evidenceMatcher {
+	pat := regexp.MustCompile(pattern)
+	return func(classifier classifier, reader source.LocationReadCloser) ([]pkg.Package, error) {
+		contents, err := getContents(reader)
+		if err != nil {
+			return nil, fmt.Errorf("unable to get read contents for file: %w", err)
+		}
+
+		matchMetadata := internal.MatchNamedCaptureGroups(pat, string(contents))
+		version, ok := matchMetadata["version"]
+		if ok {
+			return singlePackage(classifier, reader, version), nil
+		}
+		return nil, nil
+	}
+}
+
+func mustPURL(purl string) packageurl.PackageURL {
+	p, err := packageurl.FromString(purl)
+	if err != nil {
+		panic(fmt.Sprintf("invalid PURL: %s", p))
+	}
+	return p
+}
+
+func singlePackage(classifier classifier, reader source.LocationReadCloser, version string) []pkg.Package {
+	var cpes []pkg.CPE
+	for _, cpe := range classifier.CPEs {
+		cpe.Version = version
+		cpes = append(cpes, cpe)
+	}
+
+	p := pkg.Package{
+		Name:         classifier.Package,
+		Version:      version,
+		Language:     pkg.Binary,
+		Locations:    source.NewLocationSet(reader.Location),
+		Type:         pkg.BinaryPkg,
+		CPEs:         cpes,
+		MetadataType: pkg.BinaryMetadataType,
+		Metadata: pkg.BinaryMetadata{
+			Classifier:  classifier.Class,
+			RealPath:    reader.RealPath,
+			VirtualPath: reader.VirtualPath,
+		},
+	}
+
+	if classifier.Type != "" {
+		p.Type = classifier.Type
+	}
+
+	if !reflect.DeepEqual(classifier.PURL, emptyPURL) {
+		purl := classifier.PURL
+		purl.Version = version
+		p.PURL = purl.ToString()
+	}
+
+	if classifier.Language != "" {
+		p.Language = classifier.Language
+	}
+
+	p.SetID()
+
+	return []pkg.Package{p}
+}
+
+func getContents(reader source.LocationReadCloser) ([]byte, error) {
+	unionReader, err := unionreader.GetUnionReader(reader.ReadCloser)
+	if err != nil {
+		return nil, fmt.Errorf("unable to get union reader for file: %w", err)
+	}
+
+	// TODO: there may be room for improvement here, as this may use an excessive amount of memory. Alternate approach is to leverage a RuneReader.
+	contents, err := io.ReadAll(unionReader)
+	if err != nil {
+		return nil, fmt.Errorf("unable to get contents for file: %w", err)
+	}
+
+	return contents, nil
+}
+
+// singleCPE returns a []pkg.CPE based on the cpe string or panics if the CPE is invalid
+func singleCPE(cpe string) []pkg.CPE {
+	return []pkg.CPE{
+		pkg.MustCPE(cpe),
+	}
+}

syft/pkg/cataloger/binary/classifier_test.go

@@ -0,0 +1,87 @@
+package binary
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+func Test_ClassifierCPEs(t *testing.T) {
+	tests := []struct {
+		name       string
+		fixture    string
+		classifier classifier
+		cpes       []string
+	}{
+		{
+			name:    "no CPEs",
+			fixture: "test-fixtures/version.txt",
+			classifier: classifier{
+				Package:         "some-app",
+				FileGlob:        ".*/version.txt",
+				EvidenceMatcher: fileContentsVersionMatcher(`(?m)my-verison:(?P<version>[0-9.]+)`),
+				CPEs:            []pkg.CPE{},
+			},
+			cpes: nil,
+		},
+		{
+			name:    "one CPE",
+			fixture: "test-fixtures/version.txt",
+			classifier: classifier{
+				Package:         "some-app",
+				FileGlob:        ".*/version.txt",
+				EvidenceMatcher: fileContentsVersionMatcher(`(?m)my-verison:(?P<version>[0-9.]+)`),
+				CPEs: []pkg.CPE{
+					pkg.MustCPE("cpe:2.3:a:some:app:*:*:*:*:*:*:*:*"),
+				},
+			},
+			cpes: []string{
+				"cpe:2.3:a:some:app:1.8:*:*:*:*:*:*:*",
+			},
+		},
+		{
+			name:    "multiple CPEs",
+			fixture: "test-fixtures/version.txt",
+			classifier: classifier{
+				Package:         "some-app",
+				FileGlob:        ".*/version.txt",
+				EvidenceMatcher: fileContentsVersionMatcher(`(?m)my-verison:(?P<version>[0-9.]+)`),
+				CPEs: []pkg.CPE{
+					pkg.MustCPE("cpe:2.3:a:some:app:*:*:*:*:*:*:*:*"),
+					pkg.MustCPE("cpe:2.3:a:some:apps:*:*:*:*:*:*:*:*"),
+				},
+			},
+			cpes: []string{
+				"cpe:2.3:a:some:app:1.8:*:*:*:*:*:*:*",
+				"cpe:2.3:a:some:apps:1.8:*:*:*:*:*:*:*",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			resolver := source.NewMockResolverForPaths(test.fixture)
+			locations, err := resolver.FilesByPath(test.fixture)
+			require.NoError(t, err)
+			require.Len(t, locations, 1)
+			location := locations[0]
+			readCloser, err := resolver.FileContentsByLocation(location)
+			require.NoError(t, err)
+			pkgs, err := test.classifier.EvidenceMatcher(test.classifier, source.NewLocationReadCloser(location, readCloser))
+			require.NoError(t, err)
+
+			require.Len(t, pkgs, 1)
+
+			p := pkgs[0]
+
+			var cpes []string
+			for _, c := range p.CPEs {
+				cpes = append(cpes, pkg.CPEString(c))
+			}
+			require.Equal(t, test.cpes, cpes)
+		})
+	}
+}

syft/pkg/cataloger/binary/default_classifiers.go

@@ -0,0 +1,60 @@
+package binary
+
+import "github.com/anchore/syft/syft/pkg"
+
+var defaultClassifiers = []classifier{
+	{
+		Class:    "python-binary",
+		FileGlob: "**/python*",
+		EvidenceMatcher: fileNameTemplateVersionMatcher(
+			`(.*/|^)python(?P<version>[0-9]+\.[0-9]+)$`,
+			`(?m)(?P<version>{{ .version }}\.[0-9]+[-_a-zA-Z0-9]*)`),
+		Package: "python",
+	},
+	{
+		Class:    "python-binary-lib",
+		FileGlob: "**/libpython*.so*",
+		EvidenceMatcher: fileNameTemplateVersionMatcher(
+			`(.*/|^)libpython(?P<version>[0-9]+\.[0-9]+).so.*$`,
+			`(?m)(?P<version>{{ .version }}\.[0-9]+[-_a-zA-Z0-9]*)`),
+		Package: "python",
+	},
+	{
+		Class:    "cpython-source",
+		FileGlob: "**/patchlevel.h",
+		EvidenceMatcher: fileContentsVersionMatcher(
+			`(?m)#define\s+PY_VERSION\s+"?(?P<version>[0-9\.\-_a-zA-Z]+)"?`),
+		Package: "python",
+	},
+	{
+		Class:    "go-binary",
+		FileGlob: "**/go",
+		EvidenceMatcher: fileContentsVersionMatcher(
+			`(?m)go(?P<version>[0-9]+\.[0-9]+(\.[0-9]+|beta[0-9]+|alpha[0-9]+|rc[0-9]+)?)`),
+		Package: "go",
+	},
+	{
+		Class:    "nodejs-binary",
+		FileGlob: "**/node",
+		EvidenceMatcher: fileContentsVersionMatcher(
+			`(?m)node\.js\/v(?P<version>[0-9]+\.[0-9]+\.[0-9]+)`),
+		Package:  "node.js",
+		Language: pkg.JavaScript,
+		PURL:     mustPURL("pkg:generic/node@version"),
+		CPEs:     singleCPE("cpe:2.3:a:nodejs:node.js:*:*:*:*:*:*:*:*"),
+	},
+	{
+		Class:    "go-binary-hint",
+		FileGlob: "**/VERSION",
+		EvidenceMatcher: fileContentsVersionMatcher(
+			`(?m)go(?P<version>[0-9]+\.[0-9]+(\.[0-9]+|beta[0-9]+|alpha[0-9]+|rc[0-9]+)?)`),
+		Package: "go",
+	},
+	{
+		Class:    "busybox-binary",
+		FileGlob: "**/busybox",
+		EvidenceMatcher: fileContentsVersionMatcher(
+			`(?m)BusyBox\s+v(?P<version>[0-9]+\.[0-9]+\.[0-9]+)`),
+		Package: "busybox",
+	},
+}

syft/pkg/cataloger/binary/test-fixtures/classifiers/negative/.gitignore

@@ -0,0 +1 @@
+!libpython2.7.so

syft/pkg/cataloger/binary/test-fixtures/classifiers/negative/busybox

@@ -0,0 +1 @@
+another bad binary

syft/pkg/cataloger/binary/test-fixtures/classifiers/negative/go

@@ -0,0 +1 @@
+a bad go binary

syft/pkg/cataloger/binary/test-fixtures/classifiers/negative/libpython2.7.so

@@ -0,0 +1,3 @@
+# note: this should NOT match
+
+DO NOT DETECT

syft/pkg/cataloger/binary/test-fixtures/classifiers/negative/python2.6

@@ -0,0 +1,3 @@
+# note: this should NOT match
+
+just some noise

syft/pkg/cataloger/binary/test-fixtures/classifiers/positive/.gitignore

@@ -0,0 +1 @@
+!libpython3.7.so

syft/pkg/cataloger/binary/test-fixtures/classifiers/positive/VERSION

@@ -0,0 +1 @@
+go1.15-beta2

syft/pkg/cataloger/binary/test-fixtures/classifiers/positive/[

@@ -0,0 +1,3 @@
+# note: this SHOULD match as busybox 3.33.3
+
+noise!BusyBox v3.33.3!noise

syft/pkg/cataloger/binary/test-fixtures/classifiers/positive/busybox

@@ -0,0 +1 @@
+./[

syft/pkg/cataloger/binary/test-fixtures/classifiers/positive/go

@@ -0,0 +1 @@
+go1.14

syft/pkg/cataloger/binary/test-fixtures/classifiers/positive/libpython3.7.so

@@ -0,0 +1,2 @@
+# note: this SHOULD match as python 3.7
+noise3.7.4a-vZ9!morenoise

syft/pkg/cataloger/binary/test-fixtures/classifiers/positive/patchlevel.h

@@ -0,0 +1,7 @@
+# note: this SHOULD match as python 3.9
+
+some source code...
+
+#define    PY_VERSION   3.9-aZ5
+
+more source!

syft/pkg/cataloger/binary/test-fixtures/classifiers/positive/python3.6

@@ -0,0 +1,3 @@
+# note: this SHOULD match as python 3.6
+
+noise3.6.3a-vZ9!morenoise

syft/pkg/cataloger/binary/test-fixtures/image-busybox/Dockerfile

@@ -0,0 +1 @@
+FROM busybox:1.35

syft/pkg/cataloger/binary/test-fixtures/version.txt

@@ -0,0 +1 @@
+my-verison:1.8