feat: Generic Binary Cataloger (#1336)

syft/file/classification_cataloger.go

@@ -1,38 +0,0 @@
-package file
-
-import (
-	"github.com/anchore/syft/internal/log"
-	"github.com/anchore/syft/syft/source"
-)
-
-type ClassificationCataloger struct {
-	classifiers []Classifier
-}
-
-func NewClassificationCataloger(classifiers []Classifier) (*ClassificationCataloger, error) {
-	return &ClassificationCataloger{
-		classifiers: classifiers,
-	}, nil
-}
-
-func (i *ClassificationCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]Classification, error) {
-	results := make(map[source.Coordinates][]Classification)
-
-	numResults := 0
-	for _, location := range allRegularFiles(resolver) {
-		for _, classifier := range i.classifiers {
-			result, err := classifier.Classify(resolver, location)
-			if err != nil {
-				log.Warnf("file classification cataloger failed with class=%q at location=%+v: %+v", classifier.Class, location, err)
-				continue
-			}
-			if result != nil {
-				results[location.Coordinates] = append(results[location.Coordinates], *result)
-				numResults++
-			}
-		}
-	}
-	log.Debugf("file classifier discovered %d results", numResults)
-
-	return results, nil
-}

syft/file/classification_cataloger_test.go

@@ -1,210 +0,0 @@
-package file
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/anchore/stereoscope/pkg/imagetest"
-	"github.com/anchore/syft/syft/source"
-)
-
-func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
-	tests := []struct {
-		name        string
-		fixtureDir  string
-		location    string
-		expected    []Classification
-		expectedErr func(assert.TestingT, error, ...interface{}) bool
-	}{
-		{
-			name:       "positive-libpython3.7.so",
-			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "libpython3.7.so",
-			expected: []Classification{
-				{
-					Class: "python-binary",
-					Metadata: map[string]string{
-						"version": "3.7.4a-vZ9",
-					},
-				},
-			},
-			expectedErr: assert.NoError,
-		},
-		{
-			name:       "positive-python3.6",
-			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "python3.6",
-			expected: []Classification{
-				{
-					Class: "python-binary",
-					Metadata: map[string]string{
-						"version": "3.6.3a-vZ9",
-					},
-				},
-			},
-			expectedErr: assert.NoError,
-		},
-		{
-			name:       "positive-patchlevel.h",
-			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "patchlevel.h",
-			expected: []Classification{
-				{
-					Class: "cpython-source",
-					Metadata: map[string]string{
-						"version": "3.9-aZ5",
-					},
-				},
-			},
-			expectedErr: assert.NoError,
-		},
-		{
-			name:       "positive-go",
-			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "go",
-			expected: []Classification{
-				{
-					Class: "go-binary",
-					Metadata: map[string]string{
-						"version": "1.14",
-					},
-				},
-			},
-			expectedErr: assert.NoError,
-		},
-		{
-			name:       "positive-go-hint",
-			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "VERSION",
-			expected: []Classification{
-				{
-					Class: "go-binary-hint",
-					Metadata: map[string]string{
-						"version": "1.15",
-					},
-				},
-			},
-			expectedErr: assert.NoError,
-		},
-		{
-			name:       "positive-busybox",
-			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "[", // note: busybox is a link to [
-			expected: []Classification{
-				{
-					Class:       "busybox-binary",
-					VirtualPath: "busybox",
-					Metadata: map[string]string{
-						"version": "3.33.3",
-					},
-				},
-			},
-			expectedErr: assert.NoError,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-
-			c, err := NewClassificationCataloger(DefaultClassifiers)
-			test.expectedErr(t, err)
-
-			src, err := source.NewFromDirectory(test.fixtureDir)
-			test.expectedErr(t, err)
-
-			resolver, err := src.FileResolver(source.SquashedScope)
-			test.expectedErr(t, err)
-
-			actualResults, err := c.Catalog(resolver)
-			test.expectedErr(t, err)
-
-			ok := false
-			for actualLoc, actualClassification := range actualResults {
-				if test.location == actualLoc.RealPath {
-					ok = true
-					assert.Equal(t, test.expected, actualClassification)
-				}
-			}
-
-			if !ok {
-				t.Fatalf("could not find test location=%q", test.location)
-			}
-
-		})
-	}
-}
-
-func TestClassifierCataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T) {
-	tests := []struct {
-		name         string
-		fixtureImage string
-		location     string
-		expected     []Classification
-		expectedErr  func(assert.TestingT, error, ...interface{}) bool
-	}{
-		{
-			name:         "busybox-regression",
-			fixtureImage: "image-busybox",
-			location:     "/bin/[",
-			expected: []Classification{
-				{
-					Class:       "busybox-binary",
-					VirtualPath: "/bin/busybox",
-					Metadata: map[string]string{
-						"version": "1.35.0",
-					},
-				},
-			},
-			expectedErr: assert.NoError,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-
-			c, err := NewClassificationCataloger(DefaultClassifiers)
-			test.expectedErr(t, err)
-
-			img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureImage)
-			src, err := source.NewFromImage(img, "test-img")
-			test.expectedErr(t, err)
-
-			resolver, err := src.FileResolver(source.SquashedScope)
-			test.expectedErr(t, err)
-
-			actualResults, err := c.Catalog(resolver)
-			test.expectedErr(t, err)
-
-			ok := false
-			for actuaLoc, actualClassification := range actualResults {
-				if actuaLoc.RealPath == test.location {
-					ok = true
-					assert.Equal(t, test.expected, actualClassification)
-				}
-			}
-
-			if !ok {
-				t.Fatalf("could not find test location=%q", test.location)
-			}
-
-		})
-	}
-}
-
-func TestClassifierCataloger_DefaultClassifiers_NegativeCases(t *testing.T) {
-
-	c, err := NewClassificationCataloger(DefaultClassifiers)
-	assert.NoError(t, err)
-
-	src, err := source.NewFromDirectory("test-fixtures/classifiers/negative")
-	assert.NoError(t, err)
-
-	resolver, err := src.FileResolver(source.SquashedScope)
-	assert.NoError(t, err)
-
-	actualResults, err := c.Catalog(resolver)
-	assert.NoError(t, err)
-	assert.Equal(t, 0, len(actualResults))
-
-}

syft/file/classifier.go

@@ -1,153 +0,0 @@
-package file
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"regexp"
-	"text/template"
-
-	"github.com/anchore/syft/internal"
-	"github.com/anchore/syft/syft/source"
-)
-
-var DefaultClassifiers = []Classifier{
-	{
-		Class: "python-binary",
-		FilepathPatterns: []*regexp.Regexp{
-			regexp.MustCompile(`(.*/|^)python(?P<version>[0-9]+\.[0-9]+)$`),
-			regexp.MustCompile(`(.*/|^)libpython(?P<version>[0-9]+\.[0-9]+).so.*$`),
-		},
-		EvidencePatternTemplates: []string{
-			`(?m)(?P<version>{{ .version }}\.[0-9]+[-_a-zA-Z0-9]*)`,
-		},
-	},
-	{
-		Class: "cpython-source",
-		FilepathPatterns: []*regexp.Regexp{
-			regexp.MustCompile(`(.*/|^)patchlevel.h$`),
-		},
-		EvidencePatternTemplates: []string{
-			`(?m)#define\s+PY_VERSION\s+"?(?P<version>[0-9\.\-_a-zA-Z]+)"?`,
-		},
-	},
-	{
-		Class: "go-binary",
-		FilepathPatterns: []*regexp.Regexp{
-			regexp.MustCompile(`(.*/|^)go$`),
-		},
-		EvidencePatternTemplates: []string{
-			`(?m)go(?P<version>[0-9]+\.[0-9]+(\.[0-9]+|beta[0-9]+|alpha[0-9]+|rc[0-9]+)?)`,
-		},
-	},
-	{
-		Class: "nodejs-binary",
-		FilepathPatterns: []*regexp.Regexp{
-			regexp.MustCompile(`(.*/|^)node$`),
-		},
-		EvidencePatternTemplates: []string{
-			// regex that matches node.js/vx.y.z
-			`(?m)node\.js\/v(?P<version>[0-9]+\.[0-9]+\.[0-9]+)`,
-		},
-	},
-	{
-		Class: "go-binary-hint",
-		FilepathPatterns: []*regexp.Regexp{
-			regexp.MustCompile(`(.*/|^)VERSION$`),
-		},
-		EvidencePatternTemplates: []string{
-			`(?m)go(?P<version>[0-9]+\.[0-9]+(\.[0-9]+|beta[0-9]+|alpha[0-9]+|rc[0-9]+)?)`,
-		},
-	},
-	{
-		Class: "busybox-binary",
-		FilepathPatterns: []*regexp.Regexp{
-			regexp.MustCompile(`(.*/|^)busybox$`),
-		},
-		EvidencePatternTemplates: []string{
-			`(?m)BusyBox\s+v(?P<version>[0-9]+\.[0-9]+\.[0-9]+)`,
-		},
-	},
-}
-
-type Classifier struct {
-	Class                    string
-	FilepathPatterns         []*regexp.Regexp
-	EvidencePatternTemplates []string
-}
-
-type Classification struct {
-	Class       string            `json:"class"`
-	VirtualPath string            `json:"virtual_path"`
-	Metadata    map[string]string `json:"metadata"`
-}
-
-func (c Classifier) Classify(resolver source.FileResolver, location source.Location) (*Classification, error) {
-	doesFilepathMatch, filepathNamedGroupValues := FilepathMatches(c.FilepathPatterns, location)
-	if !doesFilepathMatch {
-		return nil, nil
-	}
-
-	contentReader, err := resolver.FileContentsByLocation(location)
-	if err != nil {
-		return nil, err
-	}
-	defer internal.CloseAndLogError(contentReader, location.VirtualPath)
-
-	// TODO: there is room for improvement here, as this may use an excessive amount of memory. Alternate approach is to leverage a RuneReader.
-	contents, err := io.ReadAll(contentReader)
-	if err != nil {
-		return nil, err
-	}
-
-	var result *Classification
-	for _, patternTemplate := range c.EvidencePatternTemplates {
-		tmpl, err := template.New("").Parse(patternTemplate)
-		if err != nil {
-			return nil, fmt.Errorf("unable to parse classifier template=%q : %w", patternTemplate, err)
-		}
-
-		patternBuf := &bytes.Buffer{}
-		err = tmpl.Execute(patternBuf, filepathNamedGroupValues)
-		if err != nil {
-			return nil, fmt.Errorf("unable to render template: %w", err)
-		}
-
-		pattern, err := regexp.Compile(patternBuf.String())
-		if err != nil {
-			return nil, fmt.Errorf("unable to compile rendered regex=%q: %w", patternBuf.String(), err)
-		}
-
-		if !pattern.Match(contents) {
-			continue
-		}
-
-		matchMetadata := internal.MatchNamedCaptureGroups(pattern, string(contents))
-		if result == nil {
-			result = &Classification{
-				Class:       c.Class,
-				VirtualPath: location.VirtualPath,
-				Metadata:    matchMetadata,
-			}
-		} else {
-			for key, value := range matchMetadata {
-				result.Metadata[key] = value
-			}
-		}
-	}
-	return result, nil
-}
-
-func FilepathMatches(patterns []*regexp.Regexp, location source.Location) (bool, map[string]string) {
-	for _, path := range []string{location.RealPath, location.VirtualPath} {
-		if path == "" {
-			continue
-		}
-		for _, pattern := range patterns {
-			if pattern.MatchString(path) {
-				return true, internal.MatchNamedCaptureGroups(pattern, path)
-			}
-		}
-	}
-	return false, nil
-}

syft/file/classifier_test.go

@@ -1,97 +0,0 @@
-package file
-
-import (
-	"regexp"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/anchore/syft/syft/source"
-)
-
-func TestFilepathMatches(t *testing.T) {
-	tests := []struct {
-		name                string
-		location            source.Location
-		patterns            []string
-		expectedMatches     bool
-		expectedNamedGroups map[string]string
-	}{
-		{
-			name: "simple-filename-match",
-			location: source.Location{
-				Coordinates: source.Coordinates{
-					RealPath: "python2.7",
-				},
-			},
-			patterns: []string{
-				`python([0-9]+\.[0-9]+)$`,
-			},
-			expectedMatches: true,
-		},
-		{
-			name: "filepath-match",
-			location: source.Location{
-				Coordinates: source.Coordinates{
-					RealPath: "/usr/bin/python2.7",
-				},
-			},
-			patterns: []string{
-				`python([0-9]+\.[0-9]+)$`,
-			},
-			expectedMatches: true,
-		},
-		{
-			name: "virtual-filepath-match",
-			location: source.Location{
-				VirtualPath: "/usr/bin/python2.7",
-			},
-			patterns: []string{
-				`python([0-9]+\.[0-9]+)$`,
-			},
-			expectedMatches: true,
-		},
-		{
-			name: "full-filepath-match",
-			location: source.Location{
-				VirtualPath: "/usr/bin/python2.7",
-			},
-			patterns: []string{
-				`.*/bin/python([0-9]+\.[0-9]+)$`,
-			},
-			expectedMatches: true,
-		},
-		{
-			name: "anchored-filename-match-FAILS",
-			location: source.Location{
-				Coordinates: source.Coordinates{
-					RealPath: "/usr/bin/python2.7",
-				},
-			},
-			patterns: []string{
-				`^python([0-9]+\.[0-9]+)$`,
-			},
-			expectedMatches: false,
-		},
-		{
-			name:     "empty-filename-match-FAILS",
-			location: source.Location{},
-			patterns: []string{
-				`^python([0-9]+\.[0-9]+)$`,
-			},
-			expectedMatches: false,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			var patterns []*regexp.Regexp
-			for _, p := range test.patterns {
-				patterns = append(patterns, regexp.MustCompile(p))
-			}
-			actualMatches, actualNamedGroups := FilepathMatches(patterns, test.location)
-			assert.Equal(t, test.expectedMatches, actualMatches)
-			assert.Equal(t, test.expectedNamedGroups, actualNamedGroups)
-		})
-	}
-}

syft/file/test-fixtures/classifiers/negative/.gitignore

@@ -1 +0,0 @@
-!libpython2.7.so

syft/file/test-fixtures/classifiers/negative/busybox

@@ -1 +0,0 @@
-another bad binary

syft/file/test-fixtures/classifiers/negative/go

@@ -1 +0,0 @@
-a bad go binary

syft/file/test-fixtures/classifiers/negative/libpython2.7.so

@@ -1,3 +0,0 @@
-# note: this should NOT match
-
-DO NOT DETECT

syft/file/test-fixtures/classifiers/negative/python2.6

@@ -1,3 +0,0 @@
-# note: this should NOT match
-
-just some noise

syft/file/test-fixtures/classifiers/positive/.gitignore

@@ -1 +0,0 @@
-!libpython3.7.so

syft/file/test-fixtures/classifiers/positive/VERSION

@@ -1 +0,0 @@
-go1.15-beta2

syft/file/test-fixtures/classifiers/positive/[

@@ -1,3 +0,0 @@
-# note: this SHOULD match as busybox 3.33.3
-
-noise!BusyBox v3.33.3!noise

syft/file/test-fixtures/classifiers/positive/busybox

@@ -1 +0,0 @@
-./[

syft/file/test-fixtures/classifiers/positive/go

@@ -1 +0,0 @@
-go1.14

syft/file/test-fixtures/classifiers/positive/libpython3.7.so

@@ -1,2 +0,0 @@
-# note: this SHOULD match as python 3.7
-noise3.7.4a-vZ9!morenoise

syft/file/test-fixtures/classifiers/positive/patchlevel.h

@@ -1,7 +0,0 @@
-# note: this SHOULD match as python 3.9
-
-some source code...
-
-#define    PY_VERSION   3.9-aZ5
-
-more source!

syft/file/test-fixtures/classifiers/positive/python3.6

@@ -1,3 +0,0 @@
-# note: this SHOULD match as python 3.6
-
-noise3.6.3a-vZ9!morenoise

syft/file/test-fixtures/image-busybox/Dockerfile

@@ -1 +0,0 @@
-FROM busybox:1.35