fix: more python matching support (#1667)

Signed-off-by: GitHub <noreply@github.com>
Co-authored-by: spiffcs <spiffcs@users.noreply.github.com>

Makefile

@@ -11,10 +11,10 @@ GLOW_CMD = $(TEMP_DIR)/glow
 
 # Tool versions #################################
 GOLANGCILINT_VERSION := v1.51.2
-GOSIMPORTS_VERSION := v0.3.7
+GOSIMPORTS_VERSION := v0.3.8
 BOUNCER_VERSION := v0.4.0
 CHRONICLE_VERSION := v0.6.0
-GORELEASER_VERSION := v1.16.0
+GORELEASER_VERSION := v1.16.1
 YAJSV_VERSION := v1.4.1
 COSIGN_VERSION := v1.13.1
 QUILL_VERSION := v0.2.0
@@ -252,6 +252,11 @@ install-test-ci-mac: $(SNAPSHOT_DIR)
 	cd test/install && \
 		make ci-test-mac
 
+.PHONY: generate-compare-file
+generate-compare-file:
+	$(call title,Generating compare test file)
+	go run ./cmd/syft $(COMPARE_TEST_IMAGE) -o json > $(COMPARE_DIR)/test-fixtures/acceptance-centos-8.2.2004.json
+
 # note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately
 .PHONY: compare-mac
 compare-mac: $(TEMP_DIR) $(SNAPSHOT_DIR)  ## Run compare tests on build snapshot binaries and packages (Mac)

syft/pkg/cataloger/binary/cataloger_test.go

@@ -292,8 +292,8 @@ func Test_Cataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 			fixtureDir: "test-fixtures/classifiers/positive/python-binary-lib-3.7",
 			expected: pkg.Package{
 				Name:      "python",
-				Version:   "3.7.4a-vZ9",
-				PURL:      "pkg:generic/python@3.7.4a-vZ9",
+				Version:   "3.7.4",
+				PURL:      "pkg:generic/python@3.7.4",
 				Locations: locations("libpython3.7.so"),
 				Metadata:  metadata("python-binary-lib"),
 			},
@@ -347,6 +347,34 @@ func Test_Cataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:       "positive-python-binary-3.4-alpine",
+			fixtureDir: "test-fixtures/classifiers/dynamic/python-binary-3.4-alpine",
+			expected: pkg.Package{
+				Name:      "python",
+				Version:   "3.4.10",
+				PURL:      "pkg:generic/python@3.4.10",
+				Locations: locations("python3.4", "libpython3.4m.so.1.0"),
+				Metadata: pkg.BinaryMetadata{
+					Matches: []pkg.ClassifierMatch{
+						match("python-binary", "python3.4"),
+						match("python-binary", "libpython3.4m.so.1.0"),
+						match("python-binary-lib", "libpython3.4m.so.1.0"),
+					},
+				},
+			},
+		},
+		{
+			name:       "positive-python-3.5-with-incorrect-match",
+			fixtureDir: "test-fixtures/classifiers/positive/python-3.5-with-incorrect-match",
+			expected: pkg.Package{
+				Name:      "python",
+				Version:   "3.5.3",
+				PURL:      "pkg:generic/python@3.5.3",
+				Locations: locations("python3.5"),
+				Metadata:  metadata("python-binary"),
+			},
+		},
 		{
 			name:       "positive-python3.6",
 			fixtureDir: "test-fixtures/classifiers/positive/python-binary-3.6",
@@ -515,6 +543,52 @@ func Test_Cataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 				Metadata:  metadata("rust-standard-library-linux"),
 			},
 		},
+		{
+			name:       "positive-ruby-3.2.1",
+			fixtureDir: "test-fixtures/classifiers/dynamic/ruby-library-3.2.1",
+			expected: pkg.Package{
+				Name:      "ruby",
+				Version:   "3.2.1",
+				Type:      "binary",
+				PURL:      "pkg:generic/ruby@3.2.1",
+				Locations: locations("ruby", "libruby.so.3.2.1"),
+				Metadata: pkg.BinaryMetadata{
+					Matches: []pkg.ClassifierMatch{
+						match("ruby-binary", "ruby"),
+						match("ruby-binary", "libruby.so.3.2.1"),
+					},
+				},
+			},
+		},
+		{
+			name:       "positive-ruby-2.7.7",
+			fixtureDir: "test-fixtures/classifiers/dynamic/ruby-library-2.7.7",
+			expected: pkg.Package{
+				Name:      "ruby",
+				Version:   "2.7.7p221",
+				Type:      "binary",
+				PURL:      "pkg:generic/ruby@2.7.7p221",
+				Locations: locations("ruby", "libruby.so.2.7.7"),
+				Metadata: pkg.BinaryMetadata{
+					Matches: []pkg.ClassifierMatch{
+						match("ruby-binary", "ruby"),
+						match("ruby-binary", "libruby.so.2.7.7"),
+					},
+				},
+			},
+		},
+		{
+			name:       "positive-ruby-1.9.3p551",
+			fixtureDir: "test-fixtures/classifiers/positive/ruby-1.9.3p551",
+			expected: pkg.Package{
+				Name:      "ruby",
+				Version:   "1.9.3p551",
+				Type:      "binary",
+				PURL:      "pkg:generic/ruby@1.9.3p551",
+				Locations: locations("ruby"),
+				Metadata:  metadata("ruby-binary"),
+			},
+		},
 	}
 
 	for _, test := range tests {
@@ -530,9 +604,9 @@ func Test_Cataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 			packages, _, err := c.Catalog(resolver)
 			require.NoError(t, err)
 
-			for _, p := range packages {
-				assertPackagesAreEqual(t, test.expected, p)
-			}
+			require.Len(t, packages, 1)
+
+			assertPackagesAreEqual(t, test.expected, packages[0])
 		})
 	}
 }

syft/pkg/cataloger/binary/classifier.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"reflect"
 	"regexp"
+	"strings"
 	"text/template"
 
 	"github.com/anchore/packageurl-go"
@@ -79,6 +80,11 @@ func fileNameTemplateVersionMatcher(fileNamePattern string, contentTemplate stri
 
 		filepathNamedGroupValues := internal.MatchNamedCaptureGroups(pat, location.RealPath)
 
+		// versions like 3.5 should not match any character, but explicit dot
+		for k, v := range filepathNamedGroupValues {
+			filepathNamedGroupValues[k] = strings.ReplaceAll(v, ".", "\\.")
+		}
+
 		tmpl, err := template.New("").Parse(contentTemplate)
 		if err != nil {
 			return nil, fmt.Errorf("unable to parse classifier template=%q : %w", contentTemplate, err)

syft/pkg/cataloger/binary/default_classifiers.go

@@ -12,7 +12,7 @@ var defaultClassifiers = []classifier{
 		EvidenceMatcher: evidenceMatchers(
 			// try to find version information from libpython shared libraries
 			sharedLibraryLookup(
-				`^libpython[0-9]+(?:\.[0-9]+)+\.so.*$`,
+				`^libpython[0-9]+(?:\.[0-9]+)+[a-z]?\.so.*$`,
 				libpythonMatcher),
 			// check for version information in the binary
 			fileNameTemplateVersionMatcher(
@@ -220,12 +220,31 @@ var defaultClassifiers = []classifier{
 		PURL:    mustPURL("pkg:generic/rust@version"),
 		CPEs:    singleCPE("cpe:2.3:a:rust-lang:rust:*:*:*:*:*:*:*:*"),
 	},
+	{
+		Class:    "ruby-binary",
+		FileGlob: "**/ruby",
+		EvidenceMatcher: evidenceMatchers(
+			rubyMatcher,
+			sharedLibraryLookup(
+				// try to find version information from libruby shared libraries
+				`^libruby\.so.*$`,
+				rubyMatcher),
+		),
+		Package: "ruby",
+		PURL:    mustPURL("pkg:generic/ruby@version"),
+		CPEs:    singleCPE("cpe:2.3:a:ruby-lang:ruby:*:*:*:*:*:*:*:*"),
+	},
 }
 
 // in both binaries and shared libraries, the version pattern is [NUL]3.11.2[NUL]
 var pythonVersionTemplate = `(?m)\x00(?P<version>{{ .version }}[-._a-zA-Z0-9]*)\x00`
 
 var libpythonMatcher = fileNameTemplateVersionMatcher(
-	`(?:.*/|^)libpython(?P<version>[0-9]+(?:\.[0-9]+)+)\.so.*$`,
+	`(?:.*/|^)libpython(?P<version>[0-9]+(?:\.[0-9]+)+)[a-z]?\.so.*$`,
 	pythonVersionTemplate,
 )
+
+var rubyMatcher = fileContentsVersionMatcher(
+	// ruby 3.2.1 (2023-02-08 revision 31819e82c8) [x86_64-linux]
+	// ruby 2.7.7p221 (2022-11-24 revision 168ec2b1e5) [x86_64-linux]
+	`(?m)ruby (?P<version>[0-9]\.[0-9]\.[0-9](p[0-9]+)?) `)

syft/pkg/cataloger/binary/test-fixtures/Makefile

@@ -2,7 +2,10 @@
 all: \
 	classifiers/dynamic/python-binary-shared-lib-3.11 \
 	classifiers/dynamic/python-binary-shared-lib-redhat-3.9 \
-	classifiers/dynamic/python-binary-with-version-3.9
+	classifiers/dynamic/python-binary-with-version-3.9 \
+	classifiers/dynamic/python-binary-3.4-alpine \
+	classifiers/dynamic/ruby-library-3.2.1 \
+	classifiers/dynamic/ruby-library-2.7.7
 
 classifiers/dynamic/python-binary-shared-lib-3.11:
 	$(eval $@_image := "python:3.11-slim@sha256:0b106e1d2bf485c2a41474bc9cd5103e9eea4e179f40f10741b53b127059221e")
@@ -28,6 +31,39 @@ classifiers/dynamic/python-binary-with-version-3.9:
 		/usr/bin/python3.9 \
 		$@/python3.9
 
+classifiers/dynamic/python-binary-3.4-alpine:
+	$(eval $@_image := "python:3.4-alpine@sha256:c210b660e2ea553a7afa23b41a6ed112f85dbce25cbcb567c75dfe05342a4c4b")
+	./get-image-file.sh $($@_image) \
+		/usr/local/bin/python3.4 \
+		$@/python3.4
+	./get-image-file.sh $($@_image) \
+		/usr/local/lib/libpython3.4m.so.1.0 \
+		$@/libpython3.4m.so.1.0
+
+classifiers/dynamic/ruby-library-3.2.1:
+	$(eval $@_image := "ruby:3.2.1-bullseye@sha256:b4a140656b0c5d26c0a80559b228b4d343f3fdbf56682fcbe88f6db1fa9afa6b")
+	./get-image-file.sh $($@_image) \
+		/usr/local/bin/ruby \
+		$@/ruby
+	./get-image-file.sh $($@_image) \
+		/usr/local/lib/libruby.so.3.2.1 \
+		$@/libruby.so.3.2.1
+	./get-image-file.sh $($@_image) \
+		/usr/local/lib/libruby.so.3.2 \
+		$@/libruby.so.3.2
+
+classifiers/dynamic/ruby-library-2.7.7:
+	$(eval $@_image := "ruby:2.7.7-bullseye@sha256:055191740a063f33fef1f09423e5ed8f91143aae62a3772a90910118464c5120")
+	./get-image-file.sh $($@_image) \
+		/usr/local/bin/ruby \
+		$@/ruby
+	./get-image-file.sh $($@_image) \
+		/usr/local/lib/libruby.so.2.7.7 \
+		$@/libruby.so.2.7.7
+	./get-image-file.sh $($@_image) \
+		/usr/local/lib/libruby.so.2.7 \
+		$@/libruby.so.2.7
+
 .PHONY: clean
 clean:
 	rm -rf classifiers/dynamic

syft/pkg/cataloger/binary/test-fixtures/classifiers/positive/ruby-1.9.3p551/ruby

@@ -0,0 +1,9 @@
+super from singleton method that is defined to multiple classes is not supported; this will be fixed in 1.9.3 or later
+/usr/local/lib/ruby/site_ruby/1.9.1
+/usr/local/lib/ruby/site_ruby/1.9.1/x86_64-linux
+/usr/local/lib/ruby/vendor_ruby/1.9.1
+/usr/local/lib/ruby/vendor_ruby/1.9.1/x86_64-linux
+/usr/local/lib/ruby/1.9.1
+/usr/local/lib/ruby/1.9.1/x86_64-linux
+ruby 1.9.3p551 (2014-11-13 revision 48407) [x86_64-linux]
+1.9.3