fix: align binary java detection with jvm cataloger + support IBM (#4046)

Signed-off-by: Keith Zantow <kzantow@gmail.com>

syft/pkg/cataloger/internal/binutils/branching_matcher.go

@@ -0,0 +1,43 @@
+package binutils
+
+import (
+	"io"
+
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/syft/internal/unionreader"
+	"github.com/anchore/syft/syft/pkg"
+)
+
+// BranchingEvidenceMatcher loads the contents of the reader into memory, assuming that all
+// classifier matchers are going to be reading the same file, e.g. a "java" binary, continuing to
+// try classifier EvidenceMatcher until the first set of packages is found
+func BranchingEvidenceMatcher(classifiers ...Classifier) EvidenceMatcher {
+	return func(_ Classifier, context MatcherContext) ([]pkg.Package, error) {
+		// we are scanning the same contents multiple times, so read it into memory for a reader that can reset
+		rdr, err := getReader(context)
+		if err != nil {
+			return nil, err
+		}
+		defer internal.CloseAndLogError(rdr, context.Location.RealPath)
+		if err != nil {
+			return nil, err
+		}
+		for _, c := range classifiers {
+			pkgs, err := c.EvidenceMatcher(c, MatcherContext{
+				Resolver: context.Resolver,
+				Location: context.Location,
+				GetReader: func(_ MatcherContext) (unionreader.UnionReader, error) {
+					_, err := rdr.Seek(0, io.SeekStart)
+					if err != nil {
+						return nil, err
+					}
+					return rdr, nil
+				},
+			})
+			if len(pkgs) > 0 || err != nil {
+				return pkgs, err
+			}
+		}
+		return nil, nil
+	}
+}

syft/pkg/cataloger/internal/binutils/branching_matcher_test.go

@@ -0,0 +1,95 @@
+package binutils
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/internal/unionreader"
+)
+
+func Test_BranchingMatcher(t *testing.T) {
+	matchingTest := FileContentsVersionMatcher("", `my-verison:(?<version>\d+\.\d+)`)
+	notMatchingTest := MatchPath("**/not-version*")
+
+	tests := []struct {
+		name                 string
+		matcher              EvidenceMatcher
+		expectedPackageNames []string
+	}{
+		{
+			name: "not matching",
+			matcher: BranchingEvidenceMatcher(
+				Classifier{
+					EvidenceMatcher: MatchAll(
+						notMatchingTest,
+						matchingTest,
+					),
+					Package: "a-pkg",
+				},
+			),
+			expectedPackageNames: nil,
+		},
+		{
+			name: "both match",
+			matcher: BranchingEvidenceMatcher(
+				Classifier{
+					EvidenceMatcher: matchingTest,
+					Package:         "a-pkg",
+				},
+				Classifier{
+					EvidenceMatcher: matchingTest,
+					Package:         "b-pkg",
+				},
+			),
+			expectedPackageNames: []string{"a-pkg"},
+		},
+		{
+			name: "first-does-not-match",
+			matcher: BranchingEvidenceMatcher(
+				Classifier{
+					EvidenceMatcher: MatchAll(
+						notMatchingTest,
+						matchingTest,
+					),
+					Package: "b-pkg",
+				},
+				Classifier{
+					EvidenceMatcher: matchingTest,
+					Package:         "c-pkg",
+				},
+			),
+			expectedPackageNames: []string{"c-pkg"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			resolver := file.NewMockResolverForPaths("test-fixtures/version.txt", "test-fixtures/version-parts.txt")
+			locs, err := resolver.FilesByGlob("**/version.txt")
+			require.NoError(t, err)
+			require.Len(t, locs, 1)
+			loc := locs[0]
+			rdr, err := resolver.FileContentsByLocation(loc)
+			require.NoError(t, err)
+			urdr, err := unionreader.GetUnionReader(rdr)
+			require.NoError(t, err)
+			pkgs, err := test.matcher(Classifier{
+				Package: "a-pkg",
+			}, MatcherContext{
+				Resolver: resolver,
+				Location: loc,
+				GetReader: func(resolver MatcherContext) (unionreader.UnionReader, error) {
+					return urdr, nil
+				},
+			})
+			require.NoError(t, err)
+			var got []string
+			for i := range pkgs {
+				got = append(got, pkgs[i].Name)
+			}
+			require.EqualValues(t, test.expectedPackageNames, got)
+		})
+	}
+}

syft/pkg/cataloger/internal/binutils/classifier.go

@@ -8,11 +8,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"maps"
 	"regexp"
 	"strconv"
 	"strings"
 	"text/template"
 
+	"github.com/bmatcuk/doublestar/v4"
+
 	"github.com/anchore/packageurl-go"
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/log"
@@ -71,7 +74,8 @@ func (cfg Classifier) MarshalJSON() ([]byte, error) {
 	return json.Marshal(m)
 }
 
-// EvidenceMatcher is a function called to catalog Packages that match some sort of evidence
+// EvidenceMatcher is a function called to identify based on some sort of evidence in the filesystem contents.
+// A non-nil return value indicates a successful match, regardless of packages being returned.
 type EvidenceMatcher func(classifier Classifier, context MatcherContext) ([]pkg.Package, error)
 
 type MatcherContext struct {
@@ -80,13 +84,16 @@ type MatcherContext struct {
 	GetReader func(resolver MatcherContext) (unionreader.UnionReader, error)
 }
 
-func EvidenceMatchers(matchers ...EvidenceMatcher) EvidenceMatcher {
+// MatchAny returns a combined evidence matcher that returns results from the first
+// matcher that returns results
+func MatchAny(matchers ...EvidenceMatcher) EvidenceMatcher {
 	return func(classifier Classifier, context MatcherContext) ([]pkg.Package, error) {
 		for _, matcher := range matchers {
 			match, err := matcher(classifier, context)
 			if err != nil {
 				return nil, err
 			}
+			// only return when results
 			if match != nil {
 				return match, nil
 			}
@@ -95,6 +102,23 @@ func EvidenceMatchers(matchers ...EvidenceMatcher) EvidenceMatcher {
 	}
 }
 
+// MatchAll executes all matchers until one returns nil results, only returning the final results
+func MatchAll(matchers ...EvidenceMatcher) EvidenceMatcher {
+	return func(classifier Classifier, context MatcherContext) ([]pkg.Package, error) {
+		var out []pkg.Package
+		for _, matcher := range matchers {
+			match, err := matcher(classifier, context)
+			if match == nil || err != nil {
+				return nil, err
+			}
+			if len(match) > 0 {
+				out = match
+			}
+		}
+		return out, nil
+	}
+}
+
 type ContextualEvidenceMatchers struct {
 	CatalogerName string
 }
@@ -103,8 +127,8 @@ func (c ContextualEvidenceMatchers) FileNameTemplateVersionMatcher(fileNamePatte
 	return FileNameTemplateVersionMatcher(fileNamePattern, contentTemplate, c.CatalogerName)
 }
 
-func (c ContextualEvidenceMatchers) FileContentsVersionMatcher(pattern string) EvidenceMatcher {
-	return FileContentsVersionMatcher(pattern, c.CatalogerName)
+func (c ContextualEvidenceMatchers) FileContentsVersionMatcher(patterns ...string) EvidenceMatcher {
+	return FileContentsVersionMatcher(c.CatalogerName, patterns...)
 }
 
 func FileNameTemplateVersionMatcher(fileNamePattern, contentTemplate, catalogerName string) EvidenceMatcher {
@@ -156,17 +180,36 @@ func FileNameTemplateVersionMatcher(fileNamePattern, contentTemplate, catalogerN
 	}
 }
 
-func FileContentsVersionMatcher(pattern, catalogerName string) EvidenceMatcher {
-	pat := regexp.MustCompile(pattern)
+// FileContentsVersionMatcher will match all provided patterns, extracting named capture groups from each pattern, overwriting earlier results
+func FileContentsVersionMatcher(catalogerName string, patterns ...string) EvidenceMatcher {
+	if len(patterns) == 0 {
+		panic("must specify at least one pattern")
+	}
+	var pats []*regexp.Regexp
+	for _, pattern := range patterns {
+		pats = append(pats, regexp.MustCompile(pattern))
+	}
 	return func(classifier Classifier, context MatcherContext) ([]pkg.Package, error) {
-		contents, err := getReader(context)
-		if err != nil {
-			return nil, fmt.Errorf("unable to get read contents for file: %w", err)
-		}
+		var matchMetadata map[string]string
 
-		matchMetadata, err := internal.MatchNamedCaptureGroupsFromReader(pat, contents)
-		if err != nil {
-			return nil, fmt.Errorf("unable to match version: %w", err)
+		for _, pat := range pats {
+			contents, err := getReader(context)
+			if err != nil {
+				return nil, fmt.Errorf("unable to get read contents for file: %w", err)
+			}
+
+			match, err := internal.MatchNamedCaptureGroupsFromReader(pat, contents)
+			if err != nil {
+				return nil, fmt.Errorf("unable to match version: %w", err)
+			}
+			if match == nil {
+				return nil, nil
+			}
+			if matchMetadata == nil {
+				matchMetadata = match
+			} else {
+				maps.Copy(matchMetadata, match)
+			}
 		}
 
 		// Convert {major: 1, minor: 2, patch: 3} to "1.2.3"
@@ -187,6 +230,10 @@ func FileContentsVersionMatcher(pattern, catalogerName string) EvidenceMatcher {
 
 		p := NewClassifierPackage(classifier, context.Location, matchMetadata, catalogerName)
 		if p == nil {
+			if matchMetadata != nil {
+				// if we had a successful metadata match, but no packages, return a successful match result
+				return []pkg.Package{}, nil
+			}
 			return nil, nil
 		}
 
@@ -194,29 +241,6 @@ func FileContentsVersionMatcher(pattern, catalogerName string) EvidenceMatcher {
 	}
 }
 
-// MatchExcluding tests the provided regular expressions against the file, and if matched, DOES NOT return
-// anything that the matcher would otherwise return
-func MatchExcluding(matcher EvidenceMatcher, contentPatternsToExclude ...string) EvidenceMatcher {
-	var nonMatchPatterns []*regexp.Regexp
-	for _, p := range contentPatternsToExclude {
-		nonMatchPatterns = append(nonMatchPatterns, regexp.MustCompile(p))
-	}
-	return func(classifier Classifier, context MatcherContext) ([]pkg.Package, error) {
-		contents, err := getReader(context)
-		if err != nil {
-			return nil, fmt.Errorf("unable to get read contents for file: %w", err)
-		}
-		matches, err := internal.MatchAnyFromReader(contents, nonMatchPatterns...)
-		if err != nil {
-			return nil, fmt.Errorf("unable to match content: %w", err)
-		}
-		if matches {
-			return nil, nil
-		}
-		return matcher(classifier, context)
-	}
-}
-
 func SharedLibraryLookup(sharedLibraryPattern string, sharedLibraryMatcher EvidenceMatcher) EvidenceMatcher {
 	pat := regexp.MustCompile(sharedLibraryPattern)
 	return func(classifier Classifier, context MatcherContext) (packages []pkg.Package, _ error) {
@@ -234,16 +258,19 @@ func SharedLibraryLookup(sharedLibraryPattern string, sharedLibraryMatcher Evide
 				return nil, err
 			}
 			for _, libraryLocation := range locations {
+				// create a new resolver without the cached context lookup -- this is decidedly a different file
 				newResolver := MatcherContext{
-					Resolver:  context.Resolver,
-					Location:  libraryLocation,
-					GetReader: context.GetReader,
+					Resolver: context.Resolver,
+					Location: libraryLocation,
 				}
-				newResolver.Location = libraryLocation
 				pkgs, err := sharedLibraryMatcher(classifier, newResolver)
 				if err != nil {
 					return nil, err
 				}
+				// not a successful match
+				if pkgs == nil {
+					continue
+				}
 				for _, p := range pkgs {
 					// set the source binary as the first location
 					locationSet := file.NewLocationSet(context.Location)
@@ -260,12 +287,28 @@ func SharedLibraryLookup(sharedLibraryPattern string, sharedLibraryMatcher Evide
 					}
 					packages = append(packages, p)
 				}
+				// return non-nil package results as a successful match indication if the evidence matcher returned a successful match indication
+				if packages == nil {
+					packages = pkgs
+				}
 			}
 		}
 		return packages, nil
 	}
 }
 
+func MatchPath(path string) EvidenceMatcher {
+	if !doublestar.ValidatePattern(path) {
+		panic("invalid pattern")
+	}
+	return func(_ Classifier, context MatcherContext) ([]pkg.Package, error) {
+		if doublestar.MatchUnvalidated(path, context.Location.RealPath) {
+			return []pkg.Package{}, nil // return non-nil
+		}
+		return nil, nil
+	}
+}
+
 func getReader(context MatcherContext) (unionreader.UnionReader, error) {
 	if context.GetReader != nil {
 		return context.GetReader(context)

syft/pkg/cataloger/internal/binutils/classifier_package.go

@@ -1,9 +1,12 @@
 package binutils
 
 import (
+	"bytes"
 	"reflect"
+	"text/template"
 
 	"github.com/anchore/packageurl-go"
+	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/cpe"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
@@ -21,8 +24,8 @@ func NewClassifierPackage(classifier Classifier, location file.Location, matchMe
 
 	var cpes []cpe.CPE
 	for _, c := range classifier.CPEs {
-		c.Attributes.Version = version
-		c.Attributes.Update = update
+		c.Attributes.Version = templatedUpdate(c.Attributes.Version, matchMetadata, version)
+		c.Attributes.Update = templatedUpdate(c.Attributes.Update, matchMetadata, update)
 		cpes = append(cpes, c)
 	}
 
@@ -55,3 +58,28 @@ func NewClassifierPackage(classifier Classifier, location file.Location, matchMe
 
 	return &p
 }
+
+func templatedUpdate(providedValue string, matchMetadata map[string]string, defaultValue string) string {
+	// if no template provided, just use the value directly
+	if providedValue == "" {
+		return defaultValue
+	}
+	// support templated updates
+	t, err := template.New("").Option("missingkey=zero").Parse(providedValue)
+	if err != nil {
+		log.Debugf("unable to parse classifier template=%q : %w", providedValue, err)
+	} else {
+		update := bytes.Buffer{}
+		err = t.Execute(&update, matchMetadata)
+		if err != nil {
+			log.Debugf("unable to render template: %w", err)
+		} else {
+			// only use the template result if it's non-empty
+			providedValue = update.String()
+			if providedValue != "" {
+				return providedValue
+			}
+		}
+	}
+	return defaultValue
+}

syft/pkg/cataloger/internal/binutils/classifier_test.go

@@ -27,7 +27,7 @@ func Test_ClassifierCPEs(t *testing.T) {
 			classifier: Classifier{
 				Package:         "some-app",
 				FileGlob:        "**/version.txt",
-				EvidenceMatcher: FileContentsVersionMatcher(`(?m)my-verison:(?P<version>[0-9.]+)`, "cataloger-name"),
+				EvidenceMatcher: FileContentsVersionMatcher("cataloger-name", `(?m)my-verison:(?P<version>[0-9.]+)`),
 				CPEs:            []cpe.CPE{},
 			},
 			cpes: nil,
@@ -38,7 +38,7 @@ func Test_ClassifierCPEs(t *testing.T) {
 			classifier: Classifier{
 				Package:         "some-app",
 				FileGlob:        "**/version.txt",
-				EvidenceMatcher: FileContentsVersionMatcher(`(?m)my-verison:(?P<version>[0-9.]+)`, "cataloger-name"),
+				EvidenceMatcher: FileContentsVersionMatcher("cataloger-name", `(?m)my-verison:(?P<version>[0-9.]+)`),
 				CPEs: []cpe.CPE{
 					cpe.Must("cpe:2.3:a:some:app:*:*:*:*:*:*:*:*", cpe.GeneratedSource),
 				},
@@ -53,7 +53,7 @@ func Test_ClassifierCPEs(t *testing.T) {
 			classifier: Classifier{
 				Package:         "some-app",
 				FileGlob:        "**/version.txt",
-				EvidenceMatcher: FileContentsVersionMatcher(`(?m)my-verison:(?P<version>[0-9.]+)`, "cataloger-name"),
+				EvidenceMatcher: FileContentsVersionMatcher("cataloger-name", `(?m)my-verison:(?P<version>[0-9.]+)`),
 				CPEs: []cpe.CPE{
 					cpe.Must("cpe:2.3:a:some:app:*:*:*:*:*:*:*:*", cpe.GeneratedSource),
 					cpe.Must("cpe:2.3:a:some:apps:*:*:*:*:*:*:*:*", cpe.GeneratedSource),
@@ -70,7 +70,7 @@ func Test_ClassifierCPEs(t *testing.T) {
 			classifier: Classifier{
 				Package:         "some-app",
 				FileGlob:        "**/version-parts.txt",
-				EvidenceMatcher: FileContentsVersionMatcher(`(?m)\x00(?P<major>[0-9.]+)\x00(?P<minor>[0-9.]+)\x00(?P<patch>[0-9.]+)\x00`, "cataloger-name"),
+				EvidenceMatcher: FileContentsVersionMatcher("cataloger-name", `(?m)\x00(?P<major>[0-9.]+)\x00(?P<minor>[0-9.]+)\x00(?P<patch>[0-9.]+)\x00`),
 				CPEs:            []cpe.CPE{},
 			},
 			cpes: nil,
@@ -113,7 +113,7 @@ func TestClassifier_MarshalJSON(t *testing.T) {
 			classifier: Classifier{
 				Class:           "class",
 				FileGlob:        "glob",
-				EvidenceMatcher: FileContentsVersionMatcher(".thing", "cataloger-name"),
+				EvidenceMatcher: FileContentsVersionMatcher("cataloger-name", ".thing"),
 				Package:         "pkg",
 				PURL: packageurl.PackageURL{
 					Type:       "type",
@@ -168,7 +168,7 @@ func TestFileContentsVersionMatcher(t *testing.T) {
 			mockGetContent := func(context MatcherContext) (unionreader.UnionReader, error) {
 				return unionreader.GetUnionReader(io.NopCloser(bytes.NewBufferString(tt.data)))
 			}
-			fn := FileContentsVersionMatcher(tt.pattern, "cataloger-name")
+			fn := FileContentsVersionMatcher("cataloger-name", tt.pattern)
 			p, err := fn(Classifier{}, MatcherContext{
 				GetReader: mockGetContent,
 			})