oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-01 23:16:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

chore: revert custom index in favor of key_Split

Browse Source 
Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
Keith Zantow
2026-09-20 11:22:10 -04:00
parent 99615b7dfa
commit 53cd4254e4
81 changed files with 7461 additions and 6624 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/pkg/cataloger/java/archive_contents_cataloger.go
-129
View File
@@ -1,129 +0,0 @@
package java
import (
"context"
"fmt"
"github.com/bmatcuk/doublestar/v4"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/java/internal/maven"
)
const ArchiveContentsCatalogerName = "java-archive-contents-cataloger"
// NewArchiveContentsCataloger returns a java archive cataloger for the world after extraction: it
// reads an archive's already-extracted filesystem rather than opening the archive itself.
//
// It is a second cataloger rather than a mode of the first because the two answer different
// questions. The archive-reading cataloger asks "is this file a jar?" of every file in a scan; this
// one asks "is the filesystem I have been handed a jar?", which is the only form the question can
// take once something else owns extraction. That inversion is also why it cannot be a parameter:
// when the archive cataloger task hands a jar's contents to the sub-pipeline, the jar file is one
// level up and already behind us.
//
// What it does not do is recurse. The task that extracted this archive will meet the archives inside
// it on its own, and one mechanism owning recursion is the whole point - see
// syft/java-archive-nesting#single-owner-of-archive-recursion.
func NewArchiveContentsCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger {
return archiveContentsCataloger{cfg: cfg}
}
type archiveContentsCataloger struct {
cfg ArchiveCatalogerConfig
}
func (c archiveContentsCataloger) Name() string {
return ArchiveContentsCatalogerName
}
// Catalog reports the java package of the archive whose contents this resolver holds, if it is a java
// archive at all.
//
// Nothing outside an archive is cataloged. Without a traversal on the context there is no archive to
// describe - the identity of a java archive lives half in the file (its name, its digests) and half
// in its contents (the manifest, the poms), and the file half arrives only on the traversal.
func (c archiveContentsCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
trav := archive.TraversalFromContext(ctx)
if trav == nil {
return nil, nil, nil
}
if !isJavaArchiveName(trav.Location.Path()) {
// the same rule the archive-reading cataloger applies through its globs: a .zip is an archive
// but it is not a java archive, and its contents are not a jar's contents
return nil, nil, nil
}
parser, err := newArchiveContentsParser(trav, resolver, c.cfg)
if err != nil {
return nil, nil, err
}
pkgs, relationships, err := parser.parse(ctx, nil)
if err != nil {
// attributed to the archive rather than returned bare, the way the generic cataloger attributes
// a parser's failure to the file it was parsing. An archive that turns out to hold no java
// package at all reports exactly that, and the scan carries on.
return pkgs, relationships, unknown.New(trav.Location, err)
}
return pkgs, relationships, nil
}
// newArchiveContentsParser builds the same parser the archive-reading cataloger uses, over the
// archive's extracted filesystem instead of over the archive file. Identity comes from the traversal:
// the location and virtual path it already carries, the archive's filename for the name and version
// fallbacks, and the digests taken when the archive was extracted.
func newArchiveContentsParser(trav *archive.Traversal, resolver file.Resolver, cfg ArchiveCatalogerConfig) (*archiveParser, error) {
if resolver == nil {
return nil, fmt.Errorf("no resolver for the contents of %q", trav.VirtualPath)
}
return &archiveParser{
entries: newResolverEntries(resolver, trav.Digests),
location: trav.Location,
// the colon-delimited chain the task composed, which is the identity already-published SBOMs
// carry for a nested java package
virtualPath: trav.VirtualPath,
fileInfo: newJavaArchiveFilename(archiveFileNameOf(trav)),
// recursion belongs to the task that extracted this archive
detectNested: false,
cfg: cfg,
maven: maven.NewResolver(nil, cfg.mavenConfig()),
}, nil
}
// archiveFileNameOf returns the archive's own file name, which is where a jar's name and version come
// from when its manifest does not say. The last element of the virtual path rather than of the real
// path, so a chain and a plain scan agree.
func archiveFileNameOf(trav *archive.Traversal) string {
name := trav.VirtualPath
if name == "" {
name = trav.Location.Path()
}
for i := len(name) - 1; i >= 0; i-- {
if name[i] == ':' || name[i] == '/' {
return name[i+1:]
}
}
return name
}
// isJavaArchiveName reports whether the path is one of the java archive formats, by the same glob
// list the archive-reading cataloger registers against.
func isJavaArchiveName(path string) bool {
for _, glob := range archiveFormatGlobs {
if ok, err := doublestar.Match(glob, path); err == nil && ok {
return true
}
if ok, err := doublestar.Match(glob, "/"+path); err == nil && ok {
return true
}
}
log.WithFields("path", path).Trace("archive contents are not a java archive")
return false
}
syft/pkg/cataloger/java/archive_entries.go
+39 -52
View File
@@ -16,30 +16,28 @@ import (
// archiveEntries is where an archive parser reads a java archive's entries from.
//
// The parser needs three things of an archive and nothing else: which entries match a glob, the
// contents of some of them, and the digest of the archive as a whole. Everything else it does -
// which manifest wins, how pom.properties decides identity, license discovery - is logic over those
// answers. Naming the three makes them replaceable, which is what lets the same logic run over an
// archive file and over an archive that has already been extracted to a filesystem.
// A parser needs three things of an archive: which entries match a glob, the contents of some of
// them, and the digest of the archive as a whole. Everything else - which manifest wins, how
// pom.properties decides identity, license discovery - is logic over those answers, so naming the
// three lets the same logic run over an archive file and over one already extracted to a filesystem.
//
// Entry names are archive-relative with no leading slash, matching zip entry names, because they
// become map keys and pom paths that identity is derived from. An implementation over a filesystem
// has to strip the root to match.
// Entry names are archive-relative with no leading slash, matching zip entry names, since they become
// the map keys and pom paths identity is derived from. An implementation over a filesystem must strip
// the root to match.
type archiveEntries interface {
// glob returns the names of the entries matching any of the patterns.
glob(caseInsensitive bool, patterns ...string) []string
// contents reads the named entries, returning a map keyed by the same names.
contents(ctx context.Context, names ...string) (map[string]string, error)
// digests returns the digests of the archive as a whole, which is a property of the archive file
// rather than of anything inside it.
// digests returns the digests of the archive as a whole, a property of the archive file rather
// than of anything inside it.
digests(ctx context.Context) ([]file.Digest, error)
}
// zipEntries reads entries out of a java archive on disk. This is the original source, kept for
// scans where nothing has extracted the archive: it copies the archive to a temp file, reads its
// central directory once, and opens it again for each set of entries it is asked for.
// zipEntries reads entries out of a java archive on disk, for scans where nothing has already
// extracted it. It copies the archive to a temp file, reads its central directory once, and reopens
// it for each set of entries it is asked for.
type zipEntries struct {
archivePath string
fileManifest intFile.ZipFileManifest
@@ -65,23 +63,18 @@ func (z *zipEntries) digests(ctx context.Context) ([]file.Digest, error) {
return getDigestsFromArchive(ctx, z.archivePath)
}
// resolverEntries reads entries out of an archive that has already been extracted and indexed - the
// filesystem the archive cataloger task builds for every archive it enters.
//
// Nothing is copied and nothing is re-opened: the entries are files in an indexed resolver, and the
// archive's own digests were taken by whoever extracted it, since the archive file itself is not
// present in its own contents.
// resolverEntries reads entries out of an archive already extracted and indexed by the archive
// cataloger task. Nothing is copied or reopened. The archive's own digests come from whoever
// extracted it, since the archive file is not present in its own contents.
type resolverEntries struct {
resolver file.Resolver
archiveDigests []file.Digest
locationsByName map[string]file.Location
resolver file.Resolver
archiveDigests []file.Digest
}
func newResolverEntries(resolver file.Resolver, archiveDigests []file.Digest) *resolverEntries {
return &resolverEntries{
resolver: resolver,
archiveDigests: archiveDigests,
locationsByName: make(map[string]file.Location),
resolver: resolver,
archiveDigests: archiveDigests,
}
}
@@ -90,10 +83,9 @@ func (r *resolverEntries) glob(caseInsensitive bool, patterns ...string) []strin
return r.globCaseInsensitive(patterns...)
}
// deduplicated within this call and not across calls: the parser globs the same pattern more than
// once (identity resolution asks for the poms, and so does aux package discovery), and a source
// that remembered what it had already returned would hand back nothing the second time. That
// silently cost every package derived from a bundled pom.properties.
// deduplicated within this call only: the parser globs the same pattern more than once (identity
// resolution and aux package discovery both ask for the poms), so remembering across calls would
// return nothing the second time and lose every package derived from a bundled pom.properties
seen := strset.New()
var names []string
for _, pattern := range patterns {
@@ -104,7 +96,6 @@ func (r *resolverEntries) glob(caseInsensitive bool, patterns ...string) []strin
}
for _, location := range locations {
name := strings.TrimPrefix(location.Path(), "/")
r.locationsByName[name] = location
if seen.Has(name) {
continue
}
@@ -115,10 +106,9 @@ func (r *resolverEntries) glob(caseInsensitive bool, patterns ...string) []strin
return names
}
// globCaseInsensitive matches the way the zip-backed source does, by lowercasing both sides. The
// resolver's own glob is case sensitive, so this lists the archive's files once and matches them
// here. License discovery is the only caller that asks, and what it costs is one listing of a
// filesystem holding one archive's entries.
// globCaseInsensitive matches the way zipEntries does, by lowercasing both sides. The resolver's own
// glob is case sensitive, so this lists the archive's files once and matches them here. License
// discovery is the only caller.
func (r *resolverEntries) globCaseInsensitive(patterns ...string) []string {
all, err := r.resolver.FilesByGlob("**/*")
if err != nil {
@@ -130,16 +120,13 @@ func (r *resolverEntries) globCaseInsensitive(patterns ...string) []string {
for _, pattern := range patterns {
lowered := strings.ToLower(pattern)
for _, location := range all {
// matched with a leading slash, the way the zip-backed source normalizes an entry before
// matching, because the patterns are written that way ("/META-INF/*"). An extracted
// archive's resolver reports paths without one - they are entry names, not absolute paths -
// so matching them raw silently matches nothing, which cost the license found in every
// jar that carries one as a file.
// matched with a leading slash, since the patterns are written that way ("/META-INF/*") and
// zipEntries normalizes the same way; an extracted archive's resolver reports names without
// one, so matching raw would match nothing
name := strings.TrimPrefix(location.Path(), "/")
if ok, matchErr := doublestar.Match(lowered, "/"+strings.ToLower(name)); matchErr != nil || !ok {
continue
}
r.locationsByName[name] = location
if seen.Has(name) {
continue
}
@@ -153,13 +140,14 @@ func (r *resolverEntries) globCaseInsensitive(patterns ...string) []string {
func (r *resolverEntries) contents(_ context.Context, names ...string) (map[string]string, error) {
out := make(map[string]string, len(names))
for _, name := range names {
location, ok := r.locationsByName[name]
if !ok {
// only names this source handed out can be read back; a name from anywhere else is a bug
// rather than a missing file
return nil, fmt.Errorf("entry %q was not found by any glob against this archive", name)
locations, err := r.resolver.FilesByPath(name)
if err != nil {
return nil, fmt.Errorf("unable to look up archive entry %q: %w", name, err)
}
contents, err := r.read(location)
if len(locations) == 0 {
return nil, fmt.Errorf("entry %q is not in this archive", name)
}
contents, err := r.read(locations[0])
if err != nil {
return nil, fmt.Errorf("unable to read archive entry %q: %w", name, err)
}
@@ -172,12 +160,11 @@ func (r *resolverEntries) digests(_ context.Context) ([]file.Digest, error) {
return r.archiveDigests, nil
}
// read reads one entry's content to the end and closes the stream. The parser wants entry contents as
// strings, since every consumer of them parses text.
// read reads one entry's content to the end and closes the stream, returning a string because every
// consumer parses text.
//
// The copy is bounded the same way the zip-backed source bounds its own: an entry of an extracted
// archive is still attacker-supplied content, and reading it into a string is exactly the allocation
// a decompression bomb is aiming at.
// The copy is bounded the way zipEntries bounds its own: an entry of an extracted archive is still
// attacker-supplied, and reading it into a string is the allocation a decompression bomb targets.
func (r *resolverEntries) read(location file.Location) (string, error) {
reader, err := r.resolver.FileContentsByLocation(location)
if err != nil {
syft/pkg/cataloger/java/archive_parser.go
+30 -53
View File
@@ -17,10 +17,8 @@ import (
"golang.org/x/exp/maps"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/archive"
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/tmpdir"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
@@ -58,18 +56,27 @@ var javaArchiveHashes = []crypto.Hash{
crypto.SHA1,
}
// archiveParser reads one java archive, whichever source it came from: everything it does is logic
// over an archiveSource's entries and identity, so it does not know whether this cataloger opened an
// archive file or the archive cataloger task already extracted one.
type archiveParser struct {
entries archiveEntries
location file.Location
virtualPath string
archivePath string
contentPath string
fileInfo archiveFilename
archiveSource
detectNested bool
cfg ArchiveCatalogerConfig
maven *maven.Resolver
}
// newArchiveParser builds the parser for one archive. detectNested is whether this parser recurses
// into the java archives inside this one, which only a file-backed source can do.
func newArchiveParser(src archiveSource, detectNested bool, cfg ArchiveCatalogerConfig) *archiveParser {
return &archiveParser{
archiveSource: src,
detectNested: detectNested,
cfg: cfg,
maven: maven.NewResolver(nil, cfg.mavenConfig()),
}
}
type genericArchiveParserAdapter struct {
cfg ArchiveCatalogerConfig
}
@@ -85,9 +92,9 @@ func (gap genericArchiveParserAdapter) parseJavaArchive(ctx context.Context, _ f
// processJavaArchive processes an archive for java contents, returning all Java libraries and nested archives
func (gap genericArchiveParserAdapter) processJavaArchive(ctx context.Context, reader file.LocationReadCloser, parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) {
// when the generic archive cataloger task owns recursion into archives, this parser must not
// also unarchive nested archives, otherwise they would be cataloged twice
parser, cleanupFn, err := newJavaArchiveParser(ctx, reader, !gap.cfg.nestedArchivesHandledExternally(), gap.cfg)
// when the archive cataloger task owns recursion, unarchiving here would catalog nested archives
// twice
parser, cleanupFn, err := newJavaArchiveParser(ctx, reader, !gap.cfg.NestedArchivesHandledExternally, gap.cfg)
// note: even on error, we should always run cleanup functions
defer cleanupFn()
if err != nil {
@@ -104,43 +111,14 @@ func uniquePkgKey(groupID string, p *pkg.Package) string {
return fmt.Sprintf("%s|%s|%s", groupID, p.Name, p.Version)
}
// newJavaArchiveParser returns a new java archive parser object for the given archive. Can be configured to discover
// and parse nested archives or ignore them.
// newJavaArchiveParser returns a new java archive parser for an archive file this cataloger opens
// itself. Can be configured to discover and parse nested archives or ignore them.
func newJavaArchiveParser(ctx context.Context, reader file.LocationReadCloser, detectNested bool, cfg ArchiveCatalogerConfig) (*archiveParser, func(), error) {
// establish the full virtual path of this archive: when the generic archive cataloger task drove
// us here, the context traversal supplies the containing archive chain; otherwise the reader
// path already carries any colon-delimited nesting from this cataloger's own recursion
virtualPath := archive.TraversalFromContext(ctx).VirtualPathOf(reader.Path())
// fetch the last element of the virtual path
virtualElements := strings.Split(virtualPath, ":")
currentFilepath := virtualElements[len(virtualElements)-1]
td := tmpdir.FromContext(ctx)
if td == nil {
return nil, func() {}, fmt.Errorf("no temp dir factory in context")
}
contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(td, currentFilepath, reader)
if err != nil {
return nil, cleanupFn, fmt.Errorf("unable to process java archive: %w", err)
}
entries, err := newZipEntries(ctx, archivePath)
src, cleanupFn, err := newFileArchiveSource(ctx, reader)
if err != nil {
return nil, cleanupFn, err
}
return &archiveParser{
entries: entries,
location: reader.Location,
virtualPath: virtualPath,
archivePath: archivePath,
contentPath: contentPath,
fileInfo: newJavaArchiveFilename(currentFilepath),
detectNested: detectNested,
cfg: cfg,
maven: maven.NewResolver(nil, cfg.mavenConfig()),
}, cleanupFn, nil
return newArchiveParser(src, detectNested, cfg), cleanupFn, nil
}
// parse the loaded archive and return all packages found.
@@ -199,9 +177,9 @@ func (j *archiveParser) parse(ctx context.Context, parentPkg *pkg.Package) ([]pk
}
pkgs = append(pkgs, nestedPkgs...)
relationships = append(relationships, nestedRelationships...)
} else if !j.cfg.nestedArchivesHandledExternally() {
// nested archives will not be cataloged by anything: this parser is not recursing and the
// generic archive cataloger task is disabled
} else if !j.cfg.NestedArchivesHandledExternally {
// nothing will catalog these: this parser is not recursing and the archive cataloger task is
// disabled, so record them as unknowns
// .jar and .war files are present in archives, are others? or generally just consider them top-level?
nestedArchives := j.entries.glob(true, "**/*.jar", "**/*.war")
if len(nestedArchives) > 0 {
@@ -252,7 +230,7 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package,
// parse the manifest file into a rich object
manifestContents := contents[manifestMatches[0]]
manifest, err := parseJavaManifest(j.archivePath, strings.NewReader(manifestContents))
manifest, err := parseJavaManifest(j.virtualPath, strings.NewReader(manifestContents))
if err != nil {
log.Debugf("failed to parse java manifest (%s): %+v", j.location, err)
return nil, nil
@@ -641,7 +619,7 @@ func (j *archiveParser) versionFromPropertiesFile(ctx context.Context, manifest
return ""
}
contents, err := intFile.ContentsFromZip(ctx, j.archivePath, matches...)
contents, err := j.entries.contents(ctx, matches...)
if err != nil {
log.Debugf("unable to extract version.properties (%s): %v", j.location, err)
return ""
@@ -663,10 +641,9 @@ func (j *archiveParser) versionFromPropertiesFile(ctx context.Context, manifest
return ""
}
// discoverPkgsFromNestedArchives recurses into the java archives inside this one. It exists only for
// the zip-backed source: recursion needs an archive file to unzip, and the resolver-backed source has
// no file - by the time it runs, the archive cataloger task has already extracted everything and will
// meet those nested archives itself.
// discoverPkgsFromNestedArchives recurses into the java archives inside this one. Only zipEntries
// applies, since recursion needs an archive file to unzip; with resolverEntries the archive cataloger
// task has already extracted everything and meets those nested archives itself.
func (j *archiveParser) discoverPkgsFromNestedArchives(ctx context.Context, parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) {
zipped, ok := j.entries.(*zipEntries)
if !ok {
syft/pkg/cataloger/java/archive_parser_traversal_test.go
+48 -57
View File
@@ -20,26 +20,6 @@ import (
"github.com/anchore/syft/syft/source/directorysource"
)
// makeTestZip builds a zip in memory so the parser can be exercised without the
// java-toolchain-built fixtures.
func makeTestZip(t *testing.T, entries map[string][]byte) []byte {
t.Helper()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for name, content := range entries {
w, err := zw.Create(name)
require.NoError(t, err)
_, err = w.Write(content)
require.NoError(t, err)
}
require.NoError(t, zw.Close())
return buf.Bytes()
}
func testManifest(title, version string) []byte {
return fmt.Appendf(nil, "Manifest-Version: 1.0\nImplementation-Title: %s\nImplementation-Version: %s\n", title, version)
}
func Test_virtualPathFromArchiveTraversal(t *testing.T) {
jarBytes := makeTestZip(t, map[string][]byte{
"META-INF/MANIFEST.MF": testManifest("example-app", "1.0.0"),
@@ -71,10 +51,8 @@ func Test_virtualPathFromArchiveTraversal(t *testing.T) {
t.Run("traversal prepends containing archive chain", func(t *testing.T) {
trav := &archive.Traversal{
Location: file.NewLocation("/app.war"),
VirtualPath: "/app.war",
FileSystemID: "app.war",
Depth: 1,
Location: file.NewLocation("/app.war"),
VirtualPath: "/app.war",
}
p := parse(t, trav, "/WEB-INF/lib/example-app-1.0.0.jar")
metadata, ok := p.Metadata.(pkg.JavaArchive)
@@ -86,12 +64,9 @@ func Test_virtualPathFromArchiveTraversal(t *testing.T) {
})
t.Run("multi-level traversal chains all parents", func(t *testing.T) {
outer := &archive.Traversal{VirtualPath: "/dist.zip", FileSystemID: "dist.zip", Depth: 1}
outer := &archive.Traversal{VirtualPath: "/dist.zip"}
inner := &archive.Traversal{
VirtualPath: outer.VirtualPathOf("/app.war"),
FileSystemID: "dist.zip/app.war",
Depth: 2,
Parent: outer,
VirtualPath: outer.VirtualPathOf("/app.war"),
}
p := parse(t, inner, "/WEB-INF/lib/example-app-1.0.0.jar")
metadata, ok := p.Metadata.(pkg.JavaArchive)
@@ -100,12 +75,12 @@ func Test_virtualPathFromArchiveTraversal(t *testing.T) {
})
}
// Test_virtualPathFromArchiveTraversal_colonInEntryPath is the java-cataloger-level counterpart
// of internal/archive's escaping tests: a package reached through an entry whose own path
// contains a colon must carry that colon escaped as %3A end to end in its virtual path
// (decisions.md#colon-in-an-entry-path-is-escaped), and the maven-derived
// ":groupID:artifactID" suffix that distinguishes a pom.properties package from its containing
// archive must still be appended using real, unescaped delimiters.
// Test_virtualPathFromArchiveTraversal_colonInEntryPath is the java-cataloger-level counterpart of
// internal/archive's escaping tests: a package reached through an entry whose path contains a colon
// must carry it escaped as %3A end to end in its virtual path
// (decisions.md#colon-in-an-entry-path-is-escaped), while the maven-derived ":groupID:artifactID"
// suffix that distinguishes a pom.properties package from its containing archive is still appended
// with real, unescaped delimiters.
func Test_virtualPathFromArchiveTraversal_colonInEntryPath(t *testing.T) {
jarBytes := makeTestZip(t, map[string][]byte{
"META-INF/MANIFEST.MF": testManifest("outer-app", "1.0.0"),
@@ -115,10 +90,8 @@ func Test_virtualPathFromArchiveTraversal_colonInEntryPath(t *testing.T) {
})
trav := &archive.Traversal{
Location: file.NewLocation("/outer.zip"),
VirtualPath: "outer.zip",
FileSystemID: "outer.zip",
Depth: 1,
Location: file.NewLocation("/outer.zip"),
VirtualPath: "outer.zip",
}
ctx := archive.WithTraversal(pkgtest.Context(t), trav)
@@ -201,12 +174,12 @@ func Test_nestedArchiveOwnershipSwitch(t *testing.T) {
})
t.Run("java's own archive-search depth does not hand off recursion", func(t *testing.T) {
// a library consumer setting java's squash-inlined MaxDepth must neither enable nor
// disable nested archive cataloging: that knob is Archive.MaxDepth on CreateSBOMConfig.
// Getting this wrong silently drops nested jars for every consumer that is not the CLI.
// a library consumer setting java's squash-inlined MaxDepth must neither enable nor disable
// nested archive cataloging: that knob is Archive.MaxDepth on CreateSBOMConfig. Getting it
// wrong silently drops nested jars for every consumer that is not the CLI.
cfg := DefaultArchiveCatalogerConfig()
cfg.ArchiveSearchConfig = cfg.ArchiveSearchConfig.WithMaxDepth(2)
require.False(t, cfg.nestedArchivesHandledExternally())
require.False(t, cfg.NestedArchivesHandledExternally)
pkgs := parse(t, cfg)
require.Len(t, pkgs, 2, "this cataloger must still recurse itself")
@@ -214,12 +187,11 @@ func Test_nestedArchiveOwnershipSwitch(t *testing.T) {
}
func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) {
// the wrapped-archive parsers exist only because the generic archive cataloger did not. When it
// is enabled it extracts zip and tar containers itself and this cataloger meets the same JARs
// one nesting level down, so registering them would double the work.
// the wrapped-archive parsers exist only because the archive cataloger did not. When it is enabled
// it extracts zip and tar containers itself and this cataloger meets the same jars one nesting level
// down, so registering them would double the work.
//
// asserted through the globs the cataloger actually queries, since that is the observable
// consequence of a parser being registered
// Asserted through the globs the cataloger queries, the observable consequence of a registration.
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "placeholder.txt"), []byte("x"), 0o600))
@@ -243,11 +215,11 @@ func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) {
cfg.NestedArchivesHandledExternally = true
observed := queriedGlobs(t, cfg)
// outside a traversal the cataloger opens no files at all: the task brings every archive back
// as an extracted filesystem, and each is described there
assert.False(t, observed.ObservedPathQuery("**/*.zip"), "no generic zip parser may be registered")
assert.False(t, observed.ObservedPathQuery("**/*.tar"), "no generic tar parser may be registered")
// the JAR-family globs are still registered: this cataloger is still the java parser
assert.True(t, observed.ObservedPathQuery("**/*.jar"))
assert.False(t, observed.ObservedPathQuery("**/*.jar"), "archive files are the task's to open")
})
t.Run("wrapped parsers stay registered when the feature is off", func(t *testing.T) {
@@ -272,11 +244,10 @@ func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) {
}
func Test_mavenVirtualPathSuffixUnderTraversal(t *testing.T) {
// a package derived from a pom.properties whose artifact does not match the containing archive
// gets a trailing ":<groupID>:<artifactID>" appended to the archive's virtual path. That suffix
// form is in published SBOMs, and once the archive cataloger drives the parser the base it is
// appended to comes from the traversal rather than from the reader path - so this asserts the
// two compose rather than one clobbering the other.
// a package derived from a pom.properties whose artifact does not match the containing archive gets
// a trailing ":<groupID>:<artifactID>" appended to the archive's virtual path. That suffix form is in
// published SBOMs, and once the archive cataloger drives the parser the base comes from the traversal
// rather than the reader path - so the two must compose rather than one clobbering the other.
jarBytes := makeTestZip(t, map[string][]byte{
"META-INF/MANIFEST.MF": testManifest("example-app", "1.0.0"),
"META-INF/maven/com.example.other/other-lib/pom.properties": []byte(
@@ -320,8 +291,28 @@ func Test_mavenVirtualPathSuffixUnderTraversal(t *testing.T) {
})
t.Run("under traversal: suffix hangs off the archive chain", func(t *testing.T) {
pkgs := parse(t, &archive.Traversal{VirtualPath: "app.war", Depth: 1}, "/WEB-INF/lib/example-app-1.0.0.jar")
pkgs := parse(t, &archive.Traversal{VirtualPath: "app.war"}, "/WEB-INF/lib/example-app-1.0.0.jar")
assert.Equal(t, "app.war:WEB-INF/lib/example-app-1.0.0.jar:com.example.other:other-lib",
virtualPathOf(t, pkgs, "other-lib"))
})
}
// makeTestZip builds a zip in memory so the parser can be exercised without the java-toolchain-built
// fixtures.
func makeTestZip(t *testing.T, entries map[string][]byte) []byte {
t.Helper()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for name, content := range entries {
w, err := zw.Create(name)
require.NoError(t, err)
_, err = w.Write(content)
require.NoError(t, err)
}
require.NoError(t, zw.Close())
return buf.Bytes()
}
func testManifest(title, version string) []byte {
return fmt.Appendf(nil, "Manifest-Version: 1.0\nImplementation-Title: %s\nImplementation-Version: %s\n", title, version)
}
syft/pkg/cataloger/java/archive_source.go
+119
View File
@@ -0,0 +1,119 @@
package java
import (
"context"
"fmt"
"strings"
"github.com/bmatcuk/doublestar/v4"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/tmpdir"
"github.com/anchore/syft/syft/file"
)
// archiveSource is one java archive to describe: where its entries are read from, and the identity
// that does not live inside them - the archive's own location, the colon-delimited virtual path
// SBOMs carry, and the file name that name and version fall back to.
//
// Two of these exist because two things can own extraction. When this cataloger owns it, the archive
// is a file it copies to a temp dir and reads as a zip. When the archive cataloger task owns it, the
// archive has already been extracted and indexed, and arrives as a resolver plus the traversal
// describing the file it came from. Everything past this point - which manifest wins, how
// pom.properties decides identity, license discovery - is logic over a source, so it runs unchanged
// either way.
type archiveSource struct {
entries archiveEntries
location file.Location
virtualPath string
fileInfo archiveFilename
// contentPath is the directory nested archives are extracted into. Set only when this cataloger
// opened the archive itself: recursing needs somewhere to unzip into, and an already-extracted
// archive has none. The archive file itself is held by zipEntries.
contentPath string
}
// newFileArchiveSource copies the archive to a temp dir and reads its central directory, for a scan
// where nothing has already extracted it. The returned cleanup must run even when an error is
// returned.
func newFileArchiveSource(ctx context.Context, reader file.LocationReadCloser) (archiveSource, func(), error) {
// the full virtual path of this archive: when the archive cataloger task drove us here, the context
// traversal supplies the containing chain; otherwise the reader path already carries any
// colon-delimited nesting from this cataloger's own recursion
virtualPath := archive.TraversalFromContext(ctx).VirtualPathOf(reader.Path())
// fetch the last element of the virtual path
virtualElements := strings.Split(virtualPath, ":")
currentFilepath := virtualElements[len(virtualElements)-1]
td := tmpdir.FromContext(ctx)
if td == nil {
return archiveSource{}, func() {}, fmt.Errorf("no temp dir factory in context")
}
contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(td, currentFilepath, reader)
if err != nil {
return archiveSource{}, cleanupFn, fmt.Errorf("unable to process java archive: %w", err)
}
entries, err := newZipEntries(ctx, archivePath)
if err != nil {
return archiveSource{}, cleanupFn, err
}
return archiveSource{
entries: entries,
location: reader.Location,
virtualPath: virtualPath,
fileInfo: newJavaArchiveFilename(currentFilepath),
contentPath: contentPath,
}, cleanupFn, nil
}
// newExtractedArchiveSource describes the archive whose extracted contents the given resolver holds.
// Nothing is copied or reopened, and identity comes from the traversal: the location, the chain the
// archive cataloger task composed, the file name, and the digests taken at extraction - none of which
// are inside the archive.
func newExtractedArchiveSource(trav *archive.Traversal, resolver file.Resolver) (archiveSource, error) {
if resolver == nil {
return archiveSource{}, fmt.Errorf("no resolver for the contents of %q", trav.VirtualPath)
}
return archiveSource{
entries: newResolverEntries(resolver, trav.Digests),
location: trav.Location,
virtualPath: trav.VirtualPath,
fileInfo: newJavaArchiveFilename(archiveFileNameOf(trav)),
}, nil
}
// archiveFileNameOf returns the archive's own file name, where a jar's name and version come from
// when its manifest does not say. It takes the last element of the virtual path rather than the real
// path, so a nested chain and a plain scan agree.
func archiveFileNameOf(trav *archive.Traversal) string {
name := trav.VirtualPath
if name == "" {
name = trav.Location.Path()
}
for i := len(name) - 1; i >= 0; i-- {
if name[i] == ':' || name[i] == '/' {
return name[i+1:]
}
}
return name
}
// isJavaArchiveName reports whether the path is one of the java archive formats, by the same globs
// this cataloger registers against when it is the one opening files.
func isJavaArchiveName(path string) bool {
for _, glob := range archiveFormatGlobs {
if ok, err := doublestar.Match(glob, path); err == nil && ok {
return true
}
if ok, err := doublestar.Match(glob, "/"+path); err == nil && ok {
return true
}
}
log.WithFields("path", path).Trace("archive contents are not a java archive")
return false
}
syft/pkg/cataloger/java/archive_contents_cataloger_test.go → syft/pkg/cataloger/java/archive_source_test.go
+149 -151
View File
@@ -20,9 +20,154 @@ import (
"github.com/anchore/syft/syft/pkg"
)
func Test_archiveContentsCataloger_agreesWithTheArchiveReadingCataloger(t *testing.T) {
// for the same archive, the contents-reading cataloger must report the same packages with the same
// identity as the one reading the archive file; any divergence shows up as an SBOM diff
entries := jarEntries()
cfg := DefaultArchiveCatalogerConfig()
// the archive-reading cataloger, over a directory holding the jar
archiveDir := t.TempDir()
jarPath := writeJar(t, archiveDir, "example-lib-1.2.3.jar", entries)
archiveResolver, err := fileresolver.NewFromDirectory(archiveDir, "")
require.NoError(t, err)
// the archive-reading cataloger copies the archive to a temp dir first, which the contents-reading
// one does not
archiveCtx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))
fromArchive, _, err := NewArchiveCataloger(cfg).Catalog(archiveCtx, archiveResolver)
require.NoError(t, err)
require.NotEmpty(t, fromArchive)
// the contents-reading cataloger, over the same entries laid out as a filesystem, with the traversal
// the archive cataloger task would have placed on the context
contentsDir := t.TempDir()
explode(t, contentsDir, entries)
contentsResolver, err := fileresolver.NewFromDirectory(contentsDir, "")
require.NoError(t, err)
// the traversal names the archive exactly as the resolver did for the archive-reading cataloger, so
// both sides derive identity from the same string and the comparison is about where the entries came
// from rather than how the test spelled a path
jarLocations, err := archiveResolver.FilesByGlob("**/*.jar")
require.NoError(t, err)
require.Len(t, jarLocations, 1)
archivePath := jarLocations[0].Path()
digests := digestsOfFile(t, jarPath)
trav := &archive.Traversal{
Location: file.NewLocation(archivePath),
VirtualPath: archivePath,
Digests: digests,
}
ctx := archive.WithTraversal(context.Background(), trav)
fromContents, _, err := NewArchiveCataloger(cfg).Catalog(ctx, contentsResolver)
require.NoError(t, err)
assert.Equal(t, summarize(fromArchive), summarize(fromContents))
}
func Test_archiveContentsCataloger_reportsNothingOutsideAnArchive(t *testing.T) {
// with the archive cataloger task owning extraction, every archive comes back with a traversal
// naming it. Outside one there is nothing to describe: opening archive files here as well would
// catalog each of them a second time.
cfg := DefaultArchiveCatalogerConfig()
cfg.NestedArchivesHandledExternally = true
dir := t.TempDir()
jarPath := writeJar(t, dir, "example-lib-1.2.3.jar", jarEntries())
require.FileExists(t, jarPath)
resolver, err := fileresolver.NewFromDirectory(dir, "")
require.NoError(t, err)
pkgs, _, err := NewArchiveCataloger(cfg).Catalog(context.Background(), resolver)
require.NoError(t, err)
assert.Empty(t, pkgs)
}
func Test_archiveContentsCataloger_reportsNothingForAnArchiveThatIsNotJava(t *testing.T) {
// a .zip is an archive and its contents are not a jar's; the archive-reading cataloger draws that
// line with its globs and this one draws it with the same list
dir := t.TempDir()
explode(t, dir, jarEntries())
resolver, err := fileresolver.NewFromDirectory(dir, "")
require.NoError(t, err)
ctx := archive.WithTraversal(context.Background(), &archive.Traversal{
Location: file.NewLocation("/bundle.zip"),
VirtualPath: "/bundle.zip",
})
pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver)
require.NoError(t, err)
assert.Empty(t, pkgs)
}
func Test_archiveContentsCataloger_noManifestIsNoPackage(t *testing.T) {
// unchanged from the archive-reading cataloger, and the whole coverage argument: a jar with no
// META-INF/MANIFEST.MF yields nothing today either, so requiring the contents to be traversed takes
// away less than it appears to. The maven metadata here is never consulted - without a main package
// there is nothing to attach it to.
entries := map[string]string{
"META-INF/maven/com.other/bundled-dep/pom.properties": "groupId=com.other\nartifactId=bundled-dep\nversion=4.5.6\n",
"com/example/Thing.class": "not really bytecode",
}
dir := t.TempDir()
explode(t, dir, entries)
resolver, err := fileresolver.NewFromDirectory(dir, "")
require.NoError(t, err)
ctx := archive.WithTraversal(context.Background(), &archive.Traversal{
Location: file.NewLocation("no-manifest-1.0.0.jar"),
VirtualPath: "no-manifest-1.0.0.jar",
})
pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver)
assert.Empty(t, pkgs)
// reported against the archive rather than swallowed, and as an unknown rather than a scan failure
require.Error(t, err)
coordErrs, remaining := unknown.ExtractCoordinateErrors(err)
assert.NoError(t, remaining)
require.Len(t, coordErrs, 1)
assert.Equal(t, "no-manifest-1.0.0.jar", coordErrs[0].Coordinates.RealPath)
}
func Test_archiveContentsCataloger_readsWhatWasExtractedAndNoMore(t *testing.T) {
// the sharp edge of reading contents rather than the archive: this cataloger sees exactly what the
// extraction produced. An archive truncated by a limit yields a package built from the entries that
// were written, so identity degrades rather than disappearing - the manifest is there, the maven
// metadata is not.
//
// Observed on a real artifact: jenkins-core-2.578.jar holds more than the default 10,000-entry bound,
// so its META-INF/maven pom is never extracted and its group id falls back from org.jenkins-ci.main
// to the artifact id. The archive-reading cataloger never felt this, reading the archive's own
// central directory with no bound on it.
entries := map[string]string{
"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n" +
"Implementation-Title: example-lib\nImplementation-Version: 1.2.3\n",
// the pom that would have supplied the group id is missing, as truncation would leave it
"com/example/Thing.class": "not really bytecode",
}
dir := t.TempDir()
explode(t, dir, entries)
resolver, err := fileresolver.NewFromDirectory(dir, "")
require.NoError(t, err)
ctx := archive.WithTraversal(context.Background(), &archive.Traversal{
Location: file.NewLocation("example-lib-1.2.3.jar"),
VirtualPath: "example-lib-1.2.3.jar",
})
pkgs, _, err := NewArchiveCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver)
require.NoError(t, err)
require.Len(t, pkgs, 1)
assert.Equal(t, "example-lib", pkgs[0].Name)
assert.Equal(t, "1.2.3", pkgs[0].Version)
assert.Equal(t, "pkg:maven/example-lib/example-lib@1.2.3", pkgs[0].PURL,
"with no pom to name the group, the group falls back to the artifact id")
}
// jarEntries is one java archive's worth of content, used both to build the archive the
// archive-reading cataloger parses and to lay out the filesystem the contents-reading one reads.
// The point of the pair of tests below is that these two must not disagree.
// archive-reading cataloger parses and to lay out the filesystem the contents-reading one reads. The
// tests below assert the two agree.
func jarEntries() map[string]string {
return map[string]string{
"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n" +
@@ -63,8 +208,8 @@ func writeJar(t *testing.T, dir, name string, entries map[string]string) string
return p
}
// explode lays the same entries out as files, which is what the archive cataloger task hands the
// sub-pipeline once it has extracted an archive.
// explode lays the same entries out as files, what the archive cataloger task hands the sub-pipeline
// once it has extracted an archive.
func explode(t *testing.T, dir string, entries map[string]string) {
t.Helper()
for name, contents := range entries {
@@ -102,55 +247,6 @@ func joinSorted(in []string) string {
return out
}
func Test_archiveContentsCataloger_agreesWithTheArchiveReadingCataloger(t *testing.T) {
// the whole reason a second cataloger is tolerable: for the same archive it must report the same
// packages, with the same identity, as the one that reads the archive file. Anything else is a
// divergence that will be discovered as a diff in someone's SBOM.
entries := jarEntries()
cfg := DefaultArchiveCatalogerConfig()
// the archive-reading cataloger, over a directory holding the jar
archiveDir := t.TempDir()
jarPath := writeJar(t, archiveDir, "example-lib-1.2.3.jar", entries)
archiveResolver, err := fileresolver.NewFromDirectory(archiveDir, "")
require.NoError(t, err)
// the archive-reading cataloger copies the archive to a temp dir before it can read it, which is
// the work the contents-reading one does not do
archiveCtx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))
fromArchive, _, err := NewArchiveCataloger(cfg).Catalog(archiveCtx, archiveResolver)
require.NoError(t, err)
require.NotEmpty(t, fromArchive)
// the contents-reading cataloger, over the same entries laid out as a filesystem, with the
// traversal the archive cataloger task would have placed on the context
contentsDir := t.TempDir()
explode(t, contentsDir, entries)
contentsResolver, err := fileresolver.NewFromDirectory(contentsDir, "")
require.NoError(t, err)
// the traversal names the archive exactly as the resolver did for the archive-reading cataloger,
// so both sides derive identity from the same string and the comparison is about the source of
// the entries rather than about how the test spelled a path
jarLocations, err := archiveResolver.FilesByGlob("**/*.jar")
require.NoError(t, err)
require.Len(t, jarLocations, 1)
archivePath := jarLocations[0].Path()
digests := digestsOfFile(t, jarPath)
trav := &archive.Traversal{
Location: file.NewLocation(archivePath),
VirtualPath: archivePath,
FileSystemID: "example-lib-1.2.3.jar",
Depth: 1,
Digests: digests,
}
ctx := archive.WithTraversal(context.Background(), trav)
fromContents, _, err := NewArchiveContentsCataloger(cfg).Catalog(ctx, contentsResolver)
require.NoError(t, err)
assert.Equal(t, summarize(fromArchive), summarize(fromContents))
}
func digestsOfFile(t *testing.T, path string) []file.Digest {
t.Helper()
entries, err := newZipEntries(context.Background(), path)
@@ -160,101 +256,3 @@ func digestsOfFile(t *testing.T, path string) []file.Digest {
require.NotEmpty(t, digests)
return digests
}
func Test_archiveContentsCataloger_reportsNothingOutsideAnArchive(t *testing.T) {
// no traversal means the scan is not inside an archive, and the identity of a java archive is
// half in the file - so there is nothing this cataloger can say
dir := t.TempDir()
explode(t, dir, jarEntries())
resolver, err := fileresolver.NewFromDirectory(dir, "")
require.NoError(t, err)
pkgs, _, err := NewArchiveContentsCataloger(DefaultArchiveCatalogerConfig()).
Catalog(context.Background(), resolver)
require.NoError(t, err)
assert.Empty(t, pkgs)
}
func Test_archiveContentsCataloger_reportsNothingForAnArchiveThatIsNotJava(t *testing.T) {
// a .zip is an archive, and its contents are not a jar's contents. The archive-reading cataloger
// draws that line with its globs; this one draws it with the same list.
dir := t.TempDir()
explode(t, dir, jarEntries())
resolver, err := fileresolver.NewFromDirectory(dir, "")
require.NoError(t, err)
ctx := archive.WithTraversal(context.Background(), &archive.Traversal{
Location: file.NewLocation("/bundle.zip"),
VirtualPath: "/bundle.zip",
Depth: 1,
})
pkgs, _, err := NewArchiveContentsCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver)
require.NoError(t, err)
assert.Empty(t, pkgs)
}
func Test_archiveContentsCataloger_noManifestIsNoPackage(t *testing.T) {
// unchanged from the archive-reading cataloger, and worth asserting because it is the whole
// coverage argument: a jar with no META-INF/MANIFEST.MF yields nothing today either, so requiring
// the contents to be traversed takes away less than it appears to. Note the maven metadata here is
// never consulted - without a main package there is nothing to attach it to.
entries := map[string]string{
"META-INF/maven/com.other/bundled-dep/pom.properties": "groupId=com.other\nartifactId=bundled-dep\nversion=4.5.6\n",
"com/example/Thing.class": "not really bytecode",
}
dir := t.TempDir()
explode(t, dir, entries)
resolver, err := fileresolver.NewFromDirectory(dir, "")
require.NoError(t, err)
ctx := archive.WithTraversal(context.Background(), &archive.Traversal{
Location: file.NewLocation("no-manifest-1.0.0.jar"),
VirtualPath: "no-manifest-1.0.0.jar",
Depth: 1,
})
pkgs, _, err := NewArchiveContentsCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver)
assert.Empty(t, pkgs)
// reported against the archive rather than swallowed, and as an unknown rather than a scan failure
require.Error(t, err)
coordErrs, remaining := unknown.ExtractCoordinateErrors(err)
assert.NoError(t, remaining)
require.Len(t, coordErrs, 1)
assert.Equal(t, "no-manifest-1.0.0.jar", coordErrs[0].Coordinates.RealPath)
}
func Test_archiveContentsCataloger_readsWhatWasExtractedAndNoMore(t *testing.T) {
// the sharp edge of reading contents rather than the archive: this cataloger sees exactly what the
// extraction produced. An archive truncated by a limit yields a package built from the entries
// that were written, so identity degrades rather than disappearing - the manifest is there and the
// maven metadata is not.
//
// Observed on a real artifact: jenkins-core-2.578.jar holds more than the default 10,000-entry
// bound, so its META-INF/maven pom is never extracted and its group id falls back from
// org.jenkins-ci.main to the artifact id. The archive-reading cataloger never felt this, because it
// read the archive's own central directory with no bound on it.
entries := map[string]string{
"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n" +
"Implementation-Title: example-lib\nImplementation-Version: 1.2.3\n",
// the pom that would have supplied the group id is missing, as truncation would leave it
"com/example/Thing.class": "not really bytecode",
}
dir := t.TempDir()
explode(t, dir, entries)
resolver, err := fileresolver.NewFromDirectory(dir, "")
require.NoError(t, err)
ctx := archive.WithTraversal(context.Background(), &archive.Traversal{
Location: file.NewLocation("example-lib-1.2.3.jar"),
VirtualPath: "example-lib-1.2.3.jar",
Depth: 1,
})
pkgs, _, err := NewArchiveContentsCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver)
require.NoError(t, err)
require.Len(t, pkgs, 1)
assert.Equal(t, "example-lib", pkgs[0].Name)
assert.Equal(t, "1.2.3", pkgs[0].Version)
assert.Equal(t, "pkg:maven/example-lib/example-lib@1.2.3", pkgs[0].PURL,
"with no pom to name the group, the group falls back to the artifact id")
}
syft/pkg/cataloger/java/cataloger.go
+80 -17
View File
@@ -4,34 +4,97 @@ Package java provides a concrete Cataloger implementation for packages relating
package java
import (
"context"
"github.com/anchore/syft/internal/archive"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
)
const ArchiveCatalogerName = "java-archive-cataloger"
// NewArchiveCataloger returns a new Java archive cataloger object for detecting packages with archives (jar, war, ear, par, sar, jpi, hpi, and native-image formats)
func NewArchiveCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger {
gap := newGenericArchiveParserAdapter(cfg)
c := generic.NewCataloger("java-archive-cataloger").
files := generic.NewCataloger(ArchiveCatalogerName).
WithParserByGlobs(gap.parseJavaArchive, archiveFormatGlobs...)
// when the generic archive cataloger task owns recursion into archives, it extracts zip/tar
// archives generically and this cataloger meets any java archives within them at the next
// nesting level, so the wrapped-archive parsers would only duplicate that work
if !cfg.nestedArchivesHandledExternally() {
if cfg.IncludeIndexedArchives {
// java archives wrapped within zip files
gzp := newGenericZipWrappedJavaArchiveParser(cfg)
c.WithParserByGlobs(gzp.parseZipWrappedJavaArchive, genericZipGlobs...)
}
if cfg.IncludeUnindexedArchives {
// java archives wrapped within tar files
gtp := newGenericTarWrappedJavaArchiveParser(cfg)
c.WithParserByGlobs(gtp.parseTarWrappedJavaArchive, genericTarGlobs...)
}
if cfg.IncludeIndexedArchives {
// java archives wrapped within zip files
gzp := newGenericZipWrappedJavaArchiveParser(cfg)
files.WithParserByGlobs(gzp.parseZipWrappedJavaArchive, genericZipGlobs...)
}
return c
if cfg.IncludeUnindexedArchives {
// java archives wrapped within tar files
gtp := newGenericTarWrappedJavaArchiveParser(cfg)
files.WithParserByGlobs(gtp.parseTarWrappedJavaArchive, genericTarGlobs...)
}
return &archiveCataloger{cfg: cfg, files: files}
}
// archiveCataloger describes java archives from whichever source this scan makes them available in.
//
// Which source that is turns on who owns extraction, and there are only two answers. Inside an
// archive the archive cataloger task extracted, the resolver is that archive's contents and the
// traversal names the file it came from, so the archive to describe is the one this resolver *is*.
// Everywhere else the archives are files in the resolver, and the cataloger opens each itself.
//
// One cataloger rather than one per source, so a package carries the same FoundBy and answers to the
// same --select-catalogers name however the scan was configured.
type archiveCataloger struct {
cfg ArchiveCatalogerConfig
files *generic.Cataloger
}
func (c *archiveCataloger) Name() string {
return ArchiveCatalogerName
}
func (c *archiveCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
if trav := archive.TraversalFromContext(ctx); trav != nil {
return c.catalogExtracted(ctx, trav, resolver)
}
if c.cfg.NestedArchivesHandledExternally {
// every archive in this scan comes back through the archive cataloger task with a traversal
// naming it, and is described above. Opening archive files here as well would catalog each of
// them a second time.
return nil, nil, nil
}
return c.files.Catalog(ctx, resolver)
}
// catalogExtracted reports the java package of the archive whose contents this resolver holds, if it
// is a java archive at all.
//
// It does not glob for the archives inside it: the task that extracted this one meets those on its
// own, at the next nesting level. See syft/java-archive-nesting#single-owner-of-archive-recursion.
func (c *archiveCataloger) catalogExtracted(ctx context.Context, trav *archive.Traversal, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
if !isJavaArchiveName(trav.Location.Path()) {
// the rule the globs apply when this cataloger opens files: a .zip is an archive but not a java
// archive
return nil, nil, nil
}
src, err := newExtractedArchiveSource(trav, resolver)
if err != nil {
return nil, nil, err
}
pkgs, relationships, err := newArchiveParser(src, false, c.cfg).parse(ctx, nil)
if err != nil {
// attributed to the archive rather than returned bare, as the generic cataloger attributes a
// parser's failure to the file it was parsing, so the scan carries on
return pkgs, relationships, unknown.New(trav.Location, err)
}
return pkgs, relationships, nil
}
// NewPomCataloger returns a cataloger capable of parsing dependencies from a pom.xml file.
syft/pkg/cataloger/java/config.go
+6 -13
View File
@@ -34,13 +34,13 @@ type ArchiveCatalogerConfig struct {
// app-config: java.resolve-transitive-dependencies
ResolveTransitiveDependencies bool `yaml:"resolve-transitive-dependencies" json:"resolve-transitive-dependencies" mapstructure:"resolve-transitive-dependencies"`
// NestedArchivesHandledExternally indicates that some other mechanism owns recursion into
// nested archives, so this cataloger must not unarchive them itself. It is deliberately not a
// yaml/json/mapstructure key: no config file, flag, or environment variable reaches it.
// NestedArchivesHandledExternally indicates that another mechanism owns recursion into nested
// archives, so this cataloger must not unarchive them itself. It has no yaml/json/mapstructure
// key, so no config file, flag or environment variable reaches it.
//
// It is written ONLY by syft.CreateSBOMConfig, which derives it from the nested-archive
// cataloging depth so the two sides cannot disagree. Callers building this config should leave
// it alone; the zero value is this cataloger's long-standing behavior of recursing itself.
// Only syft.CreateSBOMConfig writes it, deriving it from the nested-archive cataloging depth so
// the two cannot disagree. Callers building this config should leave it alone; the zero value
// recurses, this cataloger's default.
NestedArchivesHandledExternally bool `yaml:"-" json:"-" mapstructure:"-"`
}
@@ -90,13 +90,6 @@ func (j ArchiveCatalogerConfig) WithArchiveTraversal(search cataloging.ArchiveSe
return j
}
// nestedArchivesHandledExternally indicates another mechanism (the generic archive cataloger task)
// owns recursion into all archives, in which case this cataloger must not unarchive nested archives
// itself. See the field of the same name for why this is not derived from MaxDepth.
func (j ArchiveCatalogerConfig) nestedArchivesHandledExternally() bool {
return j.NestedArchivesHandledExternally
}
func (j ArchiveCatalogerConfig) mavenConfig() maven.Config {
return maven.Config{
UseNetwork: j.UseNetwork,
syft/pkg/cataloger/java/parse_pom_xml.go
+6 -10
View File
@@ -303,17 +303,13 @@ func pomParent(ctx context.Context, r *maven.Resolver, pom *maven.Project) *pkg.
}
}
// isArchiveMetaPom returns true if the pom.xml location is inside a META-INF/maven directory,
// indicating it is archive metadata embedded by the Maven build process rather than a project
// pom.xml. These embedded pom.xml files are verbatim copies of the original build POM with
// potentially unresolved ${property} references and dependency versions inherited from parent
// POMs that are not available inside the archive. The java-archive-cataloger already uses
// pom.properties (which always contains resolved values) to identify these archives, so the
// pom cataloger should skip them to avoid creating phantom dependency packages.
// isArchiveMetaPom reports whether a pom.xml sits under META-INF/maven, making it build metadata
// Maven embedded rather than a project pom.xml. Such a pom is a verbatim copy of the build POM, with
// unresolved ${property} references and versions inherited from parent POMs absent from the archive.
// The java-archive-cataloger identifies these archives from pom.properties, whose values are always
// resolved, so skipping them here avoids phantom dependency packages.
func isArchiveMetaPom(location file.Location) bool {
p := location.Path()
// normalize to forward slashes for consistent matching
p = filepath.ToSlash(p)
p := filepath.ToSlash(location.Path())
return strings.Contains(p, "META-INF/maven/")
}
syft/pkg/cataloger/java/parse_pom_xml_test.go
+4 -5
View File
@@ -732,8 +732,8 @@ func Test_isArchiveMetaPom(t *testing.T) {
}
func Test_pomCatalogerSkipsMetaInfPoms(t *testing.T) {
// A directory that only contains a pom.xml under META-INF/maven/ should produce
// no packages, since these are archive metadata not project files.
// a pom.xml under META-INF/maven/ alone yields no packages: that is archive metadata, not a
// project file
cat := NewPomCataloger(ArchiveCatalogerConfig{
ArchiveSearchConfig: cataloging.ArchiveSearchConfig{
IncludeIndexedArchives: true,
@@ -748,9 +748,8 @@ func Test_pomCatalogerSkipsMetaInfPoms(t *testing.T) {
}
func Test_pomCatalogerSkipsMetaInfButKeepsProjectPom(t *testing.T) {
// When a directory contains both a project pom.xml and a META-INF/maven pom.xml,
// only the project pom.xml should be cataloged. The META-INF pom.xml and its
// dependencies should be ignored.
// with both a project pom.xml and a META-INF/maven one, only the project pom.xml is
// cataloged; the META-INF pom and its dependencies are ignored
pomLocation := file.NewLocationSet(file.NewLocation("pom.xml"))
myApp := pkg.Package{
syft/pkg/cataloger/java/version_properties_test.go
+2 -3
View File
@@ -9,7 +9,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
@@ -21,10 +20,10 @@ func readManifest(t *testing.T, ctx context.Context, parser *archiveParser) *pkg
matches := parser.entries.glob(false, manifestGlob)
require.Len(t, matches, 1)
contents, err := intFile.ContentsFromZip(ctx, parser.archivePath, matches...)
contents, err := parser.entries.contents(ctx, matches...)
require.NoError(t, err)
manifest, err := parseJavaManifest(parser.archivePath, strings.NewReader(contents[matches[0]]))
manifest, err := parseJavaManifest(parser.virtualPath, strings.NewReader(contents[matches[0]]))
require.NoError(t, err)
return manifest