mirror of
https://github.com/anchore/syft.git
synced 2026-11-01 15:06:36 +01:00
feat: initial recursive archive cataloging impl
Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,129 @@
|
||||
package java
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/unknown"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java/internal/maven"
|
||||
)
|
||||
|
||||
const ArchiveContentsCatalogerName = "java-archive-contents-cataloger"
|
||||
|
||||
// NewArchiveContentsCataloger returns a java archive cataloger for the world after extraction: it
|
||||
// reads an archive's already-extracted filesystem rather than opening the archive itself.
|
||||
//
|
||||
// It is a second cataloger rather than a mode of the first because the two answer different
|
||||
// questions. The archive-reading cataloger asks "is this file a jar?" of every file in a scan; this
|
||||
// one asks "is the filesystem I have been handed a jar?", which is the only form the question can
|
||||
// take once something else owns extraction. That inversion is also why it cannot be a parameter:
|
||||
// when the archive cataloger task hands a jar's contents to the sub-pipeline, the jar file is one
|
||||
// level up and already behind us.
|
||||
//
|
||||
// What it does not do is recurse. The task that extracted this archive will meet the archives inside
|
||||
// it on its own, and one mechanism owning recursion is the whole point - see
|
||||
// syft/java-archive-nesting#single-owner-of-archive-recursion.
|
||||
func NewArchiveContentsCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger {
|
||||
return archiveContentsCataloger{cfg: cfg}
|
||||
}
|
||||
|
||||
type archiveContentsCataloger struct {
|
||||
cfg ArchiveCatalogerConfig
|
||||
}
|
||||
|
||||
func (c archiveContentsCataloger) Name() string {
|
||||
return ArchiveContentsCatalogerName
|
||||
}
|
||||
|
||||
// Catalog reports the java package of the archive whose contents this resolver holds, if it is a java
|
||||
// archive at all.
|
||||
//
|
||||
// Nothing outside an archive is cataloged. Without a traversal on the context there is no archive to
|
||||
// describe - the identity of a java archive lives half in the file (its name, its digests) and half
|
||||
// in its contents (the manifest, the poms), and the file half arrives only on the traversal.
|
||||
func (c archiveContentsCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
trav := archive.TraversalFromContext(ctx)
|
||||
if trav == nil {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
if !isJavaArchiveName(trav.Location.Path()) {
|
||||
// the same rule the archive-reading cataloger applies through its globs: a .zip is an archive
|
||||
// but it is not a java archive, and its contents are not a jar's contents
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
parser, err := newArchiveContentsParser(trav, resolver, c.cfg)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
pkgs, relationships, err := parser.parse(ctx, nil)
|
||||
if err != nil {
|
||||
// attributed to the archive rather than returned bare, the way the generic cataloger attributes
|
||||
// a parser's failure to the file it was parsing. An archive that turns out to hold no java
|
||||
// package at all reports exactly that, and the scan carries on.
|
||||
return pkgs, relationships, unknown.New(trav.Location, err)
|
||||
}
|
||||
return pkgs, relationships, nil
|
||||
}
|
||||
|
||||
// newArchiveContentsParser builds the same parser the archive-reading cataloger uses, over the
|
||||
// archive's extracted filesystem instead of over the archive file. Identity comes from the traversal:
|
||||
// the location and virtual path it already carries, the archive's filename for the name and version
|
||||
// fallbacks, and the digests taken when the archive was extracted.
|
||||
func newArchiveContentsParser(trav *archive.Traversal, resolver file.Resolver, cfg ArchiveCatalogerConfig) (*archiveParser, error) {
|
||||
if resolver == nil {
|
||||
return nil, fmt.Errorf("no resolver for the contents of %q", trav.VirtualPath)
|
||||
}
|
||||
return &archiveParser{
|
||||
entries: newResolverEntries(resolver, trav.Digests),
|
||||
location: trav.Location,
|
||||
// the colon-delimited chain the task composed, which is the identity already-published SBOMs
|
||||
// carry for a nested java package
|
||||
virtualPath: trav.VirtualPath,
|
||||
fileInfo: newJavaArchiveFilename(archiveFileNameOf(trav)),
|
||||
// recursion belongs to the task that extracted this archive
|
||||
detectNested: false,
|
||||
cfg: cfg,
|
||||
maven: maven.NewResolver(nil, cfg.mavenConfig()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// archiveFileNameOf returns the archive's own file name, which is where a jar's name and version come
|
||||
// from when its manifest does not say. The last element of the virtual path rather than of the real
|
||||
// path, so a chain and a plain scan agree.
|
||||
func archiveFileNameOf(trav *archive.Traversal) string {
|
||||
name := trav.VirtualPath
|
||||
if name == "" {
|
||||
name = trav.Location.Path()
|
||||
}
|
||||
for i := len(name) - 1; i >= 0; i-- {
|
||||
if name[i] == ':' || name[i] == '/' {
|
||||
return name[i+1:]
|
||||
}
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// isJavaArchiveName reports whether the path is one of the java archive formats, by the same glob
|
||||
// list the archive-reading cataloger registers against.
|
||||
func isJavaArchiveName(path string) bool {
|
||||
for _, glob := range archiveFormatGlobs {
|
||||
if ok, err := doublestar.Match(glob, path); err == nil && ok {
|
||||
return true
|
||||
}
|
||||
if ok, err := doublestar.Match(glob, "/"+path); err == nil && ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
log.WithFields("path", path).Trace("archive contents are not a java archive")
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
package java
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/tmpdir"
|
||||
"github.com/anchore/syft/internal/unknown"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
// jarEntries is one java archive's worth of content, used both to build the archive the
|
||||
// archive-reading cataloger parses and to lay out the filesystem the contents-reading one reads.
|
||||
// The point of the pair of tests below is that these two must not disagree.
|
||||
func jarEntries() map[string]string {
|
||||
return map[string]string{
|
||||
"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n" +
|
||||
"Implementation-Title: example-lib\n" +
|
||||
"Implementation-Version: 1.2.3\n",
|
||||
"META-INF/maven/com.example/example-lib/pom.properties": "groupId=com.example\n" +
|
||||
"artifactId=example-lib\nversion=1.2.3\n",
|
||||
"META-INF/maven/com.example/example-lib/pom.xml": `<project><modelVersion>4.0.0</modelVersion>
|
||||
<groupId>com.example</groupId><artifactId>example-lib</artifactId><version>1.2.3</version>
|
||||
<licenses><license><name>Apache License 2.0</name><url>https://www.apache.org/licenses/LICENSE-2.0</url></license></licenses>
|
||||
</project>`,
|
||||
// a bundled dependency's maven metadata, which becomes a package of its own
|
||||
"META-INF/maven/com.other/bundled-dep/pom.properties": "groupId=com.other\n" +
|
||||
"artifactId=bundled-dep\nversion=4.5.6\n",
|
||||
"com/example/Thing.class": "not really bytecode",
|
||||
}
|
||||
}
|
||||
|
||||
func writeJar(t *testing.T, dir, name string, entries map[string]string) string {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
names := make([]string, 0, len(entries))
|
||||
for n := range entries {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
w, err := zw.Create(n)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte(entries[n]))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
p := filepath.Join(dir, name)
|
||||
require.NoError(t, os.WriteFile(p, buf.Bytes(), 0o600))
|
||||
return p
|
||||
}
|
||||
|
||||
// explode lays the same entries out as files, which is what the archive cataloger task hands the
|
||||
// sub-pipeline once it has extracted an archive.
|
||||
func explode(t *testing.T, dir string, entries map[string]string) {
|
||||
t.Helper()
|
||||
for name, contents := range entries {
|
||||
p := filepath.Join(dir, filepath.FromSlash(name))
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755))
|
||||
require.NoError(t, os.WriteFile(p, []byte(contents), 0o600))
|
||||
}
|
||||
}
|
||||
|
||||
func summarize(pkgs []pkg.Package) []string {
|
||||
var out []string
|
||||
for _, p := range pkgs {
|
||||
meta, _ := p.Metadata.(pkg.JavaArchive)
|
||||
lics := p.Licenses.ToSlice()
|
||||
names := make([]string, 0, len(lics))
|
||||
for _, l := range lics {
|
||||
names = append(names, l.Value)
|
||||
}
|
||||
sort.Strings(names)
|
||||
out = append(out, p.Name+"@"+p.Version+" purl="+p.PURL+" vpath="+meta.VirtualPath+" lics="+
|
||||
joinSorted(names))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func joinSorted(in []string) string {
|
||||
out := ""
|
||||
for i, s := range in {
|
||||
if i > 0 {
|
||||
out += ","
|
||||
}
|
||||
out += s
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func Test_archiveContentsCataloger_agreesWithTheArchiveReadingCataloger(t *testing.T) {
|
||||
// the whole reason a second cataloger is tolerable: for the same archive it must report the same
|
||||
// packages, with the same identity, as the one that reads the archive file. Anything else is a
|
||||
// divergence that will be discovered as a diff in someone's SBOM.
|
||||
entries := jarEntries()
|
||||
cfg := DefaultArchiveCatalogerConfig()
|
||||
|
||||
// the archive-reading cataloger, over a directory holding the jar
|
||||
archiveDir := t.TempDir()
|
||||
jarPath := writeJar(t, archiveDir, "example-lib-1.2.3.jar", entries)
|
||||
archiveResolver, err := fileresolver.NewFromDirectory(archiveDir, "")
|
||||
require.NoError(t, err)
|
||||
// the archive-reading cataloger copies the archive to a temp dir before it can read it, which is
|
||||
// the work the contents-reading one does not do
|
||||
archiveCtx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))
|
||||
fromArchive, _, err := NewArchiveCataloger(cfg).Catalog(archiveCtx, archiveResolver)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, fromArchive)
|
||||
|
||||
// the contents-reading cataloger, over the same entries laid out as a filesystem, with the
|
||||
// traversal the archive cataloger task would have placed on the context
|
||||
contentsDir := t.TempDir()
|
||||
explode(t, contentsDir, entries)
|
||||
contentsResolver, err := fileresolver.NewFromDirectory(contentsDir, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// the traversal names the archive exactly as the resolver did for the archive-reading cataloger,
|
||||
// so both sides derive identity from the same string and the comparison is about the source of
|
||||
// the entries rather than about how the test spelled a path
|
||||
jarLocations, err := archiveResolver.FilesByGlob("**/*.jar")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, jarLocations, 1)
|
||||
archivePath := jarLocations[0].Path()
|
||||
|
||||
digests := digestsOfFile(t, jarPath)
|
||||
trav := &archive.Traversal{
|
||||
Location: file.NewLocation(archivePath),
|
||||
VirtualPath: archivePath,
|
||||
FileSystemID: "example-lib-1.2.3.jar",
|
||||
Depth: 1,
|
||||
Digests: digests,
|
||||
}
|
||||
ctx := archive.WithTraversal(context.Background(), trav)
|
||||
fromContents, _, err := NewArchiveContentsCataloger(cfg).Catalog(ctx, contentsResolver)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, summarize(fromArchive), summarize(fromContents))
|
||||
}
|
||||
|
||||
func digestsOfFile(t *testing.T, path string) []file.Digest {
|
||||
t.Helper()
|
||||
entries, err := newZipEntries(context.Background(), path)
|
||||
require.NoError(t, err)
|
||||
digests, err := entries.digests(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, digests)
|
||||
return digests
|
||||
}
|
||||
|
||||
func Test_archiveContentsCataloger_reportsNothingOutsideAnArchive(t *testing.T) {
|
||||
// no traversal means the scan is not inside an archive, and the identity of a java archive is
|
||||
// half in the file - so there is nothing this cataloger can say
|
||||
dir := t.TempDir()
|
||||
explode(t, dir, jarEntries())
|
||||
resolver, err := fileresolver.NewFromDirectory(dir, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
pkgs, _, err := NewArchiveContentsCataloger(DefaultArchiveCatalogerConfig()).
|
||||
Catalog(context.Background(), resolver)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, pkgs)
|
||||
}
|
||||
|
||||
func Test_archiveContentsCataloger_reportsNothingForAnArchiveThatIsNotJava(t *testing.T) {
|
||||
// a .zip is an archive, and its contents are not a jar's contents. The archive-reading cataloger
|
||||
// draws that line with its globs; this one draws it with the same list.
|
||||
dir := t.TempDir()
|
||||
explode(t, dir, jarEntries())
|
||||
resolver, err := fileresolver.NewFromDirectory(dir, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := archive.WithTraversal(context.Background(), &archive.Traversal{
|
||||
Location: file.NewLocation("/bundle.zip"),
|
||||
VirtualPath: "/bundle.zip",
|
||||
Depth: 1,
|
||||
})
|
||||
pkgs, _, err := NewArchiveContentsCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, pkgs)
|
||||
}
|
||||
|
||||
func Test_archiveContentsCataloger_noManifestIsNoPackage(t *testing.T) {
|
||||
// unchanged from the archive-reading cataloger, and worth asserting because it is the whole
|
||||
// coverage argument: a jar with no META-INF/MANIFEST.MF yields nothing today either, so requiring
|
||||
// the contents to be traversed takes away less than it appears to. Note the maven metadata here is
|
||||
// never consulted - without a main package there is nothing to attach it to.
|
||||
entries := map[string]string{
|
||||
"META-INF/maven/com.other/bundled-dep/pom.properties": "groupId=com.other\nartifactId=bundled-dep\nversion=4.5.6\n",
|
||||
"com/example/Thing.class": "not really bytecode",
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
explode(t, dir, entries)
|
||||
resolver, err := fileresolver.NewFromDirectory(dir, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := archive.WithTraversal(context.Background(), &archive.Traversal{
|
||||
Location: file.NewLocation("no-manifest-1.0.0.jar"),
|
||||
VirtualPath: "no-manifest-1.0.0.jar",
|
||||
Depth: 1,
|
||||
})
|
||||
pkgs, _, err := NewArchiveContentsCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver)
|
||||
assert.Empty(t, pkgs)
|
||||
// reported against the archive rather than swallowed, and as an unknown rather than a scan failure
|
||||
require.Error(t, err)
|
||||
coordErrs, remaining := unknown.ExtractCoordinateErrors(err)
|
||||
assert.NoError(t, remaining)
|
||||
require.Len(t, coordErrs, 1)
|
||||
assert.Equal(t, "no-manifest-1.0.0.jar", coordErrs[0].Coordinates.RealPath)
|
||||
}
|
||||
|
||||
func Test_archiveContentsCataloger_readsWhatWasExtractedAndNoMore(t *testing.T) {
|
||||
// the sharp edge of reading contents rather than the archive: this cataloger sees exactly what the
|
||||
// extraction produced. An archive truncated by a limit yields a package built from the entries
|
||||
// that were written, so identity degrades rather than disappearing - the manifest is there and the
|
||||
// maven metadata is not.
|
||||
//
|
||||
// Observed on a real artifact: jenkins-core-2.578.jar holds more than the default 10,000-entry
|
||||
// bound, so its META-INF/maven pom is never extracted and its group id falls back from
|
||||
// org.jenkins-ci.main to the artifact id. The archive-reading cataloger never felt this, because it
|
||||
// read the archive's own central directory with no bound on it.
|
||||
entries := map[string]string{
|
||||
"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n" +
|
||||
"Implementation-Title: example-lib\nImplementation-Version: 1.2.3\n",
|
||||
// the pom that would have supplied the group id is missing, as truncation would leave it
|
||||
"com/example/Thing.class": "not really bytecode",
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
explode(t, dir, entries)
|
||||
resolver, err := fileresolver.NewFromDirectory(dir, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx := archive.WithTraversal(context.Background(), &archive.Traversal{
|
||||
Location: file.NewLocation("example-lib-1.2.3.jar"),
|
||||
VirtualPath: "example-lib-1.2.3.jar",
|
||||
Depth: 1,
|
||||
})
|
||||
pkgs, _, err := NewArchiveContentsCataloger(DefaultArchiveCatalogerConfig()).Catalog(ctx, resolver)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, pkgs, 1)
|
||||
assert.Equal(t, "example-lib", pkgs[0].Name)
|
||||
assert.Equal(t, "1.2.3", pkgs[0].Version)
|
||||
assert.Equal(t, "pkg:maven/example-lib/example-lib@1.2.3", pkgs[0].PURL,
|
||||
"with no pom to name the group, the group falls back to the artifact id")
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package java
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// archiveEntries is where an archive parser reads a java archive's entries from.
|
||||
//
|
||||
// The parser needs three things of an archive and nothing else: which entries match a glob, the
|
||||
// contents of some of them, and the digest of the archive as a whole. Everything else it does -
|
||||
// which manifest wins, how pom.properties decides identity, license discovery - is logic over those
|
||||
// answers. Naming the three makes them replaceable, which is what lets the same logic run over an
|
||||
// archive file and over an archive that has already been extracted to a filesystem.
|
||||
//
|
||||
// Entry names are archive-relative with no leading slash, matching zip entry names, because they
|
||||
// become map keys and pom paths that identity is derived from. An implementation over a filesystem
|
||||
// has to strip the root to match.
|
||||
type archiveEntries interface {
|
||||
// glob returns the names of the entries matching any of the patterns.
|
||||
glob(caseInsensitive bool, patterns ...string) []string
|
||||
|
||||
// contents reads the named entries, returning a map keyed by the same names.
|
||||
contents(ctx context.Context, names ...string) (map[string]string, error)
|
||||
|
||||
// digests returns the digests of the archive as a whole, which is a property of the archive file
|
||||
// rather than of anything inside it.
|
||||
digests(ctx context.Context) ([]file.Digest, error)
|
||||
}
|
||||
|
||||
// zipEntries reads entries out of a java archive on disk. This is the original source, kept for
|
||||
// scans where nothing has extracted the archive: it copies the archive to a temp file, reads its
|
||||
// central directory once, and opens it again for each set of entries it is asked for.
|
||||
type zipEntries struct {
|
||||
archivePath string
|
||||
fileManifest intFile.ZipFileManifest
|
||||
}
|
||||
|
||||
func newZipEntries(ctx context.Context, archivePath string) (*zipEntries, error) {
|
||||
fileManifest, err := intFile.NewZipFileManifest(ctx, archivePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read files from java archive: %w", err)
|
||||
}
|
||||
return &zipEntries{archivePath: archivePath, fileManifest: fileManifest}, nil
|
||||
}
|
||||
|
||||
func (z *zipEntries) glob(caseInsensitive bool, patterns ...string) []string {
|
||||
return z.fileManifest.GlobMatch(caseInsensitive, patterns...)
|
||||
}
|
||||
|
||||
func (z *zipEntries) contents(ctx context.Context, names ...string) (map[string]string, error) {
|
||||
return intFile.ContentsFromZip(ctx, z.archivePath, names...)
|
||||
}
|
||||
|
||||
func (z *zipEntries) digests(ctx context.Context) ([]file.Digest, error) {
|
||||
return getDigestsFromArchive(ctx, z.archivePath)
|
||||
}
|
||||
|
||||
// resolverEntries reads entries out of an archive that has already been extracted and indexed - the
|
||||
// filesystem the archive cataloger task builds for every archive it enters.
|
||||
//
|
||||
// Nothing is copied and nothing is re-opened: the entries are files in an indexed resolver, and the
|
||||
// archive's own digests were taken by whoever extracted it, since the archive file itself is not
|
||||
// present in its own contents.
|
||||
type resolverEntries struct {
|
||||
resolver file.Resolver
|
||||
archiveDigests []file.Digest
|
||||
locationsByName map[string]file.Location
|
||||
}
|
||||
|
||||
func newResolverEntries(resolver file.Resolver, archiveDigests []file.Digest) *resolverEntries {
|
||||
return &resolverEntries{
|
||||
resolver: resolver,
|
||||
archiveDigests: archiveDigests,
|
||||
locationsByName: make(map[string]file.Location),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *resolverEntries) glob(caseInsensitive bool, patterns ...string) []string {
|
||||
if caseInsensitive {
|
||||
return r.globCaseInsensitive(patterns...)
|
||||
}
|
||||
|
||||
// deduplicated within this call and not across calls: the parser globs the same pattern more than
|
||||
// once (identity resolution asks for the poms, and so does aux package discovery), and a source
|
||||
// that remembered what it had already returned would hand back nothing the second time. That
|
||||
// silently cost every package derived from a bundled pom.properties.
|
||||
seen := strset.New()
|
||||
var names []string
|
||||
for _, pattern := range patterns {
|
||||
locations, err := r.resolver.FilesByGlob(pattern)
|
||||
if err != nil {
|
||||
log.WithFields("pattern", pattern, "error", err).Trace("unable to glob archive contents")
|
||||
continue
|
||||
}
|
||||
for _, location := range locations {
|
||||
name := strings.TrimPrefix(location.Path(), "/")
|
||||
r.locationsByName[name] = location
|
||||
if seen.Has(name) {
|
||||
continue
|
||||
}
|
||||
seen.Add(name)
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// globCaseInsensitive matches the way the zip-backed source does, by lowercasing both sides. The
|
||||
// resolver's own glob is case sensitive, so this lists the archive's files once and matches them
|
||||
// here. License discovery is the only caller that asks, and what it costs is one listing of a
|
||||
// filesystem holding one archive's entries.
|
||||
func (r *resolverEntries) globCaseInsensitive(patterns ...string) []string {
|
||||
all, err := r.resolver.FilesByGlob("**/*")
|
||||
if err != nil {
|
||||
log.WithFields("error", err).Trace("unable to list archive contents")
|
||||
return nil
|
||||
}
|
||||
seen := strset.New()
|
||||
var names []string
|
||||
for _, pattern := range patterns {
|
||||
lowered := strings.ToLower(pattern)
|
||||
for _, location := range all {
|
||||
// matched with a leading slash, the way the zip-backed source normalizes an entry before
|
||||
// matching, because the patterns are written that way ("/META-INF/*"). An extracted
|
||||
// archive's resolver reports paths without one - they are entry names, not absolute paths -
|
||||
// so matching them raw silently matches nothing, which cost the license found in every
|
||||
// jar that carries one as a file.
|
||||
name := strings.TrimPrefix(location.Path(), "/")
|
||||
if ok, matchErr := doublestar.Match(lowered, "/"+strings.ToLower(name)); matchErr != nil || !ok {
|
||||
continue
|
||||
}
|
||||
r.locationsByName[name] = location
|
||||
if seen.Has(name) {
|
||||
continue
|
||||
}
|
||||
seen.Add(name)
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func (r *resolverEntries) contents(_ context.Context, names ...string) (map[string]string, error) {
|
||||
out := make(map[string]string, len(names))
|
||||
for _, name := range names {
|
||||
location, ok := r.locationsByName[name]
|
||||
if !ok {
|
||||
// only names this source handed out can be read back; a name from anywhere else is a bug
|
||||
// rather than a missing file
|
||||
return nil, fmt.Errorf("entry %q was not found by any glob against this archive", name)
|
||||
}
|
||||
contents, err := r.read(location)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read archive entry %q: %w", name, err)
|
||||
}
|
||||
out[name] = contents
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *resolverEntries) digests(_ context.Context) ([]file.Digest, error) {
|
||||
return r.archiveDigests, nil
|
||||
}
|
||||
|
||||
// read reads one entry's content to the end and closes the stream. The parser wants entry contents as
|
||||
// strings, since every consumer of them parses text.
|
||||
//
|
||||
// The copy is bounded the same way the zip-backed source bounds its own: an entry of an extracted
|
||||
// archive is still attacker-supplied content, and reading it into a string is exactly the allocation
|
||||
// a decompression bomb is aiming at.
|
||||
func (r *resolverEntries) read(location file.Location) (string, error) {
|
||||
reader, err := r.resolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer internal.CloseAndLogError(reader, location.RealPath)
|
||||
|
||||
var contents strings.Builder
|
||||
if err := intFile.SafeCopy(&contents, reader); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return contents.String(), nil
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/tmpdir"
|
||||
@@ -57,8 +58,9 @@ var javaArchiveHashes = []crypto.Hash{
|
||||
}
|
||||
|
||||
type archiveParser struct {
|
||||
fileManifest intFile.ZipFileManifest
|
||||
entries archiveEntries
|
||||
location file.Location
|
||||
virtualPath string
|
||||
archivePath string
|
||||
contentPath string
|
||||
fileInfo archiveFilename
|
||||
@@ -82,7 +84,9 @@ func (gap genericArchiveParserAdapter) parseJavaArchive(ctx context.Context, _ f
|
||||
|
||||
// processJavaArchive processes an archive for java contents, returning all Java libraries and nested archives
|
||||
func (gap genericArchiveParserAdapter) processJavaArchive(ctx context.Context, reader file.LocationReadCloser, parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
parser, cleanupFn, err := newJavaArchiveParser(ctx, reader, true, gap.cfg)
|
||||
// when the generic archive cataloger task owns recursion into archives, this parser must not
|
||||
// also unarchive nested archives, otherwise they would be cataloged twice
|
||||
parser, cleanupFn, err := newJavaArchiveParser(ctx, reader, !gap.cfg.nestedArchivesHandledExternally(), gap.cfg)
|
||||
// note: even on error, we should always run cleanup functions
|
||||
defer cleanupFn()
|
||||
if err != nil {
|
||||
@@ -102,8 +106,13 @@ func uniquePkgKey(groupID string, p *pkg.Package) string {
|
||||
// newJavaArchiveParser returns a new java archive parser object for the given archive. Can be configured to discover
|
||||
// and parse nested archives or ignore them.
|
||||
func newJavaArchiveParser(ctx context.Context, reader file.LocationReadCloser, detectNested bool, cfg ArchiveCatalogerConfig) (*archiveParser, func(), error) {
|
||||
// establish the full virtual path of this archive: when the generic archive cataloger task drove
|
||||
// us here, the context traversal supplies the containing archive chain; otherwise the reader
|
||||
// path already carries any colon-delimited nesting from this cataloger's own recursion
|
||||
virtualPath := archive.TraversalFromContext(ctx).VirtualPathOf(reader.Path())
|
||||
|
||||
// fetch the last element of the virtual path
|
||||
virtualElements := strings.Split(reader.Path(), ":")
|
||||
virtualElements := strings.Split(virtualPath, ":")
|
||||
currentFilepath := virtualElements[len(virtualElements)-1]
|
||||
|
||||
td := tmpdir.FromContext(ctx)
|
||||
@@ -115,14 +124,15 @@ func newJavaArchiveParser(ctx context.Context, reader file.LocationReadCloser, d
|
||||
return nil, cleanupFn, fmt.Errorf("unable to process java archive: %w", err)
|
||||
}
|
||||
|
||||
fileManifest, err := intFile.NewZipFileManifest(ctx, archivePath)
|
||||
entries, err := newZipEntries(ctx, archivePath)
|
||||
if err != nil {
|
||||
return nil, cleanupFn, fmt.Errorf("unable to read files from java archive: %w", err)
|
||||
return nil, cleanupFn, err
|
||||
}
|
||||
|
||||
return &archiveParser{
|
||||
fileManifest: fileManifest,
|
||||
entries: entries,
|
||||
location: reader.Location,
|
||||
virtualPath: virtualPath,
|
||||
archivePath: archivePath,
|
||||
contentPath: contentPath,
|
||||
fileInfo: newJavaArchiveFilename(currentFilepath),
|
||||
@@ -188,9 +198,11 @@ func (j *archiveParser) parse(ctx context.Context, parentPkg *pkg.Package) ([]pk
|
||||
}
|
||||
pkgs = append(pkgs, nestedPkgs...)
|
||||
relationships = append(relationships, nestedRelationships...)
|
||||
} else {
|
||||
} else if !j.cfg.nestedArchivesHandledExternally() {
|
||||
// nested archives will not be cataloged by anything: this parser is not recursing and the
|
||||
// generic archive cataloger task is disabled
|
||||
// .jar and .war files are present in archives, are others? or generally just consider them top-level?
|
||||
nestedArchives := j.fileManifest.GlobMatch(true, "**/*.jar", "**/*.war")
|
||||
nestedArchives := j.entries.glob(true, "**/*.jar", "**/*.war")
|
||||
if len(nestedArchives) > 0 {
|
||||
slices.Sort(nestedArchives)
|
||||
errs = unknown.Appendf(errs, j.location, "nested archives not cataloged: %v", strings.Join(nestedArchives, ", "))
|
||||
@@ -223,7 +235,7 @@ func finalizePackage(p *pkg.Package) {
|
||||
// discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages.
|
||||
func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package, error) {
|
||||
// search and parse java manifest files
|
||||
manifestMatches := j.fileManifest.GlobMatch(false, manifestGlob)
|
||||
manifestMatches := j.entries.glob(false, manifestGlob)
|
||||
if len(manifestMatches) > 1 {
|
||||
return nil, fmt.Errorf("found multiple manifests in the jar: %+v", manifestMatches)
|
||||
} else if len(manifestMatches) == 0 {
|
||||
@@ -232,7 +244,7 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package,
|
||||
}
|
||||
|
||||
// fetch the manifest file
|
||||
contents, err := intFile.ContentsFromZip(ctx, j.archivePath, manifestMatches...)
|
||||
contents, err := j.entries.contents(ctx, manifestMatches...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract java manifests (%s): %w", j.location, err)
|
||||
}
|
||||
@@ -253,7 +265,7 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package,
|
||||
}
|
||||
|
||||
// grab and assign digest for the entire archive
|
||||
digests, err := getDigestsFromArchive(ctx, j.archivePath)
|
||||
digests, err := j.entries.digests(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -275,7 +287,7 @@ func (j *archiveParser) discoverMainPackage(ctx context.Context) (*pkg.Package,
|
||||
),
|
||||
Type: j.fileInfo.pkgType(),
|
||||
Metadata: pkg.JavaArchive{
|
||||
VirtualPath: j.location.Path(),
|
||||
VirtualPath: j.virtualPath,
|
||||
Manifest: manifest,
|
||||
PomProject: pkgPomProject,
|
||||
ArchiveDigests: digests,
|
||||
@@ -388,8 +400,8 @@ type parsedPomProject struct {
|
||||
// discoverMainPackageFromPomInfo attempts to resolve maven groupId, artifactId, version and other info from found pom information
|
||||
func (j *archiveParser) discoverMainPackageFromPomInfo(ctx context.Context) (group, name, version string, parsedPom *parsedPomProject) {
|
||||
// Find the pom.properties/pom.xml if the names seem like a plausible match
|
||||
properties, _ := pomPropertiesByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomPropertiesGlob))
|
||||
projects, _ := pomProjectByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob))
|
||||
properties, _ := pomPropertiesByParentPath(ctx, j.entries, j.location, j.entries.glob(false, pomPropertiesGlob))
|
||||
projects, _ := pomProjectByParentPath(ctx, j.entries, j.location, j.entries.glob(false, pomXMLGlob))
|
||||
|
||||
artifactsMap := j.buildArtifactsMap(properties)
|
||||
pomProperties, parsedPom := j.findBestPomMatch(properties, projects, artifactsMap)
|
||||
@@ -520,13 +532,13 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, paren
|
||||
var pkgs []pkg.Package
|
||||
|
||||
// pom.properties
|
||||
properties, err := pomPropertiesByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomPropertiesGlob))
|
||||
properties, err := pomPropertiesByParentPath(ctx, j.entries, j.location, j.entries.glob(false, pomPropertiesGlob))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// pom.xml
|
||||
projects, err := pomProjectByParentPath(ctx, j.archivePath, j.location, j.fileManifest.GlobMatch(false, pomXMLGlob))
|
||||
projects, err := pomProjectByParentPath(ctx, j.entries, j.location, j.entries.glob(false, pomXMLGlob))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -537,7 +549,7 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(ctx context.Context, paren
|
||||
parsedPom = proj
|
||||
}
|
||||
|
||||
pkgFromPom := newPackageFromMavenData(ctx, j.maven, propertiesObj, parsedPom, parentPkg, j.location)
|
||||
pkgFromPom := newPackageFromMavenData(ctx, j.maven, propertiesObj, parsedPom, parentPkg, j.location, j.virtualPath)
|
||||
if pkgFromPom != nil {
|
||||
pkgs = append(pkgs, *pkgFromPom)
|
||||
}
|
||||
@@ -569,14 +581,14 @@ func (j *archiveParser) getLicenseFromFileInArchive(ctx context.Context) []pkg.L
|
||||
|
||||
for _, glob := range []string{"/META-INF/*", "/*"} {
|
||||
var licenseMatches []string
|
||||
for _, f := range j.fileManifest.GlobMatch(true, glob) {
|
||||
for _, f := range j.entries.glob(true, glob) {
|
||||
if licenses.IsLicenseFile(path.Base(f)) {
|
||||
licenseMatches = append(licenseMatches, f)
|
||||
}
|
||||
}
|
||||
|
||||
if len(licenseMatches) > 0 {
|
||||
contents, err := intFile.ContentsFromZip(ctx, j.archivePath, licenseMatches...)
|
||||
contents, err := j.entries.contents(ctx, licenseMatches...)
|
||||
if err != nil {
|
||||
log.Debugf("unable to extract java license (%s): %w", j.location, err)
|
||||
continue
|
||||
@@ -609,9 +621,17 @@ func (j *archiveParser) getLicenseFromFileInArchive(ctx context.Context) []pkg.L
|
||||
return identified
|
||||
}
|
||||
|
||||
// discoverPkgsFromNestedArchives recurses into the java archives inside this one. It exists only for
|
||||
// the zip-backed source: recursion needs an archive file to unzip, and the resolver-backed source has
|
||||
// no file - by the time it runs, the archive cataloger task has already extracted everything and will
|
||||
// meet those nested archives itself.
|
||||
func (j *archiveParser) discoverPkgsFromNestedArchives(ctx context.Context, parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
zipped, ok := j.entries.(*zipEntries)
|
||||
if !ok {
|
||||
return nil, nil, nil
|
||||
}
|
||||
// we know that all java archives are zip formatted files, so we can use the shared zip helper
|
||||
return discoverPkgsFromZip(ctx, j.location, j.archivePath, j.contentPath, j.fileManifest, parentPkg, j.cfg)
|
||||
return discoverPkgsFromZip(ctx, j.location, zipped.archivePath, j.contentPath, zipped.fileManifest, parentPkg, j.cfg)
|
||||
}
|
||||
|
||||
// discoverPkgsFromZip finds Java archives within Java archives, returning all listed Java packages found and
|
||||
@@ -682,8 +702,8 @@ func discoverPkgsFromOpener(ctx context.Context, location file.Location, pathWit
|
||||
return nestedPkgs, nestedRelationships, nil
|
||||
}
|
||||
|
||||
func pomPropertiesByParentPath(ctx context.Context, archivePath string, location file.Location, extractPaths []string) (map[string]pkg.JavaPomProperties, error) {
|
||||
contentsOfMavenPropertiesFiles, err := intFile.ContentsFromZip(ctx, archivePath, extractPaths...)
|
||||
func pomPropertiesByParentPath(ctx context.Context, entries archiveEntries, location file.Location, extractPaths []string) (map[string]pkg.JavaPomProperties, error) {
|
||||
contentsOfMavenPropertiesFiles, err := entries.contents(ctx, extractPaths...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract maven files: %w", err)
|
||||
}
|
||||
@@ -711,8 +731,8 @@ func pomPropertiesByParentPath(ctx context.Context, archivePath string, location
|
||||
return propertiesByParentPath, nil
|
||||
}
|
||||
|
||||
func pomProjectByParentPath(ctx context.Context, archivePath string, location file.Location, extractPaths []string) (map[string]*parsedPomProject, error) {
|
||||
contentsOfMavenProjectFiles, err := intFile.ContentsFromZip(ctx, archivePath, extractPaths...)
|
||||
func pomProjectByParentPath(ctx context.Context, entries archiveEntries, location file.Location, extractPaths []string) (map[string]*parsedPomProject, error) {
|
||||
contentsOfMavenProjectFiles, err := entries.contents(ctx, extractPaths...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract maven files: %w", err)
|
||||
}
|
||||
@@ -739,7 +759,7 @@ func pomProjectByParentPath(ctx context.Context, archivePath string, location fi
|
||||
|
||||
// newPackageFromMavenData processes a single Maven POM properties for a given parent package, returning all listed Java packages found and
|
||||
// associating each discovered package to the given parent package. Note the pom.xml is optional, the pom.properties is not.
|
||||
func newPackageFromMavenData(ctx context.Context, r *maven.Resolver, pomProperties pkg.JavaPomProperties, parsedPom *parsedPomProject, parentPkg *pkg.Package, location file.Location) *pkg.Package {
|
||||
func newPackageFromMavenData(ctx context.Context, r *maven.Resolver, pomProperties pkg.JavaPomProperties, parsedPom *parsedPomProject, parentPkg *pkg.Package, location file.Location, virtualPath string) *pkg.Package {
|
||||
// keep the artifact name within the virtual path if this package does not match the parent package
|
||||
vPathSuffix := ""
|
||||
groupID := ""
|
||||
@@ -759,7 +779,7 @@ func newPackageFromMavenData(ctx context.Context, r *maven.Resolver, pomProperti
|
||||
// https://github.com/anchore/syft/issues/1944
|
||||
vPathSuffix += ":" + pomProperties.GroupID + ":" + pomProperties.ArtifactID
|
||||
}
|
||||
virtualPath := location.Path() + vPathSuffix
|
||||
virtualPath += vPathSuffix
|
||||
|
||||
var pkgPomProject *pkg.JavaPomProject
|
||||
|
||||
|
||||
@@ -1155,7 +1155,7 @@ func Test_newPackageFromMavenData(t *testing.T) {
|
||||
test.expectedParent.Locations = locations
|
||||
|
||||
r := maven.NewResolver(nil, maven.DefaultConfig())
|
||||
actualPackage := newPackageFromMavenData(pkgtest.Context(t), r, test.props, test.project, test.parent, file.NewLocation(virtualPath))
|
||||
actualPackage := newPackageFromMavenData(pkgtest.Context(t), r, test.props, test.project, test.parent, file.NewLocation(virtualPath), virtualPath)
|
||||
if test.expectedPackage == nil {
|
||||
require.Nil(t, actualPackage)
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
package java
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"github.com/anchore/syft/syft/source/directorysource"
|
||||
)
|
||||
|
||||
// makeTestZip builds a zip in memory so the parser can be exercised without the
|
||||
// java-toolchain-built fixtures.
|
||||
func makeTestZip(t *testing.T, entries map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for name, content := range entries {
|
||||
w, err := zw.Create(name)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write(content)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, zw.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func testManifest(title, version string) []byte {
|
||||
return fmt.Appendf(nil, "Manifest-Version: 1.0\nImplementation-Title: %s\nImplementation-Version: %s\n", title, version)
|
||||
}
|
||||
|
||||
func Test_virtualPathFromArchiveTraversal(t *testing.T) {
|
||||
jarBytes := makeTestZip(t, map[string][]byte{
|
||||
"META-INF/MANIFEST.MF": testManifest("example-app", "1.0.0"),
|
||||
})
|
||||
|
||||
parse := func(t *testing.T, traversal *archive.Traversal, readerPath string) pkg.Package {
|
||||
ctx := pkgtest.Context(t)
|
||||
if traversal != nil {
|
||||
ctx = archive.WithTraversal(ctx, traversal)
|
||||
}
|
||||
gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{})
|
||||
pkgs, _, err := gap.processJavaArchive(ctx, file.LocationReadCloser{
|
||||
Location: file.NewLocation(readerPath),
|
||||
ReadCloser: io.NopCloser(bytes.NewReader(jarBytes)),
|
||||
}, nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, pkgs, 1)
|
||||
return pkgs[0]
|
||||
}
|
||||
|
||||
t.Run("no traversal keeps reader path as virtual path", func(t *testing.T) {
|
||||
p := parse(t, nil, "some/dir/example-app-1.0.0.jar")
|
||||
metadata, ok := p.Metadata.(pkg.JavaArchive)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "some/dir/example-app-1.0.0.jar", metadata.VirtualPath)
|
||||
assert.Equal(t, "example-app", p.Name)
|
||||
assert.Equal(t, "1.0.0", p.Version)
|
||||
})
|
||||
|
||||
t.Run("traversal prepends containing archive chain", func(t *testing.T) {
|
||||
trav := &archive.Traversal{
|
||||
Location: file.NewLocation("/app.war"),
|
||||
VirtualPath: "/app.war",
|
||||
FileSystemID: "app.war",
|
||||
Depth: 1,
|
||||
}
|
||||
p := parse(t, trav, "/WEB-INF/lib/example-app-1.0.0.jar")
|
||||
metadata, ok := p.Metadata.(pkg.JavaArchive)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "/app.war:WEB-INF/lib/example-app-1.0.0.jar", metadata.VirtualPath)
|
||||
// name/version inference from the last virtual path element is unaffected by the chain
|
||||
assert.Equal(t, "example-app", p.Name)
|
||||
assert.Equal(t, "1.0.0", p.Version)
|
||||
})
|
||||
|
||||
t.Run("multi-level traversal chains all parents", func(t *testing.T) {
|
||||
outer := &archive.Traversal{VirtualPath: "/dist.zip", FileSystemID: "dist.zip", Depth: 1}
|
||||
inner := &archive.Traversal{
|
||||
VirtualPath: outer.VirtualPathOf("/app.war"),
|
||||
FileSystemID: "dist.zip/app.war",
|
||||
Depth: 2,
|
||||
Parent: outer,
|
||||
}
|
||||
p := parse(t, inner, "/WEB-INF/lib/example-app-1.0.0.jar")
|
||||
metadata, ok := p.Metadata.(pkg.JavaArchive)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "/dist.zip:app.war:WEB-INF/lib/example-app-1.0.0.jar", metadata.VirtualPath)
|
||||
})
|
||||
}
|
||||
|
||||
// Test_virtualPathFromArchiveTraversal_colonInEntryPath is the java-cataloger-level counterpart
|
||||
// of internal/archive's escaping tests: a package reached through an entry whose own path
|
||||
// contains a colon must carry that colon escaped as %3A end to end in its virtual path
|
||||
// (decisions.md#colon-in-an-entry-path-is-escaped), and the maven-derived
|
||||
// ":groupID:artifactID" suffix that distinguishes a pom.properties package from its containing
|
||||
// archive must still be appended using real, unescaped delimiters.
|
||||
func Test_virtualPathFromArchiveTraversal_colonInEntryPath(t *testing.T) {
|
||||
jarBytes := makeTestZip(t, map[string][]byte{
|
||||
"META-INF/MANIFEST.MF": testManifest("outer-app", "1.0.0"),
|
||||
"META-INF/maven/com.example/different-lib/pom.properties": []byte(
|
||||
"groupId=com.example\nartifactId=different-lib\nversion=9.9.9\n",
|
||||
),
|
||||
})
|
||||
|
||||
trav := &archive.Traversal{
|
||||
Location: file.NewLocation("/outer.zip"),
|
||||
VirtualPath: "outer.zip",
|
||||
FileSystemID: "outer.zip",
|
||||
Depth: 1,
|
||||
}
|
||||
ctx := archive.WithTraversal(pkgtest.Context(t), trav)
|
||||
|
||||
gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{})
|
||||
pkgs, _, err := gap.processJavaArchive(ctx, file.LocationReadCloser{
|
||||
Location: file.NewLocation("/lib/dep:special.jar"),
|
||||
ReadCloser: io.NopCloser(bytes.NewReader(jarBytes)),
|
||||
}, nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, pkgs, 2)
|
||||
|
||||
var mainPkg, auxPkg *pkg.Package
|
||||
for i := range pkgs {
|
||||
if pkgs[i].Name == "outer-app" {
|
||||
mainPkg = &pkgs[i]
|
||||
} else if pkgs[i].Name == "different-lib" {
|
||||
auxPkg = &pkgs[i]
|
||||
}
|
||||
}
|
||||
require.NotNil(t, mainPkg, "expected a package from the manifest")
|
||||
require.NotNil(t, auxPkg, "expected a package from pom.properties")
|
||||
|
||||
mainMetadata, ok := mainPkg.Metadata.(pkg.JavaArchive)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "outer.zip:lib/dep%3Aspecial.jar", mainMetadata.VirtualPath,
|
||||
"the entry's own colon must be escaped, and it is the only delimiter in play here")
|
||||
|
||||
auxMetadata, ok := auxPkg.Metadata.(pkg.JavaArchive)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "outer.zip:lib/dep%3Aspecial.jar:com.example:different-lib", auxMetadata.VirtualPath,
|
||||
"the maven groupID:artifactID suffix is appended with real, unescaped delimiters")
|
||||
}
|
||||
|
||||
func Test_nestedArchiveOwnershipSwitch(t *testing.T) {
|
||||
nestedJar := makeTestZip(t, map[string][]byte{
|
||||
"META-INF/MANIFEST.MF": testManifest("nested-lib", "2.0"),
|
||||
})
|
||||
outerJar := makeTestZip(t, map[string][]byte{
|
||||
"META-INF/MANIFEST.MF": testManifest("example-app", "1.0.0"),
|
||||
"lib/nested-lib-2.0.jar": nestedJar,
|
||||
})
|
||||
|
||||
parse := func(t *testing.T, cfg ArchiveCatalogerConfig) []pkg.Package {
|
||||
gap := newGenericArchiveParserAdapter(cfg)
|
||||
pkgs, _, err := gap.processJavaArchive(pkgtest.Context(t), file.LocationReadCloser{
|
||||
Location: file.NewLocation("example-app-1.0.0.jar"),
|
||||
ReadCloser: io.NopCloser(bytes.NewReader(outerJar)),
|
||||
}, nil)
|
||||
require.NoError(t, err)
|
||||
return pkgs
|
||||
}
|
||||
|
||||
t.Run("nested archive cataloging disabled: this cataloger unarchives nested jars itself", func(t *testing.T) {
|
||||
pkgs := parse(t, DefaultArchiveCatalogerConfig()) // MaxDepth defaults to 0 (disabled)
|
||||
require.Len(t, pkgs, 2)
|
||||
|
||||
var names, virtualPaths []string
|
||||
for _, p := range pkgs {
|
||||
names = append(names, p.Name)
|
||||
metadata, ok := p.Metadata.(pkg.JavaArchive)
|
||||
require.True(t, ok)
|
||||
virtualPaths = append(virtualPaths, metadata.VirtualPath)
|
||||
}
|
||||
assert.ElementsMatch(t, []string{"example-app", "nested-lib"}, names)
|
||||
assert.ElementsMatch(t, []string{
|
||||
"example-app-1.0.0.jar",
|
||||
"example-app-1.0.0.jar:lib/nested-lib-2.0.jar",
|
||||
}, virtualPaths)
|
||||
})
|
||||
|
||||
t.Run("nested archive cataloging enabled: this cataloger does not unarchive", func(t *testing.T) {
|
||||
cfg := DefaultArchiveCatalogerConfig()
|
||||
// only CreateSBOMConfig sets this in production; it is not reachable from user config
|
||||
cfg.NestedArchivesHandledExternally = true
|
||||
|
||||
// no error expected: the nested jar is deliberately left to the archive cataloger task
|
||||
pkgs := parse(t, cfg)
|
||||
require.Len(t, pkgs, 1)
|
||||
assert.Equal(t, "example-app", pkgs[0].Name)
|
||||
})
|
||||
|
||||
t.Run("java's own archive-search depth does not hand off recursion", func(t *testing.T) {
|
||||
// a library consumer setting java's squash-inlined MaxDepth must neither enable nor
|
||||
// disable nested archive cataloging: that knob is Archive.MaxDepth on CreateSBOMConfig.
|
||||
// Getting this wrong silently drops nested jars for every consumer that is not the CLI.
|
||||
cfg := DefaultArchiveCatalogerConfig()
|
||||
cfg.ArchiveSearchConfig = cfg.ArchiveSearchConfig.WithMaxDepth(2)
|
||||
require.False(t, cfg.nestedArchivesHandledExternally())
|
||||
|
||||
pkgs := parse(t, cfg)
|
||||
require.Len(t, pkgs, 2, "this cataloger must still recurse itself")
|
||||
})
|
||||
}
|
||||
|
||||
func Test_NewArchiveCataloger_wrappedParserRegistration(t *testing.T) {
|
||||
// the wrapped-archive parsers exist only because the generic archive cataloger did not. When it
|
||||
// is enabled it extracts zip and tar containers itself and this cataloger meets the same JARs
|
||||
// one nesting level down, so registering them would double the work.
|
||||
//
|
||||
// asserted through the globs the cataloger actually queries, since that is the observable
|
||||
// consequence of a parser being registered
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "placeholder.txt"), []byte("x"), 0o600))
|
||||
|
||||
queriedGlobs := func(t *testing.T, cfg ArchiveCatalogerConfig) *pkgtest.ObservingResolver {
|
||||
t.Helper()
|
||||
src, err := directorysource.NewFromPath(dir)
|
||||
require.NoError(t, err)
|
||||
resolver, err := src.FileResolver(source.AllLayersScope)
|
||||
require.NoError(t, err)
|
||||
|
||||
observed := pkgtest.NewObservingResolver(resolver)
|
||||
_, _, err = NewArchiveCataloger(cfg).Catalog(pkgtest.Context(t), observed)
|
||||
require.NoError(t, err)
|
||||
return observed
|
||||
}
|
||||
|
||||
t.Run("wrapped parsers are not registered when the archive cataloger owns recursion", func(t *testing.T) {
|
||||
cfg := DefaultArchiveCatalogerConfig()
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.IncludeUnindexedArchives = true
|
||||
cfg.NestedArchivesHandledExternally = true
|
||||
|
||||
observed := queriedGlobs(t, cfg)
|
||||
assert.False(t, observed.ObservedPathQuery("**/*.zip"), "no generic zip parser may be registered")
|
||||
assert.False(t, observed.ObservedPathQuery("**/*.tar"), "no generic tar parser may be registered")
|
||||
|
||||
// the JAR-family globs are still registered: this cataloger is still the java parser
|
||||
assert.True(t, observed.ObservedPathQuery("**/*.jar"))
|
||||
})
|
||||
|
||||
t.Run("wrapped parsers stay registered when the feature is off", func(t *testing.T) {
|
||||
cfg := DefaultArchiveCatalogerConfig()
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.IncludeUnindexedArchives = true
|
||||
|
||||
observed := queriedGlobs(t, cfg)
|
||||
assert.True(t, observed.ObservedPathQuery("**/*.zip"), "existing behavior must be unchanged")
|
||||
assert.True(t, observed.ObservedPathQuery("**/*.tar"), "existing behavior must be unchanged")
|
||||
})
|
||||
|
||||
t.Run("the existing include settings still gate them when the feature is off", func(t *testing.T) {
|
||||
cfg := DefaultArchiveCatalogerConfig()
|
||||
cfg.IncludeIndexedArchives = false
|
||||
cfg.IncludeUnindexedArchives = false
|
||||
|
||||
observed := queriedGlobs(t, cfg)
|
||||
assert.False(t, observed.ObservedPathQuery("**/*.zip"))
|
||||
assert.False(t, observed.ObservedPathQuery("**/*.tar"))
|
||||
})
|
||||
}
|
||||
|
||||
func Test_mavenVirtualPathSuffixUnderTraversal(t *testing.T) {
|
||||
// a package derived from a pom.properties whose artifact does not match the containing archive
|
||||
// gets a trailing ":<groupID>:<artifactID>" appended to the archive's virtual path. That suffix
|
||||
// form is in published SBOMs, and once the archive cataloger drives the parser the base it is
|
||||
// appended to comes from the traversal rather than from the reader path - so this asserts the
|
||||
// two compose rather than one clobbering the other.
|
||||
jarBytes := makeTestZip(t, map[string][]byte{
|
||||
"META-INF/MANIFEST.MF": testManifest("example-app", "1.0.0"),
|
||||
"META-INF/maven/com.example.other/other-lib/pom.properties": []byte(
|
||||
"groupId=com.example.other\nartifactId=other-lib\nversion=2.5.0\n",
|
||||
),
|
||||
})
|
||||
|
||||
parse := func(t *testing.T, traversal *archive.Traversal, readerPath string) []pkg.Package {
|
||||
t.Helper()
|
||||
ctx := pkgtest.Context(t)
|
||||
if traversal != nil {
|
||||
ctx = archive.WithTraversal(ctx, traversal)
|
||||
}
|
||||
gap := newGenericArchiveParserAdapter(ArchiveCatalogerConfig{})
|
||||
pkgs, _, err := gap.processJavaArchive(ctx, file.LocationReadCloser{
|
||||
Location: file.NewLocation(readerPath),
|
||||
ReadCloser: io.NopCloser(bytes.NewReader(jarBytes)),
|
||||
}, nil)
|
||||
require.NoError(t, err)
|
||||
return pkgs
|
||||
}
|
||||
|
||||
virtualPathOf := func(t *testing.T, pkgs []pkg.Package, name string) string {
|
||||
t.Helper()
|
||||
for _, p := range pkgs {
|
||||
if p.Name != name {
|
||||
continue
|
||||
}
|
||||
metadata, ok := p.Metadata.(pkg.JavaArchive)
|
||||
require.True(t, ok)
|
||||
return metadata.VirtualPath
|
||||
}
|
||||
t.Fatalf("no package named %q in %v", name, pkgs)
|
||||
return ""
|
||||
}
|
||||
|
||||
t.Run("no traversal: suffix hangs off the reader path", func(t *testing.T) {
|
||||
pkgs := parse(t, nil, "/example-app-1.0.0.jar")
|
||||
assert.Equal(t, "/example-app-1.0.0.jar:com.example.other:other-lib",
|
||||
virtualPathOf(t, pkgs, "other-lib"))
|
||||
})
|
||||
|
||||
t.Run("under traversal: suffix hangs off the archive chain", func(t *testing.T) {
|
||||
pkgs := parse(t, &archive.Traversal{VirtualPath: "app.war", Depth: 1}, "/WEB-INF/lib/example-app-1.0.0.jar")
|
||||
assert.Equal(t, "app.war:WEB-INF/lib/example-app-1.0.0.jar:com.example.other:other-lib",
|
||||
virtualPathOf(t, pkgs, "other-lib"))
|
||||
})
|
||||
}
|
||||
@@ -15,16 +15,21 @@ func NewArchiveCataloger(cfg ArchiveCatalogerConfig) pkg.Cataloger {
|
||||
c := generic.NewCataloger("java-archive-cataloger").
|
||||
WithParserByGlobs(gap.parseJavaArchive, archiveFormatGlobs...)
|
||||
|
||||
if cfg.IncludeIndexedArchives {
|
||||
// java archives wrapped within zip files
|
||||
gzp := newGenericZipWrappedJavaArchiveParser(cfg)
|
||||
c.WithParserByGlobs(gzp.parseZipWrappedJavaArchive, genericZipGlobs...)
|
||||
}
|
||||
// when the generic archive cataloger task owns recursion into archives, it extracts zip/tar
|
||||
// archives generically and this cataloger meets any java archives within them at the next
|
||||
// nesting level, so the wrapped-archive parsers would only duplicate that work
|
||||
if !cfg.nestedArchivesHandledExternally() {
|
||||
if cfg.IncludeIndexedArchives {
|
||||
// java archives wrapped within zip files
|
||||
gzp := newGenericZipWrappedJavaArchiveParser(cfg)
|
||||
c.WithParserByGlobs(gzp.parseZipWrappedJavaArchive, genericZipGlobs...)
|
||||
}
|
||||
|
||||
if cfg.IncludeUnindexedArchives {
|
||||
// java archives wrapped within tar files
|
||||
gtp := newGenericTarWrappedJavaArchiveParser(cfg)
|
||||
c.WithParserByGlobs(gtp.parseTarWrappedJavaArchive, genericTarGlobs...)
|
||||
if cfg.IncludeUnindexedArchives {
|
||||
// java archives wrapped within tar files
|
||||
gtp := newGenericTarWrappedJavaArchiveParser(cfg)
|
||||
c.WithParserByGlobs(gtp.parseTarWrappedJavaArchive, genericTarGlobs...)
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -33,6 +33,15 @@ type ArchiveCatalogerConfig struct {
|
||||
// ResolveTransitiveDependencies enables resolving transitive dependencies for java packages found within archives.
|
||||
// app-config: java.resolve-transitive-dependencies
|
||||
ResolveTransitiveDependencies bool `yaml:"resolve-transitive-dependencies" json:"resolve-transitive-dependencies" mapstructure:"resolve-transitive-dependencies"`
|
||||
|
||||
// NestedArchivesHandledExternally indicates that some other mechanism owns recursion into
|
||||
// nested archives, so this cataloger must not unarchive them itself. It is deliberately not a
|
||||
// yaml/json/mapstructure key: no config file, flag, or environment variable reaches it.
|
||||
//
|
||||
// It is written ONLY by syft.CreateSBOMConfig, which derives it from the nested-archive
|
||||
// cataloging depth so the two sides cannot disagree. Callers building this config should leave
|
||||
// it alone; the zero value is this cataloger's long-standing behavior of recursing itself.
|
||||
NestedArchivesHandledExternally bool `yaml:"-" json:"-" mapstructure:"-"`
|
||||
}
|
||||
|
||||
func DefaultArchiveCatalogerConfig() ArchiveCatalogerConfig {
|
||||
@@ -81,6 +90,13 @@ func (j ArchiveCatalogerConfig) WithArchiveTraversal(search cataloging.ArchiveSe
|
||||
return j
|
||||
}
|
||||
|
||||
// nestedArchivesHandledExternally indicates another mechanism (the generic archive cataloger task)
|
||||
// owns recursion into all archives, in which case this cataloger must not unarchive nested archives
|
||||
// itself. See the field of the same name for why this is not derived from MaxDepth.
|
||||
func (j ArchiveCatalogerConfig) nestedArchivesHandledExternally() bool {
|
||||
return j.NestedArchivesHandledExternally
|
||||
}
|
||||
|
||||
func (j ArchiveCatalogerConfig) mavenConfig() maven.Config {
|
||||
return maven.Config{
|
||||
UseNetwork: j.UseNetwork,
|
||||
|
||||
@@ -171,6 +171,37 @@ func Test_groupIDFromJavaMetadata(t *testing.T) {
|
||||
metadata: pkg.JavaArchive{},
|
||||
expect: "org.apache.ant",
|
||||
},
|
||||
{
|
||||
// regression for github.com/anchore/syft/issues/4030: spring-ldap artifacts
|
||||
// live under org.springframework.ldap, not org.springframework
|
||||
name: "known package list spring-ldap-core",
|
||||
pkgName: "spring-ldap-core",
|
||||
metadata: pkg.JavaArchive{},
|
||||
expect: "org.springframework.ldap",
|
||||
},
|
||||
{
|
||||
name: "known package list spring-ldap",
|
||||
pkgName: "spring-ldap",
|
||||
metadata: pkg.JavaArchive{},
|
||||
expect: "org.springframework.ldap",
|
||||
},
|
||||
{
|
||||
// regression for github.com/anchore/syft/issues/4598: legacy Jackson 1.x ("-asl")
|
||||
// jars built before ~2014 have no embedded pom.properties, so without the known
|
||||
// package list the group ID falls back to the artifact name itself, producing a
|
||||
// purl that doesn't match the vulnerability database's namespace (e.g. the correct
|
||||
// group for jackson-mapper-asl is org.codehaus.jackson, not jackson-mapper-asl).
|
||||
name: "known package list jackson-mapper-asl",
|
||||
pkgName: "jackson-mapper-asl",
|
||||
metadata: pkg.JavaArchive{},
|
||||
expect: "org.codehaus.jackson",
|
||||
},
|
||||
{
|
||||
name: "known package list jackson-core-asl",
|
||||
pkgName: "jackson-core-asl",
|
||||
metadata: pkg.JavaArchive{},
|
||||
expect: "org.codehaus.jackson",
|
||||
},
|
||||
{
|
||||
name: "java manifest",
|
||||
metadata: pkg.JavaArchive{
|
||||
|
||||
Reference in New Issue
Block a user