mirror of
https://github.com/anchore/syft.git
synced 2026-10-30 22:12:28 +01:00
chore: simplify and cleanup
Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
Keith Zantow
@@ -0,0 +1,825 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/scylladb/go-set/strset"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
// This file covers nesting across archive families. The other nested-archive fixtures are all zips,
|
||||
// so a zip-specific mechanism would pass them; the requirement here is that an archive of one family
|
||||
// inside another behaves as it would inside its own family - same packages, same filesystem-id chain,
|
||||
// same CONTAINS edges, same depth accounting.
|
||||
//
|
||||
// Chains are built in the test rather than committed as binaries, so each case can vary which family
|
||||
// sits at which level.
|
||||
|
||||
// archiveFamily is one archive format buildNestedArchive can produce.
|
||||
//
|
||||
// ".jar" is its own family rather than a flavour of zip: the java cataloger meets it as a jar wherever
|
||||
// it sits, while the archive task descends into it like any other zip. That differs mid-chain from at
|
||||
// the leaf, so the matrix below uses a jar as container and as payload.
|
||||
type archiveFamily string
|
||||
|
||||
const (
|
||||
familyZip archiveFamily = ".zip"
|
||||
familyJar archiveFamily = ".jar"
|
||||
familyTar archiveFamily = ".tar"
|
||||
familyTarGz archiveFamily = ".tar.gz"
|
||||
familyTgz archiveFamily = ".tgz"
|
||||
)
|
||||
|
||||
// nestPlan describes one chain to build: a family per level (outermost first), the entries for the
|
||||
// innermost archive, and extras to add beside the nested archive at a given level. The extras grade
|
||||
// the levels' sizes or entry counts, which is how the limit cases place a bound between two levels.
|
||||
type nestPlan struct {
|
||||
families []archiveFamily
|
||||
leaf map[string][]byte
|
||||
extra map[int]map[string][]byte
|
||||
}
|
||||
|
||||
// nestedArchive is one built chain: the outermost archive's bytes, and the addresses that chain
|
||||
// composes to at each level.
|
||||
type nestedArchive struct {
|
||||
// name is the outermost archive's file name, as written into the scanned directory.
|
||||
name string
|
||||
// bytes is the outermost archive.
|
||||
bytes []byte
|
||||
// sizes is each archive's size in bytes, outermost first. A container is never smaller than what it
|
||||
// holds, so these decrease, and the limit cases place a bound between two of them.
|
||||
sizes []int
|
||||
// paths is each archive's path within its parent, outermost first; paths[0] is the outermost archive
|
||||
// as the scan's resolver reports it.
|
||||
paths []string
|
||||
// fileSystemIDs is the composed chain each archive's own contents carry, outermost first.
|
||||
fileSystemIDs []string
|
||||
// virtualPaths is the same chain in java's colon-joined form, outermost first.
|
||||
virtualPaths []string
|
||||
}
|
||||
|
||||
// nestingRow is one line of the depth-3 matrix.
|
||||
type nestingRow struct {
|
||||
name string
|
||||
families []archiveFamily
|
||||
why string
|
||||
}
|
||||
|
||||
// nestingVisit is one filesystem the archive cataloger's sub-pipeline was run against.
|
||||
type nestingVisit struct {
|
||||
virtualPath string
|
||||
depth int
|
||||
// heldFiles are the basenames of the files in the live extraction work directories at this moment,
|
||||
// sampled only when the probe was given a temp dir to look in. An archive's own bytes are read where
|
||||
// they lie, so only "entries" files appear, one per archive whose entries did not fit in memory.
|
||||
heldFiles []string
|
||||
}
|
||||
|
||||
// nestingProbe records one visit per filesystem the archive cataloger runs its sub-pipeline against,
|
||||
// reading the chain off the traversal on the context.
|
||||
type nestingProbe struct {
|
||||
// tempDir, when set, is where extraction work directories live
|
||||
tempDir string
|
||||
|
||||
mu sync.Mutex
|
||||
visits []nestingVisit
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_leafIsCatalogedOnceWithTheFullChain(t *testing.T) {
|
||||
// family order must not change the result: the same package, addressed by a chain naming its own
|
||||
// containers in its own order
|
||||
for _, row := range nestingRows() {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
probe := &nestingProbe{}
|
||||
s := runNesting(t, scanDir, 3, cataloging.DefaultArchiveSearchConfig(), markerTask(), probe.task())
|
||||
|
||||
assert.Equal(t, 1, packageCount(s, "marker-pkg"), "%s: the leaf package must be cataloged exactly once", row.why)
|
||||
|
||||
locs := leafLocations(s)
|
||||
require.Len(t, locs, 1, "the leaf must be reached by exactly one path")
|
||||
// exact, never a substring: a partially composed chain passes a substring check
|
||||
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath,
|
||||
"the chain must name all three archives outer-to-inner")
|
||||
assert.Equal(t, "nested/marker.txt", locs[0].RealPath,
|
||||
"the path stays relative to the archive the file came from")
|
||||
|
||||
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(),
|
||||
"every level must be cataloged, outer to inner, whatever families they are")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) {
|
||||
// the edges make the chain navigable in the SBOM and key on coordinates the file catalogers
|
||||
// recorded, so file cataloging has to be on for the inner archives to have any
|
||||
for _, row := range nestingRows() {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
s := runNesting(t, scanDir, 3, cataloging.DefaultArchiveSearchConfig(), markerTask(), fileMetadataTask())
|
||||
|
||||
// each archive's coordinates as seen in its parent; the outermost sits in the scanned
|
||||
// directory, so it carries no filesystem id of its own
|
||||
archiveCoords := make([]file.Coordinates, len(row.families))
|
||||
for i := range row.families {
|
||||
var parentFSID string
|
||||
if i > 0 {
|
||||
parentFSID = nested.fileSystemIDs[i-1]
|
||||
}
|
||||
archiveCoords[i] = file.Coordinates{RealPath: nested.paths[i], ArchivePath: parentFSID}
|
||||
}
|
||||
|
||||
fileEdges := strset.New()
|
||||
pkgEdges := strset.New()
|
||||
for _, rel := range s.Relationships {
|
||||
if rel.Type != artifact.ContainsRelationship {
|
||||
continue
|
||||
}
|
||||
from, ok := rel.From.(file.Coordinates)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch to := rel.To.(type) {
|
||||
case file.Coordinates:
|
||||
fileEdges.Add(coordKey(from) + " -> " + coordKey(to))
|
||||
case pkg.Package:
|
||||
pkgEdges.Add(coordKey(from) + " -> " + to.Name)
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < len(archiveCoords)-1; i++ {
|
||||
assert.True(t, fileEdges.Has(coordKey(archiveCoords[i])+" -> "+coordKey(archiveCoords[i+1])),
|
||||
"expected a CONTAINS edge from %s to %s", nested.paths[i], nested.paths[i+1])
|
||||
}
|
||||
|
||||
innermost := archiveCoords[len(archiveCoords)-1]
|
||||
assert.True(t, pkgEdges.Has(coordKey(innermost)+" -> marker-pkg"),
|
||||
"expected a CONTAINS edge from %s to the leaf package", nested.paths[len(nested.paths)-1])
|
||||
assert.True(t, fileEdges.Has(coordKey(innermost)+" -> "+coordKey(file.Coordinates{
|
||||
RealPath: "nested/marker.txt",
|
||||
ArchivePath: nested.leafFileSystemID(),
|
||||
})), "expected a CONTAINS edge from the innermost archive to the leaf file")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_javaLeafVirtualPathIsTheColonJoinedChain(t *testing.T) {
|
||||
// these colon-delimited strings are in already-published SBOMs, so a jar reached through three
|
||||
// levels of mixed families must still be reported under the chain of all three
|
||||
for _, row := range nestingRows() {
|
||||
if row.families[len(row.families)-1] != familyJar {
|
||||
continue
|
||||
}
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
s := runNesting(t, scanDir, 3, cataloging.DefaultArchiveSearchConfig(), markerTask(), javaTask())
|
||||
|
||||
// the innermost archive is the jar, so its own package is the java leaf and the chain of
|
||||
// all three archives is exactly its virtual path
|
||||
leaf := fmt.Sprintf("level%d", len(row.families)-1)
|
||||
assert.Equal(t, 1, packageCount(s, leaf), "the leaf jar must be cataloged exactly once")
|
||||
assert.Equal(t, nested.virtualPaths[len(row.families)-1], javaVirtualPaths(s)[leaf])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_jarIsExercisedAsAContainer(t *testing.T) {
|
||||
// two mechanisms meet a jar: the java cataloger catalogs it as a package, the archive task descends
|
||||
// into it. A jar mid-chain must do both, exactly once each; the failure mode is a duplicate, not an
|
||||
// error.
|
||||
for _, row := range nestingRows() {
|
||||
var containerLevels []int
|
||||
for i, family := range row.families {
|
||||
if family == familyJar && i < len(row.families)-1 {
|
||||
containerLevels = append(containerLevels, i)
|
||||
}
|
||||
}
|
||||
if len(containerLevels) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
probe := &nestingProbe{}
|
||||
s := runNesting(t, scanDir, 3, cataloging.DefaultArchiveSearchConfig(), markerTask(), javaTask(), probe.task())
|
||||
|
||||
for _, level := range containerLevels {
|
||||
name := fmt.Sprintf("level%d", level)
|
||||
assert.Equal(t, 1, packageCount(s, name),
|
||||
"the container jar's own package must be cataloged exactly once")
|
||||
assert.Equal(t, nested.virtualPaths[level], javaVirtualPaths(s)[name],
|
||||
"the container jar's virtual path must be its own place in the chain")
|
||||
}
|
||||
|
||||
// and the archive task still descended through it, so the chain is complete below it
|
||||
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs())
|
||||
locs := leafLocations(s)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) {
|
||||
// two rows with different families at every level, so the bound is shown not to turn on which
|
||||
// families are involved
|
||||
for _, row := range nestingRows()[:2] {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
probe := &nestingProbe{}
|
||||
s := runNesting(t, scanDir, 2, cataloging.DefaultArchiveSearchConfig(), markerTask(), fileMetadataTask(), probe.task())
|
||||
|
||||
assert.Equal(t, nested.fileSystemIDs[:2], probe.archiveFileSystemIDs(),
|
||||
"at depth 2 the outer and middle archives are cataloged and the innermost is not")
|
||||
assert.Zero(t, packageCount(s, "marker-pkg"),
|
||||
"the leaf is one level past the bound, so its package must be absent")
|
||||
|
||||
// the innermost archive is still cataloged as a file, inside the middle archive
|
||||
innermost := file.Coordinates{
|
||||
RealPath: nested.paths[2],
|
||||
ArchivePath: nested.fileSystemIDs[1],
|
||||
}
|
||||
_, ok := s.Artifacts.FileMetadata[innermost]
|
||||
assert.True(t, ok, "the undescended archive must still be recorded as a file at %s", coordKey(innermost))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_defaultBoundsCatalogEveryRow(t *testing.T) {
|
||||
// the default bounds, pinned along with the fact that a three-level chain of any family mix sits
|
||||
// well inside them
|
||||
bounds := cataloging.DefaultArchiveSearchConfig()
|
||||
require.Equal(t, int64(2*1024*1024*1024), bounds.MaxMemoryBytes)
|
||||
require.Equal(t, int64(100*1024*1024*1024), bounds.MaxDiskBytes)
|
||||
|
||||
for _, row := range nestingRows() {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
probe := &nestingProbe{}
|
||||
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
|
||||
|
||||
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(),
|
||||
"every level of the chain must be cataloged at the default bounds")
|
||||
locs := leafLocations(s)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_memoryPressureOverflowsRatherThanFailing(t *testing.T) {
|
||||
// entries that will not fit in memory are written to disk rather than refused
|
||||
scanDir := t.TempDir()
|
||||
tempDir := isolatedTempDir(t)
|
||||
nested := writeNestedArchive(t, scanDir, gradedChain())
|
||||
|
||||
t.Run("with a generous memory limit nothing is written to disk", func(t *testing.T) {
|
||||
probe := &nestingProbe{tempDir: tempDir}
|
||||
bounds := cataloging.DefaultArchiveSearchConfig().
|
||||
WithMaxMemoryBytes(int64(2 * (nested.sizes[0] + nested.sizes[1] + nested.sizes[2])))
|
||||
runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
|
||||
|
||||
deepest := probe.deepestVisit()
|
||||
require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held")
|
||||
assert.Empty(t, deepest.heldFiles)
|
||||
})
|
||||
|
||||
t.Run("with a memory limit the outermost archive's entries do not fit in", func(t *testing.T) {
|
||||
probe := &nestingProbe{tempDir: tempDir}
|
||||
bounds := cataloging.DefaultArchiveSearchConfig().
|
||||
WithMaxMemoryBytes(int64(nested.sizes[2])).
|
||||
WithMaxDiskBytes(-1)
|
||||
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
|
||||
|
||||
deepest := probe.deepestVisit()
|
||||
require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held")
|
||||
assert.Contains(t, deepest.heldFiles, "entries", "entries that do not fit in memory are written to disk")
|
||||
|
||||
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs(), "and every level is still cataloged")
|
||||
locs := leafLocations(s)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
|
||||
})
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) {
|
||||
// an archive's own bytes are read where they lie: a scanned file from disk, a nested archive from
|
||||
// its parent's entries. Nothing copies them, so no "archive" file ever appears in a work directory
|
||||
// whatever the memory limit; only entries are held, and only they can be written out.
|
||||
scanDir := t.TempDir()
|
||||
tempDir := isolatedTempDir(t)
|
||||
nested := writeNestedArchive(t, scanDir, gradedChain())
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
memoryBytes int64
|
||||
wantHeldFiles []string
|
||||
}{
|
||||
{
|
||||
name: "with room in memory for every level's entries",
|
||||
memoryBytes: int64(2 * (nested.sizes[0] + nested.sizes[1] + nested.sizes[2])),
|
||||
},
|
||||
{
|
||||
name: "with a memory limit of zero, so every level's entries are written out",
|
||||
memoryBytes: 0,
|
||||
wantHeldFiles: []string{"entries", "entries", "entries"},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
probe := &nestingProbe{tempDir: tempDir}
|
||||
bounds := cataloging.DefaultArchiveSearchConfig().
|
||||
WithMaxMemoryBytes(tc.memoryBytes).
|
||||
WithMaxDiskBytes(-1)
|
||||
|
||||
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
|
||||
|
||||
deepest := probe.deepestVisit()
|
||||
require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held")
|
||||
assert.Equal(t, tc.wantHeldFiles, deepest.heldFiles)
|
||||
|
||||
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs())
|
||||
locs := leafLocations(s)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_diskLimitTruncatesTheLevelItCannotAdmit(t *testing.T) {
|
||||
// the walk descends before it unwinds, so the outer and middle archives still hold their entries
|
||||
// when the innermost level's do not fit in memory. With nothing admitted on disk that level is
|
||||
// cataloged from nothing, the shallower ones keep what they found, and the scan finishes. An
|
||||
// implementation that waited for capacity would deadlock, hence the timeout.
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, deepChain())
|
||||
|
||||
// a sibling of a different family, processed once the chain has been released, so cataloged in full
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(scanDir, "sibling.tgz"),
|
||||
makeTarGz(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}),
|
||||
0o644,
|
||||
))
|
||||
|
||||
// room for the outer two levels' entries (each about the size of the archive it holds) but not the
|
||||
// innermost's payload, which is half an archive larger than the slack
|
||||
memoryLimit := int64(nested.sizes[0] + nested.sizes[1] + deepChainPayloadBytes/2)
|
||||
|
||||
probe := &nestingProbe{}
|
||||
bounds := cataloging.DefaultArchiveSearchConfig().
|
||||
WithMaxMemoryBytes(memoryLimit).
|
||||
WithMaxDiskBytes(0)
|
||||
|
||||
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
|
||||
|
||||
want := append(append([]string{}, nested.fileSystemIDs...), "/sibling.tgz")
|
||||
assert.ElementsMatch(t, want, probe.archiveFileSystemIDs(),
|
||||
"every level is visited, the truncated one with nothing inside it")
|
||||
|
||||
fsIDs := strset.New()
|
||||
for _, loc := range leafLocations(s) {
|
||||
fsIDs.Add(loc.ArchivePath)
|
||||
}
|
||||
assert.False(t, fsIDs.Has(nested.leafFileSystemID()), "the truncated level's contents must be absent")
|
||||
assert.True(t, fsIDs.Has("/sibling.tgz"), "the sibling of a different family must be cataloged fully")
|
||||
|
||||
var truncated []string
|
||||
for coords, reasons := range s.Artifacts.Unknowns {
|
||||
for _, reason := range reasons {
|
||||
if strings.Contains(reason, "disk limit") {
|
||||
truncated = append(truncated, coords.ArchivePath+":"+coords.RealPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Equal(t, []string{nested.fileSystemIDs[1] + ":" + nested.paths[2]}, truncated,
|
||||
"the truncation is recorded against the archive that reached the limit")
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_unboundedLimitsEnforceNothing(t *testing.T) {
|
||||
// a caller can opt out of one bound without the other, and opting out of both must still find the
|
||||
// deepest leaf. Both opt out with a negative value, since zero means "none of that resource".
|
||||
for _, row := range nestingRows() {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
probe := &nestingProbe{}
|
||||
bounds := cataloging.DefaultArchiveSearchConfig().
|
||||
WithMaxMemoryBytes(-1).
|
||||
WithMaxDiskBytes(-1)
|
||||
|
||||
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
|
||||
|
||||
assert.Equal(t, nested.fileSystemIDs, probe.archiveFileSystemIDs())
|
||||
locs := leafLocations(s)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, nested.leafFileSystemID(), locs[0].ArchivePath)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_bothLimitsZeroCatalogsNothingInsideAnyArchive(t *testing.T) {
|
||||
// unlike negative (unbounded), zero on both limits admits no entry anywhere: the outermost archive is
|
||||
// cataloged from nothing, so the level inside it is never found, and the scan still succeeds
|
||||
for _, row := range nestingRows() {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
probe := &nestingProbe{}
|
||||
bounds := cataloging.DefaultArchiveSearchConfig().
|
||||
WithMaxMemoryBytes(0).
|
||||
WithMaxDiskBytes(0)
|
||||
|
||||
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
|
||||
|
||||
assert.Equal(t, nested.fileSystemIDs[:1], probe.archiveFileSystemIDs())
|
||||
assert.Empty(t, leafLocations(s))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// runNesting runs the archive cataloger task over scanDir with the given sub-pipeline and returns the
|
||||
// SBOM it built. It runs on its own goroutine and fails if the task has not finished in time.
|
||||
//
|
||||
// The depth-3 limit cases need the timeout: the walk descends before it unwinds, so the outer and
|
||||
// middle archives still hold their content when the inner one is refused. An implementation that
|
||||
// waited for capacity would deadlock, and a test asserting only the result would hang rather than fail.
|
||||
func runNesting(t *testing.T, scanDir string, depth int, bounds cataloging.ArchiveSearchConfig, subPipeline ...Task) *sbom.SBOM {
|
||||
t.Helper()
|
||||
tsk := newTestTask(t, bounds.WithMaxDepth(depth), subPipeline...)
|
||||
s := &sbom.SBOM{Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(),
|
||||
FileMetadata: map[file.Coordinates]file.Metadata{},
|
||||
}}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(s))
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err, "reaching a bound must not fail the scan")
|
||||
return s
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatal("the scan did not finish in time: reaching a limit must skip the archive that " +
|
||||
"reached it, never wait for capacity that nothing in the chain will release")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// markerTask emits one package per "marker.txt" found, so the matrix asserts behavior without
|
||||
// depending on any real cataloger's fixture format.
|
||||
func markerTask() Task {
|
||||
return NewTask("marker-cataloger", func(_ context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
|
||||
locations, err := resolver.FilesByGlob("**/marker.txt")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, loc := range locations {
|
||||
p := pkg.Package{
|
||||
Name: "marker-pkg",
|
||||
Version: "1.0.0",
|
||||
Type: pkg.BinaryPkg,
|
||||
Locations: file.NewLocationSet(loc),
|
||||
}
|
||||
p.SetID()
|
||||
builder.AddPackages(p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// javaTask is the java archive cataloger as the cataloging factory builds it, so a jar is met as a jar
|
||||
// wherever it sits in a chain.
|
||||
func javaTask() Task {
|
||||
return NewPackageTask(CatalogingFactoryConfig{}, java.NewArchiveCataloger(java.DefaultArchiveCatalogerConfig()))
|
||||
}
|
||||
|
||||
// fileMetadataTask records every file, which the CONTAINS chain needs: archive-to-file edges key on
|
||||
// coordinates the file catalogers recorded, so without them an inner archive has no coordinate to
|
||||
// hang an edge on.
|
||||
func fileMetadataTask() Task {
|
||||
return newFileMetadataCatalogerTask(file.AllFilesSelection)
|
||||
}
|
||||
|
||||
// pack builds one archive of this family holding the given entries.
|
||||
func (f archiveFamily) pack(t *testing.T, base string, entries map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
switch f {
|
||||
case familyZip:
|
||||
return makeZip(t, entries)
|
||||
case familyJar:
|
||||
// a manifest makes it a real jar to the java cataloger at any level; the package java reports is
|
||||
// named after the file, not the manifest title
|
||||
return makeJar(t, base, "1.0", entries)
|
||||
case familyTar:
|
||||
return makeTar(t, entries)
|
||||
case familyTarGz, familyTgz:
|
||||
return makeTarGz(t, entries)
|
||||
}
|
||||
t.Fatalf("unknown archive family %q", f)
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeJar(t *testing.T, title, version string, extra map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
entries := map[string][]byte{
|
||||
"META-INF/MANIFEST.MF": []byte(
|
||||
"Manifest-Version: 1.0\nImplementation-Title: " + title + "\nImplementation-Version: " + version + "\n",
|
||||
),
|
||||
}
|
||||
for k, v := range extra {
|
||||
entries[k] = v
|
||||
}
|
||||
return makeZip(t, entries)
|
||||
}
|
||||
|
||||
// leafFileSystemID is the chain the innermost archive's contents carry: every archive, outer to
|
||||
// inner. Assert it exactly, never as a substring: a partially composed chain passes a substring check.
|
||||
func (n nestedArchive) leafFileSystemID() string {
|
||||
return n.fileSystemIDs[len(n.fileSystemIDs)-1]
|
||||
}
|
||||
|
||||
// buildNestedArchive builds one archive per family given, each holding the next, with leaf inside
|
||||
// the innermost.
|
||||
func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive {
|
||||
t.Helper()
|
||||
require.NotEmpty(t, plan.families, "a chain needs at least one level")
|
||||
|
||||
out := nestedArchive{
|
||||
sizes: make([]int, len(plan.families)),
|
||||
paths: make([]string, len(plan.families)),
|
||||
fileSystemIDs: make([]string, len(plan.families)),
|
||||
virtualPaths: make([]string, len(plan.families)),
|
||||
}
|
||||
|
||||
// the level index goes in every name and path segment, so no two levels share a path and a chain
|
||||
// assertion cannot pass on a value composed from the wrong level
|
||||
for i, family := range plan.families {
|
||||
base := fmt.Sprintf("level%d", i)
|
||||
if i == 0 {
|
||||
// the outermost archive is a file in the scanned directory, which the resolver reports rooted
|
||||
out.name = base + string(family)
|
||||
out.paths[i] = "/" + out.name
|
||||
continue
|
||||
}
|
||||
out.paths[i] = fmt.Sprintf("nest%d/%s%s", i, base, family)
|
||||
}
|
||||
|
||||
for i := range plan.families {
|
||||
if i == 0 {
|
||||
out.fileSystemIDs[i] = out.paths[0]
|
||||
out.virtualPaths[i] = out.paths[0]
|
||||
continue
|
||||
}
|
||||
out.fileSystemIDs[i] = out.fileSystemIDs[i-1] + ":" + out.paths[i]
|
||||
out.virtualPaths[i] = out.virtualPaths[i-1] + ":" + out.paths[i]
|
||||
}
|
||||
|
||||
// built innermost first, because each level's bytes are an entry of the level above it
|
||||
var packed []byte
|
||||
for i := len(plan.families) - 1; i >= 0; i-- {
|
||||
entries := map[string][]byte{}
|
||||
if i == len(plan.families)-1 {
|
||||
for name, body := range plan.leaf {
|
||||
entries[name] = body
|
||||
}
|
||||
} else {
|
||||
entries[out.paths[i+1]] = packed
|
||||
}
|
||||
for name, body := range plan.extra[i] {
|
||||
entries[name] = body
|
||||
}
|
||||
packed = plan.families[i].pack(t, fmt.Sprintf("level%d", i), entries)
|
||||
out.sizes[i] = len(packed)
|
||||
}
|
||||
out.bytes = packed
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// writeNestedArchive writes the chain's outermost archive into dir and returns the chain.
|
||||
func writeNestedArchive(t *testing.T, dir string, plan nestPlan) nestedArchive {
|
||||
t.Helper()
|
||||
nested := buildNestedArchive(t, plan)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, nested.name), nested.bytes, 0o644))
|
||||
return nested
|
||||
}
|
||||
|
||||
// markerLeaf is the family-agnostic leaf payload: one file no archive format has an opinion about,
|
||||
// found by markerTask. It keeps the rows comparable - the same package must come out whichever
|
||||
// families the chain is made of.
|
||||
func markerLeaf() map[string][]byte {
|
||||
return map[string][]byte{"nested/marker.txt": []byte("leaf")}
|
||||
}
|
||||
|
||||
// nestingRows is the depth-3 matrix. Each row covers a case the others do not.
|
||||
func nestingRows() []nestingRow {
|
||||
return []nestingRow{
|
||||
{
|
||||
name: "tar.gz/zip/jar",
|
||||
families: []archiveFamily{familyTarGz, familyZip, familyJar},
|
||||
why: "the headline case: compressed tar outermost, jar innermost",
|
||||
},
|
||||
{
|
||||
name: "zip/tar.gz/jar",
|
||||
families: []archiveFamily{familyZip, familyTarGz, familyJar},
|
||||
why: "compressed tar in the middle, where the stream-only format is not the entry point",
|
||||
},
|
||||
{
|
||||
name: "jar/tar/zip",
|
||||
families: []archiveFamily{familyJar, familyTar, familyZip},
|
||||
why: "jar as a container rather than a payload",
|
||||
},
|
||||
{
|
||||
name: "tgz/tar/jar",
|
||||
families: []archiveFamily{familyTgz, familyTar, familyJar},
|
||||
why: "the .tgz alias resolving at a non-outermost boundary",
|
||||
},
|
||||
{
|
||||
name: "tar/jar/tar.gz",
|
||||
families: []archiveFamily{familyTar, familyJar, familyTarGz},
|
||||
why: "compressed tar as the innermost, which nothing else covers",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *nestingProbe) task() Task {
|
||||
return NewTask("nesting-probe", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
trav := archive.TraversalFromContext(ctx)
|
||||
visit := nestingVisit{}
|
||||
if trav != nil {
|
||||
visit.virtualPath = archive.VirtualPath(trav.Location)
|
||||
visit.depth = strings.Count(archive.VirtualPath(trav.Location), ":") + 1
|
||||
}
|
||||
if p.tempDir != "" {
|
||||
visit.heldFiles = heldFiles(p.tempDir)
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.visits = append(p.visits, visit)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// archiveFileSystemIDs is the chain of every archive whose contents were cataloged, in visit order.
|
||||
func (p *nestingProbe) archiveFileSystemIDs() []string {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
var out []string
|
||||
for _, v := range p.visits {
|
||||
if v.virtualPath == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, v.virtualPath)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// deepestVisit is the visit at the innermost level reached, the only moment at which every level of a
|
||||
// chain is still held and its routing observable. It returns the whole visit so a case can also assert
|
||||
// it got as deep as it meant to: an empty overflow sample from a visit that never left the scan root
|
||||
// would otherwise read as "nothing overflowed".
|
||||
func (p *nestingProbe) deepestVisit() nestingVisit {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
var best nestingVisit
|
||||
for _, v := range p.visits {
|
||||
if v.depth > best.depth {
|
||||
best = v
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// heldFiles names the files in the live extraction work directories under tempDir. Meaningful only
|
||||
// while the archives are still held, so it is sampled from inside the sub-pipeline rather than after
|
||||
// the scan.
|
||||
func heldFiles(tempDir string) (names []string) {
|
||||
_ = filepath.WalkDir(tempDir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || !d.IsDir() || !strings.HasPrefix(d.Name(), "syft-archive-") {
|
||||
return nil //nolint:nilerr // an unreadable directory holds nothing this probe can report
|
||||
}
|
||||
held, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
for _, entry := range held {
|
||||
if !entry.IsDir() {
|
||||
names = append(names, entry.Name())
|
||||
}
|
||||
}
|
||||
return filepath.SkipDir
|
||||
})
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// isolatedTempDir points archive extraction at a directory of this test's own, so an overflow sample
|
||||
// sees only this scan's work directories.
|
||||
func isolatedTempDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
t.Setenv("TMPDIR", dir)
|
||||
require.Equal(t, dir, os.TempDir(), "extraction must be writing where the probe is looking")
|
||||
return dir
|
||||
}
|
||||
|
||||
func leafLocations(s *sbom.SBOM) []file.Location {
|
||||
var out []file.Location
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if p.Name != "marker-pkg" {
|
||||
continue
|
||||
}
|
||||
out = append(out, p.Locations.ToSlice()...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func javaVirtualPaths(s *sbom.SBOM) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if metadata, ok := p.Metadata.(pkg.JavaArchive); ok {
|
||||
out[p.Name] = metadata.VirtualPath
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func packageCount(s *sbom.SBOM, name string) int {
|
||||
var n int
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if p.Name == name {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func coordKey(c file.Coordinates) string {
|
||||
return c.ArchivePath + "|" + c.RealPath
|
||||
}
|
||||
|
||||
// gradedChain is the headline row with each level padded so the three archives have clearly different
|
||||
// sizes: a container is never smaller than what it holds, so a bound can only be placed between two
|
||||
// levels if they are graded on purpose.
|
||||
func gradedChain() nestPlan {
|
||||
return nestPlan{
|
||||
families: []archiveFamily{familyTarGz, familyZip, familyJar},
|
||||
leaf: markerLeaf(),
|
||||
extra: map[int]map[string][]byte{
|
||||
0: {"pad0.bin": incompressible(24 * 1024)},
|
||||
1: {"pad1.bin": incompressible(24 * 1024)},
|
||||
2: {"pad2.bin": incompressible(8 * 1024)},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// deepChainPayloadBytes is the size of deepChain's leaf payload.
|
||||
const deepChainPayloadBytes = 64 * 1024
|
||||
|
||||
// deepChain is the headline row with one large, incompressible payload at the leaf and no other
|
||||
// padding. The payload dominates every level, so the three archives are about the same size. It sorts
|
||||
// before the marker so it is the first entry refused when the leaf archive's entries do not fit.
|
||||
func deepChain() nestPlan {
|
||||
return nestPlan{
|
||||
families: []archiveFamily{familyTarGz, familyZip, familyJar},
|
||||
leaf: map[string][]byte{
|
||||
"0-payload.bin": incompressible(deepChainPayloadBytes),
|
||||
"nested/marker.txt": []byte("leaf"),
|
||||
},
|
||||
}
|
||||
}
|
||||
+121
-270
@@ -5,7 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"maps"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
@@ -28,48 +28,37 @@ import (
|
||||
|
||||
const ArchiveCatalogerTaskName = "archive-cataloger"
|
||||
|
||||
// NewArchiveCatalogerTask returns a task that recursively extracts archives (including JAR-family
|
||||
// ones, whose cataloger skips its own unarchiving when this task is enabled), treats each as its own
|
||||
// standalone indexed filesystem, runs subPipeline against each, and records file-level CONTAINS
|
||||
// relationships from the archive to what was found inside it.
|
||||
// NewArchiveCatalogerTask returns a task that extracts every archive it finds, runs subPipeline
|
||||
// against the archive's contents as a standalone filesystem, and records CONTAINS relationships
|
||||
// from the archive to what was found inside it. It recurses into the archives it extracts up to
|
||||
// cfg.MaxDepth; the task drops itself from subPipeline so that is the only recursion.
|
||||
//
|
||||
// Returns nil when archive cataloging is disabled (MaxDepth == 0; negative means unbounded).
|
||||
//
|
||||
// subPipeline must not include this task, so recursion is driven only by the depth-limited walk here.
|
||||
// newStoreResolver is injected so this package does not depend on the internal fileresolver package.
|
||||
func NewArchiveCatalogerTask(cfg cataloging.ArchiveSearchConfig, subPipeline []Task, newStoreResolver archive.StoreResolverFactory, notify archive.Notify) Task {
|
||||
if cfg.MaxDepth == 0 || newStoreResolver == nil {
|
||||
// exclusions are the scan's exclusion patterns, applied inside each archive. Returns nil when archive
|
||||
// cataloging is disabled (MaxDepth == 0; negative means unbounded).
|
||||
func NewArchiveCatalogerTask(cfg cataloging.ArchiveSearchConfig, subPipeline []Task, exclusions []string) Task {
|
||||
if cfg.MaxDepth == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// this task drives recursion itself: in its own sub-pipeline, each nesting level would be processed
|
||||
// once by the walk and again by the nested copy
|
||||
subPipeline = withoutArchiveCataloger(subPipeline)
|
||||
if len(subPipeline) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
|
||||
// this task re-runs catalogers, so it must ensure each keeps to one row; a scan installs the
|
||||
// registry up front, this covers a pipeline assembled without one
|
||||
// catalogers run once per archive, and each must report into the one progress row it owns
|
||||
ctx = bus.WithCatalogerTaskRegistry(ctx)
|
||||
|
||||
// one progress row for the whole walk; the sub-pipeline's catalogers each report into the row
|
||||
// they already own (see bus.WithCatalogerTaskRegistry), not this one
|
||||
prog := bus.StartCatalogerTask(ctx, archiveCatalogerProgressInfo(), -1, "")
|
||||
c := &archiveCataloger{
|
||||
cfg: cfg,
|
||||
subPipeline: subPipeline,
|
||||
extractors: archive.DefaultExtractors(),
|
||||
limiter: archive.NewLimiter(archive.DefaultLimits(cfg)),
|
||||
newStoreResolver: newStoreResolver,
|
||||
notify: notify,
|
||||
prog: prog,
|
||||
maxDepth: cfg.MaxDepth,
|
||||
subPipeline: subPipeline,
|
||||
limiter: archive.NewLimiter(archive.Limits{MaxMemoryBytes: cfg.MaxMemoryBytes, MaxDiskBytes: cfg.MaxDiskBytes}),
|
||||
exclusions: archive.NewExclusions(exclusions),
|
||||
progress: bus.StartCatalogerTask(ctx, archiveCatalogerProgressInfo(), -1, ""),
|
||||
}
|
||||
err := c.catalog(archive.WithLimiter(ctx, c.limiter), resolver, nil, 0, builder)
|
||||
c.left(nil)
|
||||
err := c.catalog(ctx, resolver, 0, builder)
|
||||
c.progress.AtomicStage.Set(fmt.Sprintf("%s archives", humanize.Comma(c.progress.Current())))
|
||||
c.progress.SetCompleted()
|
||||
c.logStats()
|
||||
prog.SetCompleted()
|
||||
|
||||
return err
|
||||
}
|
||||
return NewTask(ArchiveCatalogerTaskName, fn, pkgcataloging.PackageTag, "archive")
|
||||
@@ -88,14 +77,9 @@ func archiveCatalogerProgressInfo() monitor.GenericTask {
|
||||
}
|
||||
|
||||
func withoutArchiveCataloger(tasks []Task) []Task {
|
||||
out := make([]Task, 0, len(tasks))
|
||||
var out []Task
|
||||
for _, t := range tasks {
|
||||
if t == nil {
|
||||
continue
|
||||
}
|
||||
if t.Name() == ArchiveCatalogerTaskName {
|
||||
log.WithFields("task", t.Name()).
|
||||
Debug("dropping archive cataloger task from its own sub-pipeline")
|
||||
if t == nil || t.Name() == ArchiveCatalogerTaskName {
|
||||
continue
|
||||
}
|
||||
out = append(out, t)
|
||||
@@ -104,213 +88,120 @@ func withoutArchiveCataloger(tasks []Task) []Task {
|
||||
}
|
||||
|
||||
type archiveCataloger struct {
|
||||
cfg cataloging.ArchiveSearchConfig
|
||||
subPipeline []Task
|
||||
extractors []archive.Extractor
|
||||
newStoreResolver archive.StoreResolverFactory
|
||||
maxDepth int
|
||||
subPipeline []Task
|
||||
limiter *archive.Limiter
|
||||
exclusions archive.Exclusions
|
||||
|
||||
// notify carries structured extraction events - which archive spilled to disk and why, which was
|
||||
// skipped. Nil is the no-op.
|
||||
notify archive.Notify
|
||||
// progress is the one row for the whole walk: its count is how many archives have been entered
|
||||
progress *monitor.TaskProgress
|
||||
|
||||
// limiter measures the archive content held right now, falling as each archive is released. One
|
||||
// instance per task run, since one task run is one scan.
|
||||
limiter *archive.Limiter
|
||||
|
||||
// prog is the single progress row for the whole walk: its count and stage are how many archives have
|
||||
// been entered, its context names the one being cataloged right now.
|
||||
prog *monitor.TaskProgress
|
||||
|
||||
// slowest is the archive that spent the most time on its own extraction and cataloging, by virtual
|
||||
// path. Guarded because the walk runs alongside the top-level catalogers.
|
||||
statsMu sync.Mutex
|
||||
slowest time.Duration
|
||||
slowestPath string
|
||||
archivesSeen int64
|
||||
// the archive that spent the longest on its own extraction and cataloging, excluding nested archives
|
||||
slowest time.Duration
|
||||
slowestPath string
|
||||
}
|
||||
|
||||
// recordArchiveTime keeps the longest self time seen and the archive that spent it. Self time
|
||||
// excludes nested archives; inclusive time would name the outermost archive every run.
|
||||
func (c *archiveCataloger) recordArchiveTime(virtualPath string, took time.Duration) {
|
||||
c.statsMu.Lock()
|
||||
defer c.statsMu.Unlock()
|
||||
c.archivesSeen++
|
||||
if took > c.slowest {
|
||||
c.slowest = took
|
||||
c.slowestPath = virtualPath
|
||||
}
|
||||
}
|
||||
|
||||
// logStats reports what the walk cost, once, when it is done. The peaks are high-water marks of the
|
||||
// gauges the limits are enforced against - comparable to the configured bounds, not process memory or
|
||||
// filesystem usage. One line per scan, so it logs at info; per-archive lines stay at debug.
|
||||
// logStats reports what the walk cost, once. The peaks are the most held against each limit at any
|
||||
// one moment, comparable to the configured bounds.
|
||||
func (c *archiveCataloger) logStats() {
|
||||
c.statsMu.Lock()
|
||||
slowest, slowestPath, seen := c.slowest, c.slowestPath, c.archivesSeen
|
||||
c.statsMu.Unlock()
|
||||
|
||||
peakMemory, peakDisk := c.limiter.Peak()
|
||||
fields := []any{
|
||||
"archives", seen,
|
||||
"archives", c.progress.Current(),
|
||||
"peak-memory", humanize.Bytes(uint64(peakMemory)),
|
||||
"peak-disk", humanize.Bytes(uint64(peakDisk)),
|
||||
}
|
||||
if slowestPath != "" {
|
||||
fields = append(fields, "slowest", slowestPath, "slowest-took", slowest.Round(time.Millisecond))
|
||||
if c.slowestPath != "" {
|
||||
fields = append(fields, "slowest", c.slowestPath, "slowest-took", c.slowest.Round(time.Millisecond))
|
||||
}
|
||||
log.WithFields(fields...).Info("nested archive cataloging complete")
|
||||
}
|
||||
|
||||
// catalog discovers archives in the given resolver and processes each, recursing up to MaxDepth.
|
||||
// parent is the traversal for the archive being searched (nil at the scan root).
|
||||
func (c *archiveCataloger) catalog(ctx context.Context, resolver file.Resolver, parent *archive.Traversal, depth int, builder sbomsync.Builder) error {
|
||||
if c.cfg.MaxDepth >= 0 && depth >= c.cfg.MaxDepth {
|
||||
// catalog processes every archive in the resolver, recursing into each up to maxDepth.
|
||||
func (c *archiveCataloger) catalog(ctx context.Context, resolver file.Resolver, depth int, builder sbomsync.Builder) error {
|
||||
if c.maxDepth >= 0 && depth >= c.maxDepth {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errs error
|
||||
for _, candidate := range c.discoverArchives(resolver) {
|
||||
// no between-archives check: a limit falls when an archive is released, so reaching a bound skips
|
||||
// only the archive that reached it
|
||||
if err := c.processArchive(ctx, resolver, candidate, parent, depth, builder); err != nil {
|
||||
if err := c.processArchive(ctx, resolver, candidate, depth, builder); err != nil {
|
||||
errs = unknown.Append(errs, candidate.location, err)
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver file.Resolver, candidate archiveCandidate, parent *archive.Traversal, depth int, builder sbomsync.Builder) error {
|
||||
archiveLoc := candidate.location
|
||||
content, err := parentResolver.FileContentsByLocation(archiveLoc)
|
||||
func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver file.Resolver, candidate archiveCandidate, depth int, builder sbomsync.Builder) error {
|
||||
location := candidate.location
|
||||
content, err := parentResolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer internal.CloseAndLogError(content, archiveLoc.AccessPath)
|
||||
defer internal.CloseAndLogError(content, location.AccessPath)
|
||||
|
||||
// check a non-sniffing candidate cheaply before reading it in full: an appended archive carries an
|
||||
// entry signature near the front, and almost nothing else does
|
||||
var archiveContent io.Reader = content
|
||||
if !candidate.sniffedAsArchive {
|
||||
var mayHide bool
|
||||
archiveContent, mayHide = archive.MayHideAnAppendedArchive(content)
|
||||
if !mayHide {
|
||||
if archiveContent, mayHide = archive.MayHideAnAppendedArchive(content); !mayHide {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// self time: extraction, sub-pipeline and merge, stopped before the recursive descent so a
|
||||
// containing archive is not charged for what it contains
|
||||
started := time.Now()
|
||||
|
||||
// extracted files inherit the filesystem id of where the archive was found (a layer digest for an
|
||||
// image source, empty for a directory source). The nesting chain rides separately as the archive
|
||||
// path, so same-named files in different archives - and the same archive path in different layers -
|
||||
// stay distinct in coordinate-keyed tables.
|
||||
archivePath := parent.VirtualPathOf(accessPath(archiveLoc))
|
||||
extracted, err := archive.ExtractToResolver(ctx, archiveContent, archiveLoc.AccessPath, archiveLoc.FileSystemID, archivePath, c.extractors, c.limiter, c.newStoreResolver, c.notify)
|
||||
// extracted files keep the filesystem ID of where the archive was found; the nesting chain is
|
||||
// carried separately as the archive path
|
||||
archivePath := archive.VirtualPath(location)
|
||||
extracted, err := archive.Extract(ctx, archiveContent, location.FileSystemID, archivePath, c.limiter, c.exclusions)
|
||||
if errors.Is(err, archive.ErrDiskLimitReached) {
|
||||
// terminal: nothing is released while this archive waits, so skip it and continue the scan
|
||||
log.WithFields("archive", accessPath(archiveLoc), "limit", c.cfg.MaxDiskBytes).
|
||||
Debug("skipping archive whose content would exceed the disk limit")
|
||||
// nothing is released while this archive waits, so skip it rather than block the scan
|
||||
appendUnknowns(builder, ArchiveCatalogerTaskName, []unknown.CoordinateError{{
|
||||
Coordinates: location.Coordinates,
|
||||
Reason: fmt.Errorf("archive skipped: its content would exceed the disk limit"),
|
||||
}})
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if extracted == nil {
|
||||
// nothing could open it: a format with no extractor (ar, a bare non-tar gzip), or a candidate
|
||||
// whose head only looked like it hid an archive. Logged rather than returned: an image can hold
|
||||
// thousands, and accumulating an error each into a chain later walked for coordinate errors is
|
||||
// quadratic. A corrupt archive does return an error.
|
||||
log.WithFields("archive", accessPath(archiveLoc), "sniffed-as-archive", candidate.sniffedAsArchive).
|
||||
// a format with no extractor, or a candidate whose head only looked like an appended archive.
|
||||
// Logged rather than returned: an image can hold thousands of these
|
||||
log.WithFields("archive", archivePath, "sniffed-as-archive", candidate.sniffedAsArchive).
|
||||
Trace("no extractor could read this candidate; skipping")
|
||||
return nil
|
||||
}
|
||||
// releases the extraction directory and this archive's charges, after the sub-pipeline and recursion
|
||||
defer extracted.Cleanup()
|
||||
|
||||
if extracted.Result.Truncated() {
|
||||
// catalog what was stored rather than discarding the archive, and record it in the SBOM's
|
||||
// unknowns as well as the log, since the packages found here are not the full set
|
||||
log.WithFields("archive", accessPath(archiveLoc), "limit", string(extracted.Result.Truncation)).
|
||||
Debug("archive extraction truncated by a configured limit; cataloging partial contents")
|
||||
|
||||
if extracted.Truncated {
|
||||
appendUnknowns(builder, ArchiveCatalogerTaskName, []unknown.CoordinateError{{
|
||||
Coordinates: archiveLoc.Coordinates,
|
||||
Reason: fmt.Errorf("archive cataloged from part of its contents: extraction stopped at the %s",
|
||||
string(extracted.Result.Truncation)),
|
||||
Coordinates: location.Coordinates,
|
||||
Reason: fmt.Errorf("archive cataloged from part of its contents: extraction stopped at the disk limit"),
|
||||
}})
|
||||
}
|
||||
|
||||
// expose the nesting chain to the sub-pipeline and deeper recursion, so catalogers can reconstruct
|
||||
// nesting-aware identity such as the java cataloger's colon-delimited virtual paths
|
||||
trav := &archive.Traversal{
|
||||
Location: archiveLoc,
|
||||
VirtualPath: archivePath,
|
||||
Digests: extracted.Digests,
|
||||
traversal := &archive.Traversal{Location: location, Digests: extracted.Digests}
|
||||
ctx = archive.WithTraversal(ctx, traversal)
|
||||
|
||||
c.progress.Increment()
|
||||
scratch := c.runSubPipeline(ctx, extracted.Resolver, location.Coordinates)
|
||||
mergeArchiveResults(location.Coordinates, scratch, builder)
|
||||
c.progress.AtomicStage.Set(fmt.Sprintf("%s archives (%s)", humanize.Comma(c.progress.Current()), archivePath))
|
||||
|
||||
// self time stops before descending, so a containing archive is not charged for what it contains
|
||||
if took := time.Since(started); took > c.slowest {
|
||||
c.slowest, c.slowestPath = took, archivePath
|
||||
}
|
||||
subCtx := archive.WithTraversal(ctx, trav)
|
||||
|
||||
c.entering()
|
||||
|
||||
scratch := c.runSubPipeline(subCtx, extracted.Resolver, archiveLoc.Coordinates)
|
||||
|
||||
mergeArchiveResults(archiveLoc.Coordinates, scratch, builder)
|
||||
|
||||
c.left(trav)
|
||||
c.recordArchiveTime(trav.VirtualPath, time.Since(started))
|
||||
|
||||
return c.catalog(subCtx, extracted.Resolver, trav, depth+1, builder)
|
||||
return c.catalog(ctx, extracted.Resolver, depth+1, builder)
|
||||
}
|
||||
|
||||
// entering counts this archive onto the walk's single progress row; left takes the row's stage back
|
||||
// once the archive is done, naming it unless the whole walk has finished. Both are nil-safe.
|
||||
// runSubPipeline runs every task against the resolver into a throwaway SBOM, so results can be
|
||||
// attributed to the containing archive before merging into the shared SBOM.
|
||||
//
|
||||
// The stage is reset because each sub-pipeline signs off with its own summary, which would otherwise
|
||||
// stand while the next archive is extracted. The count of archives entered is always true.
|
||||
func (c *archiveCataloger) entering() {
|
||||
if c.prog == nil {
|
||||
return
|
||||
}
|
||||
c.prog.Increment()
|
||||
}
|
||||
|
||||
func (c *archiveCataloger) left(traversal *archive.Traversal) {
|
||||
if c.prog == nil {
|
||||
return
|
||||
}
|
||||
if traversal == nil {
|
||||
c.prog.AtomicStage.Set(fmt.Sprintf("%s archives", humanize.Comma(c.prog.Current())))
|
||||
} else {
|
||||
c.prog.AtomicStage.Set(fmt.Sprintf("%s archives (%s)", humanize.Comma(c.prog.Current()), traversal.VirtualPath))
|
||||
}
|
||||
}
|
||||
|
||||
func accessPath(loc file.Location) string {
|
||||
if loc.AccessPath != "" {
|
||||
return loc.AccessPath
|
||||
}
|
||||
return loc.RealPath
|
||||
}
|
||||
|
||||
// runSubPipeline runs every cataloger task against the given resolver into a throwaway SBOM, so
|
||||
// results can be attributed to the containing archive before merging into the shared SBOM.
|
||||
//
|
||||
// Failures are recorded as RunTask records them at the top level (executor.go), so they land in the
|
||||
// SBOM's unknowns rather than only the logs: errors carrying coordinates keep them, anything left is
|
||||
// attributed to archiveCoord. A failure here never fails the archive or the scan.
|
||||
func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Resolver, archiveCoord file.Coordinates) *sbom.SBOM {
|
||||
scratch := &sbom.SBOM{
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(),
|
||||
FileMetadata: map[file.Coordinates]file.Metadata{},
|
||||
FileDigests: map[file.Coordinates][]file.Digest{},
|
||||
FileContents: map[file.Coordinates]string{},
|
||||
FileLicenses: map[file.Coordinates][]file.License{},
|
||||
Executables: map[file.Coordinates]file.Executable{},
|
||||
Unknowns: map[file.Coordinates][]string{},
|
||||
},
|
||||
}
|
||||
// Failures are recorded as RunTask records them at the top level: errors carrying coordinates keep
|
||||
// them, anything else is attributed to the archive. A failure never fails the archive or the scan.
|
||||
func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Resolver, archiveCoordinates file.Coordinates) *sbom.SBOM {
|
||||
scratch := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}}
|
||||
scratchBuilder := sbomsync.NewBuilder(scratch)
|
||||
for _, t := range c.subPipeline {
|
||||
err := runTaskSafely(ctx, t, resolver, scratchBuilder)
|
||||
@@ -321,47 +212,37 @@ func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Res
|
||||
|
||||
unknowns, remaining := unknown.ExtractCoordinateErrors(err)
|
||||
if remaining != nil {
|
||||
unknowns = append(unknowns, unknown.CoordinateError{Coordinates: archiveCoord, Reason: remaining})
|
||||
unknowns = append(unknowns, unknown.CoordinateError{Coordinates: archiveCoordinates, Reason: remaining})
|
||||
}
|
||||
appendUnknowns(scratchBuilder, t.Name(), unknowns)
|
||||
}
|
||||
return scratch
|
||||
}
|
||||
|
||||
// archiveCandidate is one file the walk will try to open, and how it came to be a candidate.
|
||||
// archiveCandidate is one file the walk will try to open.
|
||||
type archiveCandidate struct {
|
||||
location file.Location
|
||||
|
||||
// sniffedAsArchive is true when the file's own content types it as an archive. Such a file that
|
||||
// cannot be extracted is a broken archive and is reported; a candidate here only because its head
|
||||
// might hide one is passed over silently.
|
||||
// sniffedAsArchive is true when the file's content types it as an archive. Such a file that cannot
|
||||
// be extracted is a broken archive and is reported; a file that is a candidate only by name is
|
||||
// passed over silently.
|
||||
sniffedAsArchive bool
|
||||
}
|
||||
|
||||
// zipArchiveGlobs find files worth a second look: those named like zip archives whose content did not
|
||||
// sniff as one, such as a Spring Boot executable jar - a launcher stub with a zip appended, which
|
||||
// sniffs as a shell script.
|
||||
// zipArchiveGlobs find files named like zip archives whose content did not sniff as one, such as a
|
||||
// Spring Boot executable jar: a launcher script with a zip appended. The name only decides what gets
|
||||
// opened; the end-of-central-directory record decides whether it is an archive.
|
||||
//
|
||||
// Not a detection rule: the end-of-central-directory record settles whether a candidate is an
|
||||
// archive, so a `.jar` holding prose is still refused. The name decides only what gets opened, which
|
||||
// is why a self-extracting archive with no telling extension is not found.
|
||||
//
|
||||
// A glob rather than a MIME lookup because asking for every file sniffed as executable, script or
|
||||
// unrecognized returns thousands of locations that are all discarded, which tripled the scan on a
|
||||
// stock ruby image. One pattern per extension because brace alternation matched nothing.
|
||||
// Globs rather than a MIME lookup, because asking for every executable, script and unrecognized file
|
||||
// returned thousands of locations that were all discarded.
|
||||
var zipArchiveGlobs = []string{
|
||||
"**/*.jar", "**/*.war", "**/*.ear", "**/*.par", "**/*.sar", "**/*.nar", "**/*.kar",
|
||||
"**/*.hpi", "**/*.jpi", "**/*.far", "**/*.rar", "**/*.zip", "**/*.apk", "**/*.aar",
|
||||
"**/*.egg", "**/*.whl", "**/*.lpkg", "**/*.zap", "**/*.exe",
|
||||
}
|
||||
|
||||
// discoverArchives returns every location worth opening: files that are archives by their own
|
||||
// content, plus files named like zip archives whose head says script or executable but whose tail may
|
||||
// still be a whole zip. The second group is only opened; the extractor refuses anything with no
|
||||
// end-of-central-directory record.
|
||||
//
|
||||
// No exclusion filter here: exclusion patterns are applied where each filesystem is indexed, so an
|
||||
// excluded archive is not in the resolver to begin with.
|
||||
// discoverArchives returns every file worth opening: those whose content is an archive, plus those
|
||||
// named like a zip whose tail may still hold one.
|
||||
func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCandidate {
|
||||
archives, err := resolver.FilesByMIMEType(mimetype.ArchiveMIMETypeSet.List()...)
|
||||
if err != nil {
|
||||
@@ -369,23 +250,19 @@ func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCan
|
||||
return nil
|
||||
}
|
||||
|
||||
candidates := make([]archiveCandidate, 0, len(archives))
|
||||
var candidates []archiveCandidate
|
||||
seen := map[file.Coordinates]struct{}{}
|
||||
for _, loc := range archives {
|
||||
seen[loc.Coordinates] = struct{}{}
|
||||
candidates = append(candidates, archiveCandidate{location: loc, sniffedAsArchive: true})
|
||||
}
|
||||
|
||||
appended, err := resolver.FilesByGlob(zipArchiveGlobs...)
|
||||
namedLikeZips, err := resolver.FilesByGlob(zipArchiveGlobs...)
|
||||
if err != nil {
|
||||
// the archives found above are still worth walking
|
||||
log.WithFields("error", err).Debug("unable to list files that may carry an appended archive")
|
||||
return candidates
|
||||
}
|
||||
seen := make(map[file.Coordinates]struct{}, len(candidates))
|
||||
for _, c := range candidates {
|
||||
seen[c.location.Coordinates] = struct{}{}
|
||||
}
|
||||
for _, loc := range appended {
|
||||
// one file reached by both lookups must be opened once, or every package it holds is doubled
|
||||
for _, loc := range namedLikeZips {
|
||||
if _, dup := seen[loc.Coordinates]; dup {
|
||||
continue
|
||||
}
|
||||
@@ -396,78 +273,52 @@ func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCan
|
||||
}
|
||||
|
||||
// mergeArchiveResults copies the throwaway SBOM's packages, files and relationships into the shared
|
||||
// SBOM, adding a file-level CONTAINS relationship from the archive to everything found inside it.
|
||||
func mergeArchiveResults(archiveCoord file.Coordinates, scratch *sbom.SBOM, builder sbomsync.Builder) {
|
||||
// SBOM, adding a CONTAINS relationship from the archive to everything found inside it.
|
||||
func mergeArchiveResults(archiveCoordinates file.Coordinates, scratch *sbom.SBOM, builder sbomsync.Builder) {
|
||||
pkgs := scratch.Artifacts.Packages.Sorted()
|
||||
if len(pkgs) > 0 {
|
||||
builder.AddPackages(pkgs...)
|
||||
}
|
||||
|
||||
accessor, _ := builder.(sbomsync.Accessor)
|
||||
if accessor != nil {
|
||||
if accessor, ok := builder.(sbomsync.Accessor); ok {
|
||||
accessor.WriteToSBOM(func(s *sbom.SBOM) {
|
||||
mergeFileArtifacts(s, scratch)
|
||||
mergeFileArtifacts(&s.Artifacts, &scratch.Artifacts)
|
||||
})
|
||||
}
|
||||
|
||||
var rels []artifact.Relationship
|
||||
for _, p := range pkgs {
|
||||
rels = append(rels, artifact.Relationship{
|
||||
From: archiveCoord,
|
||||
To: p,
|
||||
Type: artifact.ContainsRelationship,
|
||||
})
|
||||
rels = append(rels, artifact.Relationship{From: archiveCoordinates, To: p, Type: artifact.ContainsRelationship})
|
||||
}
|
||||
for _, coord := range scratch.AllCoordinates() {
|
||||
rels = append(rels, artifact.Relationship{
|
||||
From: archiveCoord,
|
||||
To: coord,
|
||||
Type: artifact.ContainsRelationship,
|
||||
})
|
||||
for _, coordinates := range scratch.AllCoordinates() {
|
||||
rels = append(rels, artifact.Relationship{From: archiveCoordinates, To: coordinates, Type: artifact.ContainsRelationship})
|
||||
}
|
||||
// preserve relationships discovered within the archive (e.g. package evident-by file)
|
||||
rels = append(rels, scratch.Relationships...)
|
||||
|
||||
if len(rels) > 0 {
|
||||
builder.AddRelationships(rels...)
|
||||
}
|
||||
}
|
||||
|
||||
func mergeFileArtifacts(dst, src *sbom.SBOM) {
|
||||
if dst.Artifacts.FileMetadata == nil {
|
||||
dst.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
|
||||
func mergeFileArtifacts(dst, src *sbom.Artifacts) {
|
||||
mergeInto(&dst.FileMetadata, src.FileMetadata)
|
||||
mergeInto(&dst.FileDigests, src.FileDigests)
|
||||
mergeInto(&dst.FileContents, src.FileContents)
|
||||
mergeInto(&dst.FileLicenses, src.FileLicenses)
|
||||
mergeInto(&dst.Executables, src.Executables)
|
||||
if len(src.Unknowns) > 0 && dst.Unknowns == nil {
|
||||
dst.Unknowns = map[file.Coordinates][]string{}
|
||||
}
|
||||
for k, v := range src.Artifacts.FileMetadata {
|
||||
dst.Artifacts.FileMetadata[k] = v
|
||||
}
|
||||
if dst.Artifacts.FileDigests == nil {
|
||||
dst.Artifacts.FileDigests = map[file.Coordinates][]file.Digest{}
|
||||
}
|
||||
for k, v := range src.Artifacts.FileDigests {
|
||||
dst.Artifacts.FileDigests[k] = v
|
||||
}
|
||||
if dst.Artifacts.FileContents == nil {
|
||||
dst.Artifacts.FileContents = map[file.Coordinates]string{}
|
||||
}
|
||||
for k, v := range src.Artifacts.FileContents {
|
||||
dst.Artifacts.FileContents[k] = v
|
||||
}
|
||||
if dst.Artifacts.FileLicenses == nil {
|
||||
dst.Artifacts.FileLicenses = map[file.Coordinates][]file.License{}
|
||||
}
|
||||
for k, v := range src.Artifacts.FileLicenses {
|
||||
dst.Artifacts.FileLicenses[k] = v
|
||||
}
|
||||
if dst.Artifacts.Executables == nil {
|
||||
dst.Artifacts.Executables = map[file.Coordinates]file.Executable{}
|
||||
}
|
||||
for k, v := range src.Artifacts.Executables {
|
||||
dst.Artifacts.Executables[k] = v
|
||||
}
|
||||
if dst.Artifacts.Unknowns == nil {
|
||||
dst.Artifacts.Unknowns = map[file.Coordinates][]string{}
|
||||
}
|
||||
for k, v := range src.Artifacts.Unknowns {
|
||||
dst.Artifacts.Unknowns[k] = append(dst.Artifacts.Unknowns[k], v...)
|
||||
for coordinates, reasons := range src.Unknowns {
|
||||
dst.Unknowns[coordinates] = append(dst.Unknowns[coordinates], reasons...)
|
||||
}
|
||||
}
|
||||
|
||||
func mergeInto[K comparable, V any](dst *map[K]V, src map[K]V) {
|
||||
if len(src) == 0 {
|
||||
return
|
||||
}
|
||||
if *dst == nil {
|
||||
*dst = make(map[K]V, len(src))
|
||||
}
|
||||
maps.Copy(*dst, src)
|
||||
}
|
||||
|
||||
+257
-568
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user