mirror of
https://github.com/anchore/syft.git
synced 2026-10-31 06:26:30 +01:00
Add nested archive cataloging (#5260)
Signed-off-by: Kudryavcev Nikolay <kydry.nikolau@gmail.com> Signed-off-by: Thomas Bechtold <thomasbechtold@jpberlin.de> Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> Signed-off-by: Keith Zantow <kzantow@gmail.com> Co-authored-by: Kudryavcev Nikolay <kydry.nikolau@gmail.com> Co-authored-by: Thomas Bechtold <thomasbechtold@jpberlin.de> Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Keith Zantow
co-authored by
Kudryavcev Nikolay
Thomas Bechtold
Alex Goodman
Kudryavcev Nikolay
Thomas Bechtold
Alex Goodman
parent
14a0beb93c
commit
4b8a3db5f3
@@ -0,0 +1,699 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/scylladb/go-set/strset"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/internal/testutils"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
// This file covers nesting across archive families. The other nested-archive fixtures are all zips,
|
||||
// so a zip-specific mechanism would pass them; the requirement here is that an archive of one family
|
||||
// inside another behaves as it would inside its own family - same packages, same container-path chain,
|
||||
// same CONTAINS edges, same depth accounting.
|
||||
//
|
||||
// Chains are built in the test rather than committed as binaries, so each case can vary which family
|
||||
// sits at which level.
|
||||
|
||||
// archiveFamily is one archive format buildNestedArchive can produce.
|
||||
//
|
||||
// ".jar" is its own family rather than a flavour of zip: the java cataloger meets it as a jar wherever
|
||||
// it sits, while the archive task descends into it like any other zip. That differs mid-chain from at
|
||||
// the leaf, so the matrix below uses a jar as container and as payload.
|
||||
type archiveFamily string
|
||||
|
||||
const (
|
||||
familyZip archiveFamily = ".zip"
|
||||
familyJar archiveFamily = ".jar"
|
||||
familyTar archiveFamily = ".tar"
|
||||
familyTarGz archiveFamily = ".tar.gz"
|
||||
familyTgz archiveFamily = ".tgz"
|
||||
)
|
||||
|
||||
// nestPlan describes one chain to build: a family per level (outermost first), the entries for the
|
||||
// innermost archive, and extras to add beside the nested archive at a given level. The extras grade
|
||||
// the levels' sizes or entry counts, which is how the limit cases place a bound between two levels.
|
||||
type nestPlan struct {
|
||||
families []archiveFamily
|
||||
leaf map[string][]byte
|
||||
extra map[int]map[string][]byte
|
||||
}
|
||||
|
||||
// nestedArchive is one built chain: the outermost archive's bytes, and the addresses that chain
|
||||
// composes to at each level.
|
||||
type nestedArchive struct {
|
||||
// name is the outermost archive's file name, as written into the scanned directory.
|
||||
name string
|
||||
// bytes is the outermost archive.
|
||||
bytes []byte
|
||||
// sizes is each archive's size in bytes, outermost first. A container is never smaller than what it
|
||||
// holds, so these decrease, and the limit cases place a bound between two of them.
|
||||
sizes []int
|
||||
// paths is each archive's path within its parent, outermost first; paths[0] is the outermost archive
|
||||
// as the scan's resolver reports it.
|
||||
paths []string
|
||||
// containerPaths is each archive's full path, the container path its own contents carry, outermost first.
|
||||
containerPaths []string
|
||||
// virtualPaths is the same chain in java's colon-joined form, outermost first.
|
||||
virtualPaths []string
|
||||
}
|
||||
|
||||
// nestingRow is one line of the depth-3 matrix.
|
||||
type nestingRow struct {
|
||||
name string
|
||||
families []archiveFamily
|
||||
why string
|
||||
}
|
||||
|
||||
// nestingVisit is one filesystem the archive cataloger's sub-pipeline was run against.
|
||||
type nestingVisit struct {
|
||||
virtualPath string
|
||||
fullPath string
|
||||
depth int
|
||||
// heldFiles are the spill files live at this moment, sampled only when the probe was given a temp
|
||||
// dir to look in. A nested archive's own bytes are read where they lie, so one appears per archive
|
||||
// whose entries did not fit in memory.
|
||||
heldFiles []string
|
||||
}
|
||||
|
||||
// nestingProbe records one visit per filesystem the archive cataloger runs its sub-pipeline against,
|
||||
// reading the chain off the traversal on the context.
|
||||
type nestingProbe struct {
|
||||
// tempDir, when set, is where extraction work directories live
|
||||
tempDir string
|
||||
|
||||
mu sync.Mutex
|
||||
visits []nestingVisit
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_leafIsCatalogedOnceWithTheFullChain(t *testing.T) {
|
||||
// family order must not change the result: the same package, addressed by a chain naming its own
|
||||
// containers in its own order
|
||||
for _, row := range nestingRows() {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
probe := &nestingProbe{}
|
||||
s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), probe.task())
|
||||
|
||||
assert.Equal(t, 1, packageCount(s, "marker-pkg"), "%s: the leaf package must be cataloged exactly once", row.why)
|
||||
|
||||
locs := leafLocations(s)
|
||||
require.Len(t, locs, 1, "the leaf must be reached by exactly one path")
|
||||
// exact, never a substring: a partially composed chain passes a substring check
|
||||
assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates),
|
||||
"the chain must name all three archives outer-to-inner")
|
||||
assert.Equal(t, nested.leafContainerPath()+"!/nested/marker.txt", locs[0].FullNestedPath(),
|
||||
"the path stays relative to the archive the file came from")
|
||||
|
||||
assert.Equal(t, nested.containerPaths, probe.containerPaths(),
|
||||
"every level must be cataloged, outer to inner, whatever families they are")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) {
|
||||
// the edges make the chain navigable in the SBOM. The archive-to-archive edges must not depend on
|
||||
// the file catalogers having recorded the inner archives, since the default file selection does not
|
||||
for _, row := range nestingRows() {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
s := withArchiveContains(t, runNesting(t, scanDir, 3, defaultLimits(), markerTask()))
|
||||
|
||||
// each archive's coordinates as seen in its parent; the outermost sits in the scanned
|
||||
// directory, so it carries no filesystem id of its own
|
||||
archiveCoords := make([]file.Coordinates, len(row.families))
|
||||
for i := range row.families {
|
||||
archiveCoords[i] = nested.coordinates(i)
|
||||
}
|
||||
|
||||
fileEdges := strset.New()
|
||||
pkgEdges := strset.New()
|
||||
for _, rel := range s.Relationships {
|
||||
if rel.Type != artifact.ContainsRelationship {
|
||||
continue
|
||||
}
|
||||
from, ok := rel.From.(file.Coordinates)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch to := rel.To.(type) {
|
||||
case file.Coordinates:
|
||||
fileEdges.Add(coordKey(from) + " -> " + coordKey(to))
|
||||
case pkg.Package:
|
||||
pkgEdges.Add(coordKey(from) + " -> " + to.Name)
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < len(archiveCoords)-1; i++ {
|
||||
assert.True(t, fileEdges.Has(coordKey(archiveCoords[i])+" -> "+coordKey(archiveCoords[i+1])),
|
||||
"expected a CONTAINS edge from %s to %s", nested.paths[i], nested.paths[i+1])
|
||||
}
|
||||
|
||||
innermost := archiveCoords[len(archiveCoords)-1]
|
||||
assert.True(t, pkgEdges.Has(coordKey(innermost)+" -> marker-pkg"),
|
||||
"expected a CONTAINS edge from %s to the leaf package", nested.paths[len(nested.paths)-1])
|
||||
|
||||
// with file cataloging on, files inside an archive get an edge from it too
|
||||
s = withArchiveContains(t, runNesting(t, scanDir, 3, defaultLimits(), markerTask(), fileMetadataTask()))
|
||||
var leafEdge bool
|
||||
for _, rel := range s.Relationships {
|
||||
from, _ := rel.From.(file.Coordinates)
|
||||
to, _ := rel.To.(file.Coordinates)
|
||||
leafEdge = leafEdge || (from == innermost && to == file.Coordinates{RealPath: innermost.RealPath, FullPath: innermost.FullNestedPath() + "!/nested/marker.txt"})
|
||||
}
|
||||
assert.True(t, leafEdge, "expected a CONTAINS edge from the innermost archive to the leaf file")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_everyJarIsCatalogedOnceUnderItsChain(t *testing.T) {
|
||||
// two mechanisms meet a jar: the java cataloger catalogs it as a package, the archive task descends
|
||||
// into it. A jar at any level must do both, exactly once each; the failure mode is a duplicate, not an
|
||||
// error. The colon-joined virtual paths are in already-published SBOMs, so they must name the chain.
|
||||
for _, row := range nestingRows() {
|
||||
var jarLevels []int
|
||||
for i, family := range row.families {
|
||||
if family == familyJar {
|
||||
jarLevels = append(jarLevels, i)
|
||||
}
|
||||
}
|
||||
if len(jarLevels) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
probe := &nestingProbe{}
|
||||
s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), javaTask(), probe.task())
|
||||
|
||||
for _, level := range jarLevels {
|
||||
name := fmt.Sprintf("level%d", level)
|
||||
assert.Equal(t, 1, packageCount(s, name), "the jar's own package must be cataloged exactly once")
|
||||
assert.Equal(t, nested.virtualPaths[level], javaVirtualPaths(s)[name],
|
||||
"the jar's virtual path must be its own place in the chain")
|
||||
}
|
||||
|
||||
// and the archive task still descended through every level
|
||||
assert.Equal(t, nested.containerPaths, probe.containerPaths())
|
||||
locs := leafLocations(s)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) {
|
||||
// two rows with different families at every level, so the bound is shown not to turn on which
|
||||
// families are involved
|
||||
for _, row := range nestingRows()[:2] {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
probe := &nestingProbe{}
|
||||
s := runNesting(t, scanDir, 2, defaultLimits(), markerTask(), fileMetadataTask(), probe.task())
|
||||
|
||||
assert.Equal(t, nested.containerPaths[:2], probe.containerPaths(),
|
||||
"at depth 2 the outer and middle archives are cataloged and the innermost is not")
|
||||
assert.Zero(t, packageCount(s, "marker-pkg"),
|
||||
"the leaf is one level past the bound, so its package must be absent")
|
||||
|
||||
// the innermost archive is still cataloged as a file, inside the middle archive
|
||||
innermost := nested.coordinates(2)
|
||||
_, ok := s.Artifacts.FileMetadata[innermost]
|
||||
assert.True(t, ok, "the undescended archive must still be recorded as a file at %s", coordKey(innermost))
|
||||
|
||||
// and what was left unexplored is said out loud, not silently dropped
|
||||
reasons := s.Artifacts.Unknowns[innermost]
|
||||
require.Len(t, reasons, 1, "unknowns: %v", s.Artifacts.Unknowns)
|
||||
assert.Contains(t, reasons[0], "depth limit was reached")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) {
|
||||
// an archive's own bytes are read where they lie: a scanned file from disk, a nested archive from
|
||||
// its parent's entries. Nothing copies them, so at most one spill file per level appears whatever
|
||||
// the memory limit; only entries are held, and only they can be written out.
|
||||
scanDir := t.TempDir()
|
||||
tempDir := isolatedTempDir(t)
|
||||
nested := writeNestedArchive(t, scanDir, gradedChain())
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
memoryBytes int64
|
||||
wantHeldFiles []string
|
||||
}{
|
||||
{
|
||||
name: "with room in memory for every level's entries",
|
||||
memoryBytes: int64(2 * (nested.sizes[0] + nested.sizes[1] + nested.sizes[2])),
|
||||
},
|
||||
{
|
||||
name: "with a memory limit of zero, so every level's entries are written out",
|
||||
memoryBytes: 0,
|
||||
wantHeldFiles: []string{"archive-spill", "archive-spill", "archive-spill"},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
probe := &nestingProbe{tempDir: tempDir}
|
||||
bounds := archive.Limits{MaxMemoryBytes: tc.memoryBytes, MaxDiskBytes: -1}
|
||||
|
||||
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
|
||||
|
||||
deepest := probe.deepestVisit()
|
||||
require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held")
|
||||
assert.Equal(t, tc.wantHeldFiles, deepest.heldFiles)
|
||||
|
||||
assert.Equal(t, nested.containerPaths, probe.containerPaths())
|
||||
locs := leafLocations(s)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_diskLimitTruncatesTheLevelItCannotAdmit(t *testing.T) {
|
||||
// the walk descends before it unwinds, so the outer and middle archives still hold their entries
|
||||
// when the innermost level's do not fit in memory. With nothing admitted on disk that level is
|
||||
// cataloged from nothing, the shallower ones keep what they found, and the scan finishes. An
|
||||
// implementation that waited for capacity would deadlock, hence the timeout.
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, deepChain())
|
||||
|
||||
// a sibling of a different family, processed once the chain has been released, so cataloged in full
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(scanDir, "sibling.tgz"),
|
||||
testutils.TarGz(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}),
|
||||
0o644,
|
||||
))
|
||||
|
||||
// room for the outer two levels' entries (each about the size of the archive it holds) but not the
|
||||
// innermost's payload, which is half an archive larger than the slack
|
||||
memoryLimit := int64(nested.sizes[0] + nested.sizes[1] + deepChainPayloadBytes/2)
|
||||
|
||||
probe := &nestingProbe{}
|
||||
bounds := archive.Limits{MaxMemoryBytes: memoryLimit, MaxDiskBytes: 0}
|
||||
|
||||
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
|
||||
|
||||
want := append(append([]string{}, nested.containerPaths...), "/sibling.tgz")
|
||||
assert.ElementsMatch(t, want, probe.containerPaths(),
|
||||
"every level is visited, the truncated one with nothing inside it")
|
||||
|
||||
containers := strset.New()
|
||||
for _, loc := range leafLocations(s) {
|
||||
containers.Add(archive.ContainerPath(loc.Coordinates))
|
||||
}
|
||||
assert.False(t, containers.Has(nested.leafContainerPath()), "the truncated level's contents must be absent")
|
||||
assert.True(t, containers.Has("/sibling.tgz"), "the sibling of a different family must be cataloged fully")
|
||||
|
||||
var truncated []string
|
||||
for coords, reasons := range s.Artifacts.Unknowns {
|
||||
for _, reason := range reasons {
|
||||
if strings.Contains(reason, "disk limit") {
|
||||
truncated = append(truncated, coords.FullNestedPath())
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Equal(t, []string{nested.coordinates(2).FullNestedPath()}, truncated,
|
||||
"the truncation is recorded against the archive that reached the limit")
|
||||
}
|
||||
|
||||
func defaultLimits() archive.Limits {
|
||||
return archiveLimits(cataloging.DefaultArchiveSearchConfig())
|
||||
}
|
||||
|
||||
// runNesting runs the archive cataloger task over scanDir with the given sub-pipeline and returns the
|
||||
// SBOM it built. It runs on its own goroutine and fails if the task has not finished in time.
|
||||
//
|
||||
// The depth-3 limit cases need the timeout: the walk descends before it unwinds, so the outer and
|
||||
// middle archives still hold their content when the inner one is refused. An implementation that
|
||||
// waited for capacity would deadlock, and a test asserting only the result would hang rather than fail.
|
||||
func runNesting(t *testing.T, scanDir string, depth int, bounds archive.Limits, subPipeline ...Task) *sbom.SBOM {
|
||||
t.Helper()
|
||||
tsk := newLimitedTestTask(t, depth, bounds, subPipeline...)
|
||||
s := &sbom.SBOM{Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(),
|
||||
FileMetadata: map[file.Coordinates]file.Metadata{},
|
||||
}}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(s))
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err, "reaching a bound must not fail the scan")
|
||||
return s
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatal("the scan did not finish in time: reaching a limit must skip the archive that " +
|
||||
"reached it, never wait for capacity that nothing in the chain will release")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// withArchiveContains runs the task that follows every scan with archive cataloging on.
|
||||
func withArchiveContains(t *testing.T, s *sbom.SBOM) *sbom.SBOM {
|
||||
t.Helper()
|
||||
require.NoError(t, NewArchiveContainsTask().Execute(context.Background(), nil, sbomsync.NewBuilder(s)))
|
||||
return s
|
||||
}
|
||||
|
||||
// markerTask emits one package per "marker.txt" found, so the matrix asserts behavior without
|
||||
// depending on any real cataloger's fixture format.
|
||||
func markerTask() Task {
|
||||
return NewTask("marker-cataloger", func(_ context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
|
||||
locations, err := resolver.FilesByGlob("**/marker.txt")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, loc := range locations {
|
||||
p := pkg.Package{
|
||||
Name: "marker-pkg",
|
||||
Version: "1.0.0",
|
||||
Type: pkg.BinaryPkg,
|
||||
Locations: file.NewLocationSet(loc),
|
||||
}
|
||||
p.SetID()
|
||||
builder.AddPackages(p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// javaTask is the java archive cataloger as the cataloging factory builds it, so a jar is met as a jar
|
||||
// wherever it sits in a chain.
|
||||
func javaTask() Task {
|
||||
return NewPackageTask(CatalogingFactoryConfig{}, java.NewArchiveCataloger(java.DefaultArchiveCatalogerConfig()))
|
||||
}
|
||||
|
||||
// fileMetadataTask records every file, so each file inside an archive gets an archive-to-file edge.
|
||||
func fileMetadataTask() Task {
|
||||
return newFileMetadataCatalogerTask(file.AllFilesSelection)
|
||||
}
|
||||
|
||||
// pack builds one archive of this family holding the given entries.
|
||||
func (f archiveFamily) pack(t *testing.T, base string, entries map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
switch f {
|
||||
case familyZip:
|
||||
return testutils.Zip(t, entries)
|
||||
case familyJar:
|
||||
// a manifest makes it a real jar to the java cataloger at any level; the package java reports is
|
||||
// named after the file, not the manifest title
|
||||
return testutils.Jar(t, base, "1.0", entries)
|
||||
case familyTar:
|
||||
return testutils.Tar(t, entries)
|
||||
case familyTarGz, familyTgz:
|
||||
return testutils.TarGz(t, entries)
|
||||
}
|
||||
t.Fatalf("unknown archive family %q", f)
|
||||
return nil
|
||||
}
|
||||
|
||||
// leafContainerPath is the container path the innermost archive's contents carry: every archive, outer to
|
||||
// inner. Assert it exactly, never as a substring: a partially composed chain passes a substring check.
|
||||
func (n nestedArchive) leafContainerPath() string {
|
||||
return n.containerPaths[len(n.containerPaths)-1]
|
||||
}
|
||||
|
||||
// coordinates are those of the archive at the given level: the outermost archive's path, and the chain
|
||||
// of entries within it.
|
||||
func (n nestedArchive) coordinates(level int) file.Coordinates {
|
||||
return file.Coordinates{RealPath: n.paths[0], FullPath: strings.Join(n.paths[:level+1], "!/")}
|
||||
}
|
||||
|
||||
// buildNestedArchive builds one archive per family given, each holding the next, with leaf inside
|
||||
// the innermost.
|
||||
func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive {
|
||||
t.Helper()
|
||||
require.NotEmpty(t, plan.families, "a chain needs at least one level")
|
||||
|
||||
out := nestedArchive{
|
||||
sizes: make([]int, len(plan.families)),
|
||||
paths: make([]string, len(plan.families)),
|
||||
containerPaths: make([]string, len(plan.families)),
|
||||
virtualPaths: make([]string, len(plan.families)),
|
||||
}
|
||||
|
||||
// the level index goes in every name and path segment, so no two levels share a path and a chain
|
||||
// assertion cannot pass on a value composed from the wrong level
|
||||
for i, family := range plan.families {
|
||||
base := fmt.Sprintf("level%d", i)
|
||||
if i == 0 {
|
||||
// the outermost archive is a file in the scanned directory, which the resolver reports rooted
|
||||
out.name = base + string(family)
|
||||
out.paths[i] = "/" + out.name
|
||||
continue
|
||||
}
|
||||
out.paths[i] = fmt.Sprintf("nest%d/%s%s", i, base, family)
|
||||
}
|
||||
|
||||
for i := range plan.families {
|
||||
if i == 0 {
|
||||
out.containerPaths[i] = out.paths[0]
|
||||
out.virtualPaths[i] = out.paths[0]
|
||||
continue
|
||||
}
|
||||
out.containerPaths[i] = out.containerPaths[i-1] + "!/" + out.paths[i]
|
||||
out.virtualPaths[i] = out.virtualPaths[i-1] + ":" + out.paths[i]
|
||||
}
|
||||
|
||||
// built innermost first, because each level's bytes are an entry of the level above it
|
||||
var packed []byte
|
||||
for i := len(plan.families) - 1; i >= 0; i-- {
|
||||
entries := map[string][]byte{}
|
||||
if i == len(plan.families)-1 {
|
||||
for name, body := range plan.leaf {
|
||||
entries[name] = body
|
||||
}
|
||||
} else {
|
||||
entries[out.paths[i+1]] = packed
|
||||
}
|
||||
for name, body := range plan.extra[i] {
|
||||
entries[name] = body
|
||||
}
|
||||
packed = plan.families[i].pack(t, fmt.Sprintf("level%d", i), entries)
|
||||
out.sizes[i] = len(packed)
|
||||
}
|
||||
out.bytes = packed
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// writeNestedArchive writes the chain's outermost archive into dir and returns the chain.
|
||||
func writeNestedArchive(t *testing.T, dir string, plan nestPlan) nestedArchive {
|
||||
t.Helper()
|
||||
nested := buildNestedArchive(t, plan)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, nested.name), nested.bytes, 0o644))
|
||||
return nested
|
||||
}
|
||||
|
||||
// markerLeaf is the family-agnostic leaf payload: one file no archive format has an opinion about,
|
||||
// found by markerTask. It keeps the rows comparable - the same package must come out whichever
|
||||
// families the chain is made of.
|
||||
func markerLeaf() map[string][]byte {
|
||||
return map[string][]byte{"nested/marker.txt": []byte("leaf")}
|
||||
}
|
||||
|
||||
// nestingRows is the depth-3 matrix. Each row covers a case the others do not.
|
||||
func nestingRows() []nestingRow {
|
||||
return []nestingRow{
|
||||
{
|
||||
name: "tar.gz/zip/jar",
|
||||
families: []archiveFamily{familyTarGz, familyZip, familyJar},
|
||||
why: "the headline case: compressed tar outermost, jar innermost",
|
||||
},
|
||||
{
|
||||
name: "zip/tar.gz/jar",
|
||||
families: []archiveFamily{familyZip, familyTarGz, familyJar},
|
||||
why: "compressed tar in the middle, where the stream-only format is not the entry point",
|
||||
},
|
||||
{
|
||||
name: "jar/tar/zip",
|
||||
families: []archiveFamily{familyJar, familyTar, familyZip},
|
||||
why: "jar as a container rather than a payload",
|
||||
},
|
||||
{
|
||||
name: "tgz/tar/jar",
|
||||
families: []archiveFamily{familyTgz, familyTar, familyJar},
|
||||
why: "the .tgz alias resolving at a non-outermost boundary",
|
||||
},
|
||||
{
|
||||
name: "tar/jar/tar.gz",
|
||||
families: []archiveFamily{familyTar, familyJar, familyTarGz},
|
||||
why: "compressed tar as the innermost, which nothing else covers",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *nestingProbe) task() Task {
|
||||
return NewTask("nesting-probe", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
trav := archive.TraversalFromContext(ctx)
|
||||
visit := nestingVisit{}
|
||||
if trav != nil {
|
||||
visit.virtualPath = archive.VirtualPath(trav.Location)
|
||||
visit.fullPath = trav.FullTraversalArchivePath()
|
||||
visit.depth = strings.Count(archive.VirtualPath(trav.Location), ":") + 1
|
||||
}
|
||||
if p.tempDir != "" {
|
||||
visit.heldFiles = heldFiles(p.tempDir)
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.visits = append(p.visits, visit)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// containerPaths is the chain of every archive whose contents were cataloged, in visit order.
|
||||
func (p *nestingProbe) containerPaths() []string {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
var out []string
|
||||
for _, v := range p.visits {
|
||||
if v.fullPath == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, v.fullPath)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// deepestVisit is the visit at the innermost level reached, the only moment at which every level of a
|
||||
// chain is still held and its routing observable. It returns the whole visit so a case can also assert
|
||||
// it got as deep as it meant to: an empty overflow sample from a visit that never left the scan root
|
||||
// would otherwise read as "nothing overflowed".
|
||||
func (p *nestingProbe) deepestVisit() nestingVisit {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
var best nestingVisit
|
||||
for _, v := range p.visits {
|
||||
if v.depth > best.depth {
|
||||
best = v
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// heldFiles names the spill files live under tempDir, one "archive-spill" per archive holding content
|
||||
// that did not fit in memory. Meaningful only while the archives are still held, so it is sampled from
|
||||
// inside the sub-pipeline rather than after the scan.
|
||||
func heldFiles(tempDir string) (names []string) {
|
||||
_ = filepath.WalkDir(tempDir, func(_ string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || !strings.HasPrefix(d.Name(), "archive-") {
|
||||
return nil //nolint:nilerr // an unreadable directory holds nothing this probe can report
|
||||
}
|
||||
// the name is the pattern with os.CreateTemp's random suffix in place of the "*"
|
||||
names = append(names, d.Name()[:strings.LastIndexByte(d.Name(), '-')])
|
||||
return nil
|
||||
})
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// isolatedTempDir points archive extraction at a directory of this test's own, so an overflow sample
|
||||
// sees only this scan's work directories.
|
||||
func isolatedTempDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
t.Setenv("TMPDIR", dir) // unix
|
||||
t.Setenv("TMP", dir) // windows
|
||||
t.Setenv("TEMP", dir) // windows fallback
|
||||
require.Equal(t, dir, os.TempDir(), "extraction must be writing where the probe is looking")
|
||||
return dir
|
||||
}
|
||||
|
||||
func leafLocations(s *sbom.SBOM) []file.Location {
|
||||
var out []file.Location
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if p.Name != "marker-pkg" {
|
||||
continue
|
||||
}
|
||||
out = append(out, p.Locations.ToSlice()...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func javaVirtualPaths(s *sbom.SBOM) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if metadata, ok := p.Metadata.(pkg.JavaArchive); ok {
|
||||
out[p.Name] = metadata.VirtualPath
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func packageCount(s *sbom.SBOM, name string) int {
|
||||
var n int
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if p.Name == name {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func coordKey(c file.Coordinates) string {
|
||||
return c.FullNestedPath()
|
||||
}
|
||||
|
||||
// gradedChain is the headline row with each level padded so the three archives have clearly different
|
||||
// sizes: a container is never smaller than what it holds, so a bound can only be placed between two
|
||||
// levels if they are graded on purpose.
|
||||
func gradedChain() nestPlan {
|
||||
return nestPlan{
|
||||
families: []archiveFamily{familyTarGz, familyZip, familyJar},
|
||||
leaf: markerLeaf(),
|
||||
extra: map[int]map[string][]byte{
|
||||
0: {"pad0.bin": incompressible(24 * 1024)},
|
||||
1: {"pad1.bin": incompressible(24 * 1024)},
|
||||
2: {"pad2.bin": incompressible(8 * 1024)},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// deepChainPayloadBytes is the size of deepChain's leaf payload.
|
||||
const deepChainPayloadBytes = 64 * 1024
|
||||
|
||||
// deepChain is the headline row with one large, incompressible payload at the leaf and no other
|
||||
// padding. The payload dominates every level, so the three archives are about the same size. It sorts
|
||||
// before the marker so it is the first entry refused when the leaf archive's entries do not fit.
|
||||
func deepChain() nestPlan {
|
||||
return nestPlan{
|
||||
families: []archiveFamily{familyTarGz, familyZip, familyJar},
|
||||
leaf: map[string][]byte{
|
||||
"0-payload.bin": incompressible(deepChainPayloadBytes),
|
||||
"nested/marker.txt": []byte("leaf"),
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,501 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"runtime/debug"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/mimetype"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/internal/unknown"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
const ArchiveCatalogerTaskName = "archive-cataloger"
|
||||
|
||||
// NewArchiveCatalogerTask returns a task that extracts every archive it finds, runs the same tasks
|
||||
// against the archive's contents as a standalone filesystem, and records
|
||||
// CONTAINS relationships from the archive to the packages found directly inside it. It recurses into the archives
|
||||
// it extracts up to cfg.MaxDepth; the task drops itself from the tasks so that is the only recursion.
|
||||
//
|
||||
// Each archive is handled as the scan root is: pkgTasks, then the archives nested in it, then fileTasks,
|
||||
// which see the packages found in the nested archives (see Traversal.PackageFiles).
|
||||
//
|
||||
// The task is the authority on which archives were not cataloged and why; with
|
||||
// unknowns.IncludeUnexpandedArchives it also reports archives in formats it does not extract.
|
||||
//
|
||||
// exclusions are the scan's exclusion patterns, applied inside each archive. Returns nil when archive
|
||||
// cataloging is disabled (MaxDepth == 0; negative or beyond archive.MaxDepth means as deep as archive.MaxDepth).
|
||||
func NewArchiveCatalogerTask(cfg cataloging.ArchiveSearchConfig, unknowns cataloging.UnknownsConfig, pkgTasks, fileTasks []Task, exclusions []string) Task {
|
||||
return newArchiveCatalogerTask(cfg.MaxDepth, archiveLimits(cfg), unknowns.IncludeUnexpandedArchives, pkgTasks, fileTasks, exclusions)
|
||||
}
|
||||
|
||||
func archiveLimits(cfg cataloging.ArchiveSearchConfig) archive.Limits {
|
||||
return archive.Limits{MaxMemoryBytes: cfg.MaxMemoryBytes, MaxDiskBytes: cfg.MaxDiskBytes}
|
||||
}
|
||||
|
||||
func newArchiveCatalogerTask(maxDepth int, limits archive.Limits, reportUnsupportedUnknowns bool, pkgTasks, fileTasks []Task, exclusions []string) Task {
|
||||
if maxDepth == 0 {
|
||||
return nil
|
||||
}
|
||||
pkgTasks, fileTasks = withoutArchiveCataloger(pkgTasks), withoutArchiveCataloger(fileTasks)
|
||||
if len(pkgTasks)+len(fileTasks) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
|
||||
// catalogers run once per archive, and each must report into the one progress row it owns
|
||||
ctx = bus.WithCatalogerTaskRegistry(ctx)
|
||||
|
||||
c := &archiveCataloger{
|
||||
maxDepth: maxDepth,
|
||||
reportUnsupportedUnknowns: reportUnsupportedUnknowns,
|
||||
pkgTasks: pkgTasks,
|
||||
fileTasks: fileTasks,
|
||||
limiter: archive.NewLimiter(limits),
|
||||
exclusions: archive.NewExclusions(exclusions),
|
||||
progress: bus.StartCatalogerTask(ctx, archiveCatalogerProgressInfo(), -1, ""),
|
||||
}
|
||||
err := c.catalog(ctx, resolver, 0, builder)
|
||||
c.progress.AtomicStage.Set(fmt.Sprintf("%s archives", humanize.Comma(c.progress.Current())))
|
||||
c.progress.SetCompleted()
|
||||
c.logStats()
|
||||
return err
|
||||
}
|
||||
return NewTask(ArchiveCatalogerTaskName, fn, pkgcataloging.PackageTag, "archive")
|
||||
}
|
||||
|
||||
func archiveCatalogerProgressInfo() monitor.GenericTask {
|
||||
return monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "Archives",
|
||||
},
|
||||
ID: ArchiveCatalogerTaskName,
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
}
|
||||
|
||||
func withoutArchiveCataloger(tasks []Task) []Task {
|
||||
var out []Task
|
||||
for _, t := range tasks {
|
||||
if t == nil || t.Name() == ArchiveCatalogerTaskName {
|
||||
continue
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type archiveCataloger struct {
|
||||
maxDepth int
|
||||
reportUnsupportedUnknowns bool
|
||||
pkgTasks []Task
|
||||
fileTasks []Task
|
||||
limiter *archive.Limiter
|
||||
exclusions archive.Exclusions
|
||||
|
||||
// progress is the one row for the whole walk: its count is how many archives have been entered
|
||||
progress *monitor.TaskProgress
|
||||
|
||||
// ancestors are the digests of the archives the walk is currently inside. The walk is depth-first
|
||||
// and sequential, so this is a stack.
|
||||
ancestors []file.Digest
|
||||
|
||||
// the archive that spent the longest on its own extraction and cataloging, excluding nested archives
|
||||
slowest time.Duration
|
||||
slowestPath string
|
||||
}
|
||||
|
||||
// logStats reports what the walk cost, once. The peaks are the most held against each limit at any
|
||||
// one moment, comparable to the configured bounds.
|
||||
func (c *archiveCataloger) logStats() {
|
||||
peakMemory, peakDisk := c.limiter.Peak()
|
||||
fields := []any{
|
||||
"archives", c.progress.Current(),
|
||||
"peak-memory", humanize.Bytes(uint64(peakMemory)),
|
||||
"peak-disk", humanize.Bytes(uint64(peakDisk)),
|
||||
}
|
||||
if c.slowestPath != "" {
|
||||
fields = append(fields, "slowest", c.slowestPath, "slowest-took", c.slowest.Round(time.Millisecond))
|
||||
}
|
||||
log.WithFields(fields...).Info("nested archive cataloging complete")
|
||||
}
|
||||
|
||||
// catalog processes every archive in the resolver, recursing into each up to maxDepth.
|
||||
func (c *archiveCataloger) catalog(ctx context.Context, resolver file.Resolver, depth int, builder sbomsync.Builder) error {
|
||||
return c.catalogCandidates(ctx, resolver, c.discoverArchives(resolver), depth, builder)
|
||||
}
|
||||
|
||||
func (c *archiveCataloger) catalogCandidates(ctx context.Context, resolver file.Resolver, candidates []archiveCandidate, depth int, builder sbomsync.Builder) error {
|
||||
maxDepth := c.maxDepth
|
||||
// a deeper scan would write chains no decoder accepts back
|
||||
if maxDepth < 0 || maxDepth > archive.MaxDepth {
|
||||
maxDepth = archive.MaxDepth
|
||||
}
|
||||
if depth >= maxDepth {
|
||||
// java's own recursion is off while this task runs, so an archive left here is otherwise lost
|
||||
// without a trace
|
||||
for _, candidate := range candidates {
|
||||
if candidate.sniffedAsArchive {
|
||||
recordArchiveUnknown(builder, candidate.location.Coordinates, "nested archive not cataloged: the nested archive depth limit was reached")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var errs error
|
||||
for _, candidate := range candidates {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.processArchive(ctx, resolver, candidate, depth, builder); err != nil {
|
||||
errs = unknown.Append(errs, candidate.location, err)
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
// processArchive processes the given candidate archive, catching panics
|
||||
//
|
||||
//nolint:funlen
|
||||
func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver file.Resolver, candidate archiveCandidate, depth int, builder sbomsync.Builder) (err error) {
|
||||
// the decoders run on untrusted bytes; a panic in one costs this archive, not the scan
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
// the stack goes to the log, not the reason: the reason lands in the published SBOM's unknowns
|
||||
log.WithFields("path", candidate.location.FullNestedPath()).Debugf("archive cataloging panic stack:\n%s", debug.Stack())
|
||||
err = fmt.Errorf("panic while cataloging archive: %v", r)
|
||||
}
|
||||
}()
|
||||
|
||||
location := candidate.location
|
||||
content, err := parentResolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer internal.CloseAndLogError(content, location.FullNestedPath())
|
||||
|
||||
var archiveContent io.Reader = content
|
||||
if !candidate.sniffedAsArchive {
|
||||
var mayHide bool
|
||||
if archiveContent, mayHide = archive.MayHideAnAppendedArchive(content); !mayHide {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
started := time.Now()
|
||||
|
||||
if depth == 0 {
|
||||
// one budget for this archive and everything nested in it, however deep
|
||||
ctx = archive.WithBudget(ctx, archive.NewBudget(archiveTreeBudget(parentResolver, location)))
|
||||
}
|
||||
|
||||
// extracted files keep the real path and filesystem ID of the outermost archive; the chain of entries
|
||||
// to each is carried as its archive path. The chain names real paths, not the link the archive was
|
||||
// reached by, so it is stable and joins to the archive's package location by prefix.
|
||||
traversal := &archive.Traversal{Location: location, Parent: archive.TraversalFromContext(ctx)}
|
||||
archivePath := location.FullNestedPath()
|
||||
extracted, err := archive.Extract(ctx, archiveContent, location.Coordinates, c.limiter, c.exclusions)
|
||||
if errors.Is(err, archive.ErrUnsupportedFormat) {
|
||||
if c.reportUnsupportedUnknowns {
|
||||
recordArchiveUnknown(builder, location.Coordinates, "archive not cataloged: "+err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, archive.ErrDiskLimitReached) {
|
||||
// nothing is released while this archive waits, so skip it rather than block the scan
|
||||
recordArchiveUnknown(builder, location.Coordinates, "archive skipped: its content would exceed the disk limit")
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if extracted == nil {
|
||||
// a format with no extractor, or a candidate whose head only looked like an appended archive.
|
||||
// Logged rather than returned: an image can hold thousands of these
|
||||
log.WithFields("archive", archivePath, "sniffed-as-archive", candidate.sniffedAsArchive).
|
||||
Trace("no extractor could read this candidate; skipping")
|
||||
return nil
|
||||
}
|
||||
defer extracted.Cleanup()
|
||||
|
||||
// an archive whose bytes match one it is nested in (a zip quine) would recurse forever
|
||||
if c.isAncestor(extracted.Digests) {
|
||||
recordArchiveUnknown(builder, location.Coordinates, "archive not cataloged: it is identical to an archive that contains it")
|
||||
return nil
|
||||
}
|
||||
c.ancestors = append(c.ancestors, extracted.Digests...)
|
||||
defer func() { c.ancestors = c.ancestors[:len(c.ancestors)-len(extracted.Digests)] }()
|
||||
|
||||
if extracted.Truncated != nil {
|
||||
recordArchiveUnknown(builder, location.Coordinates, "archive cataloged from part of its contents: "+extracted.Truncated.Error())
|
||||
}
|
||||
|
||||
traversal.Digests = extracted.Digests
|
||||
ctx = archive.WithTraversal(ctx, traversal)
|
||||
|
||||
c.progress.Increment()
|
||||
|
||||
// everything found in this archive and the archives nested in it, merged into the parent's SBOM once complete
|
||||
found := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}}
|
||||
foundBuilder := sbomsync.NewBuilder(found)
|
||||
runArchiveTasks(ctx, c.pkgTasks, extracted, location.Coordinates, foundBuilder)
|
||||
direct := found.Artifacts.Packages.Sorted()
|
||||
traversal.Packages = packagesAt(direct, location.Coordinates)
|
||||
|
||||
// self time excludes descending, so a containing archive is not charged for what it contains
|
||||
selfTime := time.Since(started)
|
||||
|
||||
// as at the scan root, nested archives are cataloged before this archive's files, so the packages
|
||||
// describing them (located in this archive) select their files here
|
||||
errs := c.catalogCandidates(ctx, extracted, c.discoverArchives(extracted), depth+1, foundBuilder)
|
||||
|
||||
started = time.Now()
|
||||
runArchiveTasks(ctx, c.fileTasks, extracted, location.Coordinates, foundBuilder)
|
||||
// the files within it are related once the scan has decided which it describes (see NewArchiveContainsTask)
|
||||
for _, p := range direct {
|
||||
found.Relationships = append(found.Relationships, artifact.Relationship{From: location.Coordinates, To: p, Type: artifact.ContainsRelationship})
|
||||
}
|
||||
mergeArchiveResults(found, builder)
|
||||
c.progress.AtomicStage.Set(humanize.Comma(c.progress.Current()) + " archives (" + archivePath + ")")
|
||||
|
||||
if took := selfTime + time.Since(started); took > c.slowest {
|
||||
c.slowest, c.slowestPath = took, archivePath
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
// a top-level archive and everything nested in it may decompress to archiveTreeRatio times its size, or
|
||||
// archiveTreeFloorBytes if that is more. Real trees (a war of jars, a layer of tarballs) expand a few
|
||||
// times over; a recursive bomb expands by orders of magnitude at every level.
|
||||
var (
|
||||
archiveTreeRatio int64 = 100
|
||||
archiveTreeFloorBytes int64 = 1 << 30
|
||||
)
|
||||
|
||||
func archiveTreeBudget(resolver file.Resolver, location file.Location) int64 {
|
||||
var size int64
|
||||
if m, err := resolver.FileMetadataByLocation(location); err == nil && m.FileInfo != nil {
|
||||
size = m.Size()
|
||||
}
|
||||
return max(archiveTreeRatio*size, archiveTreeFloorBytes)
|
||||
}
|
||||
|
||||
func (c *archiveCataloger) isAncestor(digests []file.Digest) bool {
|
||||
for _, d := range digests {
|
||||
if slices.Contains(c.ancestors, d) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// packagesAt returns the packages located at the given coordinates, i.e. those describing the file there.
|
||||
func packagesAt(pkgs []pkg.Package, coordinates file.Coordinates) []pkg.Package {
|
||||
var out []pkg.Package
|
||||
for _, p := range pkgs {
|
||||
if locatedAt(p, coordinates) {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// locatedAt reports whether one of p's locations is at the given coordinates.
|
||||
func locatedAt(p pkg.Package, coordinates file.Coordinates) bool {
|
||||
for _, l := range p.Locations.ToUnorderedSlice() {
|
||||
if l.Coordinates == coordinates {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func recordArchiveUnknown(builder sbomsync.Builder, coordinates file.Coordinates, reason string) {
|
||||
appendUnknowns(builder, ArchiveCatalogerTaskName, []unknown.CoordinateError{{Coordinates: coordinates, Reason: errors.New(reason)}})
|
||||
}
|
||||
|
||||
// runArchiveTasks runs each task against the archive's contents.
|
||||
//
|
||||
// Failures are recorded as RunTask records them at the top level: errors carrying coordinates keep
|
||||
// them, anything else is attributed to the archive. A failure never fails the archive or the scan.
|
||||
func runArchiveTasks(ctx context.Context, tasks []Task, resolver file.Resolver, archiveCoordinates file.Coordinates, builder sbomsync.Builder) {
|
||||
for _, t := range tasks {
|
||||
err := runSubTaskSafely(ctx, t, resolver, builder)
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
log.WithFields("task", t.Name(), "error", err).Trace("archive sub-cataloger task reported an error")
|
||||
|
||||
unknowns, remaining := unknown.ExtractCoordinateErrors(err)
|
||||
if remaining != nil {
|
||||
unknowns = append(unknowns, unknown.CoordinateError{Coordinates: archiveCoordinates, Reason: remaining})
|
||||
}
|
||||
appendUnknowns(builder, t.Name(), unknowns)
|
||||
}
|
||||
}
|
||||
|
||||
// runSubTaskSafely is runTaskSafely without the task name in a panic's error, since appendUnknowns
|
||||
// already prefixes it.
|
||||
func runSubTaskSafely(ctx context.Context, t Task, resolver file.Resolver, builder sbomsync.Builder) (err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.WithFields("task", t.Name()).Debugf("archive sub-cataloger task panic stack:\n%s", debug.Stack())
|
||||
err = fmt.Errorf("panic: %v", r)
|
||||
}
|
||||
}()
|
||||
return t.Execute(ctx, resolver, builder)
|
||||
}
|
||||
|
||||
// archiveCandidate is one file the walk will try to open.
|
||||
type archiveCandidate struct {
|
||||
location file.Location
|
||||
|
||||
// sniffedAsArchive is true when the file's content types it as an archive. Such a file that cannot
|
||||
// be extracted is a broken archive and is reported; a file that is a candidate only by name is
|
||||
// passed over silently.
|
||||
sniffedAsArchive bool
|
||||
}
|
||||
|
||||
// zipArchiveGlobs find files named like zip archives whose content did not sniff as one, such as a
|
||||
// Spring Boot executable jar: a launcher script with a zip appended. The name only decides what gets
|
||||
// opened; the end-of-central-directory record decides whether it is an archive.
|
||||
//
|
||||
// Globs rather than a MIME lookup, because asking for every executable, script and unrecognized file
|
||||
// returned thousands of locations that were all discarded.
|
||||
var zipArchiveGlobs = []string{
|
||||
"**/*.jar", "**/*.war", "**/*.ear", "**/*.par", "**/*.sar", "**/*.nar", "**/*.kar",
|
||||
"**/*.hpi", "**/*.jpi", "**/*.far", "**/*.rar", "**/*.zip", "**/*.apk", "**/*.aar",
|
||||
"**/*.egg", "**/*.whl", "**/*.lpkg", "**/*.zap", "**/*.exe",
|
||||
}
|
||||
|
||||
var archiveMIMETypes = slices.Sorted(slices.Values(mimetype.ArchiveMIMETypeSet.List()))
|
||||
|
||||
// discoverArchives returns every file worth opening: those whose content is an archive, plus those
|
||||
// named like a zip whose tail may still hold one.
|
||||
func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCandidate {
|
||||
archives, err := resolver.FilesByMIMEType(archiveMIMETypes...)
|
||||
if err != nil {
|
||||
log.WithFields("error", err).Debug("unable to list archive files for recursive cataloging")
|
||||
return nil
|
||||
}
|
||||
|
||||
var candidates []archiveCandidate
|
||||
seen := map[file.Coordinates]int{}
|
||||
for _, loc := range archives {
|
||||
seen[loc.Coordinates] = len(candidates)
|
||||
candidates = append(candidates, archiveCandidate{location: loc, sniffedAsArchive: true})
|
||||
}
|
||||
|
||||
namedLikeZips, err := resolver.FilesByGlob(zipArchiveGlobs...)
|
||||
if err != nil {
|
||||
log.WithFields("error", err).Debug("unable to list files that may carry an appended archive")
|
||||
return candidates
|
||||
}
|
||||
for _, loc := range namedLikeZips {
|
||||
if i, dup := seen[loc.Coordinates]; dup {
|
||||
// a MIME hit is the bare real path; the name it is reached by (a link such as
|
||||
// lib/dep.jar -> ../store/3f9a1c) is what name-based catalogers like java key off
|
||||
if prev := candidates[i].location; prev.AccessPath == "" || prev.AccessPath == prev.FullNestedPath() {
|
||||
candidates[i].location = loc
|
||||
}
|
||||
continue
|
||||
}
|
||||
seen[loc.Coordinates] = len(candidates)
|
||||
candidates = append(candidates, archiveCandidate{location: loc})
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
// mergeArchiveResults copies everything found in an archive into its parent's SBOM.
|
||||
func mergeArchiveResults(found *sbom.SBOM, builder sbomsync.Builder) {
|
||||
if pkgs := found.Artifacts.Packages.Sorted(); len(pkgs) > 0 {
|
||||
builder.AddPackages(pkgs...)
|
||||
}
|
||||
if accessor, ok := builder.(sbomsync.Accessor); ok {
|
||||
accessor.WriteToSBOM(func(s *sbom.SBOM) {
|
||||
mergeFileArtifacts(&s.Artifacts, &found.Artifacts)
|
||||
})
|
||||
}
|
||||
if len(found.Relationships) > 0 {
|
||||
builder.AddRelationships(found.Relationships...)
|
||||
}
|
||||
}
|
||||
|
||||
// NewArchiveContainsTask relates each archive to every file the SBOM describes directly within it. It must
|
||||
// run after every other task, so the files within archives are those the scan root's configuration
|
||||
// selects (file selection, unknowns, package evidence) and no others.
|
||||
func NewArchiveContainsTask() Task {
|
||||
fn := func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error {
|
||||
var rels []artifact.Relationship
|
||||
builder.(sbomsync.Accessor).ReadFromSBOM(func(s *sbom.SBOM) {
|
||||
rels = archiveContains(s.AllCoordinates())
|
||||
})
|
||||
builder.AddRelationships(rels...)
|
||||
return nil
|
||||
}
|
||||
return NewTask("archive-contains", fn)
|
||||
}
|
||||
|
||||
// archiveContains relates the archive holding each file within one to that file, and each such archive to
|
||||
// the archive holding it, so an archive the SBOM does not otherwise describe still links what is within it.
|
||||
func archiveContains(coordinates []file.Coordinates) []artifact.Relationship {
|
||||
related := file.NewCoordinateSet(coordinates...)
|
||||
var rels []artifact.Relationship
|
||||
for _, c := range coordinates {
|
||||
for {
|
||||
container, ok := archive.ContainingArchive(c)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
rels = append(rels, artifact.Relationship{From: container, To: c, Type: artifact.ContainsRelationship})
|
||||
if related.Contains(container) {
|
||||
break
|
||||
}
|
||||
related.Add(container)
|
||||
c = container
|
||||
}
|
||||
}
|
||||
return rels
|
||||
}
|
||||
|
||||
func mergeFileArtifacts(dst, src *sbom.Artifacts) {
|
||||
mergeInto(&dst.FileMetadata, src.FileMetadata)
|
||||
mergeInto(&dst.FileDigests, src.FileDigests)
|
||||
mergeInto(&dst.FileContents, src.FileContents)
|
||||
mergeInto(&dst.FileLicenses, src.FileLicenses)
|
||||
mergeInto(&dst.Executables, src.Executables)
|
||||
if len(src.Unknowns) > 0 && dst.Unknowns == nil {
|
||||
dst.Unknowns = map[file.Coordinates][]string{}
|
||||
}
|
||||
for coordinates, reasons := range src.Unknowns {
|
||||
dst.Unknowns[coordinates] = append(dst.Unknowns[coordinates], reasons...)
|
||||
}
|
||||
}
|
||||
|
||||
func mergeInto[K comparable, V any](dst *map[K]V, src map[K]V) {
|
||||
if len(src) == 0 {
|
||||
return
|
||||
}
|
||||
if *dst == nil {
|
||||
*dst = make(map[K]V, len(src))
|
||||
}
|
||||
maps.Copy(*dst, src)
|
||||
}
|
||||
@@ -0,0 +1,957 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/wagoodman/go-partybus"
|
||||
"github.com/wagoodman/go-progress"
|
||||
|
||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/mimetype"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/internal/testutils"
|
||||
"github.com/anchore/syft/internal/unknown"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/event"
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
// dirTestResolver is a minimal file.Resolver over a directory. FilesByMIMEType sniffs each file's
|
||||
// content as a real resolver does; FileContentsByLocation opens the file, which is random-access.
|
||||
type dirTestResolver struct {
|
||||
file.Resolver
|
||||
dir string
|
||||
fsid string
|
||||
}
|
||||
|
||||
// streamingDirResolver is dirTestResolver handing out plain streams, so an archive's own bytes must be
|
||||
// held in memory or written to disk rather than read in place.
|
||||
type streamingDirResolver struct {
|
||||
dirTestResolver
|
||||
}
|
||||
|
||||
func (d streamingDirResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) {
|
||||
f, err := d.dirTestResolver.FileContentsByLocation(loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{f, f}, nil
|
||||
}
|
||||
|
||||
// panickingDirResolver is dirTestResolver whose reads of one file panic, standing in for a decoder
|
||||
// that panics on hostile bytes.
|
||||
type panickingDirResolver struct {
|
||||
dirTestResolver
|
||||
path string
|
||||
}
|
||||
|
||||
func (d panickingDirResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) {
|
||||
if loc.RealPath == d.path {
|
||||
return io.NopCloser(panicReader{}), nil
|
||||
}
|
||||
return d.dirTestResolver.FileContentsByLocation(loc)
|
||||
}
|
||||
|
||||
type panicReader struct{}
|
||||
|
||||
func (panicReader) Read([]byte) (int, error) { panic("decoder blew up") }
|
||||
|
||||
func Test_archiveCataloger_aPanicCostsOnlyThatArchive(t *testing.T) {
|
||||
rootDir := t.TempDir()
|
||||
for _, name := range []string{"bad.zip", "good.zip"} {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600))
|
||||
}
|
||||
|
||||
var seen []string
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, nil))
|
||||
|
||||
resolver := panickingDirResolver{dirTestResolver: dirTestResolver{dir: rootDir}, path: "/bad.zip"}
|
||||
err := tsk.Execute(context.Background(), resolver, sbomsync.NewBuilder(newTestSBOM()))
|
||||
|
||||
// a coordinate error becomes an unknown; a plain error (or an escaped panic) would fail the scan
|
||||
coordErrs, remaining := unknown.ExtractCoordinateErrors(err)
|
||||
require.NoError(t, remaining)
|
||||
require.Len(t, coordErrs, 1)
|
||||
assert.Equal(t, "/bad.zip", coordErrs[0].Coordinates.RealPath)
|
||||
assert.EqualError(t, coordErrs[0].Reason, "panic while cataloging archive: decoder blew up", "no stack trace in the SBOM-bound reason")
|
||||
assert.Equal(t, []string{"/good.zip"}, seen, "the sibling is still cataloged")
|
||||
}
|
||||
|
||||
func newTestTask(t *testing.T, cfg cataloging.ArchiveSearchConfig, subPipeline ...Task) Task {
|
||||
t.Helper()
|
||||
tsk := NewArchiveCatalogerTask(cfg, cataloging.UnknownsConfig{}, subPipeline, nil, nil)
|
||||
require.NotNil(t, tsk)
|
||||
return tsk
|
||||
}
|
||||
|
||||
// newLimitedTestTask builds the task with the limiter's own limits, where zero forbids the resource.
|
||||
func newLimitedTestTask(t *testing.T, maxDepth int, limits archive.Limits, subPipeline ...Task) Task {
|
||||
t.Helper()
|
||||
tsk := newArchiveCatalogerTask(maxDepth, limits, false, subPipeline, nil, nil)
|
||||
require.NotNil(t, tsk)
|
||||
return tsk
|
||||
}
|
||||
|
||||
func newTestSBOM() *sbom.SBOM {
|
||||
return &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}}
|
||||
}
|
||||
|
||||
func countingTask(ran *int) Task {
|
||||
return NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
*ran++
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// capturingTask records the virtual path of every archive the sub-pipeline is run against and how many
|
||||
// files its resolver holds.
|
||||
func capturingTask(t *testing.T, seen *[]string, fileCounts *map[string]int) Task {
|
||||
return NewTask("capture", func(ctx context.Context, r file.Resolver, _ sbomsync.Builder) error {
|
||||
trav := archive.TraversalFromContext(ctx)
|
||||
require.NotNil(t, trav)
|
||||
*seen = append(*seen, archive.VirtualPath(trav.Location))
|
||||
if fileCounts != nil {
|
||||
locs, err := r.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
(*fileCounts)[archive.VirtualPath(trav.Location)] = len(locs)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) {
|
||||
// reaching a limit is a truncation, not a failure: the sub-pipeline still runs over what was stored
|
||||
big := bytes.Repeat([]byte("x"), 4096)
|
||||
outerZip := testutils.Zip(t, map[string][]byte{
|
||||
"a/small.txt": []byte("small"),
|
||||
"b/big.txt": big,
|
||||
})
|
||||
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
// nothing is held in memory, so each entry is charged to disk: small.txt lands, big.txt is refused
|
||||
limits := archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: 6000}
|
||||
|
||||
var seen []string
|
||||
fileCounts := map[string]int{}
|
||||
tsk := newLimitedTestTask(t, 1, limits, capturingTask(t, &seen, &fileCounts))
|
||||
|
||||
s := newTestSBOM()
|
||||
err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))
|
||||
|
||||
require.NoError(t, err, "a truncated extraction must not fail the scan")
|
||||
assert.Equal(t, []string{"/outer.zip"}, seen, "the sub-pipeline must run over the partial contents")
|
||||
assert.Equal(t, 1, fileCounts["/outer.zip"], "only the entry stored before the limit is visible")
|
||||
|
||||
reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
|
||||
require.True(t, ok, "the truncation must be recorded against the archive; got %v", s.Artifacts.Unknowns)
|
||||
require.Len(t, reasons, 1)
|
||||
assert.Contains(t, reasons[0], "archive cataloged from part of its contents")
|
||||
assert.Contains(t, reasons[0], ArchiveCatalogerTaskName, "the unknown names the task that recorded it")
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_aDamagedEntryCostsOnlyThatEntry(t *testing.T) {
|
||||
rootDir := t.TempDir()
|
||||
// entry data overwritten but headers and end-of-central-directory intact: detected as an archive,
|
||||
// and data.bin fails when it is read. ok.txt comes after it and is intact.
|
||||
corrupt := testutils.Zip(t, map[string][]byte{"data.bin": incompressible(4096), "ok.txt": []byte("ok")})
|
||||
require.Greater(t, len(corrupt), 1024)
|
||||
for i := 100; i < 600; i++ {
|
||||
corrupt[i] ^= 0xff
|
||||
}
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "corrupt.zip"), corrupt, 0o600))
|
||||
|
||||
var seen []string
|
||||
fileCounts := map[string]int{}
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, &fileCounts))
|
||||
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
|
||||
|
||||
assert.Equal(t, []string{"/corrupt.zip"}, seen, "the archive is still cataloged")
|
||||
assert.Equal(t, 1, fileCounts["/corrupt.zip"], "from the entries that could be read")
|
||||
reasons := s.Artifacts.Unknowns[file.NewCoordinates("/corrupt.zip", "")]
|
||||
require.Len(t, reasons, 1)
|
||||
assert.Contains(t, reasons[0], `entry "data.bin" could not be read`)
|
||||
}
|
||||
|
||||
func Test_NewArchiveCatalogerTask_gating(t *testing.T) {
|
||||
someTask := NewTask("noop", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { return nil })
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg cataloging.ArchiveSearchConfig
|
||||
subPipeline []Task
|
||||
wantTask bool
|
||||
}{
|
||||
{
|
||||
name: "disabled at depth 0",
|
||||
cfg: cataloging.DefaultArchiveSearchConfig(),
|
||||
subPipeline: []Task{someTask},
|
||||
},
|
||||
{
|
||||
name: "no sub-pipeline",
|
||||
cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2),
|
||||
subPipeline: nil,
|
||||
},
|
||||
{
|
||||
name: "enabled",
|
||||
cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2),
|
||||
subPipeline: []Task{someTask},
|
||||
wantTask: true,
|
||||
},
|
||||
{
|
||||
name: "enabled at negative depth",
|
||||
cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(-1),
|
||||
subPipeline: []Task{someTask},
|
||||
wantTask: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := NewArchiveCatalogerTask(tt.cfg, cataloging.UnknownsConfig{}, tt.subPipeline, nil, nil)
|
||||
if tt.wantTask {
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, ArchiveCatalogerTaskName, got.Name())
|
||||
return
|
||||
}
|
||||
assert.Nil(t, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_depthStopsAtTheCeiling(t *testing.T) {
|
||||
// "unbounded" must still end, or a zip quine or recursive bomb runs forever, and a configured depth
|
||||
// past the ceiling must too, or the SBOM holds chains no decoder accepts back
|
||||
chain := testutils.Zip(t, map[string][]byte{"deep.txt": []byte("deep")})
|
||||
for range archive.MaxDepth + 4 {
|
||||
chain = testutils.Zip(t, map[string][]byte{"n.zip": chain})
|
||||
}
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), chain, 0o600))
|
||||
|
||||
for _, depth := range []int{-1, archive.MaxDepth + 2} {
|
||||
t.Run(fmt.Sprint(depth), func(t *testing.T) {
|
||||
var seen []string
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth), capturingTask(t, &seen, nil))
|
||||
_ = tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM()))
|
||||
|
||||
assert.Len(t, seen, archive.MaxDepth)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing.T) {
|
||||
// a zip quine contains itself. Building one is impractical, so the walk is started as if it were
|
||||
// already inside an archive with these exact bytes, which is the state a quine reaches one level down
|
||||
outerZip := testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")})
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
var ran int
|
||||
c := newTestCataloger(-1, archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), countingTask(&ran))
|
||||
sum := sha1.Sum(outerZip)
|
||||
c.ancestors = []file.Digest{{Algorithm: "sha1", Value: hex.EncodeToString(sum[:])}}
|
||||
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(s)))
|
||||
|
||||
assert.Zero(t, ran, "the sub-pipeline must not run inside a repeat of an ancestor")
|
||||
reasons := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
|
||||
require.Len(t, reasons, 1)
|
||||
assert.Contains(t, reasons[0], "identical to an archive that contains it")
|
||||
assert.Len(t, c.ancestors, 1, "the walk leaves the ancestor stack as it found it")
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_stopsWhenTheContextIsCancelled(t *testing.T) {
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600))
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
var ran int
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), countingTask(&ran))
|
||||
err := tsk.Execute(ctx, dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM()))
|
||||
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.Zero(t, ran)
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_decompressionBudgetIsSharedByATreeAndFreshPerTopLevelArchive(t *testing.T) {
|
||||
// a recursive bomb fits the limits one level at a time, so what bounds it is the total decompressed
|
||||
// under one top-level archive
|
||||
origRatio, origFloor := archiveTreeRatio, archiveTreeFloorBytes
|
||||
archiveTreeRatio, archiveTreeFloorBytes = 0, 100_000
|
||||
t.Cleanup(func() { archiveTreeRatio, archiveTreeFloorBytes = origRatio, origFloor })
|
||||
|
||||
zeros := make([]byte, 60_000)
|
||||
nested := testutils.Zip(t, map[string][]byte{"zeros.bin": zeros})
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bomb.zip"), testutils.Zip(t, map[string][]byte{"a.zip": nested, "b.zip": nested}), 0o600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "sibling.zip"), testutils.Zip(t, map[string][]byte{"zeros.bin": zeros}), 0o600))
|
||||
|
||||
var seen []string
|
||||
fileCounts := map[string]int{}
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), capturingTask(t, &seen, &fileCounts))
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
|
||||
|
||||
assert.Equal(t, 1, fileCounts["/bomb.zip:a.zip"], "the first nested archive fits the tree's budget")
|
||||
assert.Equal(t, 0, fileCounts["/bomb.zip:b.zip"], "the second does not: the budget is shared, not per archive")
|
||||
reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/bomb.zip", FullPath: "/bomb.zip!/b.zip"}]
|
||||
require.Len(t, reasons, 1)
|
||||
assert.Contains(t, reasons[0], "possible decompression bomb")
|
||||
assert.Equal(t, 1, fileCounts["/sibling.zip"], "another top-level archive starts with a fresh budget")
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_noSelfContainsEdge(t *testing.T) {
|
||||
// a jar java cannot identify records an unknown at its own coordinates; that must not turn into
|
||||
// "x.jar CONTAINS x.jar", which a graph walker would loop on
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "x.jar"), testutils.Zip(t, map[string][]byte{"readme.txt": []byte("no manifest")}), 0o600))
|
||||
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), javaTask(), fileMetadataTask())
|
||||
s := newTestSBOM()
|
||||
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
|
||||
_ = tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))
|
||||
withArchiveContains(t, s)
|
||||
|
||||
require.Contains(t, s.Artifacts.Unknowns, file.NewCoordinates("/x.jar", ""), "the fixture must reproduce the unknown")
|
||||
var contains int
|
||||
for _, rel := range s.Relationships {
|
||||
if rel.Type != artifact.ContainsRelationship {
|
||||
continue
|
||||
}
|
||||
contains++
|
||||
assert.NotEqual(t, rel.From.ID(), rel.To.ID(), "self-referential CONTAINS edge: %+v", rel)
|
||||
}
|
||||
assert.Positive(t, contains, "the archive still contains its files")
|
||||
}
|
||||
|
||||
func Test_NewArchiveContainsTask(t *testing.T) {
|
||||
// a file within an archive is contained only when the SBOM describes it, as a file at the scan root is
|
||||
// listed only then; an executable no package owns is recorded whatever the file selection
|
||||
in := func(fullPath string) file.Coordinates {
|
||||
return file.Coordinates{RealPath: "/outer.tar", FileSystemID: "layer", FullPath: fullPath}
|
||||
}
|
||||
outer := file.NewCoordinates("/outer.tar", "layer")
|
||||
jar := in("/outer.tar!/lib/app.jar")
|
||||
manifest := in("/outer.tar!/lib/app.jar!/META-INF/MANIFEST.MF")
|
||||
unknownExe := in("/outer.tar!/bin/setup.exe")
|
||||
|
||||
s := newTestSBOM()
|
||||
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{manifest: {}, file.NewCoordinates("/loose.txt", "layer"): {}}
|
||||
s.Artifacts.Executables = map[file.Coordinates]file.Executable{in("/outer.tar!/bin/tool"): {}}
|
||||
s.Artifacts.Unknowns = map[file.Coordinates][]string{unknownExe: {"no package identified in executable file"}}
|
||||
|
||||
withArchiveContains(t, s)
|
||||
|
||||
type edge struct{ from, to file.Coordinates }
|
||||
var edges []edge
|
||||
for _, r := range s.Relationships {
|
||||
assert.Equal(t, artifact.ContainsRelationship, r.Type)
|
||||
edges = append(edges, edge{r.From.(file.Coordinates), r.To.(file.Coordinates)})
|
||||
}
|
||||
assert.ElementsMatch(t, []edge{
|
||||
{jar, manifest},
|
||||
{outer, jar}, // the jar is not described itself, but is on the way to what is
|
||||
{outer, unknownExe},
|
||||
}, edges)
|
||||
}
|
||||
|
||||
func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) {
|
||||
// a copy of itself in the sub-pipeline would process every nesting level twice
|
||||
cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)
|
||||
|
||||
innerZip := testutils.Zip(t, map[string][]byte{"leaf.txt": []byte("leaf")})
|
||||
outerZip := testutils.Zip(t, map[string][]byte{"nested/inner.zip": innerZip})
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
var ran int
|
||||
countTask := countingTask(&ran)
|
||||
|
||||
poisoned := []Task{countTask, NewArchiveCatalogerTask(cfg, cataloging.UnknownsConfig{}, []Task{countTask}, nil, nil)}
|
||||
require.NotNil(t, poisoned[1])
|
||||
|
||||
tsk := newTestTask(t, cfg, poisoned...)
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.Equal(t, 2, ran, "two archives, one sub-pipeline task each")
|
||||
|
||||
t.Run("a sub-pipeline of nothing but itself yields no task", func(t *testing.T) {
|
||||
only := []Task{NewArchiveCatalogerTask(cfg, cataloging.UnknownsConfig{}, []Task{countTask}, nil, nil)}
|
||||
assert.Nil(t, NewArchiveCatalogerTask(cfg, cataloging.UnknownsConfig{}, only, nil, nil))
|
||||
})
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_nestedArchivesKeepTheLayerFileSystemID(t *testing.T) {
|
||||
// extracted files keep the layer digest as their FileSystemID at every level; the nesting chain
|
||||
// rides on the traversal's VirtualPath and the coordinate's FullPath
|
||||
innerZip := testutils.Zip(t, map[string][]byte{"leaf.txt": []byte("leaf")})
|
||||
outerZip := testutils.Zip(t, map[string][]byte{"nested/inner.zip": innerZip})
|
||||
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
const layerDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
|
||||
var chains, fsids []string
|
||||
captureTask := NewTask("capture-fsid", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
trav := archive.TraversalFromContext(ctx)
|
||||
require.NotNil(t, trav)
|
||||
chains = append(chains, archive.VirtualPath(trav.Location))
|
||||
fsids = append(fsids, trav.Location.FileSystemID)
|
||||
return nil
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), captureTask)
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir, fsid: layerDigest}, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.Equal(t, []string{layerDigest, layerDigest}, fsids)
|
||||
assert.ElementsMatch(t, []string{"/outer.zip", "/outer.zip:nested/inner.zip"}, chains)
|
||||
}
|
||||
|
||||
// twoLayerResolver reports the same archive path under two filesystem ids with different content, as an
|
||||
// all-layers image scan does for an archive rewritten in a later layer.
|
||||
type twoLayerResolver struct {
|
||||
file.Resolver
|
||||
path string
|
||||
contentBy map[string][]byte
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T) {
|
||||
resolver := twoLayerResolver{
|
||||
path: "/app/bundle.zip",
|
||||
contentBy: map[string][]byte{
|
||||
"layer-one": testutils.Zip(t, map[string][]byte{"lib/config.json": []byte(`{"v":1}`)}),
|
||||
"layer-two": testutils.Zip(t, map[string][]byte{"lib/config.json": []byte(`{"v":2}`)}),
|
||||
},
|
||||
}
|
||||
|
||||
var chains []string
|
||||
fileTask := NewTask("record-files", func(ctx context.Context, r file.Resolver, builder sbomsync.Builder) error {
|
||||
trav := archive.TraversalFromContext(ctx)
|
||||
require.NotNil(t, trav)
|
||||
chains = append(chains, trav.Location.FileSystemID)
|
||||
|
||||
locs, err := r.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
metadata := map[file.Coordinates]file.Metadata{}
|
||||
for _, loc := range locs {
|
||||
metadata[loc.Coordinates] = file.Metadata{Path: loc.RealPath}
|
||||
}
|
||||
builder.(sbomsync.Accessor).WriteToSBOM(func(s *sbom.SBOM) {
|
||||
s.Artifacts.FileMetadata = metadata
|
||||
})
|
||||
return nil
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), fileTask)
|
||||
|
||||
s := &sbom.SBOM{Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(),
|
||||
FileMetadata: map[file.Coordinates]file.Metadata{},
|
||||
}}
|
||||
require.NoError(t, tsk.Execute(context.Background(), resolver, sbomsync.NewBuilder(s)))
|
||||
|
||||
assert.ElementsMatch(t, []string{"layer-one", "layer-two"}, chains)
|
||||
// the tables are keyed by Coordinates, so a collision is a silently missing entry
|
||||
assert.Len(t, s.Artifacts.FileMetadata, 2)
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) {
|
||||
// runSubPipeline bypasses RunTask, where the coordinate-error to unknown conversion lives
|
||||
outerZip := testutils.Zip(t, map[string][]byte{"lib/broken.json": []byte("{")})
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1)
|
||||
|
||||
t.Run("a coordinate error keeps the coordinates the cataloger named", func(t *testing.T) {
|
||||
failing := NewTask("failing-cataloger", func(_ context.Context, r file.Resolver, _ sbomsync.Builder) error {
|
||||
locs, err := r.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, locs)
|
||||
var errs error
|
||||
for _, loc := range locs {
|
||||
errs = unknown.Append(errs, loc, errors.New("unable to parse"))
|
||||
}
|
||||
return errs
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cfg, failing)
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)),
|
||||
"a cataloger failing inside an archive must not fail the archive or the scan")
|
||||
|
||||
require.Len(t, s.Artifacts.Unknowns, 1)
|
||||
for coords, reasons := range s.Artifacts.Unknowns {
|
||||
assert.Equal(t, "/outer.zip!/lib/broken.json", coords.FullNestedPath(), "the inner coordinates must survive the merge")
|
||||
assert.Equal(t, "/outer.zip", coords.RealPath, "and must be addressable per archive")
|
||||
assert.Equal(t, []string{"failing-cataloger: unable to parse"}, reasons)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an error with no coordinates is attributed to the archive", func(t *testing.T) {
|
||||
failing := NewTask("bare-error-cataloger", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
return errors.New("cataloger blew up")
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cfg, failing)
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
|
||||
|
||||
require.Len(t, s.Artifacts.Unknowns, 1)
|
||||
reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
|
||||
require.True(t, ok, "an error with no location of its own belongs to the containing archive")
|
||||
assert.Equal(t, []string{"bare-error-cataloger: cataloger blew up"}, reasons)
|
||||
})
|
||||
|
||||
t.Run("a panicking cataloger is recovered and recorded", func(t *testing.T) {
|
||||
panicking := NewTask("panicking-cataloger", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
panic("nope")
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cfg, panicking)
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)),
|
||||
"a panic inside an archive must not fail the scan")
|
||||
|
||||
reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
|
||||
require.True(t, ok)
|
||||
require.Len(t, reasons, 1)
|
||||
assert.Equal(t, "panicking-cataloger: panic: nope", reasons[0])
|
||||
})
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) {
|
||||
outerZip := testutils.Zip(t, map[string][]byte{"lib/thing.json": []byte("{}")})
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
preexisting := file.Coordinates{RealPath: "/outer.zip", FullPath: "/outer.zip!/lib/thing.json"}
|
||||
writer := NewTask("scratch-writer", func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error {
|
||||
builder.(sbomsync.Accessor).WriteToSBOM(func(s *sbom.SBOM) {
|
||||
if s.Artifacts.Unknowns == nil {
|
||||
s.Artifacts.Unknowns = map[file.Coordinates][]string{}
|
||||
}
|
||||
s.Artifacts.Unknowns[preexisting] = append(s.Artifacts.Unknowns[preexisting], "recorded by the cataloger itself")
|
||||
})
|
||||
return nil
|
||||
})
|
||||
failing := NewTask("failing-cataloger", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
return errors.New("boom")
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), writer, failing)
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
|
||||
|
||||
assert.Equal(t, []string{"recorded by the cataloger itself"}, s.Artifacts.Unknowns[preexisting])
|
||||
assert.Equal(t, []string{"failing-cataloger: boom"}, s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")])
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) {
|
||||
entry := bytes.Repeat([]byte("z"), 400)
|
||||
entries := map[string][]byte{}
|
||||
for i := range 5 {
|
||||
entries[fmt.Sprintf("e%d.bin", i)] = entry
|
||||
}
|
||||
|
||||
rootDir := t.TempDir()
|
||||
for _, name := range []string{"one.zip", "two.zip"} {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), testutils.Zip(t, entries), 0o600))
|
||||
}
|
||||
|
||||
// room in memory for one archive's entries and their index cost, not for two at once
|
||||
limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: 20000, MaxDiskBytes: 0})
|
||||
|
||||
// sampled while the archive is still held
|
||||
var extracted []int
|
||||
var peakMemory int64
|
||||
sample := NewTask("sample", func(_ context.Context, r file.Resolver, _ sbomsync.Builder) error {
|
||||
locs, err := r.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
extracted = append(extracted, len(locs))
|
||||
if mem, _ := limiter.InUse(); mem > peakMemory {
|
||||
peakMemory = mem
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
c := newTestCataloger(1, limiter, sample)
|
||||
require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.Equal(t, []int{5, 5}, extracted,
|
||||
"both archives must extract in full: the first is released before the second is admitted")
|
||||
assert.Positive(t, peakMemory, "entries within the memory limit are held in memory")
|
||||
|
||||
mem, disk := limiter.InUse()
|
||||
assert.Zero(t, mem, "the memory limit must fall back to nothing once the walk unwinds")
|
||||
assert.Zero(t, disk, "and so must the disk limit")
|
||||
}
|
||||
|
||||
func newTestCataloger(maxDepth int, limiter *archive.Limiter, subPipeline ...Task) *archiveCataloger {
|
||||
return &archiveCataloger{
|
||||
maxDepth: maxDepth,
|
||||
pkgTasks: subPipeline,
|
||||
limiter: limiter,
|
||||
progress: bus.StartCatalogerTask(context.Background(), archiveCatalogerProgressInfo(), -1, ""),
|
||||
}
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_archiveWhoseOwnBytesExceedTheDiskLimitIsSkipped(t *testing.T) {
|
||||
// a streamed archive's own bytes must be placed before it can be opened; when they do not fit on
|
||||
// disk the archive is skipped rather than waited for, and a sibling that fits is cataloged in full
|
||||
oversized := testutils.Zip(t, map[string][]byte{"payload.bin": incompressible(8000)})
|
||||
small := testutils.Zip(t, map[string][]byte{"ok.txt": []byte("ok")})
|
||||
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "small.zip"), small, 0o600))
|
||||
|
||||
// room for the small archive's bytes, its one entry and that entry's index cost
|
||||
diskLimit := int64(len(small)) + 3000
|
||||
require.Greater(t, int64(len(oversized)), diskLimit, "the fixture must actually exceed the limit")
|
||||
|
||||
limits := archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: diskLimit}
|
||||
|
||||
var seen []string
|
||||
fileCounts := map[string]int{}
|
||||
tsk := newLimitedTestTask(t, 1, limits, capturingTask(t, &seen, &fileCounts))
|
||||
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), streamingDirResolver{dirTestResolver{dir: rootDir}}, sbomsync.NewBuilder(s)),
|
||||
"reaching a limit must not fail the scan")
|
||||
|
||||
assert.Equal(t, []string{"/small.zip"}, seen, "the oversized archive is skipped and the sibling is still cataloged")
|
||||
assert.Equal(t, 1, fileCounts["/small.zip"])
|
||||
|
||||
reasons := s.Artifacts.Unknowns[file.NewCoordinates("/oversized.zip", "")]
|
||||
require.Len(t, reasons, 1, "the skip must be recorded against the archive; got %v", s.Artifacts.Unknowns)
|
||||
assert.Contains(t, reasons[0], "archive skipped")
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) {
|
||||
marker := map[string][]byte{"nested/marker.txt": []byte("hello")}
|
||||
|
||||
scanDir := t.TempDir()
|
||||
for name, body := range map[string][]byte{
|
||||
"bundle.tar.gz": testutils.TarGz(t, marker),
|
||||
"bundle.tgz": testutils.TarGz(t, marker),
|
||||
"bundle.tar": testutils.Tar(t, marker),
|
||||
"bundle.zip": testutils.Zip(t, marker),
|
||||
"notes.txt": []byte("not an archive at all"),
|
||||
} {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), body, 0o600))
|
||||
}
|
||||
|
||||
var seen []string
|
||||
fileCounts := map[string]int{}
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, &fileCounts))
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.ElementsMatch(t, []string{"/bundle.tar.gz", "/bundle.tgz", "/bundle.tar", "/bundle.zip"}, seen,
|
||||
"every tar-family archive must be reached, and the plain text file must not be")
|
||||
for _, archivePath := range seen {
|
||||
assert.Equal(t, 1, fileCounts[archivePath], "%s must have been extracted", archivePath)
|
||||
}
|
||||
|
||||
t.Run("the MIME types the tar family is sniffed as are in the set discovery asks for", func(t *testing.T) {
|
||||
for _, mt := range []string{"application/x-tar", "application/gzip", "application/x-gtar", "application/zip"} {
|
||||
assert.True(t, mimetype.ArchiveMIMETypeSet.Has(mt), "%s must be a candidate archive type", mt)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type recordingPublisher struct {
|
||||
events []partybus.Event
|
||||
}
|
||||
|
||||
func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) {
|
||||
// a consumer keying rows by ID (as the syft CLI does) replaces a republished row and never sees it
|
||||
// complete, so each cataloger keeps to the one row it already owns
|
||||
inner := testutils.Zip(t, map[string][]byte{"inner/file.txt": []byte("hello")})
|
||||
scanDir := t.TempDir()
|
||||
for _, name := range []string{"one.zip", "two.zip"} {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), inner, 0o600))
|
||||
}
|
||||
|
||||
publisher := &recordingPublisher{}
|
||||
bus.Set(publisher)
|
||||
t.Cleanup(func() { bus.Set(nil) })
|
||||
|
||||
// the archive row's stage as each archive's sub-pipeline starts: the previous archive is done by then,
|
||||
// so the row must read the walk's own count rather than anything a nested cataloger left behind
|
||||
var rows []*monitor.TaskProgress
|
||||
var stageAtEntry []string
|
||||
noisy := NewTask("noisy", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
stageAtEntry = append(stageAtEntry, publisher.events[0].Value.(progress.StagedProgressable).Stage())
|
||||
p := bus.StartCatalogerTask(ctx, monitor.GenericTask{ID: "noisy"}, 3, "")
|
||||
rows = append(rows, p)
|
||||
p.AtomicStage.Set("working on something")
|
||||
p.Add(3)
|
||||
p.SetCompleted()
|
||||
return nil
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), noisy)
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
var started []string
|
||||
for _, e := range publisher.events {
|
||||
if e.Type != event.CatalogerTaskStarted {
|
||||
continue
|
||||
}
|
||||
info, ok := e.Source.(monitor.GenericTask)
|
||||
require.True(t, ok, "expected a generic task as the event source")
|
||||
started = append(started, info.ID)
|
||||
}
|
||||
|
||||
assert.Equal(t, []string{ArchiveCatalogerTaskName, "noisy"}, started,
|
||||
"each cataloger publishes one row, however many archives the walk enters")
|
||||
|
||||
require.Len(t, rows, 2, "the sub-pipeline ran once per archive")
|
||||
assert.Same(t, rows[0], rows[1], "both runs must report into the one row that cataloger owns")
|
||||
assert.Equal(t, int64(6), rows[0].Current(), "what every run found is counted onto that row")
|
||||
|
||||
require.Len(t, publisher.events, 2)
|
||||
prog, ok := publisher.events[0].Value.(progress.StagedProgressable)
|
||||
require.True(t, ok)
|
||||
assert.ErrorIs(t, prog.Error(), progress.ErrCompleted, "the row must be completed, or a waiting consumer never finishes")
|
||||
assert.Equal(t, int64(2), prog.Current(), "the archive row counts every archive the walk entered")
|
||||
assert.Equal(t, "2 archives", prog.Stage(), "and says so once the walk is done")
|
||||
|
||||
// one.zip sorts before two.zip, and the resolver walks the scan directory in lexical order
|
||||
assert.Equal(t, []string{"", "1 archives (/one.zip)"}, stageAtEntry,
|
||||
"the row must take its stage back from the sub-pipeline once an archive is done")
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) {
|
||||
// self time excludes nested archives, or the outer archive would be named every scan
|
||||
innerJar := testutils.Zip(t, map[string][]byte{"README.txt": []byte("inner")})
|
||||
outerZip := testutils.Zip(t, map[string][]byte{"lib/inner.jar": innerJar})
|
||||
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
const innerWork = 40 * time.Millisecond
|
||||
slow := NewTask("slow-inside-the-inner-archive", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
if trav := archive.TraversalFromContext(ctx); trav != nil && strings.HasSuffix(archive.VirtualPath(trav.Location), "inner.jar") {
|
||||
time.Sleep(innerWork)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
|
||||
c := newTestCataloger(3, limiter, slow)
|
||||
require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.Equal(t, "/outer.zip!/lib/inner.jar", c.slowestPath, "the slowest archive is named by its full chain")
|
||||
assert.GreaterOrEqual(t, c.slowest, innerWork)
|
||||
assert.Equal(t, int64(2), c.progress.Current())
|
||||
|
||||
// the peaks outlive the walk that set them
|
||||
peakMemory, _ := limiter.Peak()
|
||||
assert.Positive(t, peakMemory, "entries held in memory must show on the peak")
|
||||
memory, disk := limiter.InUse()
|
||||
assert.Zero(t, memory)
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_slowestIsUnsetWhenNoArchiveIsCataloged(t *testing.T) {
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "notes.txt"), []byte("not an archive"), 0o600))
|
||||
|
||||
noop := NewTask("noop", func(context.Context, file.Resolver, sbomsync.Builder) error { return nil })
|
||||
c := newTestCataloger(2, archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), noop)
|
||||
require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.Empty(t, c.slowestPath)
|
||||
assert.Zero(t, c.progress.Current())
|
||||
c.logStats() // must not panic with nothing to report
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError(t *testing.T) {
|
||||
// the name says archive, the content does not: it is never opened
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "notes.zip"),
|
||||
[]byte("this is prose, not an archive, whatever the extension claims\n"), 0o600))
|
||||
|
||||
var ran int
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), countingTask(&ran))
|
||||
|
||||
err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM()))
|
||||
|
||||
assert.NoError(t, err, "a file that is not an archive is not a failure to report")
|
||||
assert.Zero(t, ran, "and nothing is cataloged inside it")
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_catalogsAZipBehindALauncherScript(t *testing.T) {
|
||||
// the Spring Boot executable jar shape: content sniffing types it as a shell script, so it reaches
|
||||
// the walk only by name and qualifies only by its end-of-central-directory record
|
||||
inner := testutils.Zip(t, map[string][]byte{"nested/marker.txt": []byte("found me")})
|
||||
prefixed := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"),
|
||||
testutils.Zip(t, map[string][]byte{"BOOT-INF/lib/inner.jar": inner})...)
|
||||
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "app.jar"), prefixed, 0o600))
|
||||
|
||||
var seen []string
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3), capturingTask(t, &seen, nil))
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.Equal(t, []string{"/app.jar", "/app.jar:BOOT-INF/lib/inner.jar"}, seen,
|
||||
"the stub must not hide the archive, and the walk must carry on into what it holds")
|
||||
}
|
||||
|
||||
func (d dirTestResolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
|
||||
return filesByMIMEType(d.dir, d.fsid, types...)
|
||||
}
|
||||
|
||||
func filesByMIMEType(dir, fsid string, types ...string) ([]file.Location, error) {
|
||||
wanted := strset.New(types...)
|
||||
var locs []file.Location
|
||||
err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil || entry.IsDir() {
|
||||
return err
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mimeType := stereoscopeFile.MIMEType(f)
|
||||
if closeErr := f.Close(); closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if !wanted.Has(mimeType) {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(dir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
coords := file.Coordinates{RealPath: "/" + filepath.ToSlash(rel), FileSystemID: fsid}
|
||||
locs = append(locs, file.NewLocationFromCoordinates(coords))
|
||||
return nil
|
||||
})
|
||||
return locs, err
|
||||
}
|
||||
|
||||
func (d dirTestResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) {
|
||||
return os.Open(filepath.Join(d.dir, filepath.FromSlash(loc.RealPath)))
|
||||
}
|
||||
|
||||
func (d dirTestResolver) FileMetadataByLocation(loc file.Location) (file.Metadata, error) {
|
||||
info, err := os.Stat(filepath.Join(d.dir, filepath.FromSlash(loc.RealPath)))
|
||||
return file.Metadata{FileInfo: info}, err
|
||||
}
|
||||
|
||||
// matchesAnyGlob matches the resolver-relative path with a leading slash, the form a real resolver
|
||||
// globs over. No patterns matches everything.
|
||||
func matchesAnyGlob(root, path string, patterns []string) bool {
|
||||
if len(patterns) == 0 {
|
||||
return true
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
rel = "/" + filepath.ToSlash(rel)
|
||||
for _, pattern := range patterns {
|
||||
if ok, err := doublestar.Match(pattern, rel); err == nil && ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (r twoLayerResolver) FilesByGlob(_ ...string) ([]file.Location, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (d dirTestResolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
|
||||
var locs []file.Location
|
||||
err := filepath.WalkDir(d.dir, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil || entry.IsDir() {
|
||||
return err
|
||||
}
|
||||
if !matchesAnyGlob(d.dir, path, patterns) {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(d.dir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
coords := file.Coordinates{RealPath: "/" + filepath.ToSlash(rel), FileSystemID: d.fsid}
|
||||
locs = append(locs, file.NewLocationFromCoordinates(coords))
|
||||
return nil
|
||||
})
|
||||
return locs, err
|
||||
}
|
||||
|
||||
func (r twoLayerResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) {
|
||||
var locs []file.Location
|
||||
for _, fsid := range []string{"layer-one", "layer-two"} {
|
||||
locs = append(locs, file.NewLocationFromCoordinates(file.NewCoordinates(r.path, fsid)))
|
||||
}
|
||||
return locs, nil
|
||||
}
|
||||
|
||||
func (r twoLayerResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) {
|
||||
body, ok := r.contentBy[loc.FileSystemID]
|
||||
if !ok {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(body)), nil
|
||||
}
|
||||
|
||||
// FileMetadataByLocation reports no metadata, so the archive falls back to the minimum budget.
|
||||
func (r twoLayerResolver) FileMetadataByLocation(file.Location) (file.Metadata, error) {
|
||||
return file.Metadata{}, os.ErrNotExist
|
||||
}
|
||||
|
||||
// incompressible returns n bytes deflate cannot shrink, so an archive built from them has a
|
||||
// predictable size to set a limit against.
|
||||
func incompressible(n int) []byte {
|
||||
b := make([]byte, n)
|
||||
x := uint32(12345)
|
||||
for i := range b {
|
||||
x = x*1664525 + 1013904223
|
||||
b[i] = byte(x >> 24)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (p *recordingPublisher) Publish(e partybus.Event) {
|
||||
p.events = append(p.events, e)
|
||||
}
|
||||
@@ -47,7 +47,7 @@ func newFileDigestCatalogerTask(selection file.Selection, hashers []crypto.Hash,
|
||||
result, err := filedigest.NewCataloger(hashers).Catalog(ctx, resolver, coordinates...)
|
||||
|
||||
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
|
||||
sbom.Artifacts.FileDigests = result
|
||||
mergeInto(&sbom.Artifacts.FileDigests, result)
|
||||
})
|
||||
|
||||
return err
|
||||
@@ -78,7 +78,7 @@ func newFileMetadataCatalogerTask(selection file.Selection, tags ...string) Task
|
||||
result, err := filemetadata.NewCataloger().Catalog(ctx, resolver, coordinates...)
|
||||
|
||||
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
|
||||
sbom.Artifacts.FileMetadata = result
|
||||
mergeInto(&sbom.Artifacts.FileMetadata, result)
|
||||
})
|
||||
|
||||
return err
|
||||
@@ -104,7 +104,7 @@ func newFileContentCatalogerTask(cfg filecontent.Config, tags ...string) Task {
|
||||
result, err := filecontent.NewCataloger(cfg).Catalog(ctx, resolver)
|
||||
|
||||
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
|
||||
sbom.Artifacts.FileContents = result
|
||||
mergeInto(&sbom.Artifacts.FileContents, result)
|
||||
})
|
||||
|
||||
return err
|
||||
@@ -130,7 +130,7 @@ func newExecutableCatalogerTask(selection file.Selection, cfg executable.Config,
|
||||
result, err := executable.NewCataloger(cfg).CatalogCtx(ctx, resolver)
|
||||
|
||||
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
|
||||
sbom.Artifacts.Executables = result
|
||||
mergeInto(&sbom.Artifacts.Executables, result)
|
||||
})
|
||||
|
||||
return err
|
||||
|
||||
@@ -40,19 +40,13 @@ func Test_coordinatesForSelection(t *testing.T) {
|
||||
Relationships: []artifact.Relationship{
|
||||
{
|
||||
From: pkg.Package{},
|
||||
To: file.Coordinates{
|
||||
RealPath: "path",
|
||||
FileSystemID: "fs",
|
||||
},
|
||||
To: file.NewCoordinates("path", "fs"),
|
||||
Type: artifact.ContainsRelationship,
|
||||
},
|
||||
},
|
||||
},
|
||||
files: []file.Coordinates{
|
||||
{
|
||||
RealPath: "path",
|
||||
FileSystemID: "fs",
|
||||
},
|
||||
file.NewCoordinates("path", "fs"),
|
||||
},
|
||||
ok: true,
|
||||
},
|
||||
@@ -63,10 +57,7 @@ func Test_coordinatesForSelection(t *testing.T) {
|
||||
Relationships: []artifact.Relationship{
|
||||
{
|
||||
From: pkg.Package{},
|
||||
To: file.Coordinates{
|
||||
RealPath: "path",
|
||||
FileSystemID: "fs",
|
||||
},
|
||||
To: file.NewCoordinates("path", "fs"),
|
||||
// wrong type
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
@@ -82,10 +73,7 @@ func Test_coordinatesForSelection(t *testing.T) {
|
||||
Relationships: []artifact.Relationship{
|
||||
{
|
||||
From: file.Coordinates{}, // wrong type
|
||||
To: file.Coordinates{
|
||||
RealPath: "path",
|
||||
FileSystemID: "fs",
|
||||
},
|
||||
To: file.NewCoordinates("path", "fs"),
|
||||
Type: artifact.ContainsRelationship,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -3,9 +3,11 @@ package task
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/relationship"
|
||||
@@ -48,7 +50,7 @@ func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string
|
||||
HideOnSuccess: true,
|
||||
}
|
||||
|
||||
t := bus.StartCatalogerTask(info, -1, "")
|
||||
t := bus.StartCatalogerTask(ctx, info, -1, "")
|
||||
|
||||
pkgs, relationships, err := c.Catalog(ctx, resolver)
|
||||
|
||||
@@ -167,7 +169,7 @@ func applyComplianceRules(p *pkg.Package, cfg cataloging.ComplianceConfig) (bool
|
||||
loc := "unknown"
|
||||
locs := p.Locations.ToSlice()
|
||||
if len(locs) > 0 {
|
||||
loc = locs[0].Path()
|
||||
loc = locs[0].FullNestedPath()
|
||||
}
|
||||
switch action {
|
||||
case cataloging.ComplianceActionDrop:
|
||||
@@ -236,10 +238,10 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
|
||||
|
||||
locations := map[artifact.ID]file.Location{}
|
||||
|
||||
for _, path := range fileOwner.OwnedFiles() {
|
||||
pathRefs, err := resolver.FilesByPath(path)
|
||||
for _, ownedPath := range fileOwner.OwnedFiles() {
|
||||
pathRefs, err := resolver.FilesByPath(ownedPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to find path for path=%q: %w", path, err)
|
||||
return nil, fmt.Errorf("unable to find path for path=%q: %w", ownedPath, err)
|
||||
}
|
||||
|
||||
if len(pathRefs) == 0 {
|
||||
@@ -250,8 +252,13 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
|
||||
}
|
||||
|
||||
for _, ref := range pathRefs {
|
||||
// an archive looks a relative path up from its root, but an owned path may be relative to
|
||||
// anything (a python RECORD is relative to site-packages), so only an absolute one owns a file in one
|
||||
if archive.ContainerPath(ref.Coordinates) != "" && !path.IsAbs(ownedPath) {
|
||||
continue
|
||||
}
|
||||
if oldRef, ok := locations[ref.ID()]; ok {
|
||||
log.Debugf("found path duplicate of %s", oldRef.RealPath)
|
||||
log.Debugf("found path duplicate of %s", oldRef.FullNestedPath())
|
||||
}
|
||||
locations[ref.ID()] = ref
|
||||
}
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/testutils"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/cpe"
|
||||
@@ -64,8 +68,8 @@ func TestApplyCompliance(t *testing.T) {
|
||||
p2 := pkg.Package{Name: "", Version: "1.0"} // missing name
|
||||
p3 := pkg.Package{Name: "pkg-3", Version: ""} // missing version
|
||||
p4 := pkg.Package{Name: "pkg-4", Version: ""} // missing version
|
||||
c1 := file.Coordinates{RealPath: "/coords/1"}
|
||||
c2 := file.Coordinates{RealPath: "/coords/2"}
|
||||
c1 := file.NewCoordinates("/coords/1", "")
|
||||
c2 := file.NewCoordinates("/coords/2", "")
|
||||
|
||||
for _, p := range []*pkg.Package{&p1, &p2, &p3, &p4} {
|
||||
p.SetID()
|
||||
@@ -281,3 +285,28 @@ func TestApplyComplianceRules_DropAndStub(t *testing.T) {
|
||||
assert.False(t, isCompliant)
|
||||
assert.Equal(t, cataloging.UnknownStubValue, p.Version)
|
||||
}
|
||||
|
||||
func Test_packageFileOwnershipRelationships_exactPathsWithinAnArchive(t *testing.T) {
|
||||
// a wheel's RECORD is relative to site-packages: looked up from the wheel's root it finds the file, but
|
||||
// only an exact path owns one, as at the scan root
|
||||
data := testutils.Zip(t, map[string]string{"pip/__init__.py": "", "bin/tool": ""})
|
||||
whl, err := archive.Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("/pip.whl", ""),
|
||||
archive.NewLimiter(archive.Limits{MaxMemoryBytes: 1 << 20}), archive.NewExclusions(nil))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(whl.Cleanup)
|
||||
|
||||
found, err := whl.FilesByPath("pip/__init__.py")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, found, 1, "a relative path still finds the file")
|
||||
|
||||
p := pkg.Package{Name: "pip", Metadata: pkg.PythonPackage{Files: []pkg.PythonFileRecord{{Path: "pip/__init__.py"}, {Path: "/bin/tool"}}}}
|
||||
p.SetID()
|
||||
rels, err := packageFileOwnershipRelationships(p, whl)
|
||||
require.NoError(t, err)
|
||||
|
||||
var owned []string
|
||||
for _, r := range rels {
|
||||
owned = append(owned, r.To.(file.Coordinates).FullNestedPath())
|
||||
}
|
||||
assert.Equal(t, []string{"/pip.whl!/bin/tool"}, owned)
|
||||
}
|
||||
|
||||
@@ -40,10 +40,11 @@ func packagesToRemove(accessor sbomsync.Accessor) []artifact.ID {
|
||||
}
|
||||
case "":
|
||||
if isPrimaryEvidence {
|
||||
if exists := filterDuplicates[getKey(p, l)]; exists {
|
||||
k := getKey(p, l)
|
||||
if exists := filterDuplicates[k]; exists {
|
||||
break
|
||||
}
|
||||
filterDuplicates[getKey(p, l)] = true
|
||||
filterDuplicates[k] = true
|
||||
noPrimary = false
|
||||
break
|
||||
}
|
||||
@@ -59,5 +60,5 @@ func packagesToRemove(accessor sbomsync.Accessor) []artifact.ID {
|
||||
}
|
||||
|
||||
func getKey(pkg pkg.Package, loc file.Location) string {
|
||||
return fmt.Sprintf("%s-%s-%s-%s", pkg.Name, pkg.Version, loc.RealPath, loc.AccessPath)
|
||||
return fmt.Sprintf("%s-%s-%s-%s", pkg.Name, pkg.Version, loc.FullNestedPath(), loc.AccessPath)
|
||||
}
|
||||
|
||||
@@ -2,8 +2,11 @@ package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
@@ -36,11 +39,20 @@ func (c unknownsLabelerTask) finalize(resolver file.Resolver, s *sbom.SBOM) {
|
||||
hasPackageReference := coordinateReferenceLookup(resolver, s)
|
||||
|
||||
if c.RemoveWhenPackagesDefined {
|
||||
for coords := range s.Artifacts.Unknowns {
|
||||
for coords, reasons := range s.Artifacts.Unknowns {
|
||||
if !hasPackageReference(coords) {
|
||||
continue
|
||||
}
|
||||
delete(s.Artifacts.Unknowns, coords)
|
||||
// what the archive cataloger reports is about the archive's contents (left out, cut short),
|
||||
// which a package describing the archive does not account for
|
||||
kept := slices.DeleteFunc(reasons, func(r string) bool {
|
||||
return !strings.HasPrefix(r, formatUnknown("", ArchiveCatalogerTaskName))
|
||||
})
|
||||
if len(kept) == 0 {
|
||||
delete(s.Artifacts.Unknowns, coords)
|
||||
continue
|
||||
}
|
||||
s.Artifacts.Unknowns[coords] = kept
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,7 +71,7 @@ func (c unknownsLabelerTask) finalize(resolver file.Resolver, s *sbom.SBOM) {
|
||||
if c.IncludeUnexpandedArchives {
|
||||
ctx := context.Background()
|
||||
for coords := range s.Artifacts.FileMetadata {
|
||||
format, _, notArchiveErr := intFile.IdentifyArchive(ctx, coords.RealPath, nil)
|
||||
format, _, notArchiveErr := intFile.IdentifyArchive(ctx, coords.FullNestedPath(), nil)
|
||||
if format != nil && notArchiveErr == nil && !hasPackageReference(coords) {
|
||||
s.Artifacts.Unknowns[coords] = append(s.Artifacts.Unknowns[coords], "archive not cataloged")
|
||||
}
|
||||
@@ -71,6 +83,7 @@ func formatUnknown(err string, task ...string) string {
|
||||
return strings.Join(task, "/") + ": " + err
|
||||
}
|
||||
|
||||
//nolint:gocognit
|
||||
func coordinateReferenceLookup(resolver file.Resolver, s *sbom.SBOM) func(coords file.Coordinates) bool {
|
||||
allPackageCoords := file.NewCoordinateSet()
|
||||
|
||||
@@ -83,7 +96,28 @@ func coordinateReferenceLookup(resolver file.Resolver, s *sbom.SBOM) func(coords
|
||||
// relationships for these owned files may be disabled, but we always want to include them
|
||||
for p := range s.Artifacts.Packages.Enumerate() {
|
||||
if f, ok := p.Metadata.(pkg.FileOwner); ok {
|
||||
// owned paths are within the containers the package was found in
|
||||
var archives []file.Coordinates
|
||||
atRoot := false
|
||||
for _, container := range archive.OwnerContainers(p) {
|
||||
if container.FullPath == "" {
|
||||
atRoot = true
|
||||
} else {
|
||||
archives = append(archives, container)
|
||||
}
|
||||
}
|
||||
for _, ownedFilePath := range f.OwnedFiles() {
|
||||
// an archive's contents are not in this resolver to follow links in, and only an absolute owned
|
||||
// path names a file in one (see packageFileOwnershipRelationships)
|
||||
if path.IsAbs(ownedFilePath) {
|
||||
for _, container := range archives {
|
||||
allPackageCoords.Add(archive.OwnedCoordinates(container, ownedFilePath))
|
||||
}
|
||||
}
|
||||
if !atRoot {
|
||||
continue
|
||||
}
|
||||
|
||||
// resolve these owned files, as they may have symlinks
|
||||
// but coordinates we will test against are always absolute paths
|
||||
locations, err := resolver.FilesByPath(ownedFilePath)
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
func Test_unknownsLabeler_archiveReasonsSurvivePackages(t *testing.T) {
|
||||
jar := file.NewCoordinates("/app.jar", "")
|
||||
p := pkg.Package{Name: "app", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(jar))}
|
||||
p.SetID()
|
||||
s := &sbom.SBOM{Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(p),
|
||||
Unknowns: map[file.Coordinates][]string{jar: {
|
||||
formatUnknown("archive cataloged from part of its contents: too big", ArchiveCatalogerTaskName),
|
||||
formatUnknown("unable to read manifest", "java-archive-cataloger"),
|
||||
}},
|
||||
}}
|
||||
|
||||
unknownsLabelerTask{cataloging.UnknownsConfig{RemoveWhenPackagesDefined: true}}.finalize(file.NewMockResolverForPaths(), s)
|
||||
|
||||
assert.Equal(t, []string{formatUnknown("archive cataloged from part of its contents: too big", ArchiveCatalogerTaskName)}, s.Artifacts.Unknowns[jar],
|
||||
"a package describing the archive explains a parse failure, not contents left out")
|
||||
}
|
||||
|
||||
func Test_unknownsLabeler_archiveContentsAreReportedByFullPath(t *testing.T) {
|
||||
// a package describing an archive does not account for the files within it
|
||||
war := file.NewCoordinates("/app.war", "layer")
|
||||
in := func(fullPath string) file.Coordinates {
|
||||
return file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: fullPath}
|
||||
}
|
||||
nativeLib := in("/app.war!/WEB-INF/lib/jna.jar!/linux-x86-64/libjnidispatch.so")
|
||||
unidentifiedJar := in("/app.war!/WEB-INF/lib/javax.inject-1.jar")
|
||||
p := pkg.Package{Name: "app", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(war))}
|
||||
p.SetID()
|
||||
noPackage := formatUnknown("no package identified in archive", "java-archive-cataloger")
|
||||
|
||||
s := &sbom.SBOM{
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(p),
|
||||
Executables: map[file.Coordinates]file.Executable{nativeLib: {}},
|
||||
Unknowns: map[file.Coordinates][]string{unidentifiedJar: {noPackage}},
|
||||
},
|
||||
Relationships: []artifact.Relationship{{From: war, To: unidentifiedJar, Type: artifact.ContainsRelationship}},
|
||||
}
|
||||
|
||||
unknownsLabelerTask{cataloging.UnknownsConfig{RemoveWhenPackagesDefined: true, IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths(), s)
|
||||
|
||||
assert.Equal(t, map[file.Coordinates][]string{
|
||||
nativeLib: {formatUnknown("no package identified in executable file", unknownsLabelerTaskName)},
|
||||
unidentifiedJar: {noPackage},
|
||||
}, s.Artifacts.Unknowns)
|
||||
}
|
||||
|
||||
func Test_unknownsLabeler_relativeOwnedPathsNameNothingInAnArchive(t *testing.T) {
|
||||
// a python RECORD lists paths relative to site-packages, which resolve nowhere at the scan root either
|
||||
whl := func(fullPath string) file.Coordinates {
|
||||
return file.Coordinates{RealPath: "/pip.whl", FullPath: fullPath}
|
||||
}
|
||||
tool := whl("/pip.whl!/pip/tool")
|
||||
p := pkg.Package{
|
||||
Name: "pip",
|
||||
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(whl("/pip.whl!/pip.dist-info/METADATA"))),
|
||||
Metadata: pkg.PythonPackage{Files: []pkg.PythonFileRecord{{Path: "pip/tool"}}},
|
||||
}
|
||||
p.SetID()
|
||||
s := &sbom.SBOM{Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(p),
|
||||
Executables: map[file.Coordinates]file.Executable{tool: {}},
|
||||
}}
|
||||
|
||||
unknownsLabelerTask{cataloging.UnknownsConfig{IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths(), s)
|
||||
|
||||
assert.NotEmpty(t, s.Artifacts.Unknowns[tool])
|
||||
}
|
||||
|
||||
func Test_unknownsLabeler_ownedFilesAreWithinTheOwnersContainer(t *testing.T) {
|
||||
atRoot := file.NewCoordinates("/usr/bin/foo", "")
|
||||
inArchive := file.Coordinates{RealPath: "/rootfs.tar", FullPath: "/rootfs.tar!/usr/bin/foo"}
|
||||
owner := func(at file.Coordinates) pkg.Package {
|
||||
p := pkg.Package{
|
||||
Name: "owner",
|
||||
Type: pkg.RpmPkg,
|
||||
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(at)),
|
||||
Metadata: pkg.RpmDBEntry{Files: []pkg.RpmFileRecord{{Path: "/usr/bin/foo"}}},
|
||||
}
|
||||
p.SetID()
|
||||
return p
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
owner file.Coordinates
|
||||
referenced file.Coordinates
|
||||
}{
|
||||
{name: "an owner at the scan root", owner: file.NewCoordinates("/var/lib/rpm/Packages", ""), referenced: atRoot},
|
||||
{name: "an owner inside an archive", owner: file.Coordinates{RealPath: "/rootfs.tar", FullPath: "/rootfs.tar!/var/lib/rpm/Packages"}, referenced: inArchive},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := &sbom.SBOM{Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(owner(tt.owner)),
|
||||
Executables: map[file.Coordinates]file.Executable{atRoot: {}, inArchive: {}},
|
||||
}}
|
||||
|
||||
unknownsLabelerTask{cataloging.UnknownsConfig{IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths("/usr/bin/foo"), s)
|
||||
|
||||
for _, exe := range []file.Coordinates{atRoot, inArchive} {
|
||||
assert.Equal(t, exe != tt.referenced, len(s.Artifacts.Unknowns[exe]) > 0, exe.FullNestedPath())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user