mirror of
https://github.com/anchore/syft.git
synced 2026-11-06 17:34:21 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
173055c39e | ||
|
|
f974ef598c | ||
|
|
e6813ba7b6 | ||
|
|
1238ace393 | ||
|
|
80ecdfe63f | ||
|
|
1fa6bf6848 | ||
|
|
b0bc371473 | ||
|
|
d91462b824 | ||
|
|
181b9198c0 | ||
|
|
4614cf1d85 | ||
|
|
f46acb6e4c | ||
|
|
168722d880 | ||
|
|
4b8a3db5f3 | ||
|
|
14a0beb93c | ||
|
|
1998095808 | ||
|
|
12f0e2d837 | ||
|
|
1636e1119a | ||
|
|
fb56f1eef1 | ||
|
|
842d2b3deb | ||
|
|
1401107138 | ||
|
|
722df28ed1 | ||
|
|
4c500520d0 | ||
|
|
9f9751aade | ||
|
|
447ee20bdc | ||
|
|
eec916df6c | ||
|
|
1c4d672b28 | ||
|
|
44c215450b | ||
|
|
e1dcbd89df | ||
|
|
e61fb72e46 | ||
|
|
71fe033ab5 | ||
|
|
a5426c6642 | ||
|
|
816106f0b2 | ||
|
|
b55707f60f |
+1
-1
@@ -16,7 +16,7 @@ tools:
|
||||
# used to upload test fixture cache
|
||||
- name: oras
|
||||
version:
|
||||
want: v1.3.4
|
||||
want: v1.3.6
|
||||
method: github-release
|
||||
with:
|
||||
repo: oras-project/oras
|
||||
|
||||
@@ -6,10 +6,6 @@ description: "Bootstrap all syft tools and dependencies on top of go-make's setu
|
||||
# We add the syft-specific extras here: apt packages and the test fixture cache.
|
||||
|
||||
inputs:
|
||||
go-version:
|
||||
description: "Go version to install (passed to go-make/setup)"
|
||||
required: true
|
||||
default: "1.26.2"
|
||||
cache-key-prefix:
|
||||
description: "Prefix all cache keys with this value (passed to go-make/setup)"
|
||||
required: true
|
||||
@@ -30,9 +26,8 @@ runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Setup go + go-make tooling
|
||||
uses: anchore/go-make/.github/actions/setup@abf41688f36f2cbec13105747abbe8e60f5c56ca # v0.9.3
|
||||
uses: anchore/go-make/.github/actions/setup@cf465259f4c9f0b0b0bcb35164ede11f7dc6d9f6 # v0.9.6
|
||||
with:
|
||||
go-version: ${{ inputs.go-version }}
|
||||
cache-key-prefix: ${{ inputs.cache-key-prefix }}
|
||||
cache-enabled: ${{ inputs.cache-enabled }}
|
||||
|
||||
|
||||
@@ -89,13 +89,13 @@ jobs:
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f #v4.6.0
|
||||
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 #v4.7.0
|
||||
with:
|
||||
username: ${{ secrets.ANCHOREOSSWRITE_DH_USERNAME }}
|
||||
password: ${{ secrets.ANCHOREOSSWRITE_DH_PAT }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f #v4.6.0
|
||||
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 #v4.7.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: "Run zizmor"
|
||||
uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4
|
||||
uses: zizmorcore/zizmor-action@53290badfa369cb0bf7ad16342cb62c2fa47ac7b # v0.8.0
|
||||
with:
|
||||
# there is a pass/fail gate as a repo ruleset (if there is no ruleset configured then the action will pass by default)
|
||||
advanced-security: true
|
||||
|
||||
@@ -99,10 +99,7 @@ jobs:
|
||||
# go-make's setup only (not ./.github/actions/bootstrap): binny ships no windows release, so binny tools
|
||||
# are unavailable here (`make unit` skips the fixture refresh on windows)
|
||||
- name: Setup go + go-make tooling
|
||||
uses: anchore/go-make/.github/actions/setup@abf41688f36f2cbec13105747abbe8e60f5c56ca # v0.9.3
|
||||
with:
|
||||
# keep in sync with the go-version default in ./.github/actions/bootstrap
|
||||
go-version: "1.26.2"
|
||||
uses: anchore/go-make/.github/actions/setup@cf465259f4c9f0b0b0bcb35164ede11f7dc6d9f6 # v0.9.6
|
||||
|
||||
# the binny-free version of `make download-test-fixture-cache` (keep the oras version in sync with .binny.yaml)
|
||||
- name: Download test fixture cache
|
||||
@@ -195,10 +192,7 @@ jobs:
|
||||
|
||||
# go-make's setup only (not ./.github/actions/bootstrap): the test only needs make + the snapshot binary
|
||||
- name: Setup go + go-make tooling
|
||||
uses: anchore/go-make/.github/actions/setup@abf41688f36f2cbec13105747abbe8e60f5c56ca # v0.9.3
|
||||
with:
|
||||
# keep in sync with the go-version default in ./.github/actions/bootstrap
|
||||
go-version: "1.26.2"
|
||||
uses: anchore/go-make/.github/actions/setup@cf465259f4c9f0b0b0bcb35164ede11f7dc6d9f6 # v0.9.6
|
||||
|
||||
- name: Install Podman
|
||||
run: |
|
||||
@@ -228,10 +222,7 @@ jobs:
|
||||
|
||||
# go-make's setup only (not ./.github/actions/bootstrap): the test only needs make + the snapshot binary
|
||||
- name: Setup go + go-make tooling
|
||||
uses: anchore/go-make/.github/actions/setup@abf41688f36f2cbec13105747abbe8e60f5c56ca # v0.9.3
|
||||
with:
|
||||
# keep in sync with the go-version default in ./.github/actions/bootstrap
|
||||
go-version: "1.26.2"
|
||||
uses: anchore/go-make/.github/actions/setup@cf465259f4c9f0b0b0bcb35164ede11f7dc6d9f6 # v0.9.6
|
||||
|
||||
- name: Install Buildah
|
||||
run: |
|
||||
|
||||
+4
-4
@@ -2,11 +2,11 @@ module github.com/anchore/syft/.make
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require github.com/anchore/go-make v0.9.3
|
||||
require github.com/anchore/go-make v0.9.6
|
||||
|
||||
require (
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.2 // indirect
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.3 // indirect
|
||||
github.com/goccy/go-yaml v1.19.2 // indirect
|
||||
golang.org/x/mod v0.41.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/mod v0.42.0 // indirect
|
||||
golang.org/x/sys v0.49.0 // indirect
|
||||
)
|
||||
|
||||
+8
-8
@@ -1,10 +1,10 @@
|
||||
github.com/anchore/go-make v0.9.3 h1:D+MkpvEJsOGwkOqKN0rd2zYi9jEzwKtZreNysH0DenI=
|
||||
github.com/anchore/go-make v0.9.3/go.mod h1:4HuELygGhxex2SaCEgfCbZ0geJ3xFkqtWpmt1NrYwPg=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.2 h1:eF7W7HWKg3z9NrWV9pTLnNeoXaqq3Tq9DNKXVMfoCnw=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
|
||||
github.com/anchore/go-make v0.9.6 h1:HjO7H/M/WZ4v3B73/EuShuQHHblQJGwHDrl/MqCavWs=
|
||||
github.com/anchore/go-make v0.9.6/go.mod h1:zE6KeB+nY+sMSWHYx2uaiOznLAq3CD9JvFlzWDDpKw4=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.3 h1:1UIv1+fqH3++ViVkpnaPTB+Pi1UHycnPFAq14zgK+gI=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.3/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
|
||||
github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
|
||||
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
|
||||
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
|
||||
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
|
||||
@@ -102,6 +102,7 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
|
||||
WithComplianceConfig(cfg.ToComplianceConfig()).
|
||||
WithUnknownsConfig(cfg.ToUnknownsConfig()).
|
||||
WithSearchConfig(cfg.ToSearchConfig()).
|
||||
WithArchiveConfig(cfg.ToArchiveConfig()).
|
||||
WithPackagesConfig(cfg.ToPackagesConfig()).
|
||||
WithLicenseConfig(cfg.ToLicenseConfig()).
|
||||
WithFilesConfig(cfg.ToFilesConfig()).
|
||||
@@ -118,6 +119,15 @@ func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig {
|
||||
}
|
||||
}
|
||||
|
||||
func (cfg Catalog) ToArchiveConfig() cataloging.ArchiveSearchConfig {
|
||||
return cataloging.DefaultArchiveSearchConfig().
|
||||
WithIncludeIndexedArchives(cfg.Package.SearchIndexedArchives).
|
||||
WithIncludeUnindexedArchives(cfg.Package.SearchUnindexedArchives).
|
||||
WithMaxDepth(cfg.Package.NestedArchiveMaxDepth).
|
||||
WithMaxMemoryBytes(cfg.Package.NestedArchiveMaxMemoryBytes).
|
||||
WithMaxDiskBytes(cfg.Package.NestedArchiveMaxDiskBytes)
|
||||
}
|
||||
|
||||
func (cfg Catalog) ToRelationshipsConfig() cataloging.RelationshipsConfig {
|
||||
return cataloging.RelationshipsConfig{
|
||||
PackageFileOwnership: cfg.Relationships.PackageFileOwnership,
|
||||
@@ -136,6 +146,7 @@ func (cfg Catalog) ToComplianceConfig() cataloging.ComplianceConfig {
|
||||
|
||||
func (cfg Catalog) ToUnknownsConfig() cataloging.UnknownsConfig {
|
||||
return cataloging.UnknownsConfig{
|
||||
RemoveWhenPackagesDefined: cfg.Unknowns.RemoveWhenPackagesDefined,
|
||||
IncludeExecutablesWithoutPackages: cfg.Unknowns.ExecutablesWithoutPackages,
|
||||
IncludeUnexpandedArchives: cfg.Unknowns.UnexpandedArchives,
|
||||
}
|
||||
@@ -169,10 +180,6 @@ func (cfg Catalog) ToLicenseConfig() cataloging.LicenseConfig {
|
||||
}
|
||||
|
||||
func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
|
||||
archiveSearch := cataloging.ArchiveSearchConfig{
|
||||
IncludeIndexedArchives: cfg.Package.SearchIndexedArchives,
|
||||
IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
|
||||
}
|
||||
return pkgcataloging.Config{
|
||||
Binary: binary.DefaultClassifierCatalogerConfig(),
|
||||
Cpp: cpp.DefaultCatalogerConfig().
|
||||
@@ -219,7 +226,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
|
||||
WithMavenLocalRepositoryDir(cfg.Java.MavenLocalRepositoryDir).
|
||||
WithUseNetwork(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Java, task.Maven), cfg.Java.UseNetwork)).
|
||||
WithMavenBaseURL(cfg.Java.MavenURL).
|
||||
WithArchiveTraversal(archiveSearch, cfg.Java.MaxParentRecursiveDepth).
|
||||
WithMaxParentRecursiveDepth(cfg.Java.MaxParentRecursiveDepth).
|
||||
WithResolveTransitiveDependencies(cfg.Java.ResolveTransitiveDependencies),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,9 +6,12 @@ import (
|
||||
)
|
||||
|
||||
type packageConfig struct {
|
||||
SearchUnindexedArchives bool `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
|
||||
SearchIndexedArchives bool `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
|
||||
ExcludeBinaryOverlapByOwnership bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files
|
||||
SearchUnindexedArchives bool `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
|
||||
SearchIndexedArchives bool `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
|
||||
ExcludeBinaryOverlapByOwnership bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files
|
||||
NestedArchiveMaxDepth int `yaml:"nested-archive-max-depth" json:"nested-archive-max-depth" mapstructure:"nested-archive-max-depth"`
|
||||
NestedArchiveMaxMemoryBytes int64 `yaml:"nested-archive-max-memory-bytes" json:"nested-archive-max-memory-bytes" mapstructure:"nested-archive-max-memory-bytes"`
|
||||
NestedArchiveMaxDiskBytes int64 `yaml:"nested-archive-max-disk-bytes" json:"nested-archive-max-disk-bytes" mapstructure:"nested-archive-max-disk-bytes"`
|
||||
}
|
||||
|
||||
var _ interface {
|
||||
@@ -17,12 +20,17 @@ var _ interface {
|
||||
|
||||
func (o *packageConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
|
||||
descriptions.Add(&o.SearchIndexedArchives, `search within archives that do contain a file index to search against (zip)
|
||||
note: for now this only applies to the java package cataloger`)
|
||||
note: this only applies to the java package cataloger, and is ignored when nested-archive-max-depth is not 0 (every archive is searched)`)
|
||||
descriptions.Add(&o.SearchUnindexedArchives, `search within archives that do not contain a file index to search against (tar, tar.gz, tar.bz2, etc)
|
||||
note: enabling this may result in a performance impact since all discovered compressed tars will be decompressed
|
||||
note: for now this only applies to the java package cataloger`)
|
||||
note: this only applies to the java package cataloger, and is ignored when nested-archive-max-depth is not 0 (every archive is searched)`)
|
||||
descriptions.Add(&o.ExcludeBinaryOverlapByOwnership, `allows users to exclude synthetic binary packages from the sbom
|
||||
these packages are removed if an overlap with a non-synthetic package is found`)
|
||||
descriptions.Add(&o.NestedArchiveMaxDepth, `maximum depth to recursively catalog nested archives (0 = disabled, -1 = as deep as 16 levels, the most allowed)
|
||||
note: when enabled this replaces the java cataloger's own unbounded jar-in-jar recursion, so archives deeper than this are reported as unknowns rather than cataloged`)
|
||||
descriptions.Add(&o.NestedArchiveMaxMemoryBytes, `maximum bytes of nested archive content held in memory at once (0 = none, -1 = unbounded)`)
|
||||
descriptions.Add(&o.NestedArchiveMaxDiskBytes, `maximum bytes of nested archive content on disk at once (0 = none, -1 = unbounded)
|
||||
note: this and nested-archive-max-memory-bytes cannot both be 0 when nested-archive-max-depth is not 0`)
|
||||
}
|
||||
|
||||
func defaultPackageConfig() packageConfig {
|
||||
@@ -31,5 +39,8 @@ func defaultPackageConfig() packageConfig {
|
||||
SearchIndexedArchives: c.IncludeIndexedArchives,
|
||||
SearchUnindexedArchives: c.IncludeUnindexedArchives,
|
||||
ExcludeBinaryOverlapByOwnership: true,
|
||||
NestedArchiveMaxDepth: c.MaxDepth,
|
||||
NestedArchiveMaxMemoryBytes: c.MaxMemoryBytes,
|
||||
NestedArchiveMaxDiskBytes: c.MaxDiskBytes,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package options
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
)
|
||||
|
||||
func Test_packageConfigDefaultsMatchArchiveSearchConfig(t *testing.T) {
|
||||
want := cataloging.DefaultArchiveSearchConfig()
|
||||
got := defaultPackageConfig()
|
||||
|
||||
assert.Equal(t, want.MaxDepth, got.NestedArchiveMaxDepth)
|
||||
assert.Equal(t, want.MaxMemoryBytes, got.NestedArchiveMaxMemoryBytes)
|
||||
assert.Equal(t, want.MaxDiskBytes, got.NestedArchiveMaxDiskBytes)
|
||||
|
||||
require.Zero(t, got.NestedArchiveMaxDepth, "nested archive cataloging must be off by default")
|
||||
}
|
||||
|
||||
func Test_ToArchiveConfig(t *testing.T) {
|
||||
defaults := cataloging.DefaultArchiveSearchConfig()
|
||||
|
||||
t.Run("defaults round trip unchanged", func(t *testing.T) {
|
||||
cfg := Catalog{Package: defaultPackageConfig()}
|
||||
got := cfg.ToArchiveConfig()
|
||||
|
||||
assert.Equal(t, defaults.MaxDepth, got.MaxDepth)
|
||||
assert.Equal(t, defaults.MaxMemoryBytes, got.MaxMemoryBytes)
|
||||
assert.Equal(t, defaults.MaxDiskBytes, got.MaxDiskBytes)
|
||||
})
|
||||
|
||||
t.Run("every setting is threaded through", func(t *testing.T) {
|
||||
// a key that parses but is not carried into the archive config is invisible in a config
|
||||
// dump, so assert each one arrives
|
||||
cfg := Catalog{Package: packageConfig{
|
||||
NestedArchiveMaxDepth: 3,
|
||||
NestedArchiveMaxMemoryBytes: 111,
|
||||
NestedArchiveMaxDiskBytes: 3333,
|
||||
SearchIndexedArchives: true,
|
||||
SearchUnindexedArchives: true,
|
||||
}}
|
||||
got := cfg.ToArchiveConfig()
|
||||
|
||||
assert.Equal(t, 3, got.MaxDepth)
|
||||
assert.Equal(t, int64(111), got.MaxMemoryBytes)
|
||||
assert.Equal(t, int64(3333), got.MaxDiskBytes)
|
||||
assert.True(t, got.IncludeIndexedArchives)
|
||||
assert.True(t, got.IncludeUnindexedArchives)
|
||||
})
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -186,7 +187,12 @@ func (m *UI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
}
|
||||
|
||||
func (m UI) View() string {
|
||||
return m.frame.View()
|
||||
v := m.frame.View()
|
||||
// bubbletea erases the line the cursor rests on at exit, which would drop the frame's last line
|
||||
if !strings.HasSuffix(v, "\n") {
|
||||
return v + "\n"
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package ui
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/bubbly/bubbles/frame"
|
||||
)
|
||||
|
||||
var _ tea.Model = (*staticModel)(nil)
|
||||
|
||||
type staticModel string
|
||||
|
||||
func Test_UI_viewEndsOnAnEmptyLine(t *testing.T) {
|
||||
// bubbletea erases the line the cursor rests on when the program stops, so a view whose last line
|
||||
// carries content loses it. Unless a log line happened to be in the footer, that dropped the last row
|
||||
// of the cataloging tree on any source that logs nothing.
|
||||
subject := New(io.Discard, false)
|
||||
subject.frame.(*frame.Frame).AppendModel(staticModel("first row\nlast row"))
|
||||
|
||||
require.Equal(t, "first row\nlast row\n", subject.View())
|
||||
}
|
||||
|
||||
func (m staticModel) Init() tea.Cmd { return nil }
|
||||
|
||||
func (m staticModel) Update(tea.Msg) (tea.Model, tea.Cmd) { return m, nil }
|
||||
|
||||
func (m staticModel) View() string { return string(m) }
|
||||
@@ -12,7 +12,7 @@ require (
|
||||
github.com/acobaugh/osrelease v0.1.0
|
||||
github.com/adrg/xdg v0.5.3
|
||||
github.com/anchore/bubbly v0.2.2
|
||||
github.com/anchore/clio v0.1.1
|
||||
github.com/anchore/clio v0.1.2
|
||||
github.com/anchore/fangs v0.1.1
|
||||
github.com/anchore/go-collections v0.1.1
|
||||
github.com/anchore/go-homedir v0.1.1
|
||||
@@ -22,28 +22,28 @@ require (
|
||||
github.com/anchore/go-sync v0.1.2
|
||||
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
|
||||
github.com/anchore/packageurl-go v0.2.0
|
||||
github.com/anchore/stereoscope v0.3.4
|
||||
github.com/anchore/stereoscope v0.4.0
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
|
||||
github.com/aquasecurity/go-pep440-version v0.0.1
|
||||
github.com/bitnami/go-version v0.0.0-20250131085805-b1f57a8634ef
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.2
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.3
|
||||
github.com/cespare/xxhash/v2 v2.3.0
|
||||
github.com/charmbracelet/bubbles v1.0.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/dave/jennifer v1.7.1
|
||||
github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da
|
||||
github.com/deitch/magic v1.0.0
|
||||
github.com/diskfs/go-diskfs v1.9.4
|
||||
github.com/distribution/reference v0.6.0
|
||||
github.com/dustin/go-humanize v1.1.0
|
||||
github.com/elliotchance/phpserialize v1.4.0
|
||||
github.com/facebookincubator/nvdtools v0.1.5
|
||||
github.com/github/go-spdx/v2 v2.7.1
|
||||
github.com/gkampitakis/go-snaps v0.5.23
|
||||
github.com/github/go-spdx/v2 v2.8.0
|
||||
github.com/gkampitakis/go-snaps v0.5.24
|
||||
github.com/go-git/go-billy/v5 v5.9.2
|
||||
github.com/go-git/go-git/v5 v5.19.3
|
||||
github.com/go-test/deep v1.1.1
|
||||
github.com/go-test/deep v1.1.2
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0
|
||||
github.com/goccy/go-yaml v1.19.2
|
||||
github.com/gohugoio/hashstructure v0.6.0
|
||||
@@ -51,8 +51,8 @@ require (
|
||||
github.com/google/go-containerregistry v0.22.1
|
||||
github.com/google/licensecheck v0.3.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gookit/color v1.6.2
|
||||
github.com/gpustack/gguf-parser-go v0.26.4
|
||||
github.com/gookit/color v1.6.3
|
||||
github.com/gpustack/gguf-parser-go v0.27.0
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2
|
||||
github.com/hashicorp/go-getter v1.8.10
|
||||
github.com/hashicorp/go-multierror v1.1.1
|
||||
@@ -62,7 +62,7 @@ require (
|
||||
github.com/jedib0t/go-pretty/v6 v6.8.3
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953
|
||||
github.com/klauspost/compress v1.20.1
|
||||
github.com/klauspost/compress v1.20.3
|
||||
github.com/magiconair/properties v1.18.12
|
||||
github.com/mholt/archives v0.1.5
|
||||
github.com/moby/sys/mountinfo v0.7.2
|
||||
@@ -93,11 +93,11 @@ require (
|
||||
go.uber.org/goleak v1.3.0
|
||||
go.yaml.in/yaml/v3 v3.0.5
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
golang.org/x/mod v0.41.0
|
||||
golang.org/x/mod v0.42.0
|
||||
golang.org/x/net v0.61.0
|
||||
golang.org/x/time v0.16.0
|
||||
golang.org/x/tools v0.51.0
|
||||
modernc.org/sqlite v1.60.1
|
||||
golang.org/x/tools v0.52.0
|
||||
modernc.org/sqlite v1.61.0
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -123,9 +123,9 @@ require (
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect
|
||||
github.com/Masterminds/goutils v1.1.1 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.3-0.20260930231756-f19d9717deb0 // indirect
|
||||
github.com/Microsoft/hcsshim v0.15.0-rc.4 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.4.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.5.2 // indirect
|
||||
github.com/STARRY-S/zip v0.2.3 // indirect
|
||||
github.com/VividCortex/ewma v1.2.0 // indirect
|
||||
github.com/agext/levenshtein v1.2.1 // indirect
|
||||
@@ -156,7 +156,6 @@ require (
|
||||
github.com/aws/smithy-go v1.28.1 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/becheran/wildmatch-go v1.0.0 // indirect
|
||||
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
|
||||
github.com/bodgit/plumbing v1.3.0 // indirect
|
||||
github.com/bodgit/sevenzip v1.6.1 // indirect
|
||||
@@ -173,7 +172,7 @@ require (
|
||||
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
|
||||
github.com/containerd/cgroups/v3 v3.1.3 // indirect
|
||||
github.com/containerd/containerd/api v1.12.0 // indirect
|
||||
github.com/containerd/containerd/v2 v2.4.1 // indirect
|
||||
github.com/containerd/containerd/v2 v2.4.2 // indirect
|
||||
github.com/containerd/continuity v0.5.0 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
@@ -182,16 +181,16 @@ require (
|
||||
github.com/containerd/log/otel v0.1.0 // indirect
|
||||
github.com/containerd/platforms v1.0.0-rc.5 // indirect
|
||||
github.com/containerd/plugin v1.1.0 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.18.2 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.19.0 // indirect
|
||||
github.com/containerd/ttrpc v1.2.9 // indirect
|
||||
github.com/containerd/typeurl/v2 v2.3.0 // indirect
|
||||
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
|
||||
github.com/containers/ocicrypt v1.3.2 // indirect
|
||||
github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.7.0 // indirect
|
||||
github.com/docker/cli v29.8.2+incompatible // indirect
|
||||
github.com/docker/docker-credential-helpers v0.9.8 // indirect
|
||||
github.com/docker/go-connections v0.8.1 // indirect
|
||||
github.com/docker/cli v29.9.2+incompatible // indirect
|
||||
github.com/docker/docker-credential-helpers v0.9.10 // indirect
|
||||
github.com/docker/go-connections v0.8.2 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
@@ -202,16 +201,16 @@ require (
|
||||
github.com/felixge/fgprof v0.9.5 // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.15 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.16 // indirect
|
||||
github.com/gkampitakis/ciinfo v0.3.4 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.5 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.6 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-restruct/restruct v1.2.0-alpha // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
github.com/google/go-intervals v0.0.2 // indirect
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 // indirect
|
||||
github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 // indirect
|
||||
github.com/google/s2a-go v0.1.10 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.22 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.24.1 // indirect
|
||||
@@ -225,7 +224,7 @@ require (
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||
github.com/klauspost/pgzip v1.2.7 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
||||
@@ -233,8 +232,8 @@ require (
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.27 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.48 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.31 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.52 // indirect
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
|
||||
github.com/miekg/pkcs11 v1.1.2 // indirect
|
||||
github.com/mikelolasagasti/xz v1.0.1 // indirect
|
||||
@@ -246,8 +245,8 @@ require (
|
||||
github.com/mitchellh/reflectwalk v1.0.2 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/locker v1.0.1 // indirect
|
||||
github.com/moby/moby/api v1.56.0 // indirect
|
||||
github.com/moby/moby/client v0.6.0 // indirect
|
||||
github.com/moby/moby/api v1.56.1 // indirect
|
||||
github.com/moby/moby/client v0.6.2 // indirect
|
||||
github.com/moby/sys/capability v0.4.0 // indirect
|
||||
github.com/moby/sys/signal v0.7.1 // indirect
|
||||
github.com/moby/sys/user v0.4.1 // indirect
|
||||
@@ -257,7 +256,7 @@ require (
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/ncruces/go-strftime v1.1.0 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.4.1 // indirect
|
||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
|
||||
github.com/olekukonko/errors v1.2.0 // indirect
|
||||
@@ -284,7 +283,7 @@ require (
|
||||
github.com/sahilm/fuzzy v0.1.1 // indirect
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.11.1 // indirect
|
||||
github.com/shopspring/decimal v1.4.0 // indirect
|
||||
github.com/sigstore/fulcio v1.8.7 // indirect
|
||||
github.com/sigstore/fulcio v1.8.8 // indirect
|
||||
github.com/sigstore/protobuf-specs v0.5.2 // indirect
|
||||
github.com/sigstore/sigstore v1.10.11 // indirect
|
||||
github.com/sirupsen/logrus v1.10.2 // indirect
|
||||
@@ -308,9 +307,10 @@ require (
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/tidwall/sjson v1.2.5 // indirect
|
||||
github.com/varlink/go v0.4.1-0.20261015173642-c935be0c53f7 // indirect
|
||||
github.com/vbatts/tar-split v0.12.3 // indirect
|
||||
github.com/vbauerster/cupwriter v0.0.4 // indirect
|
||||
github.com/vbauerster/mpb/v8 v8.14.0 // indirect
|
||||
github.com/vbauerster/cupwriter v0.0.5 // indirect
|
||||
github.com/vbauerster/mpb/v8 v8.16.2 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
@@ -327,8 +327,8 @@ require (
|
||||
go.opentelemetry.io/otel/sdk v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.46.0 // indirect
|
||||
go.podman.io/image/v5 v5.41.2 // indirect
|
||||
go.podman.io/storage v1.64.1 // indirect
|
||||
go.podman.io/image/v5 v5.42.0 // indirect
|
||||
go.podman.io/storage v1.65.0 // indirect
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||
golang.org/x/crypto v0.58.0 // indirect
|
||||
golang.org/x/oauth2 v0.37.0 // indirect
|
||||
@@ -344,7 +344,7 @@ require (
|
||||
google.golang.org/grpc v1.84.0 // indirect
|
||||
google.golang.org/protobuf v1.36.12 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
modernc.org/libc v1.77.1 // indirect
|
||||
modernc.org/libc v1.79.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.12.1 // indirect
|
||||
)
|
||||
|
||||
@@ -99,15 +99,15 @@ github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lpr
|
||||
github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs=
|
||||
github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0=
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 h1:0kQAzHq8vLs7Pptv+7TxjdETLf/nIqJpIB4oC6Ba4vY=
|
||||
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29/go.mod h1:ZWa7ssZJT30CCDGJ7fk/2SBTq9BIQrrVjrcss0UW2s0=
|
||||
github.com/Microsoft/go-winio v0.6.3-0.20260930231756-f19d9717deb0 h1:71VU8kW/LCxJkx41+YYys1EgQO+AUnPa1TrxTSyYUCU=
|
||||
github.com/Microsoft/go-winio v0.6.3-0.20260930231756-f19d9717deb0/go.mod h1:ma0QpxizD4fyyzJ1C7MIblymTD2Nxz5Mho/3CepwIZg=
|
||||
github.com/Microsoft/hcsshim v0.15.0-rc.4 h1:aZFX4LH0S20Lgjq0wG61StIClj7im4yzrxIClkaR8Z8=
|
||||
github.com/Microsoft/hcsshim v0.15.0-rc.4/go.mod h1:BA9CBztgu4h/6Jsvo1O1M4qjWw09PoYpaEYgexPE578=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
|
||||
github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
|
||||
github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM=
|
||||
github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo=
|
||||
github.com/ProtonMail/go-crypto v1.5.2 h1:cucYnvqcY7UOXVD//mSyjeaPY0SSN3v5cDkYPxumINk=
|
||||
github.com/ProtonMail/go-crypto v1.5.2/go.mod h1:/RaSu30DaKO4RY+XdV/ACcCcZkGr7AhUIduq5sjzzCo=
|
||||
github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4=
|
||||
github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk=
|
||||
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
|
||||
@@ -126,8 +126,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/anchore/bubbly v0.2.2 h1:53JHRlpaNZjMcUYdz7n1U7+kOhy0zsJv9gpGEXvTcz8=
|
||||
github.com/anchore/bubbly v0.2.2/go.mod h1:M4twoFkczlfuNdU8QV7oQjVJwPbxdUWRMfKydcLba0U=
|
||||
github.com/anchore/clio v0.1.1 h1:9bkYnaFbpT0J3G0qSHyQqa/lpK5zV0C9Gtgh82H/jj0=
|
||||
github.com/anchore/clio v0.1.1/go.mod h1:CFPIopJlakJbBpKtAL/EPpj4W9hgBO7PfFc8N2FAtlE=
|
||||
github.com/anchore/clio v0.1.2 h1:kCWNUkJsRJ5A103fFSLG+CSqIP9bOIFX2X5GEnup0HM=
|
||||
github.com/anchore/clio v0.1.2/go.mod h1:LqoM9TUMDolUMaPV4hslLvuooqIjpTN8J7V/rBqk/Zc=
|
||||
github.com/anchore/fangs v0.1.1 h1:NBTHocmWM2/0gNhebpoE/wEB0Z0nxh2lESww2o/Uf5M=
|
||||
github.com/anchore/fangs v0.1.1/go.mod h1:82oZuk4+RynOvAoxnXKmI1zO5xk/kbOw+5t4fAAPSsk=
|
||||
github.com/anchore/go-collections v0.1.1 h1:kGlmSehs+ctTBZr+98F2+taHeolHy+XDx1Nczl8GZc8=
|
||||
@@ -150,8 +150,8 @@ github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZV
|
||||
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
|
||||
github.com/anchore/packageurl-go v0.2.0 h1:CkrM4RMUwrEGAiE1OVlxaZNzWj0TuHRey7o4T/EAErk=
|
||||
github.com/anchore/packageurl-go v0.2.0/go.mod h1:2JCgOQMIsqZ7TmliXG4PnUthPJAKE3mWQbsW2XHjAOE=
|
||||
github.com/anchore/stereoscope v0.3.4 h1:dT5AfjCTXUJowMiRIc+VBQVwvwnIEwnxxwA1GwAkMxw=
|
||||
github.com/anchore/stereoscope v0.3.4/go.mod h1:+HxqHTt7dvH8mea0mf3ulqkf084VvHiT1grpAQf5uEs=
|
||||
github.com/anchore/stereoscope v0.4.0 h1:MDL7YFRDaGzhCdjm/s+em1D/X9HwnZMYupfgg+mvqPI=
|
||||
github.com/anchore/stereoscope v0.4.0/go.mod h1:hYJsi3M8a8N1O66u+rMQSgv5TREGM7QIjKBjPPnwiD8=
|
||||
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
|
||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
@@ -217,8 +217,6 @@ github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3v
|
||||
github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/becheran/wildmatch-go v1.0.0 h1:mE3dGGkTmpKtT4Z+88t8RStG40yN9T+kFEGj2PZFSzA=
|
||||
github.com/becheran/wildmatch-go v1.0.0/go.mod h1:gbMvj0NtVdJ15Mg/mH9uxk2R1QCistMyU7d9KFzroX4=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
@@ -229,8 +227,8 @@ github.com/bitnami/go-version v0.0.0-20250131085805-b1f57a8634ef h1:TSFnfbbu2oAO
|
||||
github.com/bitnami/go-version v0.0.0-20250131085805-b1f57a8634ef/go.mod h1:9iglf1GG4oNRJ39bZ5AZrjgAFD2RwQbXw6Qf7Cs47wo=
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.2 h1:eF7W7HWKg3z9NrWV9pTLnNeoXaqq3Tq9DNKXVMfoCnw=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.3 h1:1UIv1+fqH3++ViVkpnaPTB+Pi1UHycnPFAq14zgK+gI=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.3/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
|
||||
github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU=
|
||||
github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs=
|
||||
github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4=
|
||||
@@ -300,8 +298,8 @@ github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6
|
||||
github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw=
|
||||
github.com/containerd/containerd/api v1.12.0 h1:kuQm82SbDrCuO4n7hf2L8zsBtZLuympyq5X/VotfX2A=
|
||||
github.com/containerd/containerd/api v1.12.0/go.mod h1:EBcSzoi9Vl18cdODaXUCskf3D2NT8lsSXeZJnU5jIUc=
|
||||
github.com/containerd/containerd/v2 v2.4.1 h1:DUx/ZJN7cEu0WuzHClDB+68H/bqMEH5pWoEjf0ae4hc=
|
||||
github.com/containerd/containerd/v2 v2.4.1/go.mod h1:vgLdtvl3prFk1d3ZVqQcsDD5Q9IKM+vAIdU54U85w+I=
|
||||
github.com/containerd/containerd/v2 v2.4.2 h1:p6Qg+xbEz8lvwdhHDx0U469xENvp4LgGBx+E7djXkGw=
|
||||
github.com/containerd/containerd/v2 v2.4.2/go.mod h1:vgLdtvl3prFk1d3ZVqQcsDD5Q9IKM+vAIdU54U85w+I=
|
||||
github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg=
|
||||
github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
@@ -318,8 +316,8 @@ github.com/containerd/platforms v1.0.0-rc.5 h1:vXd569rDrz8LeMXzAnBsy6LADV5YtsD8o
|
||||
github.com/containerd/platforms v1.0.0-rc.5/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A=
|
||||
github.com/containerd/plugin v1.1.0 h1:O+7lczNJVMy8rz0YNx3xGB8tTf5qY4i5abF041Ew19U=
|
||||
github.com/containerd/plugin v1.1.0/go.mod h1:qBTum+A8lJ6lO44A19Eo7y1OlcLj4OWFH1DA/vnHmcc=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.18.2 h1:yXkZFYIzz3eoLwlTUZKz2iQ4MrckBxJjkmD16ynUTrw=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.18.2/go.mod h1:XyVU5tcJ3PRpkA9XS2T5us6Eg35yM0214Y+wvrZTBrY=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.19.0 h1:tak4qAx648jBAgZmywkd+6kaJ5pqSOVzKYB5DcVSHxk=
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.19.0/go.mod h1:Cr0pxOkuN7Y61+Sik6X6/nPtePW7hsKWdBt+MoHQRr0=
|
||||
github.com/containerd/ttrpc v1.2.9 h1:ha0ak962T0s3CA/RoZ6S6xiWZQF24GrBaEpiGX1uihg=
|
||||
github.com/containerd/ttrpc v1.2.9/go.mod h1:jjtQRwXm4DL3KsHKW8vDiUOV6wO0hi6IPhmJhxU7aEs=
|
||||
github.com/containerd/typeurl/v2 v2.3.0 h1:HZHPhRWo5XMy3QGQoPrUzbW/2ckwjfweHmOwlkIrPAQ=
|
||||
@@ -344,8 +342,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M=
|
||||
github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da/go.mod h1:B3tI9iGHi4imdLi4Asdha1Sc6feLMTfPLXh9IUYmysk=
|
||||
github.com/deitch/magic v1.0.0 h1:Nma1YAj7EB8Mlv4pVhbPpUp0CV1luYj2NOnE2+2xPNE=
|
||||
github.com/deitch/magic v1.0.0/go.mod h1:B3tI9iGHi4imdLi4Asdha1Sc6feLMTfPLXh9IUYmysk=
|
||||
github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4=
|
||||
github.com/diskfs/go-diskfs v1.9.4 h1:0j2d7eG4IjyxL6+ChWbDPocdBCF6HQ4HBWU2WDYWVnc=
|
||||
github.com/diskfs/go-diskfs v1.9.4/go.mod h1:TePJORO83Adh5pb2SqsxAwaP0fofFxKLkxctiS/9OQc=
|
||||
@@ -353,14 +351,14 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
|
||||
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
|
||||
github.com/docker/cli v29.8.2+incompatible h1:2zgdFuoFst2T80oS42vhKGxkd0JRo305eIEKTsxR7pQ=
|
||||
github.com/docker/cli v29.8.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/cli v29.9.2+incompatible h1:trup/vUxx5uUo2t+87Cd9AmITR+DvKUe4H2tTtZN8FY=
|
||||
github.com/docker/cli v29.9.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
|
||||
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
|
||||
github.com/docker/docker-credential-helpers v0.9.8 h1:bIREROb7So6PRlq6KTtdS9MPEjC29OQRkFNlvK2OX8Q=
|
||||
github.com/docker/docker-credential-helpers v0.9.8/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c=
|
||||
github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M=
|
||||
github.com/docker/go-connections v0.8.1/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
|
||||
github.com/docker/docker-credential-helpers v0.9.10 h1:LCWwv7UmS7fQVmdKDWeMuureOeY5kkbC5kZfkzxkS6k=
|
||||
github.com/docker/docker-credential-helpers v0.9.10/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c=
|
||||
github.com/docker/go-connections v0.8.2 h1:9d9DEokx5W7qlr0shkFXIxEA7+TTD+llwzcykSlxMNc=
|
||||
github.com/docker/go-connections v0.8.2/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4=
|
||||
@@ -419,15 +417,15 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z
|
||||
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
|
||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
|
||||
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
|
||||
github.com/gabriel-vasile/mimetype v1.4.16 h1:5izNFuvA37st1dDRAcA7TtmngBmRgiIPDmTtiEvjx1E=
|
||||
github.com/gabriel-vasile/mimetype v1.4.16/go.mod h1:3oFFwxX8kTklZwzIl2aQEDZT0lKY8xH92ST88bAwba4=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/github/go-spdx/v2 v2.7.1 h1:+siV9wnsncI5pg4otH/sKvqGL8UkcLdPh16TSJYY1V4=
|
||||
github.com/github/go-spdx/v2 v2.7.1/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0=
|
||||
github.com/github/go-spdx/v2 v2.8.0 h1:WKCnF2Ee/x+zwSIEeQdm6+be0H6PiCNLQmSZ5CFmxMI=
|
||||
github.com/github/go-spdx/v2 v2.8.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0=
|
||||
github.com/gkampitakis/ciinfo v0.3.4 h1:5eBSibVuSMbb/H6Elc0IIEFbkzCJi3lm94n0+U7Z0KY=
|
||||
github.com/gkampitakis/ciinfo v0.3.4/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo=
|
||||
github.com/gkampitakis/go-snaps v0.5.23 h1:okh5QR48zpUjpWtu65AtqxdCY8huJq+dEDuUzd1PuKg=
|
||||
github.com/gkampitakis/go-snaps v0.5.23/go.mod h1:uy3lVzCCRRsAwYqSocyw5fY8xRLCYEfqoOJNxr8HonM=
|
||||
github.com/gkampitakis/go-snaps v0.5.24 h1:oyQ/CQA4s0nkVm6CLlM2GJSU8e4p6KGPZbrSCnTNBzE=
|
||||
github.com/gkampitakis/go-snaps v0.5.24/go.mod h1:uy3lVzCCRRsAwYqSocyw5fY8xRLCYEfqoOJNxr8HonM=
|
||||
github.com/glebarez/go-sqlite v1.23.0 h1:FyhIq4jqmgphQAUlY79zPldYGwISEZikaDfhiGWkkaI=
|
||||
github.com/glebarez/go-sqlite v1.23.0/go.mod h1:IIYrOH3L0rHY3jb4IXOHoWdklNajSGUN2eJcvK8WrnI=
|
||||
github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
|
||||
@@ -443,8 +441,8 @@ github.com/go-git/go-git/v5 v5.19.3/go.mod h1:Ye4C8dVigkqrv9UsB4NMdSLV4yAIkLiIhW
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-jose/go-jose/v4 v4.1.5 h1:RjgjO2LOtWOJKUC5wpwY9LR3B3vwVAz6JS2YHfYU6eA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.5/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-jose/go-jose/v4 v4.1.6 h1:sNRkftxFJxwsILiQukqp5z83QBSmntVwmgRY7AqBDO4=
|
||||
github.com/go-jose/go-jose/v4 v4.1.6/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
@@ -458,8 +456,8 @@ github.com/go-restruct/restruct v1.2.0-alpha h1:2Lp474S/9660+SJjpVxoKuWX09JsXHSr
|
||||
github.com/go-restruct/restruct v1.2.0-alpha/go.mod h1:KqrpKpn4M8OLznErihXTGLlsXFGeLxHUrLRRI/1YjGk=
|
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
|
||||
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
|
||||
github.com/go-test/deep v1.1.2 h1:JUuyv17rKrqYfLFnP9A4Yh6hw0635ugB93bMg2bwuDQ=
|
||||
github.com/go-test/deep v1.1.2/go.mod h1:lwVFiFFWPjydTsV3xGJ/YTQsnWwi8zxBFM+gOVcwINE=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
@@ -555,8 +553,8 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe
|
||||
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
|
||||
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
||||
github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 h1:PlNWJRxu5APSV0pWqfIsZ0qt9OOGHSWfZQ4zvx+WgWs=
|
||||
github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/s2a-go v0.1.10 h1:EMp+aOuXN6l8cE/gjF5Bt+vyZxsUuyCWe9chDWR/+uU=
|
||||
github.com/google/s2a-go v0.1.10/go.mod h1:pz4tyvwXvJLLbyrkh6FW1eS2zPUXMaTmyNhYtyP2tNw=
|
||||
@@ -575,12 +573,12 @@ github.com/googleapis/gax-go/v2 v2.24.1/go.mod h1:bWeBei0NVwaNZKb2y1HUBS7gLXIF3/
|
||||
github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0=
|
||||
github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E=
|
||||
github.com/gookit/color v1.2.5/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg=
|
||||
github.com/gookit/color v1.6.2 h1:M6TJRJMuDFPYkrWmjOvMj+E0xIjcHq8lMPy7+vv0z1Y=
|
||||
github.com/gookit/color v1.6.2/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
|
||||
github.com/gookit/color v1.6.3 h1:m6E49x+KftGaspcLtPLMsyci2ARhWIUW0PUV1owMlhs=
|
||||
github.com/gookit/color v1.6.3/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||
github.com/gpustack/gguf-parser-go v0.26.4 h1:B9CRrQBcIB7I/MJFZvzPZ+e+cbc1xBzQJcmpi9l/mKU=
|
||||
github.com/gpustack/gguf-parser-go v0.26.4/go.mod h1:y4TwTtDqFWTK+xvprOjRUh+dowgU2TKCX37vRKvGiZ0=
|
||||
github.com/gpustack/gguf-parser-go v0.27.0 h1:eXFU9pmJEMQHOTL9qI742lXHWSa1CtlMWUU3gFce7nM=
|
||||
github.com/gpustack/gguf-parser-go v0.27.0/go.mod h1:y4TwTtDqFWTK+xvprOjRUh+dowgU2TKCX37vRKvGiZ0=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.74 h1:mymLUKThnV9wFvogOK8NnsMP9/vlhnjXY98gr2QIGW8=
|
||||
github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.74/go.mod h1:Bh9qYL8ehmDxSg14Tk8oxFCP90XHfs6NxV1D84884xA=
|
||||
@@ -666,11 +664,11 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
|
||||
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
|
||||
github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
|
||||
github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
|
||||
github.com/klauspost/pgzip v1.2.7 h1:02QB3Ttao6zOWDnSsv3bIvjN24bX0eGjWniQ8vuBfkA=
|
||||
github.com/klauspost/pgzip v1.2.7/go.mod h1:g7E6NrOKHOzah4QwK6Ue1tNCJs8IDiNOfjiXTr85U2E=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
@@ -711,10 +709,10 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe
|
||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 h1:P8UmIzZMYDR+NGImiFvErt6VWfIRPuGM+vyjiEdkmIw=
|
||||
github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0=
|
||||
github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8=
|
||||
github.com/mattn/go-sqlite3 v1.14.48 h1:7XHIgl0a8HwOaiK4E47ozLkST78rR9+OtNGx27D/TFs=
|
||||
github.com/mattn/go-sqlite3 v1.14.48/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
|
||||
github.com/mattn/go-runewidth v0.0.31 h1:1oCV4ReTlosDBc6rYRw5cL8S/4y8iv5oRl4SJ5IBgwo=
|
||||
github.com/mattn/go-runewidth v0.0.31/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8=
|
||||
github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U=
|
||||
github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
@@ -748,10 +746,10 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
|
||||
github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
|
||||
github.com/moby/moby/api v1.56.0 h1:GQzua3NA599ASSIICx0iFgiJeO9YkdDARvQsm23ZZuQ=
|
||||
github.com/moby/moby/api v1.56.0/go.mod h1:sZ+THbVWkjOmBPPfbnzdD/G1LuIexWhqlSHHPTDQ1Uk=
|
||||
github.com/moby/moby/client v0.6.0 h1:AJjEB21QPbXSXjDsZorFBoDZPhMrfbpaPLgSMAW9Bgs=
|
||||
github.com/moby/moby/client v0.6.0/go.mod h1:OCo00wNRyA3m4lmJ228W3JbyCN4ZNNYjpOXiJydBdcQ=
|
||||
github.com/moby/moby/api v1.56.1 h1:PpWkvVPB7Fr/No8w+TfyJ/I6rWZ2YPZhT3JorIg+06c=
|
||||
github.com/moby/moby/api v1.56.1/go.mod h1:sZ+THbVWkjOmBPPfbnzdD/G1LuIexWhqlSHHPTDQ1Uk=
|
||||
github.com/moby/moby/client v0.6.2 h1:ekSPlMccrZGcw6Scb8T9D39qE0mB+XupU/yS0zNw9N4=
|
||||
github.com/moby/moby/client v0.6.2/go.mod h1:DGcnO7qmPJrqobx/ggQgQ/AUk+2wgVdQv7uxB0/mEsc=
|
||||
github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk=
|
||||
github.com/moby/sys/capability v0.4.0/go.mod h1:4g9IK291rVkms3LKCDOoYlnV8xKwoDTpIrNEE35Wq0I=
|
||||
github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg=
|
||||
@@ -777,8 +775,8 @@ github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIf
|
||||
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk=
|
||||
github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g=
|
||||
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 h1:kpt9ZfKcm+EDG4s40hMwE//d5SBgDjUOrITReV2u4aA=
|
||||
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1/go.mod h1:qgCw4bBKZX8qMgGeEZzGFVT3notl42dBjNqO2jut0M0=
|
||||
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE=
|
||||
@@ -884,8 +882,8 @@ github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7
|
||||
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||
github.com/sanity-io/litter v1.5.8 h1:uM/2lKrWdGbRXDrIq08Lh9XtVYoeGtcQxk9rtQ7+rYg=
|
||||
github.com/sanity-io/litter v1.5.8/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtCdFLPWhpg=
|
||||
github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI=
|
||||
github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e h1:7q6NSFZDeGfvvtIRwBrU/aegEYJYmvev0cHAwo17zZQ=
|
||||
@@ -901,8 +899,8 @@ github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
|
||||
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
|
||||
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
|
||||
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
|
||||
github.com/sigstore/fulcio v1.8.7 h1:d7QJQxukgXarqPCT9uDNeyzt1Bbt/biw58/VfOnwI/o=
|
||||
github.com/sigstore/fulcio v1.8.7/go.mod h1:7fX+QyigZxRGKTgHvTfOfxwwniCDxK02PVloHy5/vg4=
|
||||
github.com/sigstore/fulcio v1.8.8 h1:AtVoVQ4R7YaP6RHIH9puHyB398CHS/9WYc6d+gKf5Y0=
|
||||
github.com/sigstore/fulcio v1.8.8/go.mod h1:RaFomAGGM7VZKAR9ZGqQhcf1Peryc9E1020/kbX8Uto=
|
||||
github.com/sigstore/protobuf-specs v0.5.2 h1:RSWWUY8QrVTxbYH00jY/jg2e7YnjzrpwP+PeHTMll0E=
|
||||
github.com/sigstore/protobuf-specs v0.5.2/go.mod h1:DRBzpFuE+LnvQMN10/dU6nBeKwVLGEQ6o2FovN2Rats=
|
||||
github.com/sigstore/sigstore v1.10.11 h1:fjrwDfR0AWmRONYV/hKGJyqbT0osMtaBz9FTZ6MH9XI=
|
||||
@@ -996,14 +994,16 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri
|
||||
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/ulikunitz/xz v0.5.17 h1:flR0y/x1hgM8EGV1AW3Xll6T413G0glV8UfBwR617V4=
|
||||
github.com/ulikunitz/xz v0.5.17/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
|
||||
github.com/varlink/go v0.4.1-0.20261015173642-c935be0c53f7 h1:pl46d1ZU17758S4EQhns0po6ZPtX+EdHgMPd1uvjia4=
|
||||
github.com/varlink/go v0.4.1-0.20261015173642-c935be0c53f7/go.mod h1:Cqta0IQ13zfx9ZFugzjWDKUhyMryjOfMKoPrC3TY8fc=
|
||||
github.com/vbatts/go-mtree v0.7.0 h1:ytmOc3MTRidZiBi9VBCyZ2BHe4fZS47L5v7BVXDWW4E=
|
||||
github.com/vbatts/go-mtree v0.7.0/go.mod h1:EjdpFC+LZy1TXbRGNa1MKKgjQ+7ew3foMFJK8o4/TdY=
|
||||
github.com/vbatts/tar-split v0.12.3 h1:Cd46rkGXI3Td4yrVNwU8ripbxFaQbmesqhjBUUYAJSw=
|
||||
github.com/vbatts/tar-split v0.12.3/go.mod h1:sQOc6OlqGCr7HkGx/IDBeKiTIvqhmj8KffNhEXG4Nq0=
|
||||
github.com/vbauerster/cupwriter v0.0.4 h1:9sBPe0uXWLZuWQU5lqVbhyFlxX6c09asST/YfatFAys=
|
||||
github.com/vbauerster/cupwriter v0.0.4/go.mod h1:IFyzS6Xis5dnBH/rdAhrnuzg3c+KkUqEN6yE8lhJlDw=
|
||||
github.com/vbauerster/mpb/v8 v8.14.0 h1:55SR80dptMfASxIG/oCEkBXgBhxeSu4GrVsjl16oKmA=
|
||||
github.com/vbauerster/mpb/v8 v8.14.0/go.mod h1:HgpQPKfcWe3kbuGGPmi+jatHreMase5C3Fp5dpdAy0Q=
|
||||
github.com/vbauerster/cupwriter v0.0.5 h1:pxlPgxPLIebXh8lJPOZmn2TmjPE8+PqJIFkKkr7hI9I=
|
||||
github.com/vbauerster/cupwriter v0.0.5/go.mod h1:ptJDgOtLtGZp1gLzycD0TFnZztDhrQdsikG6EAA4xYQ=
|
||||
github.com/vbauerster/mpb/v8 v8.16.2 h1:+UH2Nhn6asKkdX58hSgN+5Jqg/JcFSqCcU+8z0IiYYo=
|
||||
github.com/vbauerster/mpb/v8 v8.16.2/go.mod h1:PFkxR+jl7Eew9ukpdUcodHBGq9ow74b6cXI7xL0t3mA=
|
||||
github.com/vifraa/gopom v1.0.0 h1:L9XlKbyvid8PAIK8nr0lihMApJQg/12OBvMA28BcWh0=
|
||||
github.com/vifraa/gopom v1.0.0/go.mod h1:oPa1dcrGrtlO37WPDBm5SqHAT+wTgF8An1Q71Z6Vv4o=
|
||||
github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651 h1:jIVmlAFIqV3d+DOxazTR9v+zgj8+VYuQBzPgBZvWBHA=
|
||||
@@ -1072,10 +1072,10 @@ go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwj
|
||||
go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
|
||||
go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.podman.io/image/v5 v5.41.2 h1:YnVaFrGLPWSBmipKfNvlOsGcBtk5bUxlPxQ+A6CS/uo=
|
||||
go.podman.io/image/v5 v5.41.2/go.mod h1:T/bzxEFpdpAzyHabPmcq5K+my3wBsht22sFkFjJSKT4=
|
||||
go.podman.io/storage v1.64.1 h1:DqbG8scW/bD3xHiSRzV7O5fzn46qVC+SwmUN6C6Fr5w=
|
||||
go.podman.io/storage v1.64.1/go.mod h1:ft0DzCRMZs1yn0rszmLYXEEwgd7IIc66JRwSRPUDwwc=
|
||||
go.podman.io/image/v5 v5.42.0 h1:C//Ix0Thj2LYOQO6++NjU3McLFLQbf4czgUyH98R9gc=
|
||||
go.podman.io/image/v5 v5.42.0/go.mod h1:mmIgf9EdpF68j8vdX1/OxhOSIknU1PO3G6/ClRkrwm4=
|
||||
go.podman.io/storage v1.65.0 h1:GgY5VibEw6coVhOGYWuxZ4F7H2AaTfPiwL7E1CJ2dnM=
|
||||
go.podman.io/storage v1.65.0/go.mod h1:ZDXsVSzdho+p4fDky+iMVCPwRtjS1kSiHcyhBwUB6M0=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
@@ -1147,8 +1147,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
|
||||
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
|
||||
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
|
||||
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -1410,8 +1410,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
|
||||
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
|
||||
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
|
||||
golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -1608,20 +1608,20 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
|
||||
howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
|
||||
modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg=
|
||||
modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE=
|
||||
modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA=
|
||||
modernc.org/cc/v4 v4.30.3 h1:aDq8gIX1X7+VFgO/JYYji6pLWL2H4tv3g8WatT3KCys=
|
||||
modernc.org/cc/v4 v4.30.3/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v4 v4.37.0 h1:BAuJCUtIHIxFIYaFhqXIw5YaoMBlNCUlFovejHfMip4=
|
||||
modernc.org/ccgo/v4 v4.37.0/go.mod h1:/8627VhwWtAbqUXmPORxFQU5SgLrAx2jl9seLw9GBhY=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
|
||||
modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/gc/v3 v3.2.0 h1:/Y3LGDMQaoYf55PebP1qsZ5FDGgAZyG1RYKEHeQzh1o=
|
||||
modernc.org/gc/v3 v3.2.0/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||
modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs=
|
||||
modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0=
|
||||
modernc.org/libc v1.79.0 h1:9Dvvv++bAMSkSuBzuOFAatbTV5OIoq6Y6FohmD3e7q8=
|
||||
modernc.org/libc v1.79.0/go.mod h1:uT3kR61NsbeKJSLqq83GJwSnZMVLlSjyAz4DfTaBy1M=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
|
||||
@@ -1630,8 +1630,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw=
|
||||
modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0=
|
||||
modernc.org/sqlite v1.61.0 h1:sQi63oLQ9uquAUA4CzOisH35Q3pEszkPBuHM0tFL8CY=
|
||||
modernc.org/sqlite v1.61.0/go.mod h1:f4gr5tbpx2umklaT/QVtbi/50hNdM+4ng+2zMo8KXJ0=
|
||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
# internal/archive
|
||||
|
||||
Opens one archive as its own filesystem so the ordinary catalogers can run against its contents.
|
||||
The archive cataloger task (`internal/task/archive_tasks.go`) drives the recursion; this package
|
||||
does everything for a single archive.
|
||||
|
||||
## Flow
|
||||
|
||||
```
|
||||
Extract(ctx, reader, archiveCoordinates, limiter, exclusions) -> *Resolver
|
||||
acquire read in place if the reader is already seekable, else hold in memory / spill to disk
|
||||
identifyFormat internal/file.IdentifyArchive first, end-of-central-directory fallback for stub-prefixed zips
|
||||
Resolver.extract every entry -> entryHeader -> Resolver.add (excluded entries are never stored)
|
||||
digestsOf SHA-1 of the archive itself
|
||||
```
|
||||
|
||||
`Extract` returns `nil` for content that is not an archive (a bare compression format such as a
|
||||
gzipped file that is not a tar counts as not an archive), `ErrDiskLimitReached` when the archive's
|
||||
own bytes cannot be placed, and otherwise a `Resolver` whose `Cleanup` releases everything.
|
||||
|
||||
## Resolver
|
||||
|
||||
One type does the work: `Resolver` is fed entries by `Extract`, holds their content, indexes them,
|
||||
and implements `file.Resolver` over the result. Each entry is a `node` carrying its header and a
|
||||
`blob`; the archive's own bytes, when they have to be held at all, are one more blob.
|
||||
|
||||
Every location it returns extends the archive's coordinates: `RealPath` and `FileSystemID` stay
|
||||
those of the outermost archive on the scanned filesystem, and `FullPath` is the `!/`-delimited
|
||||
chain down to the entry (`/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF`). `LocalPath` is the
|
||||
entry's path within this archive.
|
||||
|
||||
## Where bytes live
|
||||
|
||||
Blobs are held in memory while the memory limit admits them and in one spill file per archive
|
||||
(`archive-spill-*`) once it does not; when a chunk is refused, everything held in memory moves to
|
||||
the file first. The file is created on first use, so most archives never touch the filesystem, and
|
||||
blobs are located in it by offset with no framing of their own. It goes under the scan's temp root
|
||||
when the context carries one (`internal/tmpdir`), otherwise the system temp directory.
|
||||
|
||||
Anything already random-access (an `*os.File`, or a nested archive read out of its parent's
|
||||
resolver) is used in place and costs nothing, so only a top-level stream is ever copied. Its memory
|
||||
is refunded once it is extracted and digested; bytes it spilled wait for `Cleanup` with the entries.
|
||||
|
||||
## Limits
|
||||
|
||||
`Limits` bound what one scan holds *at once*; each falls as archives are released. For both limits,
|
||||
positive is the bound, zero forbids the resource, negative is unbounded.
|
||||
|
||||
- `Limiter` is scan-wide; a `charge` is one archive's draw on it. Bytes are charged as they land,
|
||||
never from declared sizes.
|
||||
- Memory refused → content moves to disk. Disk refused → the archive's own bytes cannot be placed
|
||||
(`ErrDiskLimitReached`, archive skipped) or an entry cannot be stored (`Resolver.Truncated`, the
|
||||
entries stored so far are still cataloged).
|
||||
- Index bookkeeping is approximated as `approxIndexBytesPerEntry` (2 KiB) per node, listed or
|
||||
implied by an entry's path, plus name length. It lives in memory, so it is charged to the memory
|
||||
limit, spilling held content to make room; only a zero memory limit lets it fall back to the disk
|
||||
budget so that configuration still indexes. An entry whose index cost is refused is dropped like
|
||||
one whose content is, and marks the resolver `Truncated`.
|
||||
- `Budget` bounds the bytes decompressed under one top-level archive, summed over everything nested
|
||||
in it. Unlike `Limits` it never refills, so a recursive bomb whose every level fits the limits
|
||||
still ends (`errBudgetSpent`, the resolver is `Truncated`).
|
||||
- `Limiter.Peak` reports the most held at once over the scan, for reporting.
|
||||
|
||||
## Hostile archives
|
||||
|
||||
Nothing an archive declares is trusted. Content is charged chunk by chunk as it decompresses, so a
|
||||
deflate or gzip bomb stops at the disk limit with the entry dropped whole. Every node is charged
|
||||
against the memory limit, including directories implied by deep paths, so an archive of millions of
|
||||
tiny entries or of a few entries with 4 KiB names stops at the memory limit. A recursive bomb stops
|
||||
at the `Budget`. Names longer than PATH_MAX are refused, and nesting stops at the task's `MaxDepth`.
|
||||
See the bomb tests in `extract_test.go`.
|
||||
|
||||
## Lookup
|
||||
|
||||
Entries are indexed in a stereoscope `filetree.FileTree` + `filetree.Index` and searched with
|
||||
`filetree.SearchContext`, the same index the directory and image resolvers use. A query may name an
|
||||
entry by its full path or by its `LocalPath` (leading slash optional); a full path into another
|
||||
archive in the scan names nothing. Directories implied by entry paths exist for `HasPath` but are
|
||||
never files. The last entry at a path wins over earlier
|
||||
duplicates, as it does on extraction; a file and a directory at one path keep the first.
|
||||
|
||||
Links resolve inside the archive only, as the file tree resolves them: a hard link names its target
|
||||
from the archive root. A lookup matching a link and its target returns one location, with
|
||||
`AccessPath` the full path matched and `FullPath` that of the file read. Dangling and cyclic links are not
|
||||
files. `AllLocations` enumerates every non-directory path, links included. Answers are sorted by path.
|
||||
|
||||
## Other pieces
|
||||
|
||||
- `Exclusions`: the scan's exclusion patterns, of which only the `**/` shape means the same thing
|
||||
re-rooted at an archive. Matched against each entry path and its parent directories at extraction
|
||||
time. `syft.CreateSBOM` passes the source's patterns to the task, which builds one `Exclusions`
|
||||
for the scan.
|
||||
- `Traversal`: put on the context by the task so a cataloger (java) knows which archive it is inside,
|
||||
what its digests are, and which packages describe its parent. `VirtualPath(location)` converts a
|
||||
location's `FullPath` to the java cataloger's colon-delimited `app.war:WEB-INF/lib/inner.jar`
|
||||
form, escaping `%` and `:` in entry names. `Container` and `EntryCoordinates` split and build the
|
||||
coordinates of files within archives (the SPDX and CycloneDX formats use them too); `OwnerContainers` and
|
||||
`OwnedCoordinates` map package-owned file paths to coordinates in the right archive.
|
||||
- `WithNestedCataloging` / `NestedCatalogingEnabled`: `syft.CreateSBOM` marks the context when the
|
||||
archive task is running, so the java cataloger leaves nested archives to the task instead of
|
||||
opening them itself.
|
||||
- `entryHeader`: keeps regular files, directories and links only (no devices or fifos), cleans entry names to
|
||||
archive-relative paths, refuses names longer than PATH_MAX or containing control characters, line
|
||||
separators or the `!/` archive path separator (marking the resolver truncated), and drops header
|
||||
fields nothing reads (PAX records, user and group names, sub-second and access times) since the
|
||||
header is held for the life of the archive.
|
||||
@@ -0,0 +1,38 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
)
|
||||
|
||||
// Exclusions are the scan's exclusion patterns that apply inside archives.
|
||||
//
|
||||
// A scan accepts three pattern shapes: `./x` anchored at the scan root, `*/x` exactly one level below
|
||||
// it, and `**/x` at any depth. Only the last means the same thing when re-rooted at an archive, so
|
||||
// only it reaches inside one.
|
||||
type Exclusions []string
|
||||
|
||||
func NewExclusions(scanPatterns []string) Exclusions {
|
||||
var out Exclusions
|
||||
for _, pattern := range scanPatterns {
|
||||
if strings.HasPrefix(pattern, "**/") {
|
||||
// a trailing slash reads as "a directory" but makes doublestar match nothing (issue #4839)
|
||||
out = append(out, strings.TrimSuffix(pattern, "/"))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Excludes reports whether an archive-relative path, or any directory above it, matches a pattern.
|
||||
// Directories are checked because an archive need not list a directory before the files in it.
|
||||
func (e Exclusions) Excludes(entryPath string) bool {
|
||||
for _, pattern := range e {
|
||||
for end := len(entryPath); end > 0; end = strings.LastIndexByte(entryPath[:end], '/') {
|
||||
if matched, _ := doublestar.Match(pattern, entryPath[:end]); matched {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestNewExclusions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
given []string
|
||||
want Exclusions
|
||||
}{
|
||||
{name: "nothing configured"},
|
||||
{name: "root-anchored does not reach inside", given: []string{"./nested/marker.txt"}},
|
||||
{name: "one level down is anchored too", given: []string{"*/vendor"}},
|
||||
{name: "any depth reaches inside", given: []string{"**/*.rpm"}, want: Exclusions{"**/*.rpm"}},
|
||||
{name: "mixed keeps only the any-depth patterns", given: []string{"./a", "**/b", "*/c", "**/d"}, want: Exclusions{"**/b", "**/d"}},
|
||||
// a trailing slash makes doublestar match nothing (issue #4839)
|
||||
{name: "a trailing slash is trimmed", given: []string{"**/vendor/"}, want: Exclusions{"**/vendor"}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, NewExclusions(tt.given))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExclusions_Excludes(t *testing.T) {
|
||||
exclusions := NewExclusions([]string{"**/vendor", "**/*.rpm"})
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
want bool
|
||||
}{
|
||||
{"vendor", true},
|
||||
{"a/vendor", true},
|
||||
{"vendor/x/y.go", true},
|
||||
{"a/vendor/y.go", true},
|
||||
{"pkg.rpm", true},
|
||||
{"deep/down/pkg.rpm", true},
|
||||
{"keep/vendored.txt", false},
|
||||
{"vendor-ish", false},
|
||||
{"keep.txt", false},
|
||||
{"PKG.RPM", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, exclusions.Excludes(tt.path))
|
||||
})
|
||||
}
|
||||
|
||||
assert.False(t, NewExclusions(nil).Excludes("anything"))
|
||||
assert.False(t, Exclusions{"["}.Excludes("keep.txt"), "a malformed pattern excludes nothing")
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// ErrUnsupportedFormat is returned for an archive in a format that is not extracted (see supportedExtractor).
|
||||
var ErrUnsupportedFormat = errors.New("archive format is not supported")
|
||||
|
||||
// Extract opens the archive read from r as its own filesystem, holding its bytes and entries within
|
||||
// the limiter's bounds. It returns nil when the content is not an archive, ErrUnsupportedFormat when it
|
||||
// is one that is not extracted, and ErrDiskLimitReached when the archive's own bytes cannot be placed
|
||||
// within the disk limit. The caller must Cleanup the result.
|
||||
//
|
||||
// archiveCoordinates are those of the archive file itself; every location the resolver returns
|
||||
// extends them with its entry path (see file.Coordinates).
|
||||
func Extract(ctx context.Context, r io.Reader, archiveCoordinates file.Coordinates, limiter *Limiter, exclusions Exclusions) (*Resolver, error) {
|
||||
charge := limiter.charge()
|
||||
resolver := newResolver(ctx, archiveCoordinates, charge)
|
||||
archivePath := archiveCoordinates.FullNestedPath()
|
||||
|
||||
// deferred so that a panic in a decoder still gives back the charge and removes any spill file
|
||||
succeeded := false
|
||||
defer func() {
|
||||
if !succeeded {
|
||||
resolver.Cleanup()
|
||||
}
|
||||
}()
|
||||
|
||||
content, release, err := resolver.acquire(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// the archive's own bytes are needed only until they are extracted and digested
|
||||
defer release()
|
||||
|
||||
format, err := identifyFormat(ctx, path.Base(archiveCoordinates.LocalPath()), content)
|
||||
if format == nil || err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := resolver.extract(ctx, format, content, exclusions); err != nil {
|
||||
return nil, fmt.Errorf("unable to extract archive %q: %w", archivePath, err)
|
||||
}
|
||||
resolver.Digests = digestsOf(ctx, content, archivePath)
|
||||
succeeded = true
|
||||
return resolver, nil
|
||||
}
|
||||
|
||||
// acquire makes the archive's own bytes random-access, which identifying, extracting and digesting
|
||||
// them needs. Bytes that already are, such as a file on disk or an entry of a parent archive, are
|
||||
// read in place and cost nothing. Anything else is held within the limits like an entry, and the
|
||||
// returned release gives back what memory it takes; bytes spilled to disk stay until Cleanup.
|
||||
// Reaching the disk limit yields ErrDiskLimitReached.
|
||||
func (r *Resolver) acquire(content io.Reader) (ReaderAtSeeker, func(), error) {
|
||||
if ras, ok := content.(ReaderAtSeeker); ok {
|
||||
return ras, func() {}, nil
|
||||
}
|
||||
var b blob
|
||||
if err := r.put(&b, content); err != nil {
|
||||
if errors.Is(err, ErrDiskLimitReached) {
|
||||
return nil, nil, err
|
||||
}
|
||||
return nil, nil, fmt.Errorf("unable to read archive content: %w", err)
|
||||
}
|
||||
return r.open(&b), func() { r.discard(&b) }, nil
|
||||
}
|
||||
|
||||
// identifyFormat returns the format able to extract this content, or nil when it is not an archive.
|
||||
//
|
||||
// Identification from the head of the stream wins: a tar holding one jar ends in that jar's central
|
||||
// directory, and reading the tar as a zip would lose the rest. The end of the stream is consulted
|
||||
// only when the head identifies nothing supported, which is how a zip appended to a launcher script (a
|
||||
// Spring Boot executable jar) is found, and how a java resource adapter (a zip named .rar) is read.
|
||||
func identifyFormat(ctx context.Context, name string, content ReaderAtSeeker) (archives.Extractor, error) {
|
||||
if _, err := content.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
format, _, err := intFile.IdentifyArchive(ctx, name, content)
|
||||
if err == nil {
|
||||
if extractor := supportedExtractor(format); extractor != nil {
|
||||
return extractor, nil
|
||||
}
|
||||
}
|
||||
if HasZipEndOfCentralDirectory(content) {
|
||||
return archives.Zip{}, nil
|
||||
}
|
||||
if err == nil && holdsFiles(format) {
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFormat, format.MediaType())
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// holdsFiles reports whether the format is an archive of files rather than a bare compression format,
|
||||
// such as a gzipped man page, which holds one stream and is not cataloged as an archive.
|
||||
func holdsFiles(format archives.Format) bool {
|
||||
switch f := format.(type) {
|
||||
case archives.CompressedArchive:
|
||||
return f.Extraction != nil
|
||||
case archives.Extractor:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// supportedExtractor admits only the zip and tar families, the tar family optionally compressed. Other
|
||||
// formats mholt identifies, such as 7z and rar, pull in decoders that have not been exercised against
|
||||
// untrusted input here, so they are not opened until they are deliberately supported. A bare
|
||||
// compression format, such as a gzipped file that is not a tar, is not extractable either.
|
||||
func supportedExtractor(format archives.Format) archives.Extractor {
|
||||
switch f := format.(type) {
|
||||
case archives.Zip:
|
||||
return f
|
||||
case archives.Tar:
|
||||
return f
|
||||
case archives.CompressedArchive:
|
||||
if _, ok := f.Extraction.(archives.Tar); ok {
|
||||
return f
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// extract adds every entry of the archive. Reaching the disk limit stops the walk and marks the
|
||||
// resolver truncated rather than failing.
|
||||
func (r *Resolver) extract(ctx context.Context, format archives.Extractor, content ReaderAtSeeker, exclusions Exclusions) error {
|
||||
if _, err := content.Seek(0, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
var source io.Reader = content
|
||||
if ca, ok := format.(archives.CompressedArchive); ok && ca.Compression != nil {
|
||||
// a compressed tar decompresses everything to reach each header, including entries skipped whole
|
||||
// and whatever an entry leaves unread, so the budget is charged for every byte the decoder
|
||||
// produces rather than for what is kept. A zip or a plain tar seeks past what it skips
|
||||
rc, err := ca.OpenReader(content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer internal.CloseAndLogError(rc, "decompressor")
|
||||
source, format, r.metered = &meteredReader{ctx: ctx, reader: rc, budget: r.budget}, ca.Extraction, true
|
||||
}
|
||||
err := format.Extract(ctx, source, func(_ context.Context, f archives.FileInfo) error {
|
||||
hdr, ok := r.entryHeader(f)
|
||||
if !ok || exclusions.Excludes(hdr.Name) {
|
||||
return nil
|
||||
}
|
||||
if !f.Mode().IsRegular() {
|
||||
return r.add(hdr, nil)
|
||||
}
|
||||
entry, err := f.Open()
|
||||
if err != nil {
|
||||
log.WithFields("entry", hdr.Name, "error", err).Trace("unable to open archive entry, skipping it")
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
if err := entry.Close(); err != nil {
|
||||
log.WithFields("entry", hdr.Name, "error", err).Trace("unable to close archive entry")
|
||||
}
|
||||
}()
|
||||
return r.add(hdr, entry)
|
||||
})
|
||||
r.finish()
|
||||
if errors.Is(err, ErrDiskLimitReached) {
|
||||
r.truncate(fmt.Errorf("extraction stopped: %w", err))
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, errBudgetSpent) {
|
||||
r.truncate(fmt.Errorf("extraction stopped, possible decompression bomb: %w", err))
|
||||
return nil
|
||||
}
|
||||
if err != nil && ctx.Err() == nil && r.entries > 0 {
|
||||
// the stream broke partway (a truncated .tar.gz); what was read before that is still good
|
||||
r.truncate(fmt.Errorf("extraction stopped early: %w", err))
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// archiveDigestHashes are the digests taken of an archive as a whole: SHA-1, matching what the java
|
||||
// cataloger reports for a jar.
|
||||
var archiveDigestHashes = []crypto.Hash{crypto.SHA1}
|
||||
|
||||
// digestsOf digests the archive's own bytes while they are in hand. A failure is logged and returns
|
||||
// nothing rather than discarding an archive that extracted fine.
|
||||
func digestsOf(ctx context.Context, content ReaderAtSeeker, archivePath string) []file.Digest {
|
||||
if _, err := content.Seek(0, io.SeekStart); err != nil {
|
||||
log.WithFields("archive", archivePath, "error", err).Trace("unable to rewind archive content for digests")
|
||||
return nil
|
||||
}
|
||||
digests, err := intFile.NewDigestsFromFile(ctx, io.NopCloser(content), archiveDigestHashes)
|
||||
if err != nil {
|
||||
log.WithFields("archive", archivePath, "error", err).Trace("unable to digest archive")
|
||||
return nil
|
||||
}
|
||||
return digests
|
||||
}
|
||||
|
||||
var (
|
||||
// zipLocalFileHeaderSignature starts every entry in a zip.
|
||||
zipLocalFileHeaderSignature = []byte{'P', 'K', 0x03, 0x04}
|
||||
|
||||
// zipEOCDSignature starts the end-of-central-directory record that closes every zip. Readers
|
||||
// locate the central directory from it, which is why a zip with data prepended still opens.
|
||||
zipEOCDSignature = []byte{'P', 'K', 0x05, 0x06}
|
||||
)
|
||||
|
||||
const (
|
||||
// maxAppendedArchiveStubBytes is how far into a file an appended zip's first entry is looked for. A
|
||||
// Spring Boot launcher stub is about 9 KB.
|
||||
maxAppendedArchiveStubBytes = 64 * 1024
|
||||
|
||||
// maxZipEOCDSearch is how far from the end the end-of-central-directory record can start: the 22
|
||||
// byte record plus a comment of up to 65535 bytes.
|
||||
maxZipEOCDSearch = 22 + 65535
|
||||
)
|
||||
|
||||
// MayHideAnAppendedArchive reports whether a zip entry signature appears in the head of the stream,
|
||||
// which a zip appended to a launcher stub does and a file that merely has an archive's extension
|
||||
// almost never does. The returned reader replays what was consumed.
|
||||
func MayHideAnAppendedArchive(r io.Reader) (io.Reader, bool) {
|
||||
head := make([]byte, maxAppendedArchiveStubBytes)
|
||||
n, err := io.ReadFull(r, head)
|
||||
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return r, false
|
||||
}
|
||||
head = head[:n]
|
||||
found := bytes.Contains(head, zipLocalFileHeaderSignature)
|
||||
|
||||
if seeker, ok := r.(io.Seeker); ok {
|
||||
if _, err := seeker.Seek(0, io.SeekStart); err == nil {
|
||||
return r, found
|
||||
}
|
||||
}
|
||||
return io.MultiReader(bytes.NewReader(head), r), found
|
||||
}
|
||||
|
||||
// HasZipEndOfCentralDirectory reports whether the content ends in a zip end-of-central-directory
|
||||
// record, the one part of a zip whose position is known without parsing it.
|
||||
func HasZipEndOfCentralDirectory(r ReaderAtSeeker) bool {
|
||||
size, err := r.Seek(0, io.SeekEnd)
|
||||
if err != nil || size < int64(len(zipEOCDSignature)) {
|
||||
return false
|
||||
}
|
||||
window := min(size, int64(maxZipEOCDSearch))
|
||||
buf := make([]byte, window)
|
||||
if _, err := r.ReadAt(buf, size-window); err != nil && !errors.Is(err, io.EOF) {
|
||||
return false
|
||||
}
|
||||
return bytes.Contains(buf, zipEOCDSignature)
|
||||
}
|
||||
@@ -0,0 +1,706 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
var sampleFiles = map[string]string{
|
||||
"file1.txt": "content1",
|
||||
"dir/file2.txt": "content2",
|
||||
}
|
||||
|
||||
func Test_identifyFormat(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
launcher := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"), zipBytes(t, sampleFiles)...)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
want archives.Extractor
|
||||
unsupported bool
|
||||
}{
|
||||
{"test.zip", zipBytes(t, sampleFiles), archives.Zip{}, false},
|
||||
{"test.tar.gz", tarGzBytes(t, sampleFiles), archives.CompressedArchive{}, false},
|
||||
{"plain.txt", []byte("just text"), nil, false},
|
||||
// identified by mholt, but not opened until their decoders are deliberately supported
|
||||
{"test.7z", []byte("7z\xbc\xaf\x27\x1c\x00\x04"), nil, true},
|
||||
{"test.rar", []byte("Rar!\x1a\x07\x01\x00"), nil, true},
|
||||
// a bare compression format holds one stream, not files
|
||||
{"page.1.gz", gzBytes(t, "a man page"), nil, false},
|
||||
// a java resource adapter archive is a zip that shares rar's extension
|
||||
{"adapter.rar", zipBytes(t, sampleFiles), archives.Zip{}, false},
|
||||
// the name can be claimed by identification, so the probe alone must not admit prose
|
||||
{"app.jar", []byte("plain text, no central directory anywhere in it"), nil, false},
|
||||
{"bundle", []byte("plain text, no central directory anywhere in it"), nil, false},
|
||||
// content sniffing types this as a shell script; only its tail says zip
|
||||
{"app.jar", launcher, archives.Zip{}, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(fmt.Sprintf("%s/%d bytes", tt.name, len(tt.data)), func(t *testing.T) {
|
||||
got, err := identifyFormat(ctx, tt.name, bytes.NewReader(tt.data))
|
||||
if tt.unsupported {
|
||||
assert.ErrorIs(t, err, ErrUnsupportedFormat)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
if tt.want == nil {
|
||||
assert.Nil(t, got)
|
||||
return
|
||||
}
|
||||
assert.IsType(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_extract_storesEveryEntryInMemoryWhenUnbounded(t *testing.T) {
|
||||
for name, data := range map[string][]byte{
|
||||
"test.zip": zipBytes(t, sampleFiles),
|
||||
"test.tar.gz": tarGzBytes(t, sampleFiles),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r, root := extractBytes(t, data, name, nil)
|
||||
assert.NoError(t, r.Truncated)
|
||||
assert.Zero(t, r.written)
|
||||
assert.Empty(t, filesIn(t, root))
|
||||
|
||||
entries := readStore(t, r)
|
||||
assert.Equal(t, "content1", entries["file1.txt"].body)
|
||||
assert.Equal(t, "content2", entries["dir/file2.txt"].body)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_extract_diskLimitTruncatesAtTheEntryThatDoesNotFit(t *testing.T) {
|
||||
// each entry costs its index estimate plus its content on disk
|
||||
record := approxIndexBytes(tar.Header{Name: "file1.txt"})
|
||||
data := zipBytes(t, map[string]string{"file1.txt": "a", "file2.txt": "b", "file3.txt": "c"})
|
||||
|
||||
r, _ := extractBytes(t, data, "test.zip", diskCharge(2*(record+1)+record))
|
||||
|
||||
assert.Error(t, r.Truncated)
|
||||
assert.Equal(t, []string{"file1.txt", "file2.txt"}, storedNames(t, r), "the refused entry is not stored")
|
||||
entries := readStore(t, r)
|
||||
assert.Equal(t, "a", entries["file1.txt"].body)
|
||||
assert.Equal(t, "b", entries["file2.txt"].body)
|
||||
}
|
||||
|
||||
func TestResolver_extract_diskLimitBitesMidEntry(t *testing.T) {
|
||||
// a tar is walked entry by entry, so the limit can bite part way through an entry
|
||||
data := tarGzBytes(t, map[string]string{
|
||||
"a/first.txt": strings.Repeat("a", 40*1024),
|
||||
"b/second.txt": strings.Repeat("b", 40*1024),
|
||||
})
|
||||
const budget = 80 * 1024
|
||||
charge := diskCharge(budget)
|
||||
|
||||
r, root := extractBytes(t, data, "test.tar.gz", charge)
|
||||
|
||||
assert.Error(t, r.Truncated)
|
||||
assert.Equal(t, []string{"a", "a/first.txt", "b"}, storedNames(t, r), "the directory a refused entry implied stays")
|
||||
assert.Greater(t, r.written, int64(40*1024), "the first entry landed whole and the second part way")
|
||||
_, disk := charge.held()
|
||||
assert.LessOrEqual(t, disk, int64(budget))
|
||||
assert.Greater(t, disk, r.written, "the index estimate is charged to disk too")
|
||||
|
||||
info, err := os.Stat(filepath.Join(root, spillFile(t, root)))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, r.written, info.Size())
|
||||
}
|
||||
|
||||
func TestResolver_extract_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testing.T) {
|
||||
var entries []testEntry
|
||||
for i := range 50 {
|
||||
entries = append(entries, testEntry{name: fmt.Sprintf("d%04d/", i), dir: true})
|
||||
}
|
||||
room := approxIndexBytes(tar.Header{Name: "d0000"}) * 10
|
||||
|
||||
for name, data := range map[string][]byte{
|
||||
"dirs.zip": zipFromEntries(t, entries),
|
||||
"dirs.tar.gz": tarGzFromEntries(t, entries),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r, _ := extractBytes(t, data, name, diskCharge(room))
|
||||
assert.Error(t, r.Truncated)
|
||||
assert.Len(t, storedNames(t, r), 10)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_extract_entryNamesAreCleanedAndNothingIsWrittenUnderThem(t *testing.T) {
|
||||
entries := []testEntry{
|
||||
{name: "../../etc/passwd", body: "evil"},
|
||||
{name: "/etc/hosts", body: "pwned"},
|
||||
{name: "META-INF/", dir: true, mode: 0o555},
|
||||
{name: "META-INF/MANIFEST.MF", body: "Manifest-Version: 1.0"},
|
||||
{name: "passwd", link: "../../../../etc/passwd"},
|
||||
{name: "shadow", link: "/etc/shadow"},
|
||||
{name: "bin/link.txt", link: "../lib/real.txt"},
|
||||
}
|
||||
|
||||
for name, data := range map[string][]byte{
|
||||
"evil.zip": zipFromEntries(t, entries),
|
||||
"evil.tar.gz": tarGzFromEntries(t, entries),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r, root := extractBytes(t, data, name, nil)
|
||||
require.NoError(t, r.Truncated)
|
||||
|
||||
assert.Equal(t, []string{"META-INF", "META-INF/MANIFEST.MF", "bin", "bin/link.txt", "etc", "etc/hosts", "etc/passwd", "passwd", "shadow"},
|
||||
storedNames(t, r))
|
||||
assert.Empty(t, filesIn(t, root), "entries live in the store, never under their own names")
|
||||
assert.NoFileExists(t, filepath.Join(t.TempDir(), "..", "etc", "passwd"))
|
||||
|
||||
got := readStore(t, r)
|
||||
assert.Equal(t, "evil", got["etc/passwd"].body)
|
||||
assert.Equal(t, "pwned", got["etc/hosts"].body)
|
||||
assert.Equal(t, stereoscopeFile.TypeDirectory, got["META-INF"].kind)
|
||||
assert.Equal(t, fs.FileMode(0o555), got["META-INF"].mode.Perm())
|
||||
assert.Equal(t, "Manifest-Version: 1.0", got["META-INF/MANIFEST.MF"].body)
|
||||
|
||||
// links are headers carrying their target verbatim; the index resolves them inside the archive
|
||||
for _, link := range []string{"passwd", "shadow", "bin/link.txt"} {
|
||||
assert.Equal(t, stereoscopeFile.TypeSymLink, got[link].kind)
|
||||
assert.Empty(t, got[link].body)
|
||||
}
|
||||
assert.Equal(t, "../../../../etc/passwd", got["passwd"].linkTarget)
|
||||
assert.Equal(t, "/etc/shadow", got["shadow"].linkTarget)
|
||||
assert.Equal(t, "../lib/real.txt", got["bin/link.txt"].linkTarget)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_extract_dataEdgeCases(t *testing.T) {
|
||||
t.Run("an empty archive is not an error", func(t *testing.T) {
|
||||
r, _ := extractBytes(t, zipBytes(t, map[string]string{}), "test.zip", nil)
|
||||
assert.Zero(t, r.entries)
|
||||
assert.NoError(t, r.Truncated)
|
||||
})
|
||||
|
||||
t.Run("a zero-byte entry costs no disk", func(t *testing.T) {
|
||||
r, _ := extractBytes(t, zipBytes(t, map[string]string{"empty.txt": ""}), "test.zip", memCharge(0))
|
||||
assert.Equal(t, []string{"empty.txt"}, storedNames(t, r))
|
||||
assert.Zero(t, r.written)
|
||||
assert.Empty(t, readStore(t, r)["empty.txt"].body)
|
||||
})
|
||||
|
||||
t.Run("unicode entry names survive", func(t *testing.T) {
|
||||
names := map[string]string{
|
||||
"café/naïve.txt": "accents",
|
||||
"日本語/ファイル.txt": "japanese",
|
||||
"emoji-🎉.txt": "emoji",
|
||||
}
|
||||
r, _ := extractBytes(t, zipBytes(t, names), "test.zip", nil)
|
||||
entries := readStore(t, r)
|
||||
for name, want := range names {
|
||||
assert.Equal(t, want, entries[name].body, name)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an entry name with too many path components is skipped and reported", func(t *testing.T) {
|
||||
atCap := strings.Repeat("a/", maxEntryNameDepth-1) + "f"
|
||||
data := zipBytes(t, map[string]string{
|
||||
atCap: "at the cap",
|
||||
strings.Repeat("b/", maxEntryNameDepth) + "f": "over the cap",
|
||||
})
|
||||
r, _ := extractBytes(t, data, "test.zip", nil)
|
||||
names := storedNames(t, r)
|
||||
assert.Contains(t, names, atCap)
|
||||
assert.Len(t, names, maxEntryNameDepth, "only the entry at the cap and its implied directories")
|
||||
assert.ErrorContains(t, r.Truncated, "path components")
|
||||
})
|
||||
|
||||
t.Run("an entry whose chain is longer than a path is skipped and reported", func(t *testing.T) {
|
||||
// the chain is the archive's own path, the separator, then the name
|
||||
fits := strings.Repeat("a", MaxPathBytes-len("test-archive!/"))
|
||||
data := zipBytes(t, map[string]string{
|
||||
fits: "at the cap",
|
||||
fits + "b": "over the cap",
|
||||
})
|
||||
r, _ := extractBytes(t, data, "test.zip", nil)
|
||||
assert.Equal(t, []string{fits}, storedNames(t, r))
|
||||
assert.ErrorContains(t, r.Truncated, "archive chain")
|
||||
})
|
||||
|
||||
t.Run("an over-long entry name is skipped", func(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{
|
||||
strings.Repeat("a/", maxEntryNameBytes) + "f": "deep",
|
||||
"ok.txt": "fine",
|
||||
})
|
||||
r, _ := extractBytes(t, data, "test.zip", nil)
|
||||
assert.Equal(t, []string{"ok.txt"}, storedNames(t, r))
|
||||
})
|
||||
}
|
||||
|
||||
func TestResolver_extract_unsafeEntryNamesAreRefusedAndReported(t *testing.T) {
|
||||
jar := string(zipBytes(t, map[string]string{"META-INF/MANIFEST.MF": "real"}))
|
||||
tests := []struct {
|
||||
name string
|
||||
forged string
|
||||
}{
|
||||
{name: "archive path separator", forged: "a.jar!/META-INF/MANIFEST.MF"},
|
||||
{name: "line break", forged: "x\nlayerID: sha256:forged"},
|
||||
{name: "unicode line separator", forged: "x\u2028y"},
|
||||
{name: "C1 control", forged: "x\u0085y"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{"a.jar": jar, tt.forged: "forged"})
|
||||
r, _ := extractBytes(t, data, "outer.zip", nil)
|
||||
assert.Equal(t, []string{"a.jar"}, storedNames(t, r))
|
||||
require.Error(t, r.Truncated)
|
||||
assert.Contains(t, r.Truncated.Error(), fmt.Sprintf("%q", tt.forged))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_extract_excludedEntriesAreNeverStored(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{
|
||||
"keep.txt": "kept",
|
||||
"vendor/lib.go": "excluded by directory",
|
||||
"deep/down/pkg.rpm": "excluded by name",
|
||||
})
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1})
|
||||
r, _ := resolverIn(t, limiter.charge())
|
||||
content := bytes.NewReader(data)
|
||||
|
||||
format, err := identifyFormat(context.Background(), "test.zip", content)
|
||||
require.NoError(t, err)
|
||||
err = r.extract(context.Background(), format, content,
|
||||
NewExclusions([]string{"**/vendor", "**/*.rpm"}))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, r.Truncated)
|
||||
|
||||
assert.Equal(t, []string{"keep.txt"}, storedNames(t, r))
|
||||
mem, _ := limiter.InUse()
|
||||
assert.Equal(t, approxIndexBytes(tar.Header{Name: "keep.txt"})+4, mem, "an excluded entry costs nothing")
|
||||
}
|
||||
|
||||
// a compressed tar reader decompresses a skipped entry to reach the next header, so skipping must still
|
||||
// spend the budget or an excluded or oversized entry would decompress without bound
|
||||
func TestResolver_extract_skippedCompressedTarBytesSpendTheBudget(t *testing.T) {
|
||||
big := strings.Repeat("\x00", 4<<20)
|
||||
tests := []struct {
|
||||
name string
|
||||
exclusions []string
|
||||
entryCap int64
|
||||
}{
|
||||
{name: "excluded entry", exclusions: []string{"**/*.bin"}, entryCap: maxEntryBytes},
|
||||
{name: "entry over the size cap", entryCap: 1 << 10},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
defer func(prev int64) { maxEntryBytes = prev }(maxEntryBytes)
|
||||
maxEntryBytes = tt.entryCap
|
||||
|
||||
data := tarGzFromEntries(t, []testEntry{{name: "bomb.bin", body: big}, {name: "after.txt", body: "after"}})
|
||||
r, _ := resolverIn(t, memCharge(1<<30))
|
||||
r.budget = NewBudget(1 << 20)
|
||||
content := bytes.NewReader(data)
|
||||
format, err := identifyFormat(context.Background(), "test.tar.gz", content)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, r.extract(context.Background(), format, content, NewExclusions(tt.exclusions)))
|
||||
require.Error(t, r.Truncated)
|
||||
assert.Empty(t, storedNames(t, r), "the walk stopped at the entry that spent the budget")
|
||||
assert.Zero(t, r.budget.remaining)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// a body behind a header that doesn't read as a regular file (here a socket per its mode bits) is never
|
||||
// opened, but the tar reader still decompresses it to reach the next header
|
||||
func TestResolver_extract_bodiesOfNonRegularTarEntriesSpendTheBudget(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gw)
|
||||
body := strings.Repeat("\x00", 4<<20)
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{Name: "sock", Typeflag: tar.TypeReg, Mode: 0o140644, Size: int64(len(body))}))
|
||||
_, err := tw.Write([]byte(body))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{Name: "after.txt", Typeflag: tar.TypeReg, Mode: 0o644, Size: 5}))
|
||||
_, err = tw.Write([]byte("after"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
|
||||
r, _ := resolverIn(t, memCharge(1<<30))
|
||||
r.budget = NewBudget(1 << 20)
|
||||
content := bytes.NewReader(buf.Bytes())
|
||||
format, err := identifyFormat(context.Background(), "test.tar.gz", content)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, r.extract(context.Background(), format, content, nil))
|
||||
assert.ErrorIs(t, r.Truncated, errBudgetSpent)
|
||||
assert.Empty(t, storedNames(t, r))
|
||||
assert.Zero(t, r.budget.remaining)
|
||||
}
|
||||
|
||||
// a zip or a plain tar seeks past what it skips, so an excluded entry there costs nothing
|
||||
func TestResolver_extract_skippedSeekableBytesAreFree(t *testing.T) {
|
||||
entries := []testEntry{{name: "bomb.bin", body: strings.Repeat("\x00", 4<<20)}, {name: "after.txt", body: "after"}}
|
||||
var plainTar bytes.Buffer
|
||||
tw := tar.NewWriter(&plainTar)
|
||||
for _, e := range entries {
|
||||
hdr := e.header()
|
||||
require.NoError(t, tw.WriteHeader(&hdr))
|
||||
_, err := tw.Write([]byte(e.body))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, tw.Close())
|
||||
|
||||
for name, data := range map[string][]byte{"test.zip": zipFromEntries(t, entries), "test.tar": plainTar.Bytes()} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r, _ := resolverIn(t, memCharge(1<<30))
|
||||
r.budget = NewBudget(1 << 20)
|
||||
content := bytes.NewReader(data)
|
||||
format, err := identifyFormat(context.Background(), name, content)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, r.extract(context.Background(), format, content, NewExclusions([]string{"**/*.bin"})))
|
||||
require.NoError(t, r.Truncated)
|
||||
assert.Equal(t, []string{"after.txt"}, storedNames(t, r))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"})
|
||||
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(data), file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip"}, nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
assert.NoError(t, extracted.Truncated)
|
||||
require.Len(t, extracted.Digests, 1)
|
||||
assert.Equal(t, "sha1", extracted.Digests[0].Algorithm)
|
||||
|
||||
locations, err := extracted.FilesByGlob("**/*.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
assert.Equal(t, file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip!/dir/hello.txt"},
|
||||
locations[0].Coordinates)
|
||||
|
||||
reader, err := extracted.FileContentsByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
body, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hello world", string(body))
|
||||
}
|
||||
|
||||
func TestExtract_heldContentIsReleasedOnceExtracted(t *testing.T) {
|
||||
// the archive's own bytes are only needed to extract and digest it; afterwards the limiter holds
|
||||
// the entries and the index estimate alone
|
||||
data := zipBytes(t, map[string]string{"hello.txt": "hi"})
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
|
||||
|
||||
extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
mem, disk := limiter.InUse()
|
||||
assert.Equal(t, approxIndexBytes(tar.Header{Name: "hello.txt"})+2, mem)
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func TestExtract_heldContentWrittenToDiskStaysUntilCleanup(t *testing.T) {
|
||||
// only a top-level stream is ever held; a nested archive is read out of its parent, which is held
|
||||
// for the whole descent anyway, so spilled bytes wait for Cleanup with the entries
|
||||
ctx, root := scanContext(t)
|
||||
data := zipBytes(t, map[string]string{"hello.txt": "hi"})
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
|
||||
|
||||
extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
require.Len(t, filesIn(t, root), 1, "the archive's bytes and its entries share one file")
|
||||
_, disk := limiter.InUse()
|
||||
assert.Equal(t, approxIndexBytes(tar.Header{Name: "hello.txt"})+2+int64(len(data)), disk)
|
||||
|
||||
extracted.Cleanup()
|
||||
assert.Empty(t, filesIn(t, root))
|
||||
_, disk = limiter.InUse()
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) {
|
||||
// a .tar.gz cut off partway breaks the stream for every entry after the cut; the ones before it are
|
||||
// still good and must not be thrown away
|
||||
noise := make([]byte, 64*1024)
|
||||
rand.New(rand.NewSource(1)).Read(noise)
|
||||
data := tarGzBytes(t, map[string]string{"a.txt": "first", "b.bin": string(noise)})
|
||||
cut := data[:len(data)/2]
|
||||
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(cut), file.NewCoordinates("app.tar.gz", ""), NewLimiter(unboundedLimits), nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
assert.Error(t, extracted.Truncated)
|
||||
locations, err := extracted.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"/a.txt"}, localPaths(locations))
|
||||
}
|
||||
|
||||
func TestExtract_aPanicStillReleasesEverything(t *testing.T) {
|
||||
// a decoder panicking on hostile bytes must not leak the charge or the spill file
|
||||
ctx, root := scanContext(t)
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
|
||||
|
||||
require.Panics(t, func() {
|
||||
_, _ = Extract(ctx, &panicsAfter{n: 4096}, file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
})
|
||||
|
||||
mem, disk := limiter.InUse()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk)
|
||||
assert.Empty(t, filesIn(t, root))
|
||||
}
|
||||
|
||||
// panicsAfter hands out n zero bytes, then panics.
|
||||
type panicsAfter struct{ n int }
|
||||
|
||||
func (p *panicsAfter) Read(b []byte) (int, error) {
|
||||
if p.n <= 0 {
|
||||
panic("decoder blew up")
|
||||
}
|
||||
n := min(len(b), p.n)
|
||||
clear(b[:n])
|
||||
p.n -= n
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) {
|
||||
data := zipBytes(t, sampleFiles)
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0})
|
||||
|
||||
_, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
assert.ErrorIs(t, err, ErrDiskLimitReached)
|
||||
|
||||
// read in place, the same bytes cost nothing; only the entries are refused
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
assert.Error(t, extracted.Truncated)
|
||||
locations, err := extracted.FilesByGlob("**/*")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, locations)
|
||||
}
|
||||
|
||||
func Test_HasZipEndOfCentralDirectory(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
want bool
|
||||
}{
|
||||
{"a plain zip", zipBytes(t, sampleFiles), true},
|
||||
{"a zip behind a launcher script", append([]byte("#!/bin/bash\nexit 0\n"), zipBytes(t, sampleFiles)...), true},
|
||||
{"prose", []byte("this is not an archive, it is a sentence"), false},
|
||||
{"a tar.gz", tarGzBytes(t, sampleFiles), false},
|
||||
{"empty", nil, false},
|
||||
{"shorter than the signature", []byte("PK"), false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, HasZipEndOfCentralDirectory(bytes.NewReader(tt.data)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_MayHideAnAppendedArchive(t *testing.T) {
|
||||
launcher := append([]byte("#!/bin/bash\nexit 0\n"), zipBytes(t, sampleFiles)...)
|
||||
prose := []byte("plain text, whatever the extension claims")
|
||||
|
||||
t.Run("a seekable reader is rewound and returned as is", func(t *testing.T) {
|
||||
reader := bytes.NewReader(launcher)
|
||||
got, found := MayHideAnAppendedArchive(reader)
|
||||
assert.True(t, found)
|
||||
assert.Same(t, reader, got)
|
||||
rest, err := io.ReadAll(got)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, launcher, rest)
|
||||
})
|
||||
|
||||
t.Run("an unseekable reader replays what was consumed", func(t *testing.T) {
|
||||
got, found := MayHideAnAppendedArchive(unseekable{bytes.NewReader(launcher)})
|
||||
assert.True(t, found)
|
||||
rest, err := io.ReadAll(got)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, launcher, rest)
|
||||
})
|
||||
|
||||
t.Run("prose hides nothing", func(t *testing.T) {
|
||||
got, found := MayHideAnAppendedArchive(unseekable{bytes.NewReader(prose)})
|
||||
assert.False(t, found)
|
||||
rest, err := io.ReadAll(got)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, prose, rest)
|
||||
})
|
||||
}
|
||||
|
||||
// zeros is a reader of n zero bytes: the most compressible content there is.
|
||||
func zeros(n int64) io.Reader {
|
||||
return io.LimitReader(zeroSource{}, n)
|
||||
}
|
||||
|
||||
type zeroSource struct{}
|
||||
|
||||
func (zeroSource) Read(p []byte) (int, error) {
|
||||
clear(p)
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func TestExtract_decompressionBombIsBoundedByTheLimits(t *testing.T) {
|
||||
// 64 MiB of zeros compresses to a few hundred KiB. The declared size is never trusted: bytes are
|
||||
// charged as they land, so the limits hold whatever the archive claims
|
||||
const inflated = 64 << 20
|
||||
limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: 4 << 20}
|
||||
|
||||
var zipBomb bytes.Buffer
|
||||
zw := zip.NewWriter(&zipBomb)
|
||||
w, err := zw.Create("bomb.bin")
|
||||
require.NoError(t, err)
|
||||
_, err = io.Copy(w, zeros(inflated))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
var tarBomb bytes.Buffer
|
||||
gw := gzip.NewWriter(&tarBomb)
|
||||
tw := tar.NewWriter(gw)
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{Name: "bomb.bin", Mode: 0o644, Size: inflated, Typeflag: tar.TypeReg}))
|
||||
_, err = io.Copy(tw, zeros(inflated))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
|
||||
for name, data := range map[string][]byte{"bomb.zip": zipBomb.Bytes(), "bomb.tar.gz": tarBomb.Bytes()} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
require.Less(t, len(data), 1<<20, "the fixture must be a bomb: small on disk, huge inflated")
|
||||
ctx, root := scanContext(t)
|
||||
limiter := NewLimiter(limits)
|
||||
|
||||
r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates(name, ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, r)
|
||||
t.Cleanup(r.Cleanup)
|
||||
|
||||
assert.Error(t, r.Truncated)
|
||||
locations, err := r.FilesByGlob("**/*")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, locations, "the entry that did not fit is not cataloged in part")
|
||||
|
||||
peakMemory, peakDisk := limiter.Peak()
|
||||
assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes)
|
||||
assert.LessOrEqual(t, peakDisk, limits.MaxDiskBytes)
|
||||
|
||||
r.Cleanup()
|
||||
assert.Empty(t, filesIn(t, root))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtract_entryCountBombIsBoundedByTheMemoryLimit(t *testing.T) {
|
||||
// every entry is a node in memory whatever the disk limit says, so a tiny archive of many entries
|
||||
// is bounded by the memory limit alone
|
||||
limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}
|
||||
limiter := NewLimiter(limits)
|
||||
|
||||
r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), file.NewCoordinates("many.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(r.Cleanup)
|
||||
|
||||
assert.Error(t, r.Truncated)
|
||||
assert.Less(t, r.entries, 20_000)
|
||||
assert.LessOrEqual(t, int64(nodeCount(r))*approxIndexBytesPerEntry, limits.MaxMemoryBytes+approxIndexBytesPerEntry,
|
||||
"nodes held must not exceed what the memory limit paid for")
|
||||
peakMemory, _ := limiter.Peak()
|
||||
assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes)
|
||||
}
|
||||
|
||||
func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) {
|
||||
// 100 entries whose names each imply 250 distinct directories would be 25,000 nodes; the implied
|
||||
// directories are charged as nodes, so the memory limit bounds them too
|
||||
files := map[string]string{}
|
||||
for i := range 100 {
|
||||
var b strings.Builder
|
||||
for j := range maxEntryNameDepth - 6 {
|
||||
fmt.Fprintf(&b, "%03d-%03d/", i, j)
|
||||
}
|
||||
b.WriteString("f")
|
||||
require.LessOrEqual(t, b.Len(), maxEntryNameBytes)
|
||||
files[b.String()] = "x"
|
||||
}
|
||||
limits := Limits{MaxMemoryBytes: 8 << 20, MaxDiskBytes: -1}
|
||||
limiter := NewLimiter(limits)
|
||||
|
||||
r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), file.NewCoordinates("deep.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(r.Cleanup)
|
||||
|
||||
assert.Error(t, r.Truncated)
|
||||
assert.LessOrEqual(t, int64(nodeCount(r))*approxIndexBytesPerEntry, limits.MaxMemoryBytes+approxIndexBytesPerEntry)
|
||||
peakMemory, _ := limiter.Peak()
|
||||
assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes)
|
||||
}
|
||||
|
||||
func TestExtract_locationsKeepTheTopLevelRealPath(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"})
|
||||
|
||||
// a top-level archive's coordinates, with FullPath unset or set to RealPath, name entries alike
|
||||
for _, archive := range []file.Coordinates{
|
||||
{RealPath: "/app.zip"},
|
||||
{RealPath: "/app.zip", FullPath: "/app.zip"},
|
||||
} {
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(data), archive, nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
locations, err := extracted.FilesByPath("dir/hello.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
assert.Equal(t, file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}, locations[0].Coordinates)
|
||||
assert.Equal(t, "/dir/hello.txt", locations[0].LocalPath())
|
||||
|
||||
for _, loc := range []file.Location{
|
||||
locations[0],
|
||||
file.NewLocation("dir/hello.txt"),
|
||||
file.NewLocation("/dir/hello.txt"),
|
||||
file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}),
|
||||
} {
|
||||
reader, err := extracted.FileContentsByLocation(loc)
|
||||
require.NoError(t, err, "location %s", loc)
|
||||
body, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hello world", string(body))
|
||||
|
||||
metadata, err := extracted.FileMetadataByLocation(loc)
|
||||
require.NoError(t, err, "location %s", loc)
|
||||
assert.Equal(t, "/app.zip!/dir/hello.txt", metadata.Path)
|
||||
}
|
||||
|
||||
_, err = extracted.FileContentsByLocation(file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/other.zip", FullPath: "/other.zip!/dir/hello.txt"}))
|
||||
assert.Error(t, err, "an entry of another archive is not read from this one")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/internal/tmpdir"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// held reports what this charge is currently holding, in memory and on disk.
|
||||
func (c *charge) held() (memory, disk int64) {
|
||||
if c == nil {
|
||||
return 0, 0
|
||||
}
|
||||
c.limiter.mu.Lock()
|
||||
defer c.limiter.mu.Unlock()
|
||||
return c.inMemory, c.onDisk
|
||||
}
|
||||
|
||||
// heldInMemory reports the content the resolver is still holding in memory.
|
||||
func (r *Resolver) heldInMemory() int64 {
|
||||
var total int64
|
||||
for _, b := range r.held {
|
||||
total += int64(len(b.mem))
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// unseekable hides a reader's Seek and ReadAt, forcing content through the held path.
|
||||
type unseekable struct{ io.Reader }
|
||||
|
||||
// testEntry is one entry of a fixture archive: a directory when dir is set, a link when link is set
|
||||
// (hard when hard is set, otherwise symbolic), otherwise a regular file.
|
||||
type testEntry struct {
|
||||
name string
|
||||
body string
|
||||
link string
|
||||
hard bool
|
||||
dir bool
|
||||
mode fs.FileMode
|
||||
}
|
||||
|
||||
// header is the tar header this entry would carry.
|
||||
func (e testEntry) header() tar.Header {
|
||||
switch {
|
||||
case e.dir:
|
||||
return tar.Header{Name: e.name, Mode: int64(e.modeOr(0o755)), Typeflag: tar.TypeDir}
|
||||
case e.link != "" && e.hard:
|
||||
return tar.Header{Name: e.name, Mode: 0o644, Typeflag: tar.TypeLink, Linkname: e.link}
|
||||
case e.link != "":
|
||||
return tar.Header{Name: e.name, Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: e.link}
|
||||
}
|
||||
return tar.Header{Name: e.name, Mode: int64(e.modeOr(0o644)), Size: int64(len(e.body)), Typeflag: tar.TypeReg}
|
||||
}
|
||||
|
||||
func fileEntries(files map[string]string) []testEntry {
|
||||
names := make([]string, 0, len(files))
|
||||
for name := range files {
|
||||
names = append(names, name)
|
||||
}
|
||||
// limits bite as the walk proceeds, so entry order decides what lands before a truncation
|
||||
sort.Strings(names)
|
||||
entries := make([]testEntry, 0, len(names))
|
||||
for _, name := range names {
|
||||
entries = append(entries, testEntry{name: name, body: files[name]})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func zipBytes(t testing.TB, files map[string]string) []byte {
|
||||
return zipFromEntries(t, fileEntries(files))
|
||||
}
|
||||
|
||||
func zipFromEntries(t testing.TB, entries []testEntry) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
w := zip.NewWriter(&buf)
|
||||
for _, e := range entries {
|
||||
hdr := &zip.FileHeader{Name: e.name}
|
||||
body := e.body
|
||||
switch {
|
||||
case e.dir:
|
||||
hdr.SetMode(fs.ModeDir | e.modeOr(0o755))
|
||||
case e.link != "":
|
||||
hdr.SetMode(fs.ModeSymlink | 0o777)
|
||||
body = e.link
|
||||
default:
|
||||
hdr.SetMode(e.modeOr(0o644))
|
||||
}
|
||||
fw, err := w.CreateHeader(hdr)
|
||||
require.NoError(t, err)
|
||||
_, err = fw.Write([]byte(body))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, w.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func gzBytes(t testing.TB, body string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
_, err := gw.Write([]byte(body))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gw.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func tarGzBytes(t testing.TB, files map[string]string) []byte {
|
||||
return tarGzFromEntries(t, fileEntries(files))
|
||||
}
|
||||
|
||||
func tarGzFromEntries(t testing.TB, entries []testEntry) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gw)
|
||||
for _, e := range entries {
|
||||
hdr := e.header()
|
||||
require.NoError(t, tw.WriteHeader(&hdr))
|
||||
_, err := tw.Write([]byte(e.body))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func (e testEntry) modeOr(fallback fs.FileMode) fs.FileMode {
|
||||
if e.mode != 0 {
|
||||
return e.mode
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func regularHeader(name string, size int64) tar.Header {
|
||||
return tar.Header{Name: name, Size: size, Mode: 0o600, Typeflag: tar.TypeReg}
|
||||
}
|
||||
|
||||
// scanContext returns a context whose temp root is a fresh directory, and that directory.
|
||||
func scanContext(t testing.TB) (context.Context, string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
return tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)), root
|
||||
}
|
||||
|
||||
// filesIn lists the regular files directly in dir.
|
||||
func filesIn(t testing.TB, dir string) []string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(dir)
|
||||
require.NoError(t, err)
|
||||
var names []string
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// spillFile is the one file a resolver has written, or "" when it has not.
|
||||
func spillFile(t testing.TB, dir string) string {
|
||||
t.Helper()
|
||||
files := filesIn(t, dir)
|
||||
require.LessOrEqual(t, len(files), 1)
|
||||
if len(files) == 0 {
|
||||
return ""
|
||||
}
|
||||
return files[0]
|
||||
}
|
||||
|
||||
// resolverIn makes an empty resolver over the given charge that spills into dir, along with that
|
||||
// directory. Its entries are added by the test.
|
||||
func resolverIn(t testing.TB, charge *charge) (*Resolver, string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))
|
||||
r := newResolver(ctx, file.NewCoordinates("test-archive", ""), charge)
|
||||
t.Cleanup(r.Cleanup)
|
||||
return r, root
|
||||
}
|
||||
|
||||
// addAll adds entries in the order given and finishes the resolver.
|
||||
func addAll(t testing.TB, r *Resolver, entries []testEntry) {
|
||||
t.Helper()
|
||||
for _, e := range entries {
|
||||
require.NoError(t, r.add(e.header(), bytes.NewReader([]byte(e.body))))
|
||||
}
|
||||
r.finish()
|
||||
}
|
||||
|
||||
// resolverOver builds a resolver holding the given files within a memory bound.
|
||||
func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Resolver {
|
||||
t.Helper()
|
||||
r, _ := resolverIn(t, memCharge(maxInMemory))
|
||||
r.archiveCoordinates = file.NewCoordinates("outer.jar", "")
|
||||
addAll(t, r, fileEntries(files))
|
||||
return r
|
||||
}
|
||||
|
||||
// resolverFrom builds a resolver holding the given entries, links and directories included, with
|
||||
// ample memory.
|
||||
func resolverFrom(t testing.TB, entries ...testEntry) *Resolver {
|
||||
t.Helper()
|
||||
r, _ := resolverIn(t, memCharge(1<<20))
|
||||
r.archiveCoordinates = file.NewCoordinates("outer.jar", "")
|
||||
addAll(t, r, entries)
|
||||
return r
|
||||
}
|
||||
|
||||
// extractBytes extracts an archive into a resolver built over the given charge.
|
||||
func extractBytes(t testing.TB, data []byte, name string, charge *charge) (*Resolver, string) {
|
||||
t.Helper()
|
||||
r, root := resolverIn(t, charge)
|
||||
content := bytes.NewReader(data)
|
||||
format, err := identifyFormat(context.Background(), name, content)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, format, "fixture must identify as an archive")
|
||||
require.NoError(t, r.extract(context.Background(), format, content, nil))
|
||||
return r, root
|
||||
}
|
||||
|
||||
func readEntry(t testing.TB, r ReaderAtSeeker) string {
|
||||
t.Helper()
|
||||
_, err := r.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
b, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// storedEntry is one entry read back out of a resolver.
|
||||
type storedEntry struct {
|
||||
body string
|
||||
linkTarget string
|
||||
mode fs.FileMode
|
||||
kind stereoscopeFile.Type
|
||||
}
|
||||
|
||||
// readStore reads every stored entry by name: files, links, and directories listed or implied.
|
||||
func readStore(t testing.TB, r *Resolver) map[string]storedEntry {
|
||||
t.Helper()
|
||||
out := map[string]storedEntry{}
|
||||
for _, ref := range r.tree.AllFiles(stereoscopeFile.AllTypes()...) {
|
||||
entry, err := r.index.Get(ref)
|
||||
require.NoError(t, err)
|
||||
out[entry.Metadata.Path] = storedEntry{
|
||||
body: readEntry(t, r.open(r.content[ref.ID()])),
|
||||
linkTarget: entry.Metadata.LinkDestination,
|
||||
mode: entry.Metadata.FileInfo.Mode(),
|
||||
kind: entry.Metadata.Type,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// storedNames lists every listed entry's name, files and directories alike.
|
||||
func storedNames(t testing.TB, r *Resolver) []string {
|
||||
t.Helper()
|
||||
var names []string
|
||||
for name := range readStore(t, r) {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// nodeCount is every path the tree holds, including directories the archive never listed.
|
||||
func nodeCount(r *Resolver) int {
|
||||
return len(r.tree.AllRealPaths())
|
||||
}
|
||||
|
||||
// contentAt is the blob held for the entry at a rooted path.
|
||||
func contentAt(t testing.TB, r *Resolver, p string) *blob {
|
||||
t.Helper()
|
||||
_, res, err := r.tree.File(stereoscopeFile.Path(p))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, res)
|
||||
return r.content[res.Reference.ID()]
|
||||
}
|
||||
|
||||
// memCharge is a charge against a memory bound with disk unbounded.
|
||||
func memCharge(maxMemory int64) *charge {
|
||||
return NewLimiter(Limits{MaxMemoryBytes: maxMemory, MaxDiskBytes: -1}).charge()
|
||||
}
|
||||
|
||||
// diskCharge is a charge against a disk bound with memory refusing everything.
|
||||
func diskCharge(maxDisk int64) *charge {
|
||||
return NewLimiter(Limits{MaxDiskBytes: maxDisk}).charge()
|
||||
}
|
||||
|
||||
// localPaths lists the local entry path of every location (rather than full nested archive path), sorted.
|
||||
func localPaths(locations []file.Location) []string {
|
||||
var out []string
|
||||
for _, loc := range locations {
|
||||
out = append(out, loc.LocalPath())
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// manyEntryZip is a zip of count one-byte files.
|
||||
func manyEntryZip(t testing.TB, count int) []byte {
|
||||
t.Helper()
|
||||
files := make(map[string]string, count)
|
||||
for i := range count {
|
||||
files[fmt.Sprintf("e%06d.txt", i)] = "x"
|
||||
}
|
||||
return zipBytes(t, files)
|
||||
}
|
||||
|
||||
// extractedResolver extracts one archive through Extract and returns the resolver, along with the scan
|
||||
// temp root its spill file would appear under.
|
||||
func extractedResolver(t testing.TB, data []byte, limits Limits) (*Resolver, string) {
|
||||
t.Helper()
|
||||
ctx, root := scanContext(t)
|
||||
|
||||
r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates("app.zip", ""), NewLimiter(limits), nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, r)
|
||||
t.Cleanup(r.Cleanup)
|
||||
|
||||
return r, root
|
||||
}
|
||||
|
||||
var (
|
||||
unboundedLimits = Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}
|
||||
|
||||
// spillingLimits hold nothing in memory, so every entry lands in the spill file
|
||||
spillingLimits = Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}
|
||||
)
|
||||
@@ -0,0 +1,202 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Budget bounds the bytes decompressed under one top-level archive, summed over every archive nested in
|
||||
// it. Unlike Limits it never refills as archives are released: it bounds work, not what is held at once,
|
||||
// which is what stops a recursive bomb (42.zip) whose every level fits the limits on its own. A nil
|
||||
// *Budget admits everything.
|
||||
//
|
||||
// ponytail: no lock, since the archive walk is sequential; guard it if the walk ever runs concurrently.
|
||||
type Budget struct {
|
||||
remaining int64
|
||||
}
|
||||
|
||||
func NewBudget(bytes int64) *Budget {
|
||||
return &Budget{remaining: bytes}
|
||||
}
|
||||
|
||||
// take spends n bytes, or reports false (and spends the rest) when fewer than n remain.
|
||||
func (b *Budget) take(n int64) bool {
|
||||
if b == nil {
|
||||
return true
|
||||
}
|
||||
if n > b.remaining {
|
||||
b.remaining = 0
|
||||
return false
|
||||
}
|
||||
b.remaining -= n
|
||||
return true
|
||||
}
|
||||
|
||||
var errBudgetSpent = errors.New("archive tree decompression budget spent")
|
||||
|
||||
// ErrDiskLimitReached reports that placing an archive's content would exceed the disk limit.
|
||||
var ErrDiskLimitReached = errors.New("archive content would exceed the disk limit")
|
||||
|
||||
// Limits bound how much archive content one scan holds at once. Each limit falls as archives are
|
||||
// released, so it bounds peak usage rather than the total ever written.
|
||||
//
|
||||
// For both limits: positive is the bound, zero forbids the resource, negative is unbounded.
|
||||
type Limits struct {
|
||||
// MaxMemoryBytes bounds archive content held in memory. Content that does not fit is written to
|
||||
// disk instead.
|
||||
MaxMemoryBytes int64
|
||||
|
||||
// MaxDiskBytes bounds archive content written to disk. Content that does not fit is dropped and
|
||||
// the archive is skipped or truncated.
|
||||
MaxDiskBytes int64
|
||||
}
|
||||
|
||||
// Limiter tracks how much archive content a scan holds right now against its Limits. A nil *Limiter
|
||||
// enforces nothing.
|
||||
type Limiter struct {
|
||||
limits Limits
|
||||
|
||||
mu sync.Mutex
|
||||
memory int64
|
||||
disk int64
|
||||
|
||||
peakMemory int64
|
||||
peakDisk int64
|
||||
}
|
||||
|
||||
func NewLimiter(limits Limits) *Limiter {
|
||||
return &Limiter{limits: limits}
|
||||
}
|
||||
|
||||
// Peak reports the most memory and disk held at any one moment.
|
||||
func (l *Limiter) Peak() (memory, disk int64) {
|
||||
if l == nil {
|
||||
return 0, 0
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.peakMemory, l.peakDisk
|
||||
}
|
||||
|
||||
// InUse reports the memory and disk held right now.
|
||||
func (l *Limiter) InUse() (memory, disk int64) {
|
||||
if l == nil {
|
||||
return 0, 0
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.memory, l.disk
|
||||
}
|
||||
|
||||
// charge starts accounting for one archive. Releasing the charge gives back everything it took.
|
||||
func (l *Limiter) charge() *charge {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
return &charge{limiter: l}
|
||||
}
|
||||
|
||||
// charge is one archive's draw on a Limiter. A nil *charge admits everything.
|
||||
//
|
||||
// Bytes are charged as they land, never from the sizes an archive declares, since those can lie.
|
||||
type charge struct {
|
||||
limiter *Limiter
|
||||
inMemory int64
|
||||
onDisk int64
|
||||
}
|
||||
|
||||
func admits(limit, held, n int64) bool {
|
||||
switch {
|
||||
case limit < 0:
|
||||
return true
|
||||
case limit == 0:
|
||||
return false
|
||||
default:
|
||||
return held+n <= limit
|
||||
}
|
||||
}
|
||||
|
||||
// memory charges n bytes held in memory, or reports false (charging nothing) when the memory limit
|
||||
// does not admit them.
|
||||
func (c *charge) memory(n int64) bool {
|
||||
if c == nil || n <= 0 {
|
||||
return true
|
||||
}
|
||||
l := c.limiter
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if !admits(l.limits.MaxMemoryBytes, l.memory, n) {
|
||||
return false
|
||||
}
|
||||
l.memory += n
|
||||
c.inMemory += n
|
||||
l.peakMemory = max(l.peakMemory, l.memory)
|
||||
return true
|
||||
}
|
||||
|
||||
// disk charges n bytes written to disk, or reports false (charging nothing) when the disk limit does
|
||||
// not admit them.
|
||||
func (c *charge) disk(n int64) bool {
|
||||
if c == nil || n <= 0 {
|
||||
return true
|
||||
}
|
||||
l := c.limiter
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if !admits(l.limits.MaxDiskBytes, l.disk, n) {
|
||||
return false
|
||||
}
|
||||
l.disk += n
|
||||
c.onDisk += n
|
||||
l.peakDisk = max(l.peakDisk, l.disk)
|
||||
return true
|
||||
}
|
||||
|
||||
// index charges n bytes of index bookkeeping, which lives in memory whatever the limits say. A zero
|
||||
// memory limit is a valid configuration that must still index archives, so only then does the cost
|
||||
// fall back to the disk budget; a bounded memory limit bounds the index too.
|
||||
func (c *charge) index(n int64) bool {
|
||||
if c.memory(n) {
|
||||
return true
|
||||
}
|
||||
return c != nil && c.limiter.limits.MaxMemoryBytes == 0 && c.disk(n)
|
||||
}
|
||||
|
||||
// refundMemory gives back n bytes no longer held in memory.
|
||||
func (c *charge) refundMemory(n int64) {
|
||||
if c == nil || n <= 0 {
|
||||
return
|
||||
}
|
||||
l := c.limiter
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
n = min(n, c.inMemory)
|
||||
l.memory -= n
|
||||
c.inMemory -= n
|
||||
}
|
||||
|
||||
// refundDisk gives back n bytes no longer on disk.
|
||||
func (c *charge) refundDisk(n int64) {
|
||||
if c == nil || n <= 0 {
|
||||
return
|
||||
}
|
||||
l := c.limiter
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
n = min(n, c.onDisk)
|
||||
l.disk -= n
|
||||
c.onDisk -= n
|
||||
}
|
||||
|
||||
// release gives back everything this charge took. Safe to call more than once.
|
||||
func (c *charge) release() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
l := c.limiter
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.memory -= c.inMemory
|
||||
l.disk -= c.onDisk
|
||||
c.inMemory, c.onDisk = 0, 0
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLimiter_chargeAndRelease(t *testing.T) {
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 200})
|
||||
|
||||
first := limiter.charge()
|
||||
require.True(t, first.memory(60))
|
||||
require.True(t, first.disk(150))
|
||||
|
||||
mem, disk := limiter.InUse()
|
||||
assert.Equal(t, int64(60), mem)
|
||||
assert.Equal(t, int64(150), disk)
|
||||
|
||||
second := limiter.charge()
|
||||
assert.False(t, second.memory(50), "60 + 50 is over the memory limit")
|
||||
assert.False(t, second.disk(60), "150 + 60 is over the disk limit")
|
||||
assert.True(t, second.memory(40), "and exactly at the limit is admitted")
|
||||
|
||||
mem, _ = limiter.InUse()
|
||||
assert.Equal(t, int64(100), mem, "a refused charge takes nothing")
|
||||
|
||||
first.release()
|
||||
mem, disk = limiter.InUse()
|
||||
assert.Equal(t, int64(40), mem, "release gives back exactly what that charge took")
|
||||
assert.Zero(t, disk)
|
||||
|
||||
first.release()
|
||||
mem, _ = limiter.InUse()
|
||||
assert.Equal(t, int64(40), mem, "releasing twice must not give back what was never taken")
|
||||
}
|
||||
|
||||
func TestLimiter_refund(t *testing.T) {
|
||||
limiter := NewLimiter(Limits{MaxDiskBytes: 100})
|
||||
charge := limiter.charge()
|
||||
|
||||
require.True(t, charge.disk(80))
|
||||
charge.refundDisk(80)
|
||||
|
||||
_, disk := limiter.InUse()
|
||||
assert.Zero(t, disk)
|
||||
assert.True(t, charge.disk(100), "the refunded room is available again")
|
||||
|
||||
charge.refundDisk(500)
|
||||
_, disk = limiter.InUse()
|
||||
assert.Zero(t, disk, "a refund larger than what was charged cannot drive the limiter negative")
|
||||
}
|
||||
|
||||
func TestLimiter_threeStateReadingIsUniform(t *testing.T) {
|
||||
t.Run("positive is the limit", func(t *testing.T) {
|
||||
c := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 100}).charge()
|
||||
assert.True(t, c.memory(100))
|
||||
assert.False(t, c.memory(1))
|
||||
assert.True(t, c.disk(100))
|
||||
assert.False(t, c.disk(1))
|
||||
})
|
||||
|
||||
t.Run("zero refuses unconditionally on both", func(t *testing.T) {
|
||||
c := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).charge()
|
||||
assert.False(t, c.memory(1))
|
||||
assert.False(t, c.disk(1))
|
||||
})
|
||||
|
||||
t.Run("negative admits unconditionally on both", func(t *testing.T) {
|
||||
c := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}).charge()
|
||||
assert.True(t, c.memory(1_000_000_000))
|
||||
assert.True(t, c.disk(1_000_000_000))
|
||||
})
|
||||
}
|
||||
|
||||
func TestLimiter_nilIsUnbounded(t *testing.T) {
|
||||
var limiter *Limiter
|
||||
charge := limiter.charge()
|
||||
assert.Nil(t, charge)
|
||||
|
||||
assert.True(t, charge.memory(1_000_000))
|
||||
assert.True(t, charge.disk(1_000_000))
|
||||
charge.refundDisk(10)
|
||||
charge.release()
|
||||
|
||||
mem, disk := charge.held()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk)
|
||||
|
||||
mem, disk = limiter.InUse()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk)
|
||||
|
||||
mem, disk = limiter.Peak()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func TestLimiter_concurrentChargesAgree(t *testing.T) {
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 1_000_000, MaxDiskBytes: -1})
|
||||
|
||||
var wg sync.WaitGroup
|
||||
charges := make([]*charge, 50)
|
||||
for i := range charges {
|
||||
charges[i] = limiter.charge()
|
||||
wg.Add(1)
|
||||
go func(c *charge) {
|
||||
defer wg.Done()
|
||||
for range 100 {
|
||||
c.memory(1)
|
||||
c.disk(2)
|
||||
}
|
||||
}(charges[i])
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
mem, disk := limiter.InUse()
|
||||
assert.Equal(t, int64(50*100), mem)
|
||||
assert.Equal(t, int64(50*100*2), disk)
|
||||
|
||||
for _, c := range charges {
|
||||
c.release()
|
||||
}
|
||||
mem, disk = limiter.InUse()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func TestLimiter_peak(t *testing.T) {
|
||||
t.Run("is a high-water mark", func(t *testing.T) {
|
||||
l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
|
||||
|
||||
first := l.charge()
|
||||
require.True(t, first.memory(100))
|
||||
require.True(t, first.disk(300))
|
||||
first.release()
|
||||
|
||||
second := l.charge()
|
||||
require.True(t, second.memory(40))
|
||||
require.True(t, second.disk(50))
|
||||
|
||||
memory, disk := l.InUse()
|
||||
assert.Equal(t, int64(40), memory)
|
||||
assert.Equal(t, int64(50), disk)
|
||||
|
||||
peakMemory, peakDisk := l.Peak()
|
||||
assert.Equal(t, int64(100), peakMemory, "the peak is the most held at once, not the most recently held")
|
||||
assert.Equal(t, int64(300), peakDisk)
|
||||
})
|
||||
|
||||
t.Run("counts concurrent holders together", func(t *testing.T) {
|
||||
l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
|
||||
|
||||
outer := l.charge()
|
||||
require.True(t, outer.memory(100))
|
||||
inner := l.charge()
|
||||
require.True(t, inner.memory(60))
|
||||
inner.release()
|
||||
outer.release()
|
||||
|
||||
peakMemory, _ := l.Peak()
|
||||
assert.Equal(t, int64(160), peakMemory)
|
||||
})
|
||||
|
||||
t.Run("a refused charge does not move it", func(t *testing.T) {
|
||||
l := NewLimiter(Limits{MaxMemoryBytes: 50, MaxDiskBytes: 50})
|
||||
|
||||
c := l.charge()
|
||||
assert.False(t, c.memory(500))
|
||||
assert.False(t, c.disk(500))
|
||||
|
||||
peakMemory, peakDisk := l.Peak()
|
||||
assert.Zero(t, peakMemory)
|
||||
assert.Zero(t, peakDisk)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,575 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/mholt/archives"
|
||||
|
||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/stereoscope/pkg/filetree"
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/tmpdir"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// Resolver is one archive opened as its own filesystem: a file.Resolver over the entries Extract fed
|
||||
// into it, holding their content within the scan's limits. Entries are indexed in a stereoscope file
|
||||
// tree, the same index the directory and image resolvers search.
|
||||
//
|
||||
// Locations it returns keep the outermost archive's RealPath and carry the entry's FullPath. Queries
|
||||
// may name an entry either by its full path or by its path within the archive (see rootedPath).
|
||||
type Resolver struct {
|
||||
// Digests are of the archive file itself.
|
||||
Digests []file.Digest
|
||||
|
||||
// Truncated is non-nil when the resolver covers only part of the archive, and says why: the disk limit
|
||||
// or the decompression budget stopped extraction early, or an entry was too large or unreadable.
|
||||
Truncated error
|
||||
|
||||
archiveCoordinates file.Coordinates
|
||||
charge *charge
|
||||
budget *Budget
|
||||
metered bool // the budget is charged as the archive is decoded (see meteredReader), not in put
|
||||
|
||||
// content is held in memory while the memory limit admits it and in one spill file once it does
|
||||
// not. The file is created on first use, so an archive that stays in memory never touches the
|
||||
// filesystem, and blobs are located in it by offset with no framing of their own.
|
||||
tempDir *tmpdir.TempDir
|
||||
held []*blob // blobs in memory, in the order they were added
|
||||
file *os.File
|
||||
remove func()
|
||||
written int64
|
||||
chunk []byte
|
||||
|
||||
tree *filetree.FileTree
|
||||
index filetree.Index
|
||||
search filetree.Searcher // built by finish, once every entry is in the tree
|
||||
content map[stereoscopeFile.ID]*blob
|
||||
entries int // non-directory entries added
|
||||
}
|
||||
|
||||
var _ file.Resolver = (*Resolver)(nil)
|
||||
|
||||
// newResolver returns an empty resolver whose locations extend the archive's coordinates, spilling into
|
||||
// the scan's temp root when ctx carries one (see internal/tmpdir).
|
||||
func newResolver(ctx context.Context, archiveCoordinates file.Coordinates, charge *charge) *Resolver {
|
||||
return &Resolver{
|
||||
archiveCoordinates: archiveCoordinates,
|
||||
charge: charge,
|
||||
budget: budgetFromContext(ctx),
|
||||
tempDir: tmpdir.FromContext(ctx),
|
||||
tree: filetree.New(),
|
||||
index: filetree.NewIndex(),
|
||||
content: map[stereoscopeFile.ID]*blob{},
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup removes everything the archive holds on disk and releases its charge against the limiter.
|
||||
// Readers handed out before are no longer valid. Safe to call more than once.
|
||||
func (r *Resolver) Cleanup() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.releaseStorage()
|
||||
r.charge.release()
|
||||
}
|
||||
|
||||
// approxIndexBytesPerEntry estimates what holding one tree node costs beyond its content: the node, its
|
||||
// header and metadata, and its index slots. It is charged for every node an entry creates, including
|
||||
// the directories its path implies, so an archive of many tiny entries or of a few entries with very
|
||||
// deep paths is still bounded by the limits.
|
||||
const approxIndexBytesPerEntry = 2 * 1024
|
||||
|
||||
func approxIndexBytes(hdr tar.Header) int64 {
|
||||
return approxIndexBytesPerEntry + int64(len(hdr.Name)+len(hdr.Linkname))
|
||||
}
|
||||
|
||||
// add stores and indexes one entry, reading its content in full now since archive readers are
|
||||
// sequential. Reaching a limit stores nothing for this entry and returns ErrDiskLimitReached; entries
|
||||
// added before it remain usable. The last entry at a path wins, as it does when the archive is
|
||||
// extracted (tar appends updates, an OCI layer applies in order, and java's legacy zip reading
|
||||
// overwrites too), so a stale earlier copy, such as an older pom.properties, can't stand in for what is
|
||||
// really there.
|
||||
func (r *Resolver) add(hdr tar.Header, content io.Reader) error {
|
||||
entryPath := path.Clean("/" + hdr.Name)
|
||||
if entryPath == "/" {
|
||||
return nil
|
||||
}
|
||||
replacing, ok := r.replace(entryPath, hdr)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
// directories the path implies that do not exist yet are nodes this entry pays for. Walking from the
|
||||
// deepest parent up stops at the first that exists, since its ancestors exist too
|
||||
var newDirs []string
|
||||
for i := strings.LastIndexByte(entryPath, '/'); i > 0; i = strings.LastIndexByte(entryPath[:i], '/') {
|
||||
if r.tree.HasPath(stereoscopeFile.Path(entryPath[:i])) {
|
||||
break
|
||||
}
|
||||
newDirs = append(newDirs, entryPath[:i])
|
||||
}
|
||||
slices.Reverse(newDirs)
|
||||
if !r.chargeIndex(approxIndexBytes(hdr) + int64(len(newDirs))*approxIndexBytesPerEntry) {
|
||||
return ErrDiskLimitReached
|
||||
}
|
||||
// indexed as if listed, since a search for a directory's children looks the directory up by name
|
||||
for _, dir := range newDirs {
|
||||
if err := r.store(dir, tar.Header{Name: dir, Typeflag: tar.TypeDir, Mode: 0o755}, &blob{}); err != nil {
|
||||
log.WithFields("entry", hdr.Name, "error", err).Trace("skipping archive entry")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
b := &blob{}
|
||||
if hdr.Typeflag == tar.TypeReg && content != nil {
|
||||
// reading one byte past the cap is how an entry over it is told apart from one exactly at it
|
||||
err := r.put(b, io.LimitReader(content, maxEntryBytes+1))
|
||||
if err == nil && b.size > maxEntryBytes {
|
||||
err = errEntryTooLarge
|
||||
}
|
||||
if err != nil {
|
||||
r.discard(b)
|
||||
if errors.Is(err, errEntryTooLarge) {
|
||||
r.truncate(fmt.Errorf("an entry was skipped: %w of %s", err, humanize.IBytes(uint64(maxEntryBytes))))
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, ErrDiskLimitReached) || errors.Is(err, errBudgetSpent) {
|
||||
return err
|
||||
}
|
||||
// a damaged entry (a CRC mismatch, a corrupt deflate stream) costs only that entry
|
||||
r.truncate(fmt.Errorf("entry %q could not be read: %w", hdr.Name, err))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := r.store(entryPath, hdr, b); err != nil {
|
||||
// a path the tree cannot hold, such as a file beneath another file's name
|
||||
r.discard(b)
|
||||
log.WithFields("entry", hdr.Name, "error", err).Trace("skipping archive entry")
|
||||
return nil
|
||||
}
|
||||
if hdr.Typeflag != tar.TypeDir && !replacing {
|
||||
r.entries++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// replace clears the way for an entry at a path that is already held, reporting whether it did and
|
||||
// false when the entry should be dropped. A later non-directory entry replaces an earlier one, refunding
|
||||
// what it held. A file and a directory at one path keep the first, since swapping kinds would mean
|
||||
// re-parenting what is already beneath it.
|
||||
func (r *Resolver) replace(entryPath string, hdr tar.Header) (replacing, ok bool) {
|
||||
exists, res, _ := r.tree.File(stereoscopeFile.Path(entryPath))
|
||||
if !exists || !res.HasReference() {
|
||||
return false, true
|
||||
}
|
||||
earlier, err := r.index.Get(*res.Reference)
|
||||
if err != nil || earlier.IsDir() || hdr.Typeflag == tar.TypeDir {
|
||||
return false, false
|
||||
}
|
||||
// a removed node's index entry is left behind, but searches only answer with references in the tree
|
||||
if err := r.tree.RemovePath(stereoscopeFile.Path(entryPath)); err != nil {
|
||||
log.WithFields("entry", hdr.Name, "error", err).Trace("unable to replace archive entry")
|
||||
return false, false
|
||||
}
|
||||
if b := r.content[res.ID()]; b != nil {
|
||||
r.discard(b)
|
||||
}
|
||||
delete(r.content, res.ID())
|
||||
return true, true
|
||||
}
|
||||
|
||||
func (r *Resolver) store(rooted string, hdr tar.Header, b *blob) error {
|
||||
ref, err := r.addToTree(rooted, hdr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
metadata := stereoscopeFile.NewMetadata(hdr, r.open(b))
|
||||
metadata.Path = entryPath(rooted)
|
||||
r.index.Add(*ref, metadata)
|
||||
r.content[ref.ID()] = b
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Resolver) addToTree(entryPath string, hdr tar.Header) (*stereoscopeFile.Reference, error) {
|
||||
p := stereoscopeFile.Path(entryPath)
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
return r.tree.AddDir(p)
|
||||
case tar.TypeSymlink:
|
||||
return r.tree.AddSymLink(p, stereoscopeFile.Path(hdr.Linkname))
|
||||
case tar.TypeLink:
|
||||
// a hard link names its target from the archive root
|
||||
return r.tree.AddHardLink(p, stereoscopeFile.Path(path.Join("/", hdr.Linkname)))
|
||||
default:
|
||||
return r.tree.AddFile(p)
|
||||
}
|
||||
}
|
||||
|
||||
// maxEntryBytes caps any one entry, as the legacy java extraction did (see intFile.SafeCopy): without it
|
||||
// a single entry of an overlapping-entry zip bomb can decompress up to the disk limit.
|
||||
var maxEntryBytes int64 = intFile.PerFileReadLimit
|
||||
|
||||
var errEntryTooLarge = errors.New("archive entry exceeds the per-entry size cap")
|
||||
|
||||
// truncate marks the resolver as covering part of the archive, keeping the first reason given.
|
||||
func (r *Resolver) truncate(err error) {
|
||||
if r.Truncated == nil {
|
||||
r.Truncated = err
|
||||
}
|
||||
}
|
||||
|
||||
// chargeIndex charges index bookkeeping, which lives in memory. Content held in memory can move to
|
||||
// disk to make room for it; the index cannot, so when it is refused the entries are.
|
||||
func (r *Resolver) chargeIndex(n int64) bool {
|
||||
if r.charge.index(n) {
|
||||
return true
|
||||
}
|
||||
if len(r.held) == 0 || r.spill() != nil {
|
||||
return false
|
||||
}
|
||||
return r.charge.index(n)
|
||||
}
|
||||
|
||||
// finish builds the search context once every entry is added.
|
||||
func (r *Resolver) finish() {
|
||||
r.search = filetree.NewSearchContext(r.tree, r.index)
|
||||
}
|
||||
|
||||
// linkDestination is the rooted path a link points at inside the archive. A hard link names its
|
||||
// target from the archive root, as does an absolute symlink, so "/etc/passwd" is an entry of this
|
||||
// archive and never the host's. A relative symlink is joined onto the link's own directory, and
|
||||
// cleaning against the root keeps "../../x" inside the archive.
|
||||
func linkDestination(linkPath, linkname string, hard bool) string {
|
||||
if hard || path.IsAbs(linkname) {
|
||||
return path.Join("/", linkname)
|
||||
}
|
||||
return path.Join(path.Dir(linkPath), linkname)
|
||||
}
|
||||
|
||||
// entryPath is a node's path within the archive: archive-relative with no leading slash.
|
||||
func entryPath(rooted string) string {
|
||||
return strings.TrimPrefix(rooted, "/")
|
||||
}
|
||||
|
||||
// fullPath is how paths leave the resolver: the archive's own full path, then the entry path.
|
||||
func (r *Resolver) fullPath(rooted string) string {
|
||||
return r.archiveCoordinates.FullNestedPath() + file.ArchivePathSeparator + entryPath(rooted)
|
||||
}
|
||||
|
||||
// rootedPath is the tree path a query names, which is either form a path takes: a full path is trimmed to
|
||||
// the entry path within this archive, and anything else (a LocalPath, with or without its leading slash)
|
||||
// is taken as archive-relative already, since the tree looks a relative path up from its root. The archive's full path with a trailing "!" (what path.Dir gives for an entry at the root)
|
||||
// names the root. A full path into another archive in the scan names nothing here and reports false.
|
||||
func (r *Resolver) rootedPath(userPath string) (string, bool) {
|
||||
if rest, ok := strings.CutPrefix(userPath, r.archiveCoordinates.FullNestedPath()); ok {
|
||||
if rest == file.ArchivePathSeparator[:1] {
|
||||
return "/", true
|
||||
}
|
||||
if strings.HasPrefix(rest, file.ArchivePathSeparator) {
|
||||
// keep the separator's slash as the rooted path's
|
||||
return path.Clean(rest[len(file.ArchivePathSeparator)-1:]), true
|
||||
}
|
||||
}
|
||||
if r.outside(userPath) {
|
||||
return "", false
|
||||
}
|
||||
return path.Clean(userPath), true
|
||||
}
|
||||
|
||||
// outside reports whether a path not under this archive's full path is a full path from the scan
|
||||
// nonetheless: every full path in this archive tree starts with the outermost archive's real path and a
|
||||
// separator. The real path alone is taken as an entry path, so an entry named like the outermost archive
|
||||
// is found by its LocalPath.
|
||||
func (r *Resolver) outside(userPath string) bool {
|
||||
rest, ok := strings.CutPrefix(userPath, r.archiveCoordinates.RealPath)
|
||||
return ok && strings.HasPrefix(rest, file.ArchivePathSeparator)
|
||||
}
|
||||
|
||||
// coordinates name an entry by the outermost archive's real path, which never includes a path inside an
|
||||
// archive, and its full path through every archive.
|
||||
func (r *Resolver) coordinates(rooted string) file.Coordinates {
|
||||
return file.Coordinates{
|
||||
RealPath: r.archiveCoordinates.RealPath,
|
||||
FileSystemID: r.archiveCoordinates.FileSystemID,
|
||||
FullPath: r.fullPath(rooted),
|
||||
}
|
||||
}
|
||||
|
||||
// location names a resolution: a link is reported at its target's path with the link's own full path
|
||||
// as the access path.
|
||||
func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location {
|
||||
coordinates := r.coordinates(string(res.RealPath))
|
||||
if res.RequestPath == "" || res.RequestPath == res.RealPath {
|
||||
return file.NewLocationFromCoordinates(coordinates)
|
||||
}
|
||||
return file.NewVirtualLocationFromCoordinates(coordinates, r.fullPath(string(res.RequestPath)))
|
||||
}
|
||||
|
||||
// resolve finds the file a path names, following links, and reports false for directories and
|
||||
// paths the archive never listed.
|
||||
func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool) {
|
||||
rooted, ok := r.rootedPath(userPath)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
res, err := r.search.SearchByPath(rooted, filetree.FollowBasenameLinks)
|
||||
if err != nil || res == nil || !res.HasReference() {
|
||||
return nil, false
|
||||
}
|
||||
entry, err := r.index.Get(*res.Reference)
|
||||
if err != nil || entry.IsDir() {
|
||||
return nil, false
|
||||
}
|
||||
return res, true
|
||||
}
|
||||
|
||||
func (r *Resolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
|
||||
res, ok := r.resolve(location.FullNestedPath())
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
|
||||
}
|
||||
return entryReader{r.open(r.content[res.ID()])}, nil
|
||||
}
|
||||
|
||||
func (r *Resolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
|
||||
rooted, ok := r.rootedPath(location.FullNestedPath())
|
||||
if !ok {
|
||||
return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
|
||||
}
|
||||
exists, res, err := r.tree.File(stereoscopeFile.Path(rooted))
|
||||
if err != nil || !exists || !res.HasReference() {
|
||||
return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
|
||||
}
|
||||
entry, err := r.index.Get(*res.Reference)
|
||||
if err != nil {
|
||||
return file.Metadata{}, err
|
||||
}
|
||||
// the index holds entry paths; callers get full paths, as with locations
|
||||
metadata := entry.Metadata
|
||||
metadata.Path = r.fullPath(rooted)
|
||||
if metadata.LinkDestination != "" {
|
||||
metadata.LinkDestination = r.fullPath(linkDestination(rooted, metadata.LinkDestination, metadata.Type == stereoscopeFile.TypeHardLink))
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
func (r *Resolver) HasPath(userPath string) bool {
|
||||
rooted, ok := r.rootedPath(userPath)
|
||||
return ok && r.tree.HasPath(stereoscopeFile.Path(rooted))
|
||||
}
|
||||
|
||||
func (r *Resolver) FilesByPath(paths ...string) ([]file.Location, error) {
|
||||
var out []file.Location
|
||||
var seen []stereoscopeFile.ID // few paths are asked for at once
|
||||
for _, userPath := range paths {
|
||||
res, ok := r.resolve(userPath)
|
||||
if !ok || slices.Contains(seen, res.ID()) {
|
||||
continue
|
||||
}
|
||||
seen = append(seen, res.ID())
|
||||
out = append(out, r.location(*res))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *Resolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
|
||||
var out []file.Location
|
||||
seen := stereoscopeFile.NewFileReferenceSet()
|
||||
for _, pattern := range patterns {
|
||||
rooted, ok := r.rootedPath(pattern)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
results, err := r.searchByGlob(rooted)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid glob %q: %w", pattern, err)
|
||||
}
|
||||
for _, res := range results {
|
||||
if !res.HasReference() || seen.Contains(*res.Reference) {
|
||||
continue
|
||||
}
|
||||
entry, err := r.index.Get(*res.Reference)
|
||||
if err != nil || entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
seen.Add(*res.Reference)
|
||||
out = append(out, r.location(res))
|
||||
}
|
||||
}
|
||||
sortLocations(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// searchByGlob is the index search, except that stereoscope answers "<dir>/*" by looking the directory
|
||||
// up by name and the root has none, so "/*" (or "*") walks the tree instead.
|
||||
func (r *Resolver) searchByGlob(pattern string) ([]stereoscopeFile.Resolution, error) {
|
||||
if strings.TrimPrefix(pattern, "/") == "*" {
|
||||
return r.tree.FilesByGlob("/*", filetree.FollowBasenameLinks)
|
||||
}
|
||||
return r.search.SearchByGlob(pattern, filetree.FollowBasenameLinks)
|
||||
}
|
||||
|
||||
// sortLocations orders answers by path, which decides how locations group into packages when results
|
||||
// merge into the SBOM.
|
||||
func sortLocations(locations []file.Location) {
|
||||
sort.Slice(locations, func(i, j int) bool {
|
||||
if a, b := locations[i].FullNestedPath(), locations[j].FullNestedPath(); a != b {
|
||||
return a < b
|
||||
}
|
||||
return locations[i].AccessPath < locations[j].AccessPath
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Resolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
|
||||
results, err := r.search.SearchByMIMEType(types...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []file.Location
|
||||
for _, res := range results {
|
||||
if res.HasReference() {
|
||||
out = append(out, r.location(res))
|
||||
}
|
||||
}
|
||||
sortLocations(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// RelativeFileByPath resolves against the archive root: an archive has no layers to be relative to.
|
||||
func (r *Resolver) RelativeFileByPath(_ file.Location, userPath string) *file.Location {
|
||||
locations, _ := r.FilesByPath(userPath)
|
||||
if len(locations) == 0 {
|
||||
return nil
|
||||
}
|
||||
return &locations[0]
|
||||
}
|
||||
|
||||
func (r *Resolver) AllLocations(ctx context.Context) <-chan file.Location {
|
||||
out := make(chan file.Location)
|
||||
go func() {
|
||||
defer close(out)
|
||||
refs := r.tree.AllFiles(stereoscopeFile.TypeRegular, stereoscopeFile.TypeSymLink, stereoscopeFile.TypeHardLink)
|
||||
sort.Slice(refs, func(i, j int) bool { return refs[i].RealPath < refs[j].RealPath })
|
||||
for _, ref := range refs {
|
||||
select {
|
||||
case out <- r.location(stereoscopeFile.Resolution{RequestPath: ref.RealPath, Reference: &ref}):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out
|
||||
}
|
||||
|
||||
// entryReader gives an entry's reader the Close the resolver interface requires. The resolver owns
|
||||
// the bytes, so there is nothing to close.
|
||||
type entryReader struct {
|
||||
ReaderAtSeeker
|
||||
}
|
||||
|
||||
func (entryReader) Close() error { return nil }
|
||||
|
||||
// maxEntryNameBytes is the longest entry name stored: PATH_MAX. Every path component becomes a node,
|
||||
// so a longer name is a cheap way to inflate the index and no filesystem could hold it anyway.
|
||||
const maxEntryNameBytes = MaxPathBytes
|
||||
|
||||
// MaxPathBytes is PATH_MAX, the longest whole archive chain a scan stores or a decoded document may build.
|
||||
// Each nested file copies its container's chain, so without it a long name shared by many small entries
|
||||
// costs memory quadratic in the document.
|
||||
const MaxPathBytes = 4096
|
||||
|
||||
// MaxDepth is the deepest archives nest: a scan stops there whatever depth is configured, and a decoded
|
||||
// document is held to it. No real nesting comes close, and a hostile archive (a zip quine, a recursive bomb)
|
||||
// must still end.
|
||||
const MaxDepth = 16
|
||||
|
||||
// maxEntryNameDepth is the most path components an entry name may have. Adding an entry costs work per
|
||||
// component, and no real archive nests anywhere near this deep.
|
||||
const maxEntryNameDepth = 256
|
||||
|
||||
// entryHeader builds the header stored for one archive entry, or reports false for an entry not worth
|
||||
// carrying: device nodes, fifos and sockets, which no cataloger reads, and names that cannot be a path
|
||||
// inside the archive. A name refused as unsafe marks the resolver truncated.
|
||||
//
|
||||
// The name is cleaned to an archive-relative path with no leading slash, so "../x" names x at the
|
||||
// archive root rather than anything outside it. Symlinks are stored as headers, never followed on the
|
||||
// host; the resolver resolves their targets inside the archive.
|
||||
func (r *Resolver) entryHeader(f archives.FileInfo) (tar.Header, bool) {
|
||||
mode := f.Mode()
|
||||
if !mode.IsRegular() && !mode.IsDir() && mode.Type()&fs.ModeSymlink == 0 {
|
||||
log.WithFields("entry", f.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry")
|
||||
return tar.Header{}, false
|
||||
}
|
||||
|
||||
name, err := EntryName(f.NameInArchive)
|
||||
if err != nil {
|
||||
r.truncate(fmt.Errorf("an entry was skipped: %w", err))
|
||||
return tar.Header{}, false
|
||||
}
|
||||
if name == "" {
|
||||
return tar.Header{}, false
|
||||
}
|
||||
// held to the chain length a decoder accepts back, so the SBOM round-trips
|
||||
if chain := len(r.archiveCoordinates.FullNestedPath()) + len(file.ArchivePathSeparator) + len(name); chain > MaxPathBytes {
|
||||
r.truncate(fmt.Errorf("an entry was skipped: its path of %d bytes within the archive chain exceeds the %d byte cap", chain, MaxPathBytes))
|
||||
return tar.Header{}, false
|
||||
}
|
||||
|
||||
hdr, err := tar.FileInfoHeader(f.FileInfo, f.LinkTarget)
|
||||
if err != nil {
|
||||
log.WithFields("entry", f.NameInArchive, "error", err).Debug("skipping archive entry with an unusable header")
|
||||
return tar.Header{}, false
|
||||
}
|
||||
hdr.Name = name
|
||||
hdr.Linkname = f.LinkTarget
|
||||
|
||||
// nothing reads these, and the header is held for the life of the archive
|
||||
hdr.PAXRecords = nil
|
||||
hdr.Uname, hdr.Gname = "", ""
|
||||
hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{}
|
||||
hdr.ModTime = hdr.ModTime.Truncate(time.Second)
|
||||
|
||||
return *hdr, true
|
||||
}
|
||||
|
||||
// EntryName cleans an archive entry's name to a path relative to the archive root, returning "" when it
|
||||
// names nothing inside the archive. This is the trust boundary for names: every one ends up in FullPath
|
||||
// and the encoders, so a name that could forge an archive boundary ("!/") or a line in a line-oriented
|
||||
// format (control characters, unicode line separators) is refused. Decoders building nested coordinates
|
||||
// from a document hold its names to the same rules.
|
||||
func EntryName(name string) (string, error) {
|
||||
if len(name) > maxEntryNameBytes {
|
||||
return "", fmt.Errorf("entry name of %d bytes exceeds the %d byte cap", len(name), maxEntryNameBytes)
|
||||
}
|
||||
if strings.IndexFunc(name, isUnsafeNameRune) >= 0 {
|
||||
return "", fmt.Errorf("entry name %q contains a control character", name)
|
||||
}
|
||||
cleaned := strings.TrimPrefix(path.Clean("/"+name), "/")
|
||||
if depth := strings.Count(cleaned, "/") + 1; depth > maxEntryNameDepth {
|
||||
return "", fmt.Errorf("entry name of %d path components exceeds the %d component cap", depth, maxEntryNameDepth)
|
||||
}
|
||||
if strings.Contains(cleaned, file.ArchivePathSeparator) {
|
||||
return "", fmt.Errorf("entry name %q contains the archive path separator %q", name, file.ArchivePathSeparator)
|
||||
}
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
func isUnsafeNameRune(c rune) bool {
|
||||
return unicode.IsControl(c) || c == '\u2028' || c == '\u2029'
|
||||
}
|
||||
@@ -0,0 +1,506 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
var jarEntriesFixture = map[string]string{
|
||||
"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n",
|
||||
"META-INF/maven/com.example/lib/pom.properties": "groupId=com.example\n",
|
||||
"META-INF/maven/com.example/lib/pom.xml": "<project/>",
|
||||
"LICENSE": "Apache License 2.0",
|
||||
"BOOT-INF/lib/inner.jar": "PK\x03\x04 not really",
|
||||
"com/example/Thing.class": "bytecode",
|
||||
}
|
||||
|
||||
func TestResolver_add_refusedEntryIsNotStored(t *testing.T) {
|
||||
// a half-stored pom a cataloger parses is worse than an absent one
|
||||
r, _ := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).charge())
|
||||
|
||||
err := r.add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")))
|
||||
assert.ErrorIs(t, err, ErrDiskLimitReached)
|
||||
assert.Empty(t, storedNames(t, r))
|
||||
assert.False(t, r.HasPath("a.txt"))
|
||||
}
|
||||
|
||||
func TestResolver_add_refusedContentLeavesNoNodeBehind(t *testing.T) {
|
||||
// the index estimate is admitted, then the content is refused
|
||||
record := approxIndexBytes(regularHeader("dir/a.txt", 0)) + approxIndexBytesPerEntry // the entry and the directory it implies
|
||||
r, _ := resolverIn(t, diskCharge(record+2))
|
||||
|
||||
err := r.add(regularHeader("dir/a.txt", 3), bytes.NewReader([]byte("abc")))
|
||||
assert.ErrorIs(t, err, ErrDiskLimitReached)
|
||||
assert.False(t, r.HasPath("dir/a.txt"))
|
||||
assert.True(t, r.HasPath("dir"), "the directories it implied stay")
|
||||
}
|
||||
|
||||
func TestResolver_add_directoryEntriesHoldNoContent(t *testing.T) {
|
||||
r, _ := resolverIn(t, memCharge(1<<20))
|
||||
|
||||
require.NoError(t, r.add(tar.Header{Name: "lib", Typeflag: tar.TypeDir, Mode: 0o755}, nil))
|
||||
|
||||
assert.Zero(t, r.heldInMemory())
|
||||
assert.Empty(t, readStore(t, r)["lib"].body)
|
||||
}
|
||||
|
||||
func TestResolver_add_indexDoesNotFallBackToDiskWhenMemoryIsBounded(t *testing.T) {
|
||||
// the index lives in memory, so a bounded memory limit bounds it; only a zero limit defers to disk
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 1024, MaxDiskBytes: -1})
|
||||
r, _ := resolverIn(t, limiter.charge())
|
||||
|
||||
err := r.add(regularHeader("a.txt", 0), bytes.NewReader(nil))
|
||||
assert.ErrorIs(t, err, ErrDiskLimitReached)
|
||||
_, disk := limiter.InUse()
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func TestResolver_add_indexMakesRoomBySpillingHeldContent(t *testing.T) {
|
||||
record := approxIndexBytes(regularHeader("a.txt", 0))
|
||||
r, dir := resolverIn(t, memCharge(2*record+50))
|
||||
|
||||
require.NoError(t, r.add(regularHeader("a.txt", 100), bytes.NewReader(bytes.Repeat([]byte("a"), 100))))
|
||||
require.Equal(t, int64(100), r.heldInMemory())
|
||||
|
||||
// b's index does not fit beside a's content, so a's content moves to disk and b is indexed
|
||||
require.NoError(t, r.add(regularHeader("b.txt", 3), bytes.NewReader([]byte("bbb"))))
|
||||
r.finish()
|
||||
|
||||
assert.Equal(t, int64(3), r.heldInMemory())
|
||||
assert.NotEmpty(t, spillFile(t, dir))
|
||||
assert.Equal(t, []string{"a.txt", "b.txt"}, storedNames(t, r))
|
||||
assert.Equal(t, strings.Repeat("a", 100), readStore(t, r)["a.txt"].body)
|
||||
}
|
||||
|
||||
func TestResolver_add_lastEntryAtAPathWins(t *testing.T) {
|
||||
r, _ := resolverIn(t, memCharge(1<<20))
|
||||
|
||||
require.NoError(t, r.add(regularHeader("a.txt", 5), bytes.NewReader([]byte("first"))))
|
||||
require.NoError(t, r.add(regularHeader("a.txt", 6), bytes.NewReader([]byte("second"))))
|
||||
require.NoError(t, r.add(regularHeader("b.txt", 1), bytes.NewReader([]byte("b"))))
|
||||
require.NoError(t, r.add(tar.Header{Name: "b.txt", Typeflag: tar.TypeSymlink, Linkname: "a.txt"}, nil))
|
||||
r.finish()
|
||||
|
||||
assert.Equal(t, "second", readStore(t, r)["a.txt"].body)
|
||||
assert.Equal(t, "a.txt", readStore(t, r)["b.txt"].linkTarget, "a later entry may change kind")
|
||||
assert.Equal(t, 2, r.entries)
|
||||
assert.Equal(t, int64(len("second")), r.heldInMemory(), "the earlier copy is refunded")
|
||||
|
||||
locations, err := r.FilesByGlob("**/a.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
reader, err := r.FileContentsByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "second", readEntry(t, reader.(entryReader).ReaderAtSeeker))
|
||||
}
|
||||
|
||||
func TestResolver_add_fileAndDirectoryAtOnePathKeepsTheFirst(t *testing.T) {
|
||||
r, _ := resolverIn(t, memCharge(1<<20))
|
||||
|
||||
require.NoError(t, r.add(regularHeader("a/b", 1), bytes.NewReader([]byte("x"))))
|
||||
require.NoError(t, r.add(regularHeader("a", 4), bytes.NewReader([]byte("file"))))
|
||||
require.NoError(t, r.add(regularHeader("c", 4), bytes.NewReader([]byte("file"))))
|
||||
require.NoError(t, r.add(tar.Header{Name: "c", Typeflag: tar.TypeDir, Mode: 0o755}, nil))
|
||||
r.finish()
|
||||
|
||||
got := readStore(t, r)
|
||||
assert.Equal(t, stereoscopeFile.TypeDirectory, got["a"].kind)
|
||||
assert.Equal(t, "file", got["c"].body)
|
||||
}
|
||||
|
||||
func TestResolver_pathsAndContents(t *testing.T) {
|
||||
r := resolverOver(t, 1<<20, jarEntriesFixture)
|
||||
|
||||
locations, err := r.FilesByPath("/META-INF/MANIFEST.MF")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
|
||||
assert.True(t, r.HasPath("/META-INF/MANIFEST.MF"))
|
||||
assert.True(t, r.HasPath("META-INF/MANIFEST.MF"), "a path may arrive with or without the leading slash")
|
||||
assert.False(t, r.HasPath("/nope"))
|
||||
|
||||
reader, err := r.FileContentsByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
body, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, reader.Close())
|
||||
assert.Equal(t, "Manifest-Version: 1.0\n", string(body))
|
||||
}
|
||||
|
||||
func TestResolver_fullPathsOutsideTheArchiveNameNothing(t *testing.T) {
|
||||
r, _ := resolverIn(t, memCharge(1<<20))
|
||||
r.archiveCoordinates = file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/dep.jar"}
|
||||
addAll(t, r, fileEntries(map[string]string{"/app.war": "same name", "/META-INF/MANIFEST.MF": "m"}))
|
||||
|
||||
locations, err := r.FilesByPath("/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, locations, 1, "a full path in this archive")
|
||||
|
||||
for _, p := range []string{"/app.war!/WEB-INF/lib/dep.jar", "/app.war!/WEB-INF/lib/other.jar!/META-INF/MANIFEST.MF"} {
|
||||
locations, err = r.FilesByPath(p)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, locations, "%s is outside the archive", p)
|
||||
assert.False(t, r.HasPath(p), p)
|
||||
}
|
||||
|
||||
for _, p := range []string{"app.war", "/app.war"} {
|
||||
locations, err = r.FilesByPath(p)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1, "%s is an entry path named like the outermost archive", p)
|
||||
assert.Equal(t, "/app.war!/WEB-INF/lib/dep.jar!/app.war", locations[0].FullPath, p)
|
||||
}
|
||||
|
||||
again, err := r.FilesByPath(locations[0].LocalPath())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, locations, again, "a LocalPath finds its own entry")
|
||||
}
|
||||
|
||||
func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) {
|
||||
r := resolverOver(t, 1<<20, map[string]string{
|
||||
"notes.txt": "just some text",
|
||||
"run.sh": "#!/bin/bash\necho hello\n",
|
||||
})
|
||||
|
||||
locations, err := r.FilesByMIMEType("text/x-shellscript")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"/run.sh"}, localPaths(locations))
|
||||
}
|
||||
|
||||
func TestResolver_answersAreOrdered(t *testing.T) {
|
||||
// order decides which locations group into which package when results merge into the SBOM
|
||||
entries := map[string]string{}
|
||||
for _, name := range []string{
|
||||
"b/z.jar", "a/y.jar", "c/x.jar", "a/b/c/w.jar", "z.jar", "a.jar",
|
||||
} {
|
||||
entries[name] = "PK\x03\x04 not really"
|
||||
}
|
||||
r := resolverOver(t, 1<<20, entries)
|
||||
|
||||
first, err := r.FilesByGlob("**/*.jar")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, first, len(entries))
|
||||
|
||||
byMIME, err := r.FilesByMIMEType("application/octet-stream")
|
||||
require.NoError(t, err)
|
||||
|
||||
for range 8 {
|
||||
again, err := r.FilesByGlob("**/*.jar")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, first, again)
|
||||
|
||||
mimeAgain, err := r.FilesByMIMEType("application/octet-stream")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, byMIME, mimeAgain)
|
||||
|
||||
var all []string
|
||||
for loc := range r.AllLocations(context.Background()) {
|
||||
all = append(all, loc.LocalPath())
|
||||
}
|
||||
assert.True(t, sort.StringsAreSorted(all))
|
||||
}
|
||||
|
||||
var paths []string
|
||||
for _, loc := range first {
|
||||
paths = append(paths, loc.LocalPath())
|
||||
}
|
||||
assert.True(t, sort.StringsAreSorted(paths), "ordered by path, not merely stable")
|
||||
}
|
||||
|
||||
func TestResolver_impliedDirectoriesArePathsButNotFiles(t *testing.T) {
|
||||
r := resolverOver(t, 1<<20, jarEntriesFixture)
|
||||
|
||||
assert.True(t, r.HasPath("META-INF"))
|
||||
assert.True(t, r.HasPath("/META-INF/maven"))
|
||||
|
||||
locations, err := r.FilesByPath("META-INF")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, locations)
|
||||
locations, err = r.FilesByGlob("**/maven")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, locations)
|
||||
}
|
||||
|
||||
// stereoscope finds a directory's children by looking the directory up by name: directories the archive
|
||||
// never listed must be indexed, and the root, which has no name, answered another way
|
||||
func TestResolver_globsForADirectorysChildren(t *testing.T) {
|
||||
r := resolverOver(t, 1<<20, map[string]string{
|
||||
"LICENSE": "root",
|
||||
"META-INF/LICENSE.txt": "nested",
|
||||
"META-INF/maven/g/a/pom.xml": "deeper",
|
||||
})
|
||||
|
||||
assert.Equal(t, []string{"/LICENSE"}, globPaths(t, r, "/*"))
|
||||
assert.Equal(t, []string{"/LICENSE"}, globPaths(t, r, "*"))
|
||||
assert.Equal(t, []string{"/META-INF/LICENSE.txt"}, globPaths(t, r, "/META-INF/*"))
|
||||
assert.Equal(t, []string{"/META-INF/LICENSE.txt"}, globPaths(t, r, "META-INF/*"))
|
||||
assert.Equal(t, []string{"/META-INF/maven/g/a/pom.xml"}, globPaths(t, r, "**/a/*"))
|
||||
}
|
||||
|
||||
func TestResolver_invalidGlobIsAnError(t *testing.T) {
|
||||
r := resolverOver(t, 1<<20, jarEntriesFixture)
|
||||
_, err := r.FilesByGlob("**/[")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func globPaths(t *testing.T, r *Resolver, pattern string) []string {
|
||||
t.Helper()
|
||||
locations, err := r.FilesByGlob(pattern)
|
||||
require.NoError(t, err)
|
||||
return localPaths(locations)
|
||||
}
|
||||
|
||||
// several paths to one file answer as one location: the file's own path when it matched, otherwise
|
||||
// the lowest-sorting link
|
||||
func TestResolver_linksCollapseToOnePath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
entries []testEntry
|
||||
pattern string
|
||||
want []string
|
||||
reason string
|
||||
}{
|
||||
{
|
||||
name: "the file itself beats every link to it",
|
||||
entries: []testEntry{
|
||||
{name: "opt/real.jar", body: "PK\x03\x04"},
|
||||
{name: "opt/a-sym.jar", link: "real.jar"},
|
||||
{name: "opt/z-sym.jar", link: "real.jar"},
|
||||
{name: "opt/hard.jar", link: "opt/real.jar", hard: true},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: []string{"/opt/real.jar"},
|
||||
reason: "one file, so one location - and it is named by the path that holds it",
|
||||
},
|
||||
{
|
||||
name: "with no candidate naming the file, the lowest-sorting link wins",
|
||||
entries: []testEntry{
|
||||
{name: "opt/real.bin", body: "PK\x03\x04"},
|
||||
{name: "opt/z-sym.jar", link: "real.bin"},
|
||||
{name: "opt/a-sym.jar", link: "real.bin"},
|
||||
{name: "opt/m-sym.jar", link: "real.bin"},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: []string{"/opt/real.bin"},
|
||||
reason: "the content is the target's, so that is the real path the answer carries",
|
||||
},
|
||||
{
|
||||
name: "distinct files stay distinct",
|
||||
entries: []testEntry{
|
||||
{name: "opt/one.jar", body: "one"},
|
||||
{name: "opt/two.jar", body: "two"},
|
||||
{name: "opt/one-sym.jar", link: "one.jar"},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: []string{"/opt/one.jar", "/opt/two.jar"},
|
||||
reason: "collapsing is per file, not per pattern",
|
||||
},
|
||||
{
|
||||
name: "a dangling link is not a file",
|
||||
entries: []testEntry{
|
||||
{name: "opt/real.jar", body: "PK\x03\x04"},
|
||||
{name: "opt/gone.jar", link: "missing.jar"},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: []string{"/opt/real.jar"},
|
||||
reason: "it names no content",
|
||||
},
|
||||
{
|
||||
name: "two dangling links are two paths, not one",
|
||||
entries: []testEntry{
|
||||
{name: "opt/a.jar", link: "missing.jar"},
|
||||
{name: "opt/b.jar", link: "missing.jar"},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: nil,
|
||||
reason: "a link to nothing is not a file",
|
||||
},
|
||||
{
|
||||
name: "a chain of links collapses onto the file at its end",
|
||||
entries: []testEntry{
|
||||
{name: "opt/real.jar", body: "PK\x03\x04"},
|
||||
{name: "opt/mid.jar", link: "real.jar"},
|
||||
{name: "opt/outer.jar", link: "mid.jar"},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: []string{"/opt/real.jar"},
|
||||
reason: "following one hop at a time reaches the same file from every path",
|
||||
},
|
||||
{
|
||||
name: "a cycle resolves to nothing rather than looping",
|
||||
entries: []testEntry{
|
||||
{name: "opt/a.jar", link: "b.jar"},
|
||||
{name: "opt/b.jar", link: "a.jar"},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: nil,
|
||||
reason: "neither reaches a file",
|
||||
},
|
||||
{
|
||||
name: "a hard link resolves from the archive root, not from its own directory",
|
||||
entries: []testEntry{
|
||||
{name: "opt/real.jar", body: "PK\x03\x04"},
|
||||
{name: "opt/nested/hard.jar", link: "opt/real.jar", hard: true},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: []string{"/opt/real.jar"},
|
||||
reason: "read as relative it would resolve to opt/nested/opt/real.jar and dangle",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := resolverFrom(t, tt.entries...)
|
||||
assert.Equal(t, tt.want, globPaths(t, r, tt.pattern), tt.reason)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) {
|
||||
r := resolverFrom(t,
|
||||
testEntry{name: "opt/real.bin", body: "PK\x03\x04 real contents"},
|
||||
testEntry{name: "opt/a-sym.jar", link: "real.bin"},
|
||||
testEntry{name: "opt/z-sym.jar", link: "real.bin"},
|
||||
)
|
||||
|
||||
locations, err := r.FilesByGlob("**/*.jar")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
|
||||
assert.Equal(t, "/opt/real.bin", locations[0].LocalPath(), "content comes from the file")
|
||||
assert.Equal(t, "outer.jar!/opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched")
|
||||
|
||||
reader, err := r.FileContentsByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, reader.Close()) })
|
||||
|
||||
body, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "PK\x03\x04 real contents", string(body),
|
||||
"a location standing for a link must still read the file's bytes, not the link's none")
|
||||
}
|
||||
|
||||
func TestResolver_fullPathsAreTrimmedToEntryPaths(t *testing.T) {
|
||||
// catalogers build queries from the full paths the resolver hands out, e.g. path.Dir plus a sibling
|
||||
r := resolverOver(t, 1<<20, map[string]string{"pom.xml": "a", "lib/dep.jar": "b"})
|
||||
|
||||
locations, err := r.FilesByPath("outer.jar!/lib/dep.jar")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
assert.Equal(t, "outer.jar!/lib/dep.jar", locations[0].AccessPath)
|
||||
|
||||
locations, err = r.FilesByPath(path.Join(path.Dir(locations[0].AccessPath), "../pom.xml"))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1, "a sibling at the archive root is reached through the full path")
|
||||
assert.Equal(t, "outer.jar!/pom.xml", locations[0].FullNestedPath())
|
||||
|
||||
assert.True(t, r.HasPath(path.Dir("outer.jar!/pom.xml")), "the directory of a root entry is the root")
|
||||
assert.True(t, r.HasPath("lib/dep.jar"), "an entry path is still accepted")
|
||||
|
||||
locations, err = r.FilesByGlob("outer.jar!/**/*.jar")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
}
|
||||
|
||||
func TestResolver_danglingLinkHasNoContent(t *testing.T) {
|
||||
r := resolverFrom(t, testEntry{name: "opt/gone.jar", link: "missing.jar"})
|
||||
|
||||
var locations []file.Location
|
||||
for l := range r.AllLocations(context.Background()) {
|
||||
locations = append(locations, l)
|
||||
}
|
||||
require.Len(t, locations, 1)
|
||||
assert.Equal(t, "/opt/gone.jar", locations[0].LocalPath())
|
||||
|
||||
_, err := r.FileContentsByLocation(locations[0])
|
||||
assert.Error(t, err, "a link to nothing has no content to read")
|
||||
|
||||
metadata, err := r.FileMetadataByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "outer.jar!/opt/missing.jar", metadata.LinkDestination,
|
||||
"where it pointed is still reported, resolved against the link's own directory")
|
||||
}
|
||||
|
||||
func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) {
|
||||
r := resolverFrom(t,
|
||||
testEntry{name: "opt/real.jar", body: "PK\x03\x04 zip contents here"},
|
||||
testEntry{name: "opt/sym.jar", link: "real.jar"},
|
||||
testEntry{name: "opt/hard.jar", link: "opt/real.jar", hard: true},
|
||||
testEntry{name: "opt/gone.jar", link: "missing.jar"},
|
||||
testEntry{name: "opt/notes.txt", body: "hello world"},
|
||||
)
|
||||
|
||||
byMIME := func(types ...string) []string {
|
||||
t.Helper()
|
||||
locations, err := r.FilesByMIMEType(types...)
|
||||
require.NoError(t, err)
|
||||
return localPaths(locations)
|
||||
}
|
||||
|
||||
assert.Equal(t, []string{"/opt/real.jar"}, byMIME("application/zip"),
|
||||
"the file answers, not the two links to it")
|
||||
assert.Equal(t, []string{"/opt/notes.txt"}, byMIME("text/plain"))
|
||||
assert.Empty(t, byMIME(""),
|
||||
"a link sniffs to no type, and must not be reachable by asking for that")
|
||||
assert.Equal(t, []string{"/opt/notes.txt", "/opt/real.jar"}, byMIME("application/zip", "text/plain"))
|
||||
}
|
||||
|
||||
// AllLocations enumerates what the archive holds rather than searching for files: the file metadata
|
||||
// cataloger records a row per path, links included
|
||||
func TestResolver_allLocationsKeepsEveryPath(t *testing.T) {
|
||||
r := resolverFrom(t,
|
||||
testEntry{name: "opt/real.jar", body: "PK\x03\x04"},
|
||||
testEntry{name: "opt/sym.jar", link: "real.jar"},
|
||||
testEntry{name: "opt/gone.jar", link: "missing.jar"},
|
||||
)
|
||||
|
||||
var got []string
|
||||
for l := range r.AllLocations(context.Background()) {
|
||||
got = append(got, l.LocalPath())
|
||||
}
|
||||
assert.Equal(t, []string{"/opt/gone.jar", "/opt/real.jar", "/opt/sym.jar"}, got,
|
||||
"every path the archive names, links included")
|
||||
}
|
||||
|
||||
func TestResolver_add_anEntryOverTheCapIsSkippedAndReadNoFurther(t *testing.T) {
|
||||
// one entry of an overlapping-entry bomb can inflate without bound; it must stop at the cap, cost
|
||||
// the archive only that entry, and say so
|
||||
orig := maxEntryBytes
|
||||
maxEntryBytes = 1024
|
||||
t.Cleanup(func() { maxEntryBytes = orig })
|
||||
|
||||
r, _ := resolverIn(t, memCharge(-1))
|
||||
endless := &countingReader{r: zeroSource{}}
|
||||
require.NoError(t, r.add(tar.Header{Name: "bomb.bin", Typeflag: tar.TypeReg, Mode: 0o644}, endless))
|
||||
require.NoError(t, r.add(tar.Header{Name: "ok.txt", Typeflag: tar.TypeReg, Mode: 0o644}, strings.NewReader("ok")))
|
||||
r.finish()
|
||||
|
||||
assert.LessOrEqual(t, endless.n, 1024+copyChunkSize, "reading stops just past the cap")
|
||||
assert.ErrorIs(t, r.Truncated, errEntryTooLarge)
|
||||
assert.ErrorContains(t, r.Truncated, "per-entry size cap of 1.0 KiB")
|
||||
locations, err := r.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"/ok.txt"}, localPaths(locations))
|
||||
}
|
||||
|
||||
type countingReader struct {
|
||||
r io.Reader
|
||||
n int
|
||||
}
|
||||
|
||||
func (c *countingReader) Read(p []byte) (int, error) {
|
||||
n, err := c.r.Read(p)
|
||||
c.n += n
|
||||
return n, err
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
)
|
||||
|
||||
// copyChunkSize is how many bytes are read and charged at a time, bounding how far a charge runs
|
||||
// ahead of what is actually held.
|
||||
const copyChunkSize = 32 * 1024
|
||||
|
||||
// spillFilePattern names the one file an archive's content spills into, as in os.CreateTemp.
|
||||
const spillFilePattern = "archive-spill-*"
|
||||
|
||||
// ReaderAtSeeker is the random access an archive format needs: a zip is read from its central
|
||||
// directory at the end of the stream.
|
||||
type ReaderAtSeeker interface {
|
||||
io.Reader
|
||||
io.ReaderAt
|
||||
io.Seeker
|
||||
}
|
||||
|
||||
// blob is one piece of content the resolver holds: in memory, or at an offset in its spill file once
|
||||
// memory refused it. Moving changes nothing that points at the blob.
|
||||
type blob struct {
|
||||
mem []byte // content while held in memory
|
||||
off int64 // where content starts in the spill file once written there
|
||||
size int64
|
||||
}
|
||||
|
||||
// put reads content in full into b, since archive readers are sequential. Each chunk is charged before it
|
||||
// is held, so a stream larger than the memory limit is never held in full. Reaching the disk limit
|
||||
// returns ErrDiskLimitReached; what landed in the file before that stays there and stays charged.
|
||||
func (r *Resolver) put(b *blob, content io.Reader) error {
|
||||
if r.chunk == nil {
|
||||
r.chunk = make([]byte, copyChunkSize)
|
||||
}
|
||||
for {
|
||||
n, readErr := content.Read(r.chunk)
|
||||
if n > 0 {
|
||||
// content read through a meteredReader was charged as it was decoded
|
||||
if !r.metered && !r.budget.take(int64(n)) {
|
||||
return errBudgetSpent
|
||||
}
|
||||
if err := r.write(b, r.chunk[:n]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if readErr == nil {
|
||||
continue
|
||||
}
|
||||
if errors.Is(readErr, io.EOF) {
|
||||
break
|
||||
}
|
||||
return readErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// meteredReader charges the decompression budget for every byte read through it and stops once ctx is
|
||||
// done, so nothing a decoder produces gets around the budget.
|
||||
type meteredReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
budget *Budget
|
||||
}
|
||||
|
||||
func (m *meteredReader) Read(p []byte) (int, error) {
|
||||
if err := m.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, err := m.reader.Read(p)
|
||||
if n > 0 && !m.budget.take(int64(n)) {
|
||||
return 0, errBudgetSpent
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// write appends p to b: into memory while the memory limit admits it, otherwise into the spill file
|
||||
// after everything held in memory is moved there first.
|
||||
func (r *Resolver) write(b *blob, p []byte) error {
|
||||
n := int64(len(p))
|
||||
onDisk := b.mem == nil && b.size > 0
|
||||
if !onDisk {
|
||||
if r.charge.memory(n) {
|
||||
if b.mem == nil {
|
||||
r.held = append(r.held, b)
|
||||
}
|
||||
b.mem = append(b.mem, p...)
|
||||
b.size += n
|
||||
return nil
|
||||
}
|
||||
if err := r.spill(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := r.openFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !r.charge.disk(n) {
|
||||
return ErrDiskLimitReached
|
||||
}
|
||||
if b.size == 0 {
|
||||
b.off = r.written
|
||||
}
|
||||
// b is always the blob most recently written, so it ends where the file does
|
||||
w, err := r.file.WriteAt(p, r.written)
|
||||
r.written += int64(w)
|
||||
b.size += int64(w)
|
||||
r.charge.refundDisk(n - int64(w))
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to write archive content to disk: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// spill moves every blob held in memory into the spill file, charging the disk limit as bytes land
|
||||
// and refunding the memory they leave.
|
||||
func (r *Resolver) spill() error {
|
||||
if len(r.held) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := r.openFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "held", len(r.held), "bytes", r.written).
|
||||
Debug("archive content does not fit in memory; writing it to disk")
|
||||
|
||||
for len(r.held) > 0 {
|
||||
b := r.held[0]
|
||||
n := int64(len(b.mem))
|
||||
if !r.charge.disk(n) {
|
||||
return ErrDiskLimitReached
|
||||
}
|
||||
if _, err := r.file.WriteAt(b.mem, r.written); err != nil {
|
||||
return fmt.Errorf("unable to write archive content to disk: %w", err)
|
||||
}
|
||||
b.off, b.mem = r.written, nil
|
||||
r.written += n
|
||||
r.charge.refundMemory(n)
|
||||
r.held = r.held[1:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Resolver) openFile() error {
|
||||
if r.file != nil {
|
||||
return nil
|
||||
}
|
||||
var err error
|
||||
if r.tempDir != nil {
|
||||
r.file, r.remove, err = r.tempDir.NewFile(spillFilePattern)
|
||||
} else {
|
||||
r.file, err = os.CreateTemp("", spillFilePattern)
|
||||
r.remove = func() {
|
||||
if err := os.Remove(r.file.Name()); err != nil {
|
||||
log.WithFields("path", r.file.Name(), "error", err).Trace("unable to remove archive temp file")
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create archive temp file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// discard drops a blob's bytes held in memory and refunds them. Bytes it holds on disk stay until
|
||||
// Cleanup.
|
||||
func (r *Resolver) discard(b *blob) {
|
||||
for i, held := range r.held {
|
||||
if held == b {
|
||||
r.held = append(r.held[:i], r.held[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
r.charge.refundMemory(int64(len(b.mem)))
|
||||
b.mem, b.size = nil, 0
|
||||
}
|
||||
|
||||
// open returns a reader over a blob's content, wherever it is. The reader supports Read, Seek and
|
||||
// ReadAt, so a nested archive can be read where it lies.
|
||||
func (r *Resolver) open(b *blob) ReaderAtSeeker {
|
||||
switch {
|
||||
case b.mem != nil:
|
||||
return bytes.NewReader(b.mem)
|
||||
case b.size == 0:
|
||||
return bytes.NewReader(nil)
|
||||
default:
|
||||
return io.NewSectionReader(r.file, b.off, b.size)
|
||||
}
|
||||
}
|
||||
|
||||
// releaseStorage refunds every blob held in memory and closes, removes and refunds the spill file.
|
||||
// Readers from open must not be used after this. Safe to call more than once.
|
||||
func (r *Resolver) releaseStorage() {
|
||||
for _, b := range r.held {
|
||||
r.charge.refundMemory(int64(len(b.mem)))
|
||||
b.mem = nil
|
||||
}
|
||||
r.held = nil
|
||||
if r.file == nil {
|
||||
return
|
||||
}
|
||||
if err := r.file.Close(); err != nil {
|
||||
log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "error", err).Trace("unable to close archive temp file")
|
||||
}
|
||||
r.remove()
|
||||
r.charge.refundDisk(r.written)
|
||||
r.file, r.remove, r.written = nil, nil, 0
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestResolver_put_holdsSmallContentInMemory(t *testing.T) {
|
||||
s, dir := resolverIn(t, memCharge(1024))
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader([]byte("hello"))))
|
||||
|
||||
assert.Equal(t, int64(5), s.heldInMemory())
|
||||
assert.Empty(t, filesIn(t, dir))
|
||||
assert.Equal(t, "hello", readEntry(t, s.open(&b)))
|
||||
}
|
||||
|
||||
func TestResolver_put_spillsToDiskAndKeepsTheSameBlobs(t *testing.T) {
|
||||
// nodes pointing at a blob are not rebuilt when its content moves
|
||||
s, dir := resolverIn(t, memCharge(8))
|
||||
|
||||
var first, second blob
|
||||
require.NoError(t, s.put(&first, bytes.NewReader([]byte("aaaa"))))
|
||||
require.Equal(t, int64(4), s.heldInMemory())
|
||||
|
||||
require.NoError(t, s.put(&second, bytes.NewReader([]byte("bbbbbb"))))
|
||||
|
||||
assert.Zero(t, s.heldInMemory(), "everything moves out, not just the blob that did not fit")
|
||||
assert.NotEmpty(t, spillFile(t, dir))
|
||||
assert.Equal(t, int64(10), s.written)
|
||||
assert.Equal(t, "aaaa", readEntry(t, s.open(&first)))
|
||||
assert.Equal(t, "bbbbbb", readEntry(t, s.open(&second)))
|
||||
}
|
||||
|
||||
func TestResolver_put_threeStateLimits(t *testing.T) {
|
||||
body := []byte(strings.Repeat("e", 500))
|
||||
|
||||
t.Run("a zero memory limit sends everything to disk", func(t *testing.T) {
|
||||
charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 10_000}).charge()
|
||||
s, dir := resolverIn(t, charge)
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader(body)))
|
||||
|
||||
assert.Zero(t, s.heldInMemory())
|
||||
assert.NotEmpty(t, spillFile(t, dir))
|
||||
assert.Equal(t, body, []byte(readEntry(t, s.open(&b))))
|
||||
mem, disk := charge.held()
|
||||
assert.Zero(t, mem)
|
||||
assert.Equal(t, int64(500), disk)
|
||||
})
|
||||
|
||||
t.Run("a negative memory limit holds content regardless of how much is already held", func(t *testing.T) {
|
||||
charge := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: 10_000}).charge()
|
||||
require.True(t, charge.memory(1_000_000))
|
||||
s, dir := resolverIn(t, charge)
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader(body)))
|
||||
|
||||
assert.Empty(t, filesIn(t, dir))
|
||||
mem, disk := charge.held()
|
||||
assert.Equal(t, int64(1_000_000+len(body)), mem)
|
||||
assert.Zero(t, disk)
|
||||
})
|
||||
|
||||
t.Run("a zero disk limit still holds what memory admits", func(t *testing.T) {
|
||||
charge := NewLimiter(Limits{MaxMemoryBytes: 10_000, MaxDiskBytes: 0}).charge()
|
||||
s, dir := resolverIn(t, charge)
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader(body)))
|
||||
|
||||
assert.Empty(t, filesIn(t, dir))
|
||||
mem, disk := charge.held()
|
||||
assert.Equal(t, int64(len(body)), mem)
|
||||
assert.Zero(t, disk)
|
||||
})
|
||||
|
||||
t.Run("a negative disk limit writes with no ceiling", func(t *testing.T) {
|
||||
s, dir := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}).charge())
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader(body)))
|
||||
|
||||
assert.NotEmpty(t, spillFile(t, dir))
|
||||
assert.Equal(t, body, []byte(readEntry(t, s.open(&b))))
|
||||
})
|
||||
|
||||
t.Run("a nil charge holds everything in memory", func(t *testing.T) {
|
||||
s, dir := resolverIn(t, nil)
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader(body)))
|
||||
|
||||
assert.Empty(t, filesIn(t, dir))
|
||||
assert.Equal(t, body, []byte(readEntry(t, s.open(&b))))
|
||||
})
|
||||
}
|
||||
|
||||
func TestResolver_put_bytesReadBeforeTheLimitRefusedAreNotLost(t *testing.T) {
|
||||
body := []byte(strings.Repeat("b", 250))
|
||||
charge := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 10_000}).charge()
|
||||
s, _ := resolverIn(t, charge)
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader(body)))
|
||||
|
||||
assert.Equal(t, body, []byte(readEntry(t, s.open(&b))))
|
||||
mem, disk := charge.held()
|
||||
assert.Zero(t, mem, "nothing stays charged to a limit that refused")
|
||||
assert.Equal(t, int64(250), disk)
|
||||
}
|
||||
|
||||
// peakTrackingReader records the highest memory charge observed while being read.
|
||||
type peakTrackingReader struct {
|
||||
r io.Reader
|
||||
charge *charge
|
||||
peak int64
|
||||
}
|
||||
|
||||
func (p *peakTrackingReader) Read(b []byte) (int, error) {
|
||||
n, err := p.r.Read(b)
|
||||
if mem, _ := p.charge.held(); mem > p.peak {
|
||||
p.peak = mem
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func TestResolver_put_neverBuffersBeyondTheMemoryLimit(t *testing.T) {
|
||||
const limit = int64(3 * copyChunkSize)
|
||||
body := []byte(strings.Repeat("c", 10*copyChunkSize))
|
||||
charge := NewLimiter(Limits{MaxMemoryBytes: limit, MaxDiskBytes: int64(len(body)) + 1}).charge()
|
||||
tracked := &peakTrackingReader{r: bytes.NewReader(body), charge: charge}
|
||||
s, _ := resolverIn(t, charge)
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, tracked))
|
||||
|
||||
assert.Equal(t, body, []byte(readEntry(t, s.open(&b))))
|
||||
assert.LessOrEqual(t, tracked.peak, limit, "memory charged must never exceed the limit, even transiently")
|
||||
mem, disk := charge.held()
|
||||
assert.Zero(t, mem)
|
||||
assert.Equal(t, int64(len(body)), disk)
|
||||
}
|
||||
|
||||
func TestResolver_put_diskLimitIsTerminal(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
limits Limits
|
||||
}{
|
||||
{"a disk limit that is exceeded", Limits{MaxDiskBytes: 100}},
|
||||
{"a zero disk limit", Limits{MaxMemoryBytes: 10, MaxDiskBytes: 0}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
charge := NewLimiter(tt.limits).charge()
|
||||
s, dir := resolverIn(t, charge)
|
||||
|
||||
var b blob
|
||||
err := s.put(&b, bytes.NewReader([]byte(strings.Repeat("d", 500))))
|
||||
require.ErrorIs(t, err, ErrDiskLimitReached)
|
||||
|
||||
s.Cleanup()
|
||||
assert.Empty(t, filesIn(t, dir), "cleanup removes the partial file")
|
||||
mem, disk := charge.held()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk, "and refunds it")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_cleanupRefundsEverythingAndIsIdempotent(t *testing.T) {
|
||||
charge := NewLimiter(Limits{MaxMemoryBytes: 8, MaxDiskBytes: -1}).charge()
|
||||
s, dir := resolverIn(t, charge)
|
||||
|
||||
var spilled, held blob
|
||||
require.NoError(t, s.put(&spilled, bytes.NewReader([]byte("spilled to disk"))))
|
||||
require.NoError(t, s.put(&held, bytes.NewReader([]byte("held"))))
|
||||
mem, disk := charge.held()
|
||||
require.Equal(t, int64(4), mem)
|
||||
require.Equal(t, int64(15), disk)
|
||||
require.Equal(t, "held", readEntry(t, s.open(&held)))
|
||||
|
||||
s.Cleanup()
|
||||
|
||||
assert.Empty(t, filesIn(t, dir))
|
||||
mem, disk = charge.held()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk)
|
||||
_, err := io.ReadAll(s.open(&spilled))
|
||||
assert.Error(t, err, "readers over the spill file are no longer valid")
|
||||
s.Cleanup()
|
||||
mem, disk = charge.held()
|
||||
assert.Zero(t, mem, "a second cleanup refunds nothing twice")
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func TestResolver_put_discardRefundsWhatIsHeldInMemory(t *testing.T) {
|
||||
charge := memCharge(1024)
|
||||
s, _ := resolverIn(t, charge)
|
||||
|
||||
var keep, drop blob
|
||||
require.NoError(t, s.put(&keep, bytes.NewReader([]byte("keep"))))
|
||||
require.NoError(t, s.put(&drop, bytes.NewReader([]byte("dropped"))))
|
||||
|
||||
s.discard(&drop)
|
||||
|
||||
mem, _ := charge.held()
|
||||
assert.Equal(t, int64(4), mem)
|
||||
assert.Equal(t, int64(4), s.heldInMemory())
|
||||
assert.Empty(t, readEntry(t, s.open(&drop)))
|
||||
assert.Equal(t, "keep", readEntry(t, s.open(&keep)))
|
||||
}
|
||||
|
||||
func TestResolver_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
|
||||
small, smallRoot := extractedResolver(t, manyEntryZip(t, 2), spillingLimits)
|
||||
large, largeRoot := extractedResolver(t, manyEntryZip(t, 5000), spillingLimits)
|
||||
|
||||
assert.Len(t, filesIn(t, smallRoot), 1)
|
||||
assert.Len(t, filesIn(t, largeRoot), 1, "5000 entries cost the same filesystem entries as 2")
|
||||
|
||||
assert.Equal(t, 2, small.entries)
|
||||
assert.Equal(t, 5000, large.entries)
|
||||
|
||||
// entries cost their content and nothing else on disk: no headers, no padding
|
||||
assert.Equal(t, int64(2), small.written)
|
||||
assert.Equal(t, int64(5000), large.written)
|
||||
}
|
||||
|
||||
func TestResolver_anEntryAtTheEndIsReadBySeeking(t *testing.T) {
|
||||
// the reader over an entry is a bounded window at its offset, not a re-read of what precedes it
|
||||
const filler = 64 * 1024
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for i := range 8 {
|
||||
w, err := zw.Create(fmt.Sprintf("filler%d.bin", i))
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write(bytes.Repeat([]byte{byte(i)}, filler))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
last, err := zw.Create("zz-last.txt")
|
||||
require.NoError(t, err)
|
||||
wanted := strings.Repeat("last", 64)
|
||||
_, err = last.Write([]byte(wanted))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
r, root := extractedResolver(t, buf.Bytes(), spillingLimits)
|
||||
|
||||
body, err := os.ReadFile(filepath.Join(root, spillFile(t, root)))
|
||||
require.NoError(t, err)
|
||||
offset := int64(bytes.Index(body, []byte(wanted)))
|
||||
require.Greater(t, offset, int64(len(body))*9/10, "the last entry must really be at the end of the file")
|
||||
|
||||
reader := r.open(contentAt(t, r, "/zz-last.txt"))
|
||||
end, err := reader.Seek(0, io.SeekEnd)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(wanted)), end)
|
||||
|
||||
head := make([]byte, 8)
|
||||
_, err = reader.ReadAt(head, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wanted[:8], string(head))
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
// Traversal describes the archive whose contents a cataloger is running against. The archive
|
||||
// cataloger task puts one on the context before running catalogers inside an archive, so a cataloger
|
||||
// such as java can describe the archive it is inside without owning the recursion.
|
||||
type Traversal struct {
|
||||
// Location is the archive file's location within its parent filesystem, including the access path
|
||||
// it was reached by, so VirtualPath(Location) is the chain from the scan root as seen through that
|
||||
// access path. The files inside it extend its real coordinates instead (see ContainerPath).
|
||||
Location file.Location
|
||||
|
||||
// Digests are of the archive file itself, taken during extraction; a cataloger running inside the
|
||||
// archive cannot compute them since the archive is not inside itself.
|
||||
Digests []file.Digest
|
||||
|
||||
// Parent is the traversal of the archive this one is nested in, nil for an archive at the scan root.
|
||||
Parent *Traversal
|
||||
|
||||
// Packages are those the catalogers found describing this archive (located at it), filled in once
|
||||
// they have run. Catalogers inside a nested archive read them through Parent, as java does to link a
|
||||
// nested jar to the jar that holds it.
|
||||
Packages []pkg.Package
|
||||
}
|
||||
|
||||
// FullPathIn is the full path of the entry at entryPath inside the archive whose full path is
|
||||
// containerPath, or entryPath itself for a blank containerPath (the scan root).
|
||||
func FullPathIn(containerPath, entryPath string) string {
|
||||
if containerPath == "" {
|
||||
return entryPath
|
||||
}
|
||||
// cleaning an already clean path allocates nothing
|
||||
return containerPath + file.ArchivePathSeparator + strings.TrimPrefix(path.Clean(entryPath), "/")
|
||||
}
|
||||
|
||||
// OwnerContainers are the containers a package owns files within: the container of each of its
|
||||
// locations, each once, as coordinates whose FullPath is the container's full path (blank at the scan
|
||||
// root). A package without locations owns files at the scan root. See OwnedCoordinates.
|
||||
func OwnerContainers(p pkg.Package) []file.Coordinates {
|
||||
var out []file.Coordinates
|
||||
for _, l := range p.Locations.ToUnorderedSlice() {
|
||||
c := file.Coordinates{FileSystemID: l.FileSystemID}
|
||||
if container := ContainerPath(l.Coordinates); container != "" {
|
||||
c.RealPath, c.FullPath = l.RealPath, container
|
||||
}
|
||||
if !slices.Contains(out, c) {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
out = append(out, file.Coordinates{})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// OwnedCoordinates are the coordinates of the file at ownedPath within container, one of
|
||||
// OwnerContainers. At the scan root the path may still need resolving (it may pass through links).
|
||||
func OwnedCoordinates(container file.Coordinates, ownedPath string) file.Coordinates {
|
||||
if container.FullPath == "" {
|
||||
return file.Coordinates{RealPath: ownedPath, FileSystemID: container.FileSystemID}
|
||||
}
|
||||
return file.Coordinates{
|
||||
RealPath: container.RealPath,
|
||||
FileSystemID: container.FileSystemID,
|
||||
FullPath: FullPathIn(container.FullPath, ownedPath),
|
||||
}
|
||||
}
|
||||
|
||||
// FullTraversalArchivePath is the archive's own full path, which is the ContainerPath of every file inside it. Outside
|
||||
// any archive (a nil Traversal) it is blank, as for files on the scanned filesystem.
|
||||
func (t *Traversal) FullTraversalArchivePath() string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
return t.Location.FullNestedPath()
|
||||
}
|
||||
|
||||
// ContainerPath returns the full path of the archive directly holding the file at c, or blank for a
|
||||
// file on the scanned filesystem, whether or not its FullPath is set.
|
||||
func ContainerPath(c file.Coordinates) string {
|
||||
entries, ok := archiveEntries(c)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
if i := strings.LastIndex(entries, file.ArchivePathSeparator); i >= 0 {
|
||||
return c.RealPath + file.ArchivePathSeparator + entries[:i]
|
||||
}
|
||||
return c.RealPath
|
||||
}
|
||||
|
||||
// ContainingArchive returns the coordinates of the archive directly holding the file at c, reporting false
|
||||
// for a file on the scanned filesystem.
|
||||
func ContainingArchive(c file.Coordinates) (file.Coordinates, bool) {
|
||||
containerPath := ContainerPath(c)
|
||||
if containerPath == "" {
|
||||
return file.Coordinates{}, false
|
||||
}
|
||||
container := file.Coordinates{RealPath: c.RealPath, FileSystemID: c.FileSystemID}
|
||||
if containerPath != c.RealPath {
|
||||
container.FullPath = containerPath
|
||||
}
|
||||
return container, true
|
||||
}
|
||||
|
||||
// archiveEntries is the "!/"-delimited chain of entry paths after RealPath, reporting false for a file
|
||||
// not inside an archive. RealPath is split off first, so a "!/" in it is not taken for a boundary.
|
||||
func archiveEntries(c file.Coordinates) (string, bool) {
|
||||
remain, ok := strings.CutPrefix(c.FullNestedPath(), c.RealPath)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
return strings.CutPrefix(remain, file.ArchivePathSeparator)
|
||||
}
|
||||
|
||||
// VirtualPath returns the colon-delimited chain of archives from the scan root to loc, ending in loc
|
||||
// itself as accessed: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path
|
||||
// unchanged. In an entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':'
|
||||
// recovers the archive boundaries and no two entry names share an escaped form. Leading slashes are
|
||||
// dropped to match the java cataloger, which joins slash-less zip entry names onto the containing
|
||||
// archive's path.
|
||||
func VirtualPath(loc file.Location) string {
|
||||
accessPath := loc.Path()
|
||||
entries, ok := archiveEntries(loc.Coordinates)
|
||||
if !ok {
|
||||
return accessPath
|
||||
}
|
||||
segments := strings.Split(entries, file.ArchivePathSeparator)
|
||||
if rest, ok := strings.CutPrefix(accessPath, ContainerPath(loc.Coordinates)); ok && strings.HasPrefix(rest, file.ArchivePathSeparator) {
|
||||
accessPath = rest[len(file.ArchivePathSeparator):]
|
||||
}
|
||||
segments[len(segments)-1] = accessPath
|
||||
for i, s := range segments {
|
||||
segments[i] = strings.TrimPrefix(escapeEntryPath(s), "/")
|
||||
}
|
||||
return loc.RealPath + ":" + strings.Join(segments, ":")
|
||||
}
|
||||
|
||||
var entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A")
|
||||
|
||||
func escapeEntryPath(p string) string {
|
||||
if !strings.ContainsAny(p, "%:") {
|
||||
return p
|
||||
}
|
||||
return entryPathEscaper.Replace(p)
|
||||
}
|
||||
|
||||
type traversalCtxKey struct{}
|
||||
|
||||
func WithTraversal(ctx context.Context, t *Traversal) context.Context {
|
||||
return context.WithValue(ctx, traversalCtxKey{}, t)
|
||||
}
|
||||
|
||||
// TraversalFromContext returns the archive traversal on the context, or nil outside an archive.
|
||||
func TraversalFromContext(ctx context.Context) *Traversal {
|
||||
t, _ := ctx.Value(traversalCtxKey{}).(*Traversal)
|
||||
return t
|
||||
}
|
||||
|
||||
type budgetCtxKey struct{}
|
||||
|
||||
// WithBudget sets the decompression budget that Extract spends from, for this archive and every archive
|
||||
// extracted beneath it.
|
||||
func WithBudget(ctx context.Context, b *Budget) context.Context {
|
||||
return context.WithValue(ctx, budgetCtxKey{}, b)
|
||||
}
|
||||
|
||||
func budgetFromContext(ctx context.Context) *Budget {
|
||||
b, _ := ctx.Value(budgetCtxKey{}).(*Budget)
|
||||
return b
|
||||
}
|
||||
|
||||
type nestedCatalogingCtxKey struct{}
|
||||
|
||||
// WithNestedCataloging marks the context of a scan whose archive cataloger task will extract every
|
||||
// archive and run the catalogers inside it. syft.CreateSBOM sets it before any task runs, so a
|
||||
// cataloger that would otherwise open archives itself can leave them to the task instead.
|
||||
func WithNestedCataloging(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, nestedCatalogingCtxKey{}, true)
|
||||
}
|
||||
|
||||
// NestedCatalogingEnabled reports whether the archive cataloger task is extracting archives in this scan.
|
||||
func NestedCatalogingEnabled(ctx context.Context) bool {
|
||||
enabled, _ := ctx.Value(nestedCatalogingCtxKey{}).(bool)
|
||||
return enabled
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
func TestTraversalContextRoundTrip(t *testing.T) {
|
||||
assert.Nil(t, TraversalFromContext(context.Background()))
|
||||
|
||||
trav := &Traversal{Location: file.NewLocation("app.war")}
|
||||
ctx := WithTraversal(context.Background(), trav)
|
||||
require.Same(t, trav, TraversalFromContext(ctx))
|
||||
|
||||
child := &Traversal{Location: locationIn("app.war", "WEB-INF/lib/dep.jar")}
|
||||
ctx = WithTraversal(ctx, child)
|
||||
require.Same(t, child, TraversalFromContext(ctx))
|
||||
}
|
||||
|
||||
// locationIn is a location as the Resolver returns it: the outermost archive's real path, and the chain
|
||||
// of entry paths within it.
|
||||
func locationIn(realPath, archivePath string) file.Location {
|
||||
return file.NewLocationFromCoordinates(file.Coordinates{RealPath: realPath, FullPath: realPath + file.ArchivePathSeparator + archivePath})
|
||||
}
|
||||
|
||||
func TestContainerPath(t *testing.T) {
|
||||
assert.Equal(t, "", ContainerPath(file.NewCoordinates("/app.war", "")))
|
||||
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/app.war", FullPath: "/app.war"}), "a top-level file whose FullPath is set")
|
||||
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/odd!/name"}), "a separator in a top-level real path is not an archive boundary")
|
||||
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/odd!/name", FullPath: "/odd!/name"}), "nor when FullPath is set")
|
||||
assert.Equal(t, "/odd!/name.zip", ContainerPath(locationIn("/odd!/name.zip", "a.txt").Coordinates))
|
||||
assert.Equal(t, "/odd!/name.zip!/b.jar", ContainerPath(locationIn("/odd!/name.zip", "b.jar!/a.txt").Coordinates))
|
||||
assert.Equal(t, "/app.war", ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar").Coordinates))
|
||||
assert.Equal(t, "/app.war!/WEB-INF/lib/dep.jar",
|
||||
ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF").Coordinates))
|
||||
}
|
||||
|
||||
func TestVirtualPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
location file.Location
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "outside an archive returns the path unchanged",
|
||||
location: file.NewLocation("/WEB-INF/lib/dep.jar"),
|
||||
expected: "/WEB-INF/lib/dep.jar",
|
||||
},
|
||||
{
|
||||
name: "outside an archive preserves a relative path",
|
||||
location: file.NewLocation("some/dir/example.jar"),
|
||||
expected: "some/dir/example.jar",
|
||||
},
|
||||
{
|
||||
name: "outside an archive with FullPath set returns the path unchanged",
|
||||
location: file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/odd!/name.jar", FullPath: "/odd!/name.jar"}),
|
||||
expected: "/odd!/name.jar",
|
||||
},
|
||||
{
|
||||
name: "single level trims leading slash",
|
||||
location: locationIn("app.war", "/WEB-INF/lib/dep.jar"),
|
||||
expected: "app.war:WEB-INF/lib/dep.jar",
|
||||
},
|
||||
{
|
||||
name: "single level with slash-less entry",
|
||||
location: locationIn("app.war", "WEB-INF/lib/dep.jar"),
|
||||
expected: "app.war:WEB-INF/lib/dep.jar",
|
||||
},
|
||||
{
|
||||
name: "multi-level chain",
|
||||
location: locationIn("app.war", "WEB-INF/lib/dep.jar!/inner/most.jar"),
|
||||
expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar",
|
||||
},
|
||||
{
|
||||
// a colon in the entry name must not be mistaken for a chain delimiter
|
||||
name: "colon in entry path is escaped",
|
||||
location: locationIn("app.war", "WEB-INF/lib/dep:special.jar"),
|
||||
expected: "app.war:WEB-INF/lib/dep%3Aspecial.jar",
|
||||
},
|
||||
{
|
||||
name: "every colon in the entry path is escaped",
|
||||
location: locationIn("app.war", "lib/a:b:c.jar"),
|
||||
expected: "app.war:lib/a%3Ab%3Ac.jar",
|
||||
},
|
||||
{
|
||||
// or the literal text %3A would collide with an escaped colon
|
||||
name: "the escape character is itself escaped",
|
||||
location: locationIn("outer.zip", "a%3Ab.jar"),
|
||||
expected: "outer.zip:a%253Ab.jar",
|
||||
},
|
||||
{
|
||||
name: "access path is preferred over real path",
|
||||
location: file.NewVirtualLocation("/real/dep.jar", "/link/dep.jar"),
|
||||
expected: "/link/dep.jar",
|
||||
},
|
||||
{
|
||||
name: "access path within an archive replaces the last entry",
|
||||
location: file.NewVirtualLocationFromCoordinates(
|
||||
file.Coordinates{RealPath: "app.war", FullPath: "app.war!/WEB-INF/lib/real.jar"}, "app.war!/WEB-INF/lib/link.jar"),
|
||||
expected: "app.war:WEB-INF/lib/link.jar",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.expected, VirtualPath(tt.location))
|
||||
})
|
||||
}
|
||||
}
|
||||
+79
-1
@@ -1,6 +1,9 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/wagoodman/go-partybus"
|
||||
"github.com/wagoodman/go-progress"
|
||||
|
||||
@@ -36,7 +39,82 @@ func Notify(message string) {
|
||||
})
|
||||
}
|
||||
|
||||
func StartCatalogerTask(info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress {
|
||||
type catalogerTaskRegistryKey struct{}
|
||||
|
||||
// catalogerTaskRegistry remembers the progress row each cataloger publishes. A cataloger runs once
|
||||
// over the scan root and again inside every archive the archive cataloger walks; a re-run reports
|
||||
// into the row it already owns rather than publishing another, and republishing an ID would replace
|
||||
// the live row with one the UI never sees completed.
|
||||
type catalogerTaskRegistry struct {
|
||||
mu sync.Mutex
|
||||
rows map[string]*monitor.TaskProgress
|
||||
}
|
||||
|
||||
// WithCatalogerTaskRegistry returns a context under which each cataloger publishes one progress row
|
||||
// however many times it runs. A context that already carries a registry is returned unchanged.
|
||||
func WithCatalogerTaskRegistry(ctx context.Context) context.Context {
|
||||
if catalogerTaskRegistryFromContext(ctx) != nil {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, catalogerTaskRegistryKey{}, &catalogerTaskRegistry{
|
||||
rows: make(map[string]*monitor.TaskProgress),
|
||||
})
|
||||
}
|
||||
|
||||
func catalogerTaskRegistryFromContext(ctx context.Context) *catalogerTaskRegistry {
|
||||
reg, _ := ctx.Value(catalogerTaskRegistryKey{}).(*catalogerTaskRegistry)
|
||||
return reg
|
||||
}
|
||||
|
||||
// catalogerTaskKey identifies the row a task owns: its ID when it has one, otherwise its parent and
|
||||
// title (file catalogers publish no ID).
|
||||
func catalogerTaskKey(info monitor.GenericTask) string {
|
||||
if info.ID != "" {
|
||||
return "id:" + info.ID
|
||||
}
|
||||
return "title:" + info.ParentID + "/" + info.Title.Default
|
||||
}
|
||||
|
||||
func StartCatalogerTask(ctx context.Context, info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress {
|
||||
reg := catalogerTaskRegistryFromContext(ctx)
|
||||
if reg == nil {
|
||||
return publishCatalogerTask(info, size, initialStage)
|
||||
}
|
||||
|
||||
reg.mu.Lock()
|
||||
defer reg.mu.Unlock()
|
||||
|
||||
key := catalogerTaskKey(info)
|
||||
if row, ok := reg.rows[key]; ok {
|
||||
resumeCatalogerTask(row, size, initialStage)
|
||||
return row
|
||||
}
|
||||
|
||||
t := publishCatalogerTask(info, size, initialStage)
|
||||
reg.rows[key] = t
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
// resumeCatalogerTask reopens a row an earlier run completed and grows its total by what this run
|
||||
// adds, so the row does not read as finished mid-walk. A row carrying a real failure keeps it.
|
||||
func resumeCatalogerTask(row *monitor.TaskProgress, size int64, initialStage string) {
|
||||
if size < 0 || row.Size() < 0 {
|
||||
row.SetTotal(-1)
|
||||
} else {
|
||||
row.SetTotal(row.Size() + size)
|
||||
}
|
||||
|
||||
if progress.IsErrCompleted(row.Error()) {
|
||||
row.SetError(nil)
|
||||
}
|
||||
|
||||
if initialStage != "" {
|
||||
row.AtomicStage.Set(initialStage)
|
||||
}
|
||||
}
|
||||
|
||||
func publishCatalogerTask(info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress {
|
||||
t := &monitor.TaskProgress{
|
||||
AtomicStage: progress.NewAtomicStage(initialStage),
|
||||
Manual: progress.NewManual(size),
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/wagoodman/go-progress"
|
||||
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
)
|
||||
|
||||
func digestsTask() monitor.GenericTask {
|
||||
return monitor.GenericTask{
|
||||
Title: monitor.Title{Default: "File digests"},
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartCatalogerTask_withoutARegistryEveryRunPublishes(t *testing.T) {
|
||||
// the right answer for a caller driving a single pass: nothing re-runs, so nothing should be reused
|
||||
ctx := context.Background()
|
||||
|
||||
first := StartCatalogerTask(ctx, digestsTask(), 2, "")
|
||||
second := StartCatalogerTask(ctx, digestsTask(), 3, "")
|
||||
|
||||
assert.NotSame(t, first, second)
|
||||
assert.Equal(t, int64(2), first.Size())
|
||||
assert.Equal(t, int64(3), second.Size())
|
||||
}
|
||||
|
||||
func TestStartCatalogerTask_reRunReportsIntoTheRowItAlreadyOwns(t *testing.T) {
|
||||
ctx := WithCatalogerTaskRegistry(context.Background())
|
||||
|
||||
first := StartCatalogerTask(ctx, digestsTask(), 2, "")
|
||||
first.Increment()
|
||||
first.Increment()
|
||||
first.SetCompleted()
|
||||
|
||||
// the archive walk reaches the same cataloger again, over an archive's contents this time
|
||||
second := StartCatalogerTask(ctx, digestsTask(), 3, "")
|
||||
require.Same(t, first, second, "a re-run must find the row it already published")
|
||||
|
||||
assert.Equal(t, int64(5), second.Size(), "the total must cover both runs, not just the first")
|
||||
assert.False(t, progress.IsCompleted(second), "a row with work still to do must not read as complete")
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
second.Increment()
|
||||
}
|
||||
second.SetCompleted()
|
||||
|
||||
assert.Equal(t, int64(5), second.Current())
|
||||
assert.Equal(t, int64(5), second.Size(), "the count must not run past the total")
|
||||
assert.True(t, progress.IsCompleted(second))
|
||||
}
|
||||
|
||||
func TestStartCatalogerTask_aTotalResolvedOnCompletionIsAddable(t *testing.T) {
|
||||
// a run that could not say how much it would do says so when it signs off (Manual.SetCompleted
|
||||
// backfills the total from the count), so by the time a re-run arrives there is a real number to
|
||||
// add to. This is the file metadata cataloger's shape: it publishes -1 every run.
|
||||
ctx := WithCatalogerTaskRegistry(context.Background())
|
||||
|
||||
row := StartCatalogerTask(ctx, digestsTask(), -1, "")
|
||||
row.Increment()
|
||||
row.SetCompleted()
|
||||
require.Equal(t, int64(1), row.Size(), "signing off must resolve the total to what the run did")
|
||||
|
||||
require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), -1, ""))
|
||||
assert.Equal(t, int64(-1), row.Size(), "a re-run that cannot say its own size makes the row indeterminate again")
|
||||
|
||||
row.Increment()
|
||||
row.SetCompleted()
|
||||
assert.Equal(t, int64(2), row.Current())
|
||||
assert.Equal(t, int64(2), row.Size(), "and signing off resolves it again, over both runs")
|
||||
}
|
||||
|
||||
func TestStartCatalogerTask_indeterminateTotalsStayIndeterminate(t *testing.T) {
|
||||
// a run whose size is unknown cannot be added to one that is known
|
||||
tests := []struct {
|
||||
name string
|
||||
first, next int64
|
||||
complete func(row *monitor.TaskProgress)
|
||||
}{
|
||||
{
|
||||
name: "the re-run cannot say its own size",
|
||||
first: 2, next: -1,
|
||||
complete: func(row *monitor.TaskProgress) { row.Increment(); row.SetCompleted() },
|
||||
},
|
||||
{
|
||||
name: "the first run did nothing, so completing resolved nothing",
|
||||
first: -1, next: 3,
|
||||
complete: func(row *monitor.TaskProgress) { row.SetCompleted() },
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx := WithCatalogerTaskRegistry(context.Background())
|
||||
|
||||
row := StartCatalogerTask(ctx, digestsTask(), tt.first, "")
|
||||
tt.complete(row)
|
||||
|
||||
require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), tt.next, ""))
|
||||
assert.Equal(t, int64(-1), row.Size())
|
||||
assert.False(t, progress.IsCompleted(row), "an indeterminate row is not complete until it says so")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartCatalogerTask_aFailedRowKeepsItsFailure(t *testing.T) {
|
||||
// a later run finding nothing wrong does not mean the earlier one succeeded
|
||||
ctx := WithCatalogerTaskRegistry(context.Background())
|
||||
failure := errors.New("unable to read the thing")
|
||||
|
||||
row := StartCatalogerTask(ctx, digestsTask(), 2, "")
|
||||
row.SetError(failure)
|
||||
|
||||
require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), 3, ""))
|
||||
assert.ErrorIs(t, row.Error(), failure)
|
||||
}
|
||||
|
||||
func TestStartCatalogerTask_rowsAreKeyedByTaskIdentity(t *testing.T) {
|
||||
ctx := WithCatalogerTaskRegistry(context.Background())
|
||||
|
||||
byID := func(id string) monitor.GenericTask {
|
||||
return monitor.GenericTask{ID: id, Title: monitor.Title{Default: "shared title"}}
|
||||
}
|
||||
|
||||
assert.Same(t, StartCatalogerTask(ctx, byID("a"), -1, ""), StartCatalogerTask(ctx, byID("a"), -1, ""))
|
||||
assert.NotSame(t, StartCatalogerTask(ctx, byID("a"), -1, ""), StartCatalogerTask(ctx, byID("b"), -1, ""),
|
||||
"an ID is the identity where there is one, even when two tasks share a title")
|
||||
|
||||
// a task publishing without an ID, as every file cataloger does, is keyed by where it sits and what
|
||||
// it is called
|
||||
underCataloging := monitor.GenericTask{Title: monitor.Title{Default: "Files"}, ParentID: "cataloging"}
|
||||
underSomethingElse := monitor.GenericTask{Title: monitor.Title{Default: "Files"}, ParentID: "other"}
|
||||
|
||||
assert.Same(t, StartCatalogerTask(ctx, underCataloging, -1, ""), StartCatalogerTask(ctx, underCataloging, -1, ""))
|
||||
assert.NotSame(t, StartCatalogerTask(ctx, underCataloging, -1, ""), StartCatalogerTask(ctx, underSomethingElse, -1, ""))
|
||||
}
|
||||
|
||||
func TestWithCatalogerTaskRegistry_isIdempotent(t *testing.T) {
|
||||
// anything that re-runs catalogers may ask for a registry; a second one would hold none of the rows
|
||||
// already published and every re-run would publish again
|
||||
ctx := WithCatalogerTaskRegistry(context.Background())
|
||||
row := StartCatalogerTask(ctx, digestsTask(), 2, "")
|
||||
|
||||
nested := WithCatalogerTaskRegistry(ctx)
|
||||
assert.Same(t, row, StartCatalogerTask(nested, digestsTask(), 2, ""))
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal/capabilities/internal"
|
||||
@@ -31,6 +32,10 @@ type ConfigInfo struct {
|
||||
|
||||
var appConfigAnnotationPattern = regexp.MustCompile(`^//\s*app-config:\s*(.+)$`)
|
||||
|
||||
// embeddedFieldsAnnotationPattern matches "// capabilities-fields: A, B" on an embedded struct, which limits
|
||||
// the embedded fields listed to the ones named (for a shared type the cataloger reads only part of).
|
||||
var embeddedFieldsAnnotationPattern = regexp.MustCompile(`^//\s*capabilities-fields:\s*(.+)$`)
|
||||
|
||||
// DiscoverConfigs walks the cataloger directory and discovers all configuration structs
|
||||
// Returns map where key is "packageName.StructName" (e.g., "golang.CatalogerConfig")
|
||||
func DiscoverConfigs(repoRoot string) (map[string]ConfigInfo, error) {
|
||||
@@ -160,6 +165,9 @@ func extractCatalogerConfigFieldsRecursive(structType *ast.StructType, visitedTy
|
||||
if len(field.Names) == 0 {
|
||||
// this is an embedded field - resolve and extract its fields
|
||||
embeddedFields := resolveEmbeddedStructFields(field.Type, visitedTypes)
|
||||
if only := embeddedFieldAllowlist(field.Doc); only != nil {
|
||||
embeddedFields = slices.DeleteFunc(embeddedFields, func(f ConfigField) bool { return !only[f.Name] })
|
||||
}
|
||||
fields = append(fields, embeddedFields...)
|
||||
continue
|
||||
}
|
||||
@@ -195,6 +203,26 @@ func extractCatalogerConfigFieldsRecursive(structType *ast.StructType, visitedTy
|
||||
return fields
|
||||
}
|
||||
|
||||
// embeddedFieldAllowlist returns the field names from a "capabilities-fields:" annotation, or nil when there
|
||||
// is none (every exported embedded field is listed).
|
||||
func embeddedFieldAllowlist(commentGroup *ast.CommentGroup) map[string]bool {
|
||||
if commentGroup == nil {
|
||||
return nil
|
||||
}
|
||||
for _, comment := range commentGroup.List {
|
||||
matches := embeddedFieldsAnnotationPattern.FindStringSubmatch(comment.Text)
|
||||
if len(matches) < 2 {
|
||||
continue
|
||||
}
|
||||
only := make(map[string]bool)
|
||||
for _, name := range strings.Split(matches[1], ",") {
|
||||
only[strings.TrimSpace(name)] = true
|
||||
}
|
||||
return only
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveEmbeddedStructFields resolves an embedded struct type and extracts its fields recursively
|
||||
func resolveEmbeddedStructFields(fieldType ast.Expr, visitedTypes map[string]bool) []ConfigField {
|
||||
// extract the type name from the expression
|
||||
|
||||
@@ -457,3 +457,32 @@ func TestDiscoverAllowedConfigStructs(t *testing.T) {
|
||||
t.Logf(" - %s", configType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbeddedFieldAllowlist(t *testing.T) {
|
||||
comments := func(lines ...string) *ast.CommentGroup {
|
||||
g := &ast.CommentGroup{}
|
||||
for _, l := range lines {
|
||||
g.List = append(g.List, &ast.Comment{Text: l})
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
doc *ast.CommentGroup
|
||||
want map[string]bool
|
||||
}{
|
||||
{name: "no doc lists every field", doc: nil, want: nil},
|
||||
{name: "no annotation lists every field", doc: comments("// Deprecated: use something else."), want: nil},
|
||||
{
|
||||
name: "annotation limits to the named fields",
|
||||
doc: comments("// Deprecated: use something else.", "// capabilities-fields: IncludeIndexedArchives, IncludeUnindexedArchives"),
|
||||
want: map[string]bool{"IncludeIndexedArchives": true, "IncludeUnindexedArchives": true},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
require.Equal(t, tt.want, embeddedFieldAllowlist(tt.doc))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,5 +69,5 @@ func locationSorter(a, b file.Location) int {
|
||||
return strings.Compare(a.AccessPath, b.AccessPath)
|
||||
}
|
||||
|
||||
return strings.Compare(a.RealPath, b.RealPath)
|
||||
return strings.Compare(a.FullNestedPath(), b.FullNestedPath())
|
||||
}
|
||||
|
||||
@@ -14,5 +14,5 @@ func DefaultLocationComparer(x, y file.Location) bool {
|
||||
}
|
||||
|
||||
func LocationComparerWithoutLayer(x, y file.Location) bool {
|
||||
return cmp.Equal(x.RealPath, y.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath)
|
||||
return cmp.Equal(x.FullNestedPath(), y.FullNestedPath()) && cmp.Equal(x.AccessPath, y.AccessPath)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ package internal
|
||||
const (
|
||||
// JSONSchemaVersion is the current schema version output by the JSON encoder
|
||||
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
|
||||
JSONSchemaVersion = "16.1.11"
|
||||
JSONSchemaVersion = "16.1.12"
|
||||
|
||||
// Changelog
|
||||
// 16.1.0 - reformulated the python pdm fields (added "URL" and removed the unused "path" field).
|
||||
@@ -18,4 +18,5 @@ const (
|
||||
// 16.1.9 - add Symbols (grouped by owning package import path) to GolangBinaryBuildinfoEntry metadata
|
||||
// 16.1.10 - add SafeTensorsModelInfo metadata type for the safetensors AI model cataloger
|
||||
// 16.1.11 - add CpanDistribution and CpanUnpackedRelease metadata types for the perl catalogers
|
||||
// 16.1.12 - add fullPath to file coordinates, naming the nested archive chain a file was found in (blank for files not within an archive)
|
||||
)
|
||||
|
||||
@@ -6,13 +6,13 @@ import (
|
||||
"io"
|
||||
)
|
||||
|
||||
const perFileReadLimit = 2 * GB
|
||||
// PerFileReadLimit is the most read from any one archive entry, guarding against decompression bombs.
|
||||
const PerFileReadLimit = 2 * GB
|
||||
|
||||
// safeCopy limits the copy from the reader. This is useful when extracting files from archives to
|
||||
// protect against decompression bomb attacks.
|
||||
func safeCopy(writer io.Writer, reader io.Reader) error {
|
||||
numBytes, err := io.Copy(writer, io.LimitReader(reader, perFileReadLimit))
|
||||
if numBytes >= perFileReadLimit {
|
||||
// SafeCopy bounds the copy to guard against decompression bombs when extracting from archives.
|
||||
func SafeCopy(writer io.Writer, reader io.Reader) error {
|
||||
numBytes, err := io.Copy(writer, io.LimitReader(reader, PerFileReadLimit))
|
||||
if numBytes >= PerFileReadLimit {
|
||||
return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
|
||||
}
|
||||
// Propagate decompression / read errors up to the caller. io.Copy
|
||||
|
||||
@@ -29,7 +29,7 @@ func (r *errReader) Read(p []byte) (int, error) {
|
||||
func TestSafeCopy(t *testing.T) {
|
||||
t.Run("clean copy returns nil", func(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
if err := safeCopy(&buf, strings.NewReader("hello")); err != nil {
|
||||
if err := SafeCopy(&buf, strings.NewReader("hello")); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := buf.String(); got != "hello" {
|
||||
@@ -38,12 +38,12 @@ func TestSafeCopy(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("propagates decompression error", func(t *testing.T) {
|
||||
// #4806: safeCopy used to drop non-EOF errors, so the caller
|
||||
// #4806: SafeCopy used to drop non-EOF errors, so the caller
|
||||
// would persist a partial buffer as a successful extract and
|
||||
// downstream catalogers silently read empty manifests.
|
||||
sentinel := errors.New("flate: corrupt input before offset 42")
|
||||
var buf bytes.Buffer
|
||||
err := safeCopy(&buf, &errReader{data: []byte("partial"), err: sentinel})
|
||||
err := SafeCopy(&buf, &errReader{data: []byte("partial"), err: sentinel})
|
||||
if err == nil {
|
||||
t.Fatalf("expected error to be returned, got nil")
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func TestSafeCopy(t *testing.T) {
|
||||
// The old code had a dead io.EOF branch that labelled clean
|
||||
// reads as decompression bombs; keep the happy path clean.
|
||||
var buf bytes.Buffer
|
||||
err := safeCopy(&buf, io.LimitReader(strings.NewReader("abc"), 3))
|
||||
err := SafeCopy(&buf, io.LimitReader(strings.NewReader("abc"), 3))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error on clean copy: %v", err)
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ func ExtractGlobsFromTarToUniqueTempFile(ctx context.Context, archivePath, dir s
|
||||
}
|
||||
defer internal.CloseAndLogError(packedFile, archivePath)
|
||||
|
||||
if err := safeCopy(tempFile, packedFile); err != nil {
|
||||
if err := SafeCopy(tempFile, packedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for tar=%q: %w", file.Name(), archivePath, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -101,7 +101,7 @@ func ExtractFromZipToUniqueTempFile(ctx context.Context, archivePath, dir string
|
||||
return fmt.Errorf("unable to extract directories, only files: %s", file.NameInArchive)
|
||||
}
|
||||
|
||||
if err := safeCopy(tempFile, zippedFile); err != nil {
|
||||
if err := SafeCopy(tempFile, zippedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
|
||||
@@ -138,7 +138,7 @@ func ContentsFromZip(ctx context.Context, archivePath string, paths ...string) (
|
||||
}
|
||||
|
||||
var buffer bytes.Buffer
|
||||
if err := safeCopy(&buffer, zippedFile); err != nil {
|
||||
if err := SafeCopy(&buffer, zippedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
|
||||
@@ -210,7 +210,7 @@ func extractSingleFile(file archives.FileInfo, expandedFilePath, archivePath str
|
||||
}
|
||||
}()
|
||||
|
||||
if err := safeCopy(outputFile, zippedFile); err != nil {
|
||||
if err := SafeCopy(outputFile, zippedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q to dest=%q for zip=%q: %w", file.NameInArchive, outputFile.Name(), archivePath, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,7 +49,7 @@ func findRhelFeatures(resolver file.Resolver, release *linux.Release) error {
|
||||
for _, contentManifestFile := range contentManifestFiles {
|
||||
found, err := fileMatchesAny(resolver, contentManifestFile, rhelEUSPatterns...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.RealPath, err)
|
||||
return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.FullNestedPath(), err)
|
||||
}
|
||||
if found {
|
||||
release.ExtendedSupport = true
|
||||
@@ -163,9 +163,9 @@ func hasActiveUbuntuESMService(resolver file.Resolver) (bool, error) {
|
||||
func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location) (bool, error) {
|
||||
contents, err := resolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.RealPath, err)
|
||||
return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.FullNestedPath(), err)
|
||||
}
|
||||
defer internal.CloseAndLogError(contents, location.RealPath)
|
||||
defer internal.CloseAndLogError(contents, location.FullNestedPath())
|
||||
|
||||
var status struct {
|
||||
Services []struct {
|
||||
@@ -174,7 +174,7 @@ func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location
|
||||
} `json:"services"`
|
||||
}
|
||||
if err := json.NewDecoder(contents).Decode(&status); err != nil {
|
||||
return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.RealPath, err)
|
||||
return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.FullNestedPath(), err)
|
||||
}
|
||||
|
||||
// pro status values are enabled/disabled/n-a/warning/—; only "enabled" proves the stream is active.
|
||||
@@ -200,8 +200,8 @@ func hasInstalledESMPackage(packages *pkg.Collection) bool {
|
||||
func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) {
|
||||
contents, err := resolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("unable to read %s: %w", location.RealPath, err)
|
||||
return false, fmt.Errorf("unable to read %s: %w", location.FullNestedPath(), err)
|
||||
}
|
||||
defer internal.CloseAndLogError(contents, location.RealPath)
|
||||
defer internal.CloseAndLogError(contents, location.FullNestedPath())
|
||||
return internal.MatchAnyFromReader(contents, patterns...)
|
||||
}
|
||||
|
||||
@@ -71,7 +71,7 @@ func compareElfBinaryPackages(s *sbom.SBOM) []artifact.ID {
|
||||
pkgsToDelete := make([]artifact.ID, 0)
|
||||
for _, elfPkg := range allElfPackages(s) {
|
||||
for _, loc := range onlyPrimaryEvidenceLocations(elfPkg) {
|
||||
for _, otherPkg := range s.Artifacts.Packages.PackagesByPath(loc.RealPath) {
|
||||
for _, otherPkg := range s.Artifacts.Packages.PackagesByPath(loc.FullNestedPath()) {
|
||||
// we only care about comparing binary packages to each other (not other types)
|
||||
if otherPkg.Type != pkg.BinaryPkg {
|
||||
continue
|
||||
@@ -130,7 +130,7 @@ func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver
|
||||
|
||||
for _, loc := range libLocations {
|
||||
// are you in our index?
|
||||
realBaseName := path.Base(loc.RealPath)
|
||||
realBaseName := path.Base(loc.FullNestedPath())
|
||||
pkgCollection := index.owningLibraryPackage(realBaseName)
|
||||
if pkgCollection.PackageCount() < 1 {
|
||||
addRelationship(
|
||||
|
||||
@@ -3,6 +3,7 @@ package binary
|
||||
import (
|
||||
"path"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
@@ -63,7 +64,7 @@ func locationsThatProvideLibraries(accessor sbomsync.Accessor) (map[string]file.
|
||||
continue
|
||||
}
|
||||
|
||||
basename := path.Base(coord.RealPath)
|
||||
basename := path.Base(coord.FullNestedPath())
|
||||
set := libLocationsByBasename[basename]
|
||||
set.Add(coord)
|
||||
allLibLocations.Add(coord)
|
||||
@@ -82,6 +83,11 @@ func packagesWithLibraryOwnership(resolver file.Resolver, accessor sbomsync.Acce
|
||||
var ownedFilePaths []string
|
||||
if p.Type == pkg.BinaryPkg {
|
||||
for _, loc := range p.Locations.ToSlice() {
|
||||
if archive.ContainerPath(loc.Coordinates) != "" {
|
||||
// a path within an archive cannot be resolved from the scan root; the location is the library
|
||||
addLibraryOwner(packagesByLibraryPath, allLibLocations, loc.Coordinates, p)
|
||||
continue
|
||||
}
|
||||
ownedFilePaths = append(ownedFilePaths, loc.Path())
|
||||
}
|
||||
} else {
|
||||
@@ -114,18 +120,19 @@ func populatePackagesByLibraryPath(
|
||||
}
|
||||
|
||||
for _, loc := range ownedLocation {
|
||||
// if the location is a library, add the package to the set of packages that own the library
|
||||
if !allLibLocations.Contains(loc.Coordinates) {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, ok := packagesByLibraryPath[loc.Coordinates]; !ok {
|
||||
packagesByLibraryPath[loc.Coordinates] = pkg.NewCollection()
|
||||
}
|
||||
|
||||
// we have a library path, add the package to the set of packages that own the library
|
||||
packagesByLibraryPath[loc.Coordinates].Add(p)
|
||||
addLibraryOwner(packagesByLibraryPath, allLibLocations, loc.Coordinates, p)
|
||||
}
|
||||
}
|
||||
return packagesByLibraryPath
|
||||
}
|
||||
|
||||
// addLibraryOwner records p as an owner of the file at coordinates when that file is a library.
|
||||
func addLibraryOwner(packagesByLibraryPath map[file.Coordinates]*pkg.Collection, allLibLocations file.CoordinateSet, coordinates file.Coordinates, p pkg.Package) {
|
||||
if !allLibLocations.Contains(coordinates) {
|
||||
return
|
||||
}
|
||||
if _, ok := packagesByLibraryPath[coordinates]; !ok {
|
||||
packagesByLibraryPath[coordinates] = pkg.NewCollection()
|
||||
}
|
||||
packagesByLibraryPath[coordinates].Add(p)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
@@ -100,7 +101,7 @@ func findOwnershipByFilesRelationships(resolver file.Resolver, catalog *pkg.Coll
|
||||
// Now establish relationships, considering symlinks
|
||||
for _, candidateOwnerPkg := range catalog.Sorted() {
|
||||
id := candidateOwnerPkg.ID()
|
||||
if candidateOwnerPkg.Metadata == nil {
|
||||
if candidateOwnerPkg.Metadata == nil || withinArchive(candidateOwnerPkg) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -164,7 +165,7 @@ func directOwnersByPath(catalog *pkg.Collection) map[pkg.Type]*strset.Set {
|
||||
|
||||
// First, identify direct ownership of all files
|
||||
for _, p := range catalog.Sorted() {
|
||||
if p.Metadata == nil {
|
||||
if p.Metadata == nil || withinArchive(p) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -192,6 +193,21 @@ func directOwnersByPath(catalog *pkg.Collection) map[pkg.Type]*strset.Set {
|
||||
return directOwnership
|
||||
}
|
||||
|
||||
// withinArchive reports whether every location of p is inside an archive. Such a package's owned files
|
||||
// are paths within its archive, not the scan root, so it is not treated as a file owner.
|
||||
func withinArchive(p pkg.Package) bool {
|
||||
locations := p.Locations.ToSlice()
|
||||
if len(locations) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, l := range locations {
|
||||
if archive.ContainerPath(l.Coordinates) == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func resolvePaths(ownedFilePath string, resolver file.Resolver) []string {
|
||||
// though we have a string path, we need to resolve symlinks and other filesystem oddities since we cannot assume this is a real path
|
||||
var locs []file.Location
|
||||
@@ -206,6 +222,8 @@ func resolvePaths(ownedFilePath string, resolver file.Resolver) []string {
|
||||
|
||||
ownedFilePaths := strset.New(ownedFilePath)
|
||||
for _, loc := range locs {
|
||||
// Owned file paths are currently based on the top-level resolver, which is always the RealPath and includes
|
||||
// all files found within archives at that location
|
||||
ownedFilePaths.Add(loc.RealPath)
|
||||
}
|
||||
return ownedFilePaths.List()
|
||||
|
||||
@@ -78,6 +78,56 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
|
||||
return []pkg.Package{parent, child}, []artifact.Relationship{relationship}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "package-within-archive-is-not-a-file-owner",
|
||||
setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) {
|
||||
nestedOwner := pkg.Package{
|
||||
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(file.Coordinates{
|
||||
RealPath: "/rootfs.tar",
|
||||
FullPath: "/rootfs.tar!/var/lib/rpm/Packages",
|
||||
})),
|
||||
Type: pkg.RpmPkg,
|
||||
Metadata: pkg.RpmDBEntry{
|
||||
Files: []pkg.RpmFileRecord{{Path: "/d/path"}},
|
||||
},
|
||||
}
|
||||
nestedOwner.SetID()
|
||||
|
||||
child := pkg.Package{
|
||||
Locations: file.NewLocationSet(file.NewLocation("/d/path")),
|
||||
Type: pkg.BinaryPkg,
|
||||
}
|
||||
child.SetID()
|
||||
|
||||
return []pkg.Package{nestedOwner, child}, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
// the archive's own package (when it has one) is found by its path; what is inside it is
|
||||
// linked to that package by CONTAINS, not owned by whoever owns the archive file
|
||||
name: "top-level-owner-of-an-archive-does-not-own-packages-within-it",
|
||||
setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) {
|
||||
parent := pkg.Package{
|
||||
Locations: file.NewLocationSet(file.NewLocation("/var/lib/rpm/Packages")),
|
||||
Type: pkg.RpmPkg,
|
||||
Metadata: pkg.RpmDBEntry{
|
||||
Files: []pkg.RpmFileRecord{{Path: "/opt/bundle.tar"}},
|
||||
},
|
||||
}
|
||||
parent.SetID()
|
||||
|
||||
child := pkg.Package{
|
||||
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(file.Coordinates{
|
||||
RealPath: "/opt/bundle.tar",
|
||||
FullPath: "/opt/bundle.tar!/lib/thing.jar",
|
||||
})),
|
||||
Type: pkg.JavaPkg,
|
||||
}
|
||||
child.SetID()
|
||||
|
||||
return []pkg.Package{parent, child}, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "only-real-file-owner-gets-ownership-relationship-not-symlink-owner",
|
||||
resolver: mockFR{
|
||||
|
||||
@@ -15,18 +15,9 @@ func TestRelationshipsEvidentBy(t *testing.T) {
|
||||
|
||||
c := pkg.NewCollection()
|
||||
|
||||
coordA := file.Coordinates{
|
||||
RealPath: "/somewhere/real",
|
||||
FileSystemID: "abc",
|
||||
}
|
||||
coordC := file.Coordinates{
|
||||
RealPath: "/somewhere/real",
|
||||
FileSystemID: "abc",
|
||||
}
|
||||
coordD := file.Coordinates{
|
||||
RealPath: "/somewhere/real",
|
||||
FileSystemID: "abc",
|
||||
}
|
||||
coordA := file.NewCoordinates("/somewhere/real", "abc")
|
||||
coordC := file.NewCoordinates("/somewhere/real", "abc")
|
||||
coordD := file.NewCoordinates("/somewhere/real", "abc")
|
||||
pkgA := pkg.Package{
|
||||
Locations: file.NewLocationSet(
|
||||
// added!
|
||||
@@ -39,10 +30,7 @@ func TestRelationshipsEvidentBy(t *testing.T) {
|
||||
pkgA.SetID()
|
||||
c.Add(pkgA)
|
||||
|
||||
coordB := file.Coordinates{
|
||||
RealPath: "/somewhere-else/real",
|
||||
FileSystemID: "def",
|
||||
}
|
||||
coordB := file.NewCoordinates("/somewhere-else/real", "def")
|
||||
pkgB := pkg.Package{
|
||||
Locations: file.NewLocationSet(
|
||||
// added!
|
||||
|
||||
@@ -21,12 +21,8 @@ func Test_Index(t *testing.T) {
|
||||
p3 := pkg.Package{
|
||||
Name: "pkg-3",
|
||||
}
|
||||
c1 := file.Coordinates{
|
||||
RealPath: "/coords/1",
|
||||
}
|
||||
c2 := file.Coordinates{
|
||||
RealPath: "/coords/2",
|
||||
}
|
||||
c1 := file.NewCoordinates("/coords/1", "")
|
||||
c2 := file.NewCoordinates("/coords/2", "")
|
||||
|
||||
for _, p := range []*pkg.Package{&p1, &p2, &p3} {
|
||||
p.SetID()
|
||||
@@ -156,18 +152,10 @@ func Test_Coordinates(t *testing.T) {
|
||||
p3 := pkg.Package{
|
||||
Name: "pkg-3",
|
||||
}
|
||||
c1 := file.Coordinates{
|
||||
RealPath: "/coords/1",
|
||||
}
|
||||
c2 := file.Coordinates{
|
||||
RealPath: "/coords/2",
|
||||
}
|
||||
c3 := file.Coordinates{
|
||||
RealPath: "/coords/3",
|
||||
}
|
||||
c4 := file.Coordinates{
|
||||
RealPath: "/coords/4",
|
||||
}
|
||||
c1 := file.NewCoordinates("/coords/1", "")
|
||||
c2 := file.NewCoordinates("/coords/2", "")
|
||||
c3 := file.NewCoordinates("/coords/3", "")
|
||||
c4 := file.NewCoordinates("/coords/4", "")
|
||||
|
||||
for _, p := range []*pkg.Package{&p1, &p2, &p3} {
|
||||
p.SetID()
|
||||
@@ -237,10 +225,10 @@ func TestRemove(t *testing.T) {
|
||||
p1 := pkg.Package{Name: "pkg-1"}
|
||||
p2 := pkg.Package{Name: "pkg-2"}
|
||||
p3 := pkg.Package{Name: "pkg-3"}
|
||||
c1 := file.Coordinates{RealPath: "/coords/1"}
|
||||
c2 := file.Coordinates{RealPath: "/coords/2"}
|
||||
c3 := file.Coordinates{RealPath: "/coords/3"}
|
||||
c4 := file.Coordinates{RealPath: "/coords/4"}
|
||||
c1 := file.NewCoordinates("/coords/1", "")
|
||||
c2 := file.NewCoordinates("/coords/2", "")
|
||||
c3 := file.NewCoordinates("/coords/3", "")
|
||||
c4 := file.NewCoordinates("/coords/4", "")
|
||||
|
||||
for _, p := range []*pkg.Package{&p1, &p2, &p3} {
|
||||
p.SetID()
|
||||
|
||||
@@ -2131,40 +2131,40 @@ var urlToLicense = map[string]string{
|
||||
"raw.githubusercontent.com/NTIA/scos-sensor/refs/heads/master/LICENSE.md": "NTIA-PD",
|
||||
"raw.githubusercontent.com/OpenMDW/OpenMDW/refs/heads/main/1.0/LICENSE.openmdw": "OpenMDW-1.0",
|
||||
"raw.githubusercontent.com/blackhole89/notekit/master/data/fonts/Charter%20license.txt": "Bitstream-Charter",
|
||||
"raw.githubusercontent.com/landtuna/open-mtools/6c04c2882eca7c4313ec267766ea53ac9bfe129b/mdump.c": "Informatica",
|
||||
"raw.githubusercontent.com/landtuna/open-mtools/6c04c2882eca7c4313ec267766ea53ac9bfe129b/msend.c": "Informatica",
|
||||
"rc.quest.com/topics/openssh/license.php#tcpwrappers": "TCP-wrappers",
|
||||
"research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1",
|
||||
"research.scea.com/scea_shared_source_license.html": "SCEA",
|
||||
"romanrm.net/mit-zero": "MIT-0",
|
||||
"sam.zoy.org/wtfpl/COPYING": "WTFPL",
|
||||
"scancode-licensedb.aboutcode.org/bsd-unchanged": "BSD-2-Clause-pos-unchanged",
|
||||
"scripts.sil.org/cms/scripts/page.php?item_id=OFL10_web": "OFL-1.0-no-RFN",
|
||||
"scripts.sil.org/cms/scripts/page.php?item_id=OFL_web": "OFL-1.1-no-RFN",
|
||||
"slicer.org/LICENSE": "3D-Slicer-1.0",
|
||||
"solderpad.org/licenses/SHL-0.5/": "SHL-0.5",
|
||||
"solderpad.org/licenses/SHL-0.51/": "SHL-0.51",
|
||||
"source.icu-project.org/repos/icu/icu/trunk/license.html": "ICU",
|
||||
"sourceforge.net/p/buddy/gitcode/ci/master/tree/README": "Buddy",
|
||||
"sourceforge.net/p/ctrio/git/ci/master/tree/README": "ISC-Veillard",
|
||||
"sourceforge.net/p/mcj/xfig/ci/master/tree/src/Makefile.am": "Xfig",
|
||||
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/pnmtorle.c": "URT-RLE",
|
||||
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/rletopnm.c": "URT-RLE",
|
||||
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/ppm/ppmtompeg/jrevdct.c#l1189": "MPEG-SSG",
|
||||
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/editor/pnmstitch.c#l2": "pnmstitch",
|
||||
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/netpbm.c#l8": "HPND-Pbmplus",
|
||||
"sourceforge.net/p/sbcl/sbcl/ci/master/tree/src/code/loop.lisp": "LOOP",
|
||||
"sourceforge.net/p/trustedqsl/tqsl/ci/master/tree/LICENSE.txt": "TrustedQSL",
|
||||
"sourceforge.net/p/xmedcon/code/ci/master/tree/libs/dicom/README": "OFFIS",
|
||||
"sourceforge.net/p/xmedcon/code/ci/master/tree/libs/ljpg/": "IJG-short",
|
||||
"sourceforge.net/projects/sevenzip/files/LZMA%20SDK/": "LZMA-SDK-9.22",
|
||||
"sources.debian.org/copyright/license/debianutils/4.11.2/": "SMAIL-GPL",
|
||||
"sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl=19#L19": "mpi-permissive",
|
||||
"sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1=1.1&r2=1.1.1.1&diff_format=s": "BitTorrent-1.0",
|
||||
"sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html": "bzip2-1.0.6",
|
||||
"sourceware.org/cgit/binutils-gdb/tree/include/coff/sym.h#n11": "MIPS",
|
||||
"sourceware.org/cgit/binutils-gdb/tree/include/coff/symconst.h#n11": "ThirdEye",
|
||||
"sourceware.org/cgit/valgrind/tree/mpi/libmpiwrap.c": "bzip2-1.0.6",
|
||||
"raw.githubusercontent.com/landtuna/open-mtools/6c04c2882eca7c4313ec267766ea53ac9bfe129b/mdump.c": "Informatica",
|
||||
"raw.githubusercontent.com/landtuna/open-mtools/6c04c2882eca7c4313ec267766ea53ac9bfe129b/msend.c": "Informatica",
|
||||
"rc.quest.com/topics/openssh/license.php#tcpwrappers": "TCP-wrappers",
|
||||
"research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1",
|
||||
"research.scea.com/scea_shared_source_license.html": "SCEA",
|
||||
"romanrm.net/mit-zero": "MIT-0",
|
||||
"sam.zoy.org/wtfpl/COPYING": "WTFPL",
|
||||
"scancode-licensedb.aboutcode.org/bsd-unchanged": "BSD-2-Clause-pos-unchanged",
|
||||
"scripts.sil.org/cms/scripts/page.php?item_id=OFL10_web": "OFL-1.0-no-RFN",
|
||||
"scripts.sil.org/cms/scripts/page.php?item_id=OFL_web": "OFL-1.1-no-RFN",
|
||||
"slicer.org/LICENSE": "3D-Slicer-1.0",
|
||||
"solderpad.org/licenses/SHL-0.5/": "SHL-0.5",
|
||||
"solderpad.org/licenses/SHL-0.51/": "SHL-0.51",
|
||||
"source.icu-project.org/repos/icu/icu/trunk/license.html": "ICU",
|
||||
"sourceforge.net/p/buddy/gitcode/ci/master/tree/README": "Buddy",
|
||||
"sourceforge.net/p/ctrio/git/ci/master/tree/README": "ISC-Veillard",
|
||||
"sourceforge.net/p/mcj/xfig/ci/master/tree/src/Makefile.am": "Xfig",
|
||||
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/pnmtorle.c": "URT-RLE",
|
||||
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/rletopnm.c": "URT-RLE",
|
||||
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/ppm/ppmtompeg/jrevdct.c#l1189": "MPEG-SSG",
|
||||
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/editor/pnmstitch.c#l2": "pnmstitch",
|
||||
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/netpbm.c#l8": "HPND-Pbmplus",
|
||||
"sourceforge.net/p/sbcl/sbcl/ci/master/tree/src/code/loop.lisp": "LOOP",
|
||||
"sourceforge.net/p/trustedqsl/tqsl/ci/master/tree/LICENSE.txt": "TrustedQSL",
|
||||
"sourceforge.net/p/xmedcon/code/ci/master/tree/libs/dicom/README": "OFFIS",
|
||||
"sourceforge.net/p/xmedcon/code/ci/master/tree/libs/ljpg/": "IJG-short",
|
||||
"sourceforge.net/projects/sevenzip/files/LZMA%20SDK/": "LZMA-SDK-9.22",
|
||||
"sources.debian.org/copyright/license/debianutils/4.11.2/": "SMAIL-GPL",
|
||||
"sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl=19#L19": "mpi-permissive",
|
||||
"sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1=1.1&r2=1.1.1.1&diff_format=s": "BitTorrent-1.0",
|
||||
"sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html": "bzip2-1.0.6",
|
||||
"sourceware.org/cgit/binutils-gdb/tree/include/coff/sym.h#n11": "MIPS",
|
||||
"sourceware.org/cgit/binutils-gdb/tree/include/coff/symconst.h#n11": "ThirdEye",
|
||||
"sourceware.org/cgit/valgrind/tree/mpi/libmpiwrap.c": "bzip2-1.0.6",
|
||||
"sourceware.org/git/?p=binutils-gdb.git;a=blob;f=libiberty/strcasecmp.c;h=131d81c2ce7881fa48c363dc5bf5fb302c61ce0b;hb=HEAD": "BSD-4.3RENO",
|
||||
"sourceware.org/git/?p=binutils-gdb.git;a=blob;f=readline/readline/support/wcwidth.c;h=0f5ec995796f4813abbcf4972aec0378ab74722a;hb=HEAD#l55": "HPND-Markus-Kuhn",
|
||||
"sourceware.org/git/?p=bzip2.git;a=blob;f=LICENSE;hb=bzip2-1.0.6": "bzip2-1.0.6",
|
||||
|
||||
@@ -0,0 +1,699 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/scylladb/go-set/strset"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/internal/testutils"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
// This file covers nesting across archive families. The other nested-archive fixtures are all zips,
|
||||
// so a zip-specific mechanism would pass them; the requirement here is that an archive of one family
|
||||
// inside another behaves as it would inside its own family - same packages, same container-path chain,
|
||||
// same CONTAINS edges, same depth accounting.
|
||||
//
|
||||
// Chains are built in the test rather than committed as binaries, so each case can vary which family
|
||||
// sits at which level.
|
||||
|
||||
// archiveFamily is one archive format buildNestedArchive can produce.
|
||||
//
|
||||
// ".jar" is its own family rather than a flavour of zip: the java cataloger meets it as a jar wherever
|
||||
// it sits, while the archive task descends into it like any other zip. That differs mid-chain from at
|
||||
// the leaf, so the matrix below uses a jar as container and as payload.
|
||||
type archiveFamily string
|
||||
|
||||
const (
|
||||
familyZip archiveFamily = ".zip"
|
||||
familyJar archiveFamily = ".jar"
|
||||
familyTar archiveFamily = ".tar"
|
||||
familyTarGz archiveFamily = ".tar.gz"
|
||||
familyTgz archiveFamily = ".tgz"
|
||||
)
|
||||
|
||||
// nestPlan describes one chain to build: a family per level (outermost first), the entries for the
|
||||
// innermost archive, and extras to add beside the nested archive at a given level. The extras grade
|
||||
// the levels' sizes or entry counts, which is how the limit cases place a bound between two levels.
|
||||
type nestPlan struct {
|
||||
families []archiveFamily
|
||||
leaf map[string][]byte
|
||||
extra map[int]map[string][]byte
|
||||
}
|
||||
|
||||
// nestedArchive is one built chain: the outermost archive's bytes, and the addresses that chain
|
||||
// composes to at each level.
|
||||
type nestedArchive struct {
|
||||
// name is the outermost archive's file name, as written into the scanned directory.
|
||||
name string
|
||||
// bytes is the outermost archive.
|
||||
bytes []byte
|
||||
// sizes is each archive's size in bytes, outermost first. A container is never smaller than what it
|
||||
// holds, so these decrease, and the limit cases place a bound between two of them.
|
||||
sizes []int
|
||||
// paths is each archive's path within its parent, outermost first; paths[0] is the outermost archive
|
||||
// as the scan's resolver reports it.
|
||||
paths []string
|
||||
// containerPaths is each archive's full path, the container path its own contents carry, outermost first.
|
||||
containerPaths []string
|
||||
// virtualPaths is the same chain in java's colon-joined form, outermost first.
|
||||
virtualPaths []string
|
||||
}
|
||||
|
||||
// nestingRow is one line of the depth-3 matrix.
|
||||
type nestingRow struct {
|
||||
name string
|
||||
families []archiveFamily
|
||||
why string
|
||||
}
|
||||
|
||||
// nestingVisit is one filesystem the archive cataloger's sub-pipeline was run against.
|
||||
type nestingVisit struct {
|
||||
virtualPath string
|
||||
fullPath string
|
||||
depth int
|
||||
// heldFiles are the spill files live at this moment, sampled only when the probe was given a temp
|
||||
// dir to look in. A nested archive's own bytes are read where they lie, so one appears per archive
|
||||
// whose entries did not fit in memory.
|
||||
heldFiles []string
|
||||
}
|
||||
|
||||
// nestingProbe records one visit per filesystem the archive cataloger runs its sub-pipeline against,
|
||||
// reading the chain off the traversal on the context.
|
||||
type nestingProbe struct {
|
||||
// tempDir, when set, is where extraction work directories live
|
||||
tempDir string
|
||||
|
||||
mu sync.Mutex
|
||||
visits []nestingVisit
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_leafIsCatalogedOnceWithTheFullChain(t *testing.T) {
|
||||
// family order must not change the result: the same package, addressed by a chain naming its own
|
||||
// containers in its own order
|
||||
for _, row := range nestingRows() {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
probe := &nestingProbe{}
|
||||
s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), probe.task())
|
||||
|
||||
assert.Equal(t, 1, packageCount(s, "marker-pkg"), "%s: the leaf package must be cataloged exactly once", row.why)
|
||||
|
||||
locs := leafLocations(s)
|
||||
require.Len(t, locs, 1, "the leaf must be reached by exactly one path")
|
||||
// exact, never a substring: a partially composed chain passes a substring check
|
||||
assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates),
|
||||
"the chain must name all three archives outer-to-inner")
|
||||
assert.Equal(t, nested.leafContainerPath()+"!/nested/marker.txt", locs[0].FullNestedPath(),
|
||||
"the path stays relative to the archive the file came from")
|
||||
|
||||
assert.Equal(t, nested.containerPaths, probe.containerPaths(),
|
||||
"every level must be cataloged, outer to inner, whatever families they are")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_containsEdgesChainAtEveryLevel(t *testing.T) {
|
||||
// the edges make the chain navigable in the SBOM. The archive-to-archive edges must not depend on
|
||||
// the file catalogers having recorded the inner archives, since the default file selection does not
|
||||
for _, row := range nestingRows() {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
s := withArchiveContains(t, runNesting(t, scanDir, 3, defaultLimits(), markerTask()))
|
||||
|
||||
// each archive's coordinates as seen in its parent; the outermost sits in the scanned
|
||||
// directory, so it carries no filesystem id of its own
|
||||
archiveCoords := make([]file.Coordinates, len(row.families))
|
||||
for i := range row.families {
|
||||
archiveCoords[i] = nested.coordinates(i)
|
||||
}
|
||||
|
||||
fileEdges := strset.New()
|
||||
pkgEdges := strset.New()
|
||||
for _, rel := range s.Relationships {
|
||||
if rel.Type != artifact.ContainsRelationship {
|
||||
continue
|
||||
}
|
||||
from, ok := rel.From.(file.Coordinates)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch to := rel.To.(type) {
|
||||
case file.Coordinates:
|
||||
fileEdges.Add(coordKey(from) + " -> " + coordKey(to))
|
||||
case pkg.Package:
|
||||
pkgEdges.Add(coordKey(from) + " -> " + to.Name)
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < len(archiveCoords)-1; i++ {
|
||||
assert.True(t, fileEdges.Has(coordKey(archiveCoords[i])+" -> "+coordKey(archiveCoords[i+1])),
|
||||
"expected a CONTAINS edge from %s to %s", nested.paths[i], nested.paths[i+1])
|
||||
}
|
||||
|
||||
innermost := archiveCoords[len(archiveCoords)-1]
|
||||
assert.True(t, pkgEdges.Has(coordKey(innermost)+" -> marker-pkg"),
|
||||
"expected a CONTAINS edge from %s to the leaf package", nested.paths[len(nested.paths)-1])
|
||||
|
||||
// with file cataloging on, files inside an archive get an edge from it too
|
||||
s = withArchiveContains(t, runNesting(t, scanDir, 3, defaultLimits(), markerTask(), fileMetadataTask()))
|
||||
var leafEdge bool
|
||||
for _, rel := range s.Relationships {
|
||||
from, _ := rel.From.(file.Coordinates)
|
||||
to, _ := rel.To.(file.Coordinates)
|
||||
leafEdge = leafEdge || (from == innermost && to == file.Coordinates{RealPath: innermost.RealPath, FullPath: innermost.FullNestedPath() + "!/nested/marker.txt"})
|
||||
}
|
||||
assert.True(t, leafEdge, "expected a CONTAINS edge from the innermost archive to the leaf file")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_everyJarIsCatalogedOnceUnderItsChain(t *testing.T) {
|
||||
// two mechanisms meet a jar: the java cataloger catalogs it as a package, the archive task descends
|
||||
// into it. A jar at any level must do both, exactly once each; the failure mode is a duplicate, not an
|
||||
// error. The colon-joined virtual paths are in already-published SBOMs, so they must name the chain.
|
||||
for _, row := range nestingRows() {
|
||||
var jarLevels []int
|
||||
for i, family := range row.families {
|
||||
if family == familyJar {
|
||||
jarLevels = append(jarLevels, i)
|
||||
}
|
||||
}
|
||||
if len(jarLevels) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
probe := &nestingProbe{}
|
||||
s := runNesting(t, scanDir, 3, defaultLimits(), markerTask(), javaTask(), probe.task())
|
||||
|
||||
for _, level := range jarLevels {
|
||||
name := fmt.Sprintf("level%d", level)
|
||||
assert.Equal(t, 1, packageCount(s, name), "the jar's own package must be cataloged exactly once")
|
||||
assert.Equal(t, nested.virtualPaths[level], javaVirtualPaths(s)[name],
|
||||
"the jar's virtual path must be its own place in the chain")
|
||||
}
|
||||
|
||||
// and the archive task still descended through every level
|
||||
assert.Equal(t, nested.containerPaths, probe.containerPaths())
|
||||
locs := leafLocations(s)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_depthCountsLevelsNotFamilies(t *testing.T) {
|
||||
// two rows with different families at every level, so the bound is shown not to turn on which
|
||||
// families are involved
|
||||
for _, row := range nestingRows()[:2] {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, nestPlan{families: row.families, leaf: markerLeaf()})
|
||||
|
||||
probe := &nestingProbe{}
|
||||
s := runNesting(t, scanDir, 2, defaultLimits(), markerTask(), fileMetadataTask(), probe.task())
|
||||
|
||||
assert.Equal(t, nested.containerPaths[:2], probe.containerPaths(),
|
||||
"at depth 2 the outer and middle archives are cataloged and the innermost is not")
|
||||
assert.Zero(t, packageCount(s, "marker-pkg"),
|
||||
"the leaf is one level past the bound, so its package must be absent")
|
||||
|
||||
// the innermost archive is still cataloged as a file, inside the middle archive
|
||||
innermost := nested.coordinates(2)
|
||||
_, ok := s.Artifacts.FileMetadata[innermost]
|
||||
assert.True(t, ok, "the undescended archive must still be recorded as a file at %s", coordKey(innermost))
|
||||
|
||||
// and what was left unexplored is said out loud, not silently dropped
|
||||
reasons := s.Artifacts.Unknowns[innermost]
|
||||
require.Len(t, reasons, 1, "unknowns: %v", s.Artifacts.Unknowns)
|
||||
assert.Contains(t, reasons[0], "depth limit was reached")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_aNestedArchiveIsReadWhereItLies(t *testing.T) {
|
||||
// an archive's own bytes are read where they lie: a scanned file from disk, a nested archive from
|
||||
// its parent's entries. Nothing copies them, so at most one spill file per level appears whatever
|
||||
// the memory limit; only entries are held, and only they can be written out.
|
||||
scanDir := t.TempDir()
|
||||
tempDir := isolatedTempDir(t)
|
||||
nested := writeNestedArchive(t, scanDir, gradedChain())
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
memoryBytes int64
|
||||
wantHeldFiles []string
|
||||
}{
|
||||
{
|
||||
name: "with room in memory for every level's entries",
|
||||
memoryBytes: int64(2 * (nested.sizes[0] + nested.sizes[1] + nested.sizes[2])),
|
||||
},
|
||||
{
|
||||
name: "with a memory limit of zero, so every level's entries are written out",
|
||||
memoryBytes: 0,
|
||||
wantHeldFiles: []string{"archive-spill", "archive-spill", "archive-spill"},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
probe := &nestingProbe{tempDir: tempDir}
|
||||
bounds := archive.Limits{MaxMemoryBytes: tc.memoryBytes, MaxDiskBytes: -1}
|
||||
|
||||
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
|
||||
|
||||
deepest := probe.deepestVisit()
|
||||
require.Equal(t, 3, deepest.depth, "the sample must be taken with all three levels still held")
|
||||
assert.Equal(t, tc.wantHeldFiles, deepest.heldFiles)
|
||||
|
||||
assert.Equal(t, nested.containerPaths, probe.containerPaths())
|
||||
locs := leafLocations(s)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, nested.leafContainerPath(), archive.ContainerPath(locs[0].Coordinates))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_mixedFamilyNesting_diskLimitTruncatesTheLevelItCannotAdmit(t *testing.T) {
|
||||
// the walk descends before it unwinds, so the outer and middle archives still hold their entries
|
||||
// when the innermost level's do not fit in memory. With nothing admitted on disk that level is
|
||||
// cataloged from nothing, the shallower ones keep what they found, and the scan finishes. An
|
||||
// implementation that waited for capacity would deadlock, hence the timeout.
|
||||
scanDir := t.TempDir()
|
||||
nested := writeNestedArchive(t, scanDir, deepChain())
|
||||
|
||||
// a sibling of a different family, processed once the chain has been released, so cataloged in full
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(scanDir, "sibling.tgz"),
|
||||
testutils.TarGz(t, map[string][]byte{"nested/marker.txt": []byte("sibling")}),
|
||||
0o644,
|
||||
))
|
||||
|
||||
// room for the outer two levels' entries (each about the size of the archive it holds) but not the
|
||||
// innermost's payload, which is half an archive larger than the slack
|
||||
memoryLimit := int64(nested.sizes[0] + nested.sizes[1] + deepChainPayloadBytes/2)
|
||||
|
||||
probe := &nestingProbe{}
|
||||
bounds := archive.Limits{MaxMemoryBytes: memoryLimit, MaxDiskBytes: 0}
|
||||
|
||||
s := runNesting(t, scanDir, 3, bounds, markerTask(), probe.task())
|
||||
|
||||
want := append(append([]string{}, nested.containerPaths...), "/sibling.tgz")
|
||||
assert.ElementsMatch(t, want, probe.containerPaths(),
|
||||
"every level is visited, the truncated one with nothing inside it")
|
||||
|
||||
containers := strset.New()
|
||||
for _, loc := range leafLocations(s) {
|
||||
containers.Add(archive.ContainerPath(loc.Coordinates))
|
||||
}
|
||||
assert.False(t, containers.Has(nested.leafContainerPath()), "the truncated level's contents must be absent")
|
||||
assert.True(t, containers.Has("/sibling.tgz"), "the sibling of a different family must be cataloged fully")
|
||||
|
||||
var truncated []string
|
||||
for coords, reasons := range s.Artifacts.Unknowns {
|
||||
for _, reason := range reasons {
|
||||
if strings.Contains(reason, "disk limit") {
|
||||
truncated = append(truncated, coords.FullNestedPath())
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Equal(t, []string{nested.coordinates(2).FullNestedPath()}, truncated,
|
||||
"the truncation is recorded against the archive that reached the limit")
|
||||
}
|
||||
|
||||
func defaultLimits() archive.Limits {
|
||||
return archiveLimits(cataloging.DefaultArchiveSearchConfig())
|
||||
}
|
||||
|
||||
// runNesting runs the archive cataloger task over scanDir with the given sub-pipeline and returns the
|
||||
// SBOM it built. It runs on its own goroutine and fails if the task has not finished in time.
|
||||
//
|
||||
// The depth-3 limit cases need the timeout: the walk descends before it unwinds, so the outer and
|
||||
// middle archives still hold their content when the inner one is refused. An implementation that
|
||||
// waited for capacity would deadlock, and a test asserting only the result would hang rather than fail.
|
||||
func runNesting(t *testing.T, scanDir string, depth int, bounds archive.Limits, subPipeline ...Task) *sbom.SBOM {
|
||||
t.Helper()
|
||||
tsk := newLimitedTestTask(t, depth, bounds, subPipeline...)
|
||||
s := &sbom.SBOM{Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(),
|
||||
FileMetadata: map[file.Coordinates]file.Metadata{},
|
||||
}}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(s))
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err, "reaching a bound must not fail the scan")
|
||||
return s
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatal("the scan did not finish in time: reaching a limit must skip the archive that " +
|
||||
"reached it, never wait for capacity that nothing in the chain will release")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// withArchiveContains runs the task that follows every scan with archive cataloging on.
|
||||
func withArchiveContains(t *testing.T, s *sbom.SBOM) *sbom.SBOM {
|
||||
t.Helper()
|
||||
require.NoError(t, NewArchiveContainsTask().Execute(context.Background(), nil, sbomsync.NewBuilder(s)))
|
||||
return s
|
||||
}
|
||||
|
||||
// markerTask emits one package per "marker.txt" found, so the matrix asserts behavior without
|
||||
// depending on any real cataloger's fixture format.
|
||||
func markerTask() Task {
|
||||
return NewTask("marker-cataloger", func(_ context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
|
||||
locations, err := resolver.FilesByGlob("**/marker.txt")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, loc := range locations {
|
||||
p := pkg.Package{
|
||||
Name: "marker-pkg",
|
||||
Version: "1.0.0",
|
||||
Type: pkg.BinaryPkg,
|
||||
Locations: file.NewLocationSet(loc),
|
||||
}
|
||||
p.SetID()
|
||||
builder.AddPackages(p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// javaTask is the java archive cataloger as the cataloging factory builds it, so a jar is met as a jar
|
||||
// wherever it sits in a chain.
|
||||
func javaTask() Task {
|
||||
return NewPackageTask(CatalogingFactoryConfig{}, java.NewArchiveCataloger(java.DefaultArchiveCatalogerConfig()))
|
||||
}
|
||||
|
||||
// fileMetadataTask records every file, so each file inside an archive gets an archive-to-file edge.
|
||||
func fileMetadataTask() Task {
|
||||
return newFileMetadataCatalogerTask(file.AllFilesSelection)
|
||||
}
|
||||
|
||||
// pack builds one archive of this family holding the given entries.
|
||||
func (f archiveFamily) pack(t *testing.T, base string, entries map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
switch f {
|
||||
case familyZip:
|
||||
return testutils.Zip(t, entries)
|
||||
case familyJar:
|
||||
// a manifest makes it a real jar to the java cataloger at any level; the package java reports is
|
||||
// named after the file, not the manifest title
|
||||
return testutils.Jar(t, base, "1.0", entries)
|
||||
case familyTar:
|
||||
return testutils.Tar(t, entries)
|
||||
case familyTarGz, familyTgz:
|
||||
return testutils.TarGz(t, entries)
|
||||
}
|
||||
t.Fatalf("unknown archive family %q", f)
|
||||
return nil
|
||||
}
|
||||
|
||||
// leafContainerPath is the container path the innermost archive's contents carry: every archive, outer to
|
||||
// inner. Assert it exactly, never as a substring: a partially composed chain passes a substring check.
|
||||
func (n nestedArchive) leafContainerPath() string {
|
||||
return n.containerPaths[len(n.containerPaths)-1]
|
||||
}
|
||||
|
||||
// coordinates are those of the archive at the given level: the outermost archive's path, and the chain
|
||||
// of entries within it.
|
||||
func (n nestedArchive) coordinates(level int) file.Coordinates {
|
||||
return file.Coordinates{RealPath: n.paths[0], FullPath: strings.Join(n.paths[:level+1], "!/")}
|
||||
}
|
||||
|
||||
// buildNestedArchive builds one archive per family given, each holding the next, with leaf inside
|
||||
// the innermost.
|
||||
func buildNestedArchive(t *testing.T, plan nestPlan) nestedArchive {
|
||||
t.Helper()
|
||||
require.NotEmpty(t, plan.families, "a chain needs at least one level")
|
||||
|
||||
out := nestedArchive{
|
||||
sizes: make([]int, len(plan.families)),
|
||||
paths: make([]string, len(plan.families)),
|
||||
containerPaths: make([]string, len(plan.families)),
|
||||
virtualPaths: make([]string, len(plan.families)),
|
||||
}
|
||||
|
||||
// the level index goes in every name and path segment, so no two levels share a path and a chain
|
||||
// assertion cannot pass on a value composed from the wrong level
|
||||
for i, family := range plan.families {
|
||||
base := fmt.Sprintf("level%d", i)
|
||||
if i == 0 {
|
||||
// the outermost archive is a file in the scanned directory, which the resolver reports rooted
|
||||
out.name = base + string(family)
|
||||
out.paths[i] = "/" + out.name
|
||||
continue
|
||||
}
|
||||
out.paths[i] = fmt.Sprintf("nest%d/%s%s", i, base, family)
|
||||
}
|
||||
|
||||
for i := range plan.families {
|
||||
if i == 0 {
|
||||
out.containerPaths[i] = out.paths[0]
|
||||
out.virtualPaths[i] = out.paths[0]
|
||||
continue
|
||||
}
|
||||
out.containerPaths[i] = out.containerPaths[i-1] + "!/" + out.paths[i]
|
||||
out.virtualPaths[i] = out.virtualPaths[i-1] + ":" + out.paths[i]
|
||||
}
|
||||
|
||||
// built innermost first, because each level's bytes are an entry of the level above it
|
||||
var packed []byte
|
||||
for i := len(plan.families) - 1; i >= 0; i-- {
|
||||
entries := map[string][]byte{}
|
||||
if i == len(plan.families)-1 {
|
||||
for name, body := range plan.leaf {
|
||||
entries[name] = body
|
||||
}
|
||||
} else {
|
||||
entries[out.paths[i+1]] = packed
|
||||
}
|
||||
for name, body := range plan.extra[i] {
|
||||
entries[name] = body
|
||||
}
|
||||
packed = plan.families[i].pack(t, fmt.Sprintf("level%d", i), entries)
|
||||
out.sizes[i] = len(packed)
|
||||
}
|
||||
out.bytes = packed
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// writeNestedArchive writes the chain's outermost archive into dir and returns the chain.
|
||||
func writeNestedArchive(t *testing.T, dir string, plan nestPlan) nestedArchive {
|
||||
t.Helper()
|
||||
nested := buildNestedArchive(t, plan)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, nested.name), nested.bytes, 0o644))
|
||||
return nested
|
||||
}
|
||||
|
||||
// markerLeaf is the family-agnostic leaf payload: one file no archive format has an opinion about,
|
||||
// found by markerTask. It keeps the rows comparable - the same package must come out whichever
|
||||
// families the chain is made of.
|
||||
func markerLeaf() map[string][]byte {
|
||||
return map[string][]byte{"nested/marker.txt": []byte("leaf")}
|
||||
}
|
||||
|
||||
// nestingRows is the depth-3 matrix. Each row covers a case the others do not.
|
||||
func nestingRows() []nestingRow {
|
||||
return []nestingRow{
|
||||
{
|
||||
name: "tar.gz/zip/jar",
|
||||
families: []archiveFamily{familyTarGz, familyZip, familyJar},
|
||||
why: "the headline case: compressed tar outermost, jar innermost",
|
||||
},
|
||||
{
|
||||
name: "zip/tar.gz/jar",
|
||||
families: []archiveFamily{familyZip, familyTarGz, familyJar},
|
||||
why: "compressed tar in the middle, where the stream-only format is not the entry point",
|
||||
},
|
||||
{
|
||||
name: "jar/tar/zip",
|
||||
families: []archiveFamily{familyJar, familyTar, familyZip},
|
||||
why: "jar as a container rather than a payload",
|
||||
},
|
||||
{
|
||||
name: "tgz/tar/jar",
|
||||
families: []archiveFamily{familyTgz, familyTar, familyJar},
|
||||
why: "the .tgz alias resolving at a non-outermost boundary",
|
||||
},
|
||||
{
|
||||
name: "tar/jar/tar.gz",
|
||||
families: []archiveFamily{familyTar, familyJar, familyTarGz},
|
||||
why: "compressed tar as the innermost, which nothing else covers",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *nestingProbe) task() Task {
|
||||
return NewTask("nesting-probe", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
trav := archive.TraversalFromContext(ctx)
|
||||
visit := nestingVisit{}
|
||||
if trav != nil {
|
||||
visit.virtualPath = archive.VirtualPath(trav.Location)
|
||||
visit.fullPath = trav.FullTraversalArchivePath()
|
||||
visit.depth = strings.Count(archive.VirtualPath(trav.Location), ":") + 1
|
||||
}
|
||||
if p.tempDir != "" {
|
||||
visit.heldFiles = heldFiles(p.tempDir)
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.visits = append(p.visits, visit)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// containerPaths is the chain of every archive whose contents were cataloged, in visit order.
|
||||
func (p *nestingProbe) containerPaths() []string {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
var out []string
|
||||
for _, v := range p.visits {
|
||||
if v.fullPath == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, v.fullPath)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// deepestVisit is the visit at the innermost level reached, the only moment at which every level of a
|
||||
// chain is still held and its routing observable. It returns the whole visit so a case can also assert
|
||||
// it got as deep as it meant to: an empty overflow sample from a visit that never left the scan root
|
||||
// would otherwise read as "nothing overflowed".
|
||||
func (p *nestingProbe) deepestVisit() nestingVisit {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
var best nestingVisit
|
||||
for _, v := range p.visits {
|
||||
if v.depth > best.depth {
|
||||
best = v
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// heldFiles names the spill files live under tempDir, one "archive-spill" per archive holding content
|
||||
// that did not fit in memory. Meaningful only while the archives are still held, so it is sampled from
|
||||
// inside the sub-pipeline rather than after the scan.
|
||||
func heldFiles(tempDir string) (names []string) {
|
||||
_ = filepath.WalkDir(tempDir, func(_ string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || !strings.HasPrefix(d.Name(), "archive-") {
|
||||
return nil //nolint:nilerr // an unreadable directory holds nothing this probe can report
|
||||
}
|
||||
// the name is the pattern with os.CreateTemp's random suffix in place of the "*"
|
||||
names = append(names, d.Name()[:strings.LastIndexByte(d.Name(), '-')])
|
||||
return nil
|
||||
})
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// isolatedTempDir points archive extraction at a directory of this test's own, so an overflow sample
|
||||
// sees only this scan's work directories.
|
||||
func isolatedTempDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
t.Setenv("TMPDIR", dir) // unix
|
||||
t.Setenv("TMP", dir) // windows
|
||||
t.Setenv("TEMP", dir) // windows fallback
|
||||
require.Equal(t, dir, os.TempDir(), "extraction must be writing where the probe is looking")
|
||||
return dir
|
||||
}
|
||||
|
||||
func leafLocations(s *sbom.SBOM) []file.Location {
|
||||
var out []file.Location
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if p.Name != "marker-pkg" {
|
||||
continue
|
||||
}
|
||||
out = append(out, p.Locations.ToSlice()...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func javaVirtualPaths(s *sbom.SBOM) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if metadata, ok := p.Metadata.(pkg.JavaArchive); ok {
|
||||
out[p.Name] = metadata.VirtualPath
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func packageCount(s *sbom.SBOM, name string) int {
|
||||
var n int
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if p.Name == name {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func coordKey(c file.Coordinates) string {
|
||||
return c.FullNestedPath()
|
||||
}
|
||||
|
||||
// gradedChain is the headline row with each level padded so the three archives have clearly different
|
||||
// sizes: a container is never smaller than what it holds, so a bound can only be placed between two
|
||||
// levels if they are graded on purpose.
|
||||
func gradedChain() nestPlan {
|
||||
return nestPlan{
|
||||
families: []archiveFamily{familyTarGz, familyZip, familyJar},
|
||||
leaf: markerLeaf(),
|
||||
extra: map[int]map[string][]byte{
|
||||
0: {"pad0.bin": incompressible(24 * 1024)},
|
||||
1: {"pad1.bin": incompressible(24 * 1024)},
|
||||
2: {"pad2.bin": incompressible(8 * 1024)},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// deepChainPayloadBytes is the size of deepChain's leaf payload.
|
||||
const deepChainPayloadBytes = 64 * 1024
|
||||
|
||||
// deepChain is the headline row with one large, incompressible payload at the leaf and no other
|
||||
// padding. The payload dominates every level, so the three archives are about the same size. It sorts
|
||||
// before the marker so it is the first entry refused when the leaf archive's entries do not fit.
|
||||
func deepChain() nestPlan {
|
||||
return nestPlan{
|
||||
families: []archiveFamily{familyTarGz, familyZip, familyJar},
|
||||
leaf: map[string][]byte{
|
||||
"0-payload.bin": incompressible(deepChainPayloadBytes),
|
||||
"nested/marker.txt": []byte("leaf"),
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,501 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"runtime/debug"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/mimetype"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/internal/unknown"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
const ArchiveCatalogerTaskName = "archive-cataloger"
|
||||
|
||||
// NewArchiveCatalogerTask returns a task that extracts every archive it finds, runs the same tasks
|
||||
// against the archive's contents as a standalone filesystem, and records
|
||||
// CONTAINS relationships from the archive to the packages found directly inside it. It recurses into the archives
|
||||
// it extracts up to cfg.MaxDepth; the task drops itself from the tasks so that is the only recursion.
|
||||
//
|
||||
// Each archive is handled as the scan root is: pkgTasks, then the archives nested in it, then fileTasks,
|
||||
// which see the packages found in the nested archives (see Traversal.PackageFiles).
|
||||
//
|
||||
// The task is the authority on which archives were not cataloged and why; with
|
||||
// unknowns.IncludeUnexpandedArchives it also reports archives in formats it does not extract.
|
||||
//
|
||||
// exclusions are the scan's exclusion patterns, applied inside each archive. Returns nil when archive
|
||||
// cataloging is disabled (MaxDepth == 0; negative or beyond archive.MaxDepth means as deep as archive.MaxDepth).
|
||||
func NewArchiveCatalogerTask(cfg cataloging.ArchiveSearchConfig, unknowns cataloging.UnknownsConfig, pkgTasks, fileTasks []Task, exclusions []string) Task {
|
||||
return newArchiveCatalogerTask(cfg.MaxDepth, archiveLimits(cfg), unknowns.IncludeUnexpandedArchives, pkgTasks, fileTasks, exclusions)
|
||||
}
|
||||
|
||||
func archiveLimits(cfg cataloging.ArchiveSearchConfig) archive.Limits {
|
||||
return archive.Limits{MaxMemoryBytes: cfg.MaxMemoryBytes, MaxDiskBytes: cfg.MaxDiskBytes}
|
||||
}
|
||||
|
||||
func newArchiveCatalogerTask(maxDepth int, limits archive.Limits, reportUnsupportedUnknowns bool, pkgTasks, fileTasks []Task, exclusions []string) Task {
|
||||
if maxDepth == 0 {
|
||||
return nil
|
||||
}
|
||||
pkgTasks, fileTasks = withoutArchiveCataloger(pkgTasks), withoutArchiveCataloger(fileTasks)
|
||||
if len(pkgTasks)+len(fileTasks) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
|
||||
// catalogers run once per archive, and each must report into the one progress row it owns
|
||||
ctx = bus.WithCatalogerTaskRegistry(ctx)
|
||||
|
||||
c := &archiveCataloger{
|
||||
maxDepth: maxDepth,
|
||||
reportUnsupportedUnknowns: reportUnsupportedUnknowns,
|
||||
pkgTasks: pkgTasks,
|
||||
fileTasks: fileTasks,
|
||||
limiter: archive.NewLimiter(limits),
|
||||
exclusions: archive.NewExclusions(exclusions),
|
||||
progress: bus.StartCatalogerTask(ctx, archiveCatalogerProgressInfo(), -1, ""),
|
||||
}
|
||||
err := c.catalog(ctx, resolver, 0, builder)
|
||||
c.progress.AtomicStage.Set(fmt.Sprintf("%s archives", humanize.Comma(c.progress.Current())))
|
||||
c.progress.SetCompleted()
|
||||
c.logStats()
|
||||
return err
|
||||
}
|
||||
return NewTask(ArchiveCatalogerTaskName, fn, pkgcataloging.PackageTag, "archive")
|
||||
}
|
||||
|
||||
func archiveCatalogerProgressInfo() monitor.GenericTask {
|
||||
return monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "Archives",
|
||||
},
|
||||
ID: ArchiveCatalogerTaskName,
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
}
|
||||
|
||||
func withoutArchiveCataloger(tasks []Task) []Task {
|
||||
var out []Task
|
||||
for _, t := range tasks {
|
||||
if t == nil || t.Name() == ArchiveCatalogerTaskName {
|
||||
continue
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type archiveCataloger struct {
|
||||
maxDepth int
|
||||
reportUnsupportedUnknowns bool
|
||||
pkgTasks []Task
|
||||
fileTasks []Task
|
||||
limiter *archive.Limiter
|
||||
exclusions archive.Exclusions
|
||||
|
||||
// progress is the one row for the whole walk: its count is how many archives have been entered
|
||||
progress *monitor.TaskProgress
|
||||
|
||||
// ancestors are the digests of the archives the walk is currently inside. The walk is depth-first
|
||||
// and sequential, so this is a stack.
|
||||
ancestors []file.Digest
|
||||
|
||||
// the archive that spent the longest on its own extraction and cataloging, excluding nested archives
|
||||
slowest time.Duration
|
||||
slowestPath string
|
||||
}
|
||||
|
||||
// logStats reports what the walk cost, once. The peaks are the most held against each limit at any
|
||||
// one moment, comparable to the configured bounds.
|
||||
func (c *archiveCataloger) logStats() {
|
||||
peakMemory, peakDisk := c.limiter.Peak()
|
||||
fields := []any{
|
||||
"archives", c.progress.Current(),
|
||||
"peak-memory", humanize.Bytes(uint64(peakMemory)),
|
||||
"peak-disk", humanize.Bytes(uint64(peakDisk)),
|
||||
}
|
||||
if c.slowestPath != "" {
|
||||
fields = append(fields, "slowest", c.slowestPath, "slowest-took", c.slowest.Round(time.Millisecond))
|
||||
}
|
||||
log.WithFields(fields...).Info("nested archive cataloging complete")
|
||||
}
|
||||
|
||||
// catalog processes every archive in the resolver, recursing into each up to maxDepth.
|
||||
func (c *archiveCataloger) catalog(ctx context.Context, resolver file.Resolver, depth int, builder sbomsync.Builder) error {
|
||||
return c.catalogCandidates(ctx, resolver, c.discoverArchives(resolver), depth, builder)
|
||||
}
|
||||
|
||||
func (c *archiveCataloger) catalogCandidates(ctx context.Context, resolver file.Resolver, candidates []archiveCandidate, depth int, builder sbomsync.Builder) error {
|
||||
maxDepth := c.maxDepth
|
||||
// a deeper scan would write chains no decoder accepts back
|
||||
if maxDepth < 0 || maxDepth > archive.MaxDepth {
|
||||
maxDepth = archive.MaxDepth
|
||||
}
|
||||
if depth >= maxDepth {
|
||||
// java's own recursion is off while this task runs, so an archive left here is otherwise lost
|
||||
// without a trace
|
||||
for _, candidate := range candidates {
|
||||
if candidate.sniffedAsArchive {
|
||||
recordArchiveUnknown(builder, candidate.location.Coordinates, "nested archive not cataloged: the nested archive depth limit was reached")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var errs error
|
||||
for _, candidate := range candidates {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.processArchive(ctx, resolver, candidate, depth, builder); err != nil {
|
||||
errs = unknown.Append(errs, candidate.location, err)
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
// processArchive processes the given candidate archive, catching panics
|
||||
//
|
||||
//nolint:funlen
|
||||
func (c *archiveCataloger) processArchive(ctx context.Context, parentResolver file.Resolver, candidate archiveCandidate, depth int, builder sbomsync.Builder) (err error) {
|
||||
// the decoders run on untrusted bytes; a panic in one costs this archive, not the scan
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
// the stack goes to the log, not the reason: the reason lands in the published SBOM's unknowns
|
||||
log.WithFields("path", candidate.location.FullNestedPath()).Debugf("archive cataloging panic stack:\n%s", debug.Stack())
|
||||
err = fmt.Errorf("panic while cataloging archive: %v", r)
|
||||
}
|
||||
}()
|
||||
|
||||
location := candidate.location
|
||||
content, err := parentResolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer internal.CloseAndLogError(content, location.FullNestedPath())
|
||||
|
||||
var archiveContent io.Reader = content
|
||||
if !candidate.sniffedAsArchive {
|
||||
var mayHide bool
|
||||
if archiveContent, mayHide = archive.MayHideAnAppendedArchive(content); !mayHide {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
started := time.Now()
|
||||
|
||||
if depth == 0 {
|
||||
// one budget for this archive and everything nested in it, however deep
|
||||
ctx = archive.WithBudget(ctx, archive.NewBudget(archiveTreeBudget(parentResolver, location)))
|
||||
}
|
||||
|
||||
// extracted files keep the real path and filesystem ID of the outermost archive; the chain of entries
|
||||
// to each is carried as its archive path. The chain names real paths, not the link the archive was
|
||||
// reached by, so it is stable and joins to the archive's package location by prefix.
|
||||
traversal := &archive.Traversal{Location: location, Parent: archive.TraversalFromContext(ctx)}
|
||||
archivePath := location.FullNestedPath()
|
||||
extracted, err := archive.Extract(ctx, archiveContent, location.Coordinates, c.limiter, c.exclusions)
|
||||
if errors.Is(err, archive.ErrUnsupportedFormat) {
|
||||
if c.reportUnsupportedUnknowns {
|
||||
recordArchiveUnknown(builder, location.Coordinates, "archive not cataloged: "+err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, archive.ErrDiskLimitReached) {
|
||||
// nothing is released while this archive waits, so skip it rather than block the scan
|
||||
recordArchiveUnknown(builder, location.Coordinates, "archive skipped: its content would exceed the disk limit")
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if extracted == nil {
|
||||
// a format with no extractor, or a candidate whose head only looked like an appended archive.
|
||||
// Logged rather than returned: an image can hold thousands of these
|
||||
log.WithFields("archive", archivePath, "sniffed-as-archive", candidate.sniffedAsArchive).
|
||||
Trace("no extractor could read this candidate; skipping")
|
||||
return nil
|
||||
}
|
||||
defer extracted.Cleanup()
|
||||
|
||||
// an archive whose bytes match one it is nested in (a zip quine) would recurse forever
|
||||
if c.isAncestor(extracted.Digests) {
|
||||
recordArchiveUnknown(builder, location.Coordinates, "archive not cataloged: it is identical to an archive that contains it")
|
||||
return nil
|
||||
}
|
||||
c.ancestors = append(c.ancestors, extracted.Digests...)
|
||||
defer func() { c.ancestors = c.ancestors[:len(c.ancestors)-len(extracted.Digests)] }()
|
||||
|
||||
if extracted.Truncated != nil {
|
||||
recordArchiveUnknown(builder, location.Coordinates, "archive cataloged from part of its contents: "+extracted.Truncated.Error())
|
||||
}
|
||||
|
||||
traversal.Digests = extracted.Digests
|
||||
ctx = archive.WithTraversal(ctx, traversal)
|
||||
|
||||
c.progress.Increment()
|
||||
|
||||
// everything found in this archive and the archives nested in it, merged into the parent's SBOM once complete
|
||||
found := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}}
|
||||
foundBuilder := sbomsync.NewBuilder(found)
|
||||
runArchiveTasks(ctx, c.pkgTasks, extracted, location.Coordinates, foundBuilder)
|
||||
direct := found.Artifacts.Packages.Sorted()
|
||||
traversal.Packages = packagesAt(direct, location.Coordinates)
|
||||
|
||||
// self time excludes descending, so a containing archive is not charged for what it contains
|
||||
selfTime := time.Since(started)
|
||||
|
||||
// as at the scan root, nested archives are cataloged before this archive's files, so the packages
|
||||
// describing them (located in this archive) select their files here
|
||||
errs := c.catalogCandidates(ctx, extracted, c.discoverArchives(extracted), depth+1, foundBuilder)
|
||||
|
||||
started = time.Now()
|
||||
runArchiveTasks(ctx, c.fileTasks, extracted, location.Coordinates, foundBuilder)
|
||||
// the files within it are related once the scan has decided which it describes (see NewArchiveContainsTask)
|
||||
for _, p := range direct {
|
||||
found.Relationships = append(found.Relationships, artifact.Relationship{From: location.Coordinates, To: p, Type: artifact.ContainsRelationship})
|
||||
}
|
||||
mergeArchiveResults(found, builder)
|
||||
c.progress.AtomicStage.Set(humanize.Comma(c.progress.Current()) + " archives (" + archivePath + ")")
|
||||
|
||||
if took := selfTime + time.Since(started); took > c.slowest {
|
||||
c.slowest, c.slowestPath = took, archivePath
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
// a top-level archive and everything nested in it may decompress to archiveTreeRatio times its size, or
|
||||
// archiveTreeFloorBytes if that is more. Real trees (a war of jars, a layer of tarballs) expand a few
|
||||
// times over; a recursive bomb expands by orders of magnitude at every level.
|
||||
var (
|
||||
archiveTreeRatio int64 = 100
|
||||
archiveTreeFloorBytes int64 = 1 << 30
|
||||
)
|
||||
|
||||
func archiveTreeBudget(resolver file.Resolver, location file.Location) int64 {
|
||||
var size int64
|
||||
if m, err := resolver.FileMetadataByLocation(location); err == nil && m.FileInfo != nil {
|
||||
size = m.Size()
|
||||
}
|
||||
return max(archiveTreeRatio*size, archiveTreeFloorBytes)
|
||||
}
|
||||
|
||||
func (c *archiveCataloger) isAncestor(digests []file.Digest) bool {
|
||||
for _, d := range digests {
|
||||
if slices.Contains(c.ancestors, d) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// packagesAt returns the packages located at the given coordinates, i.e. those describing the file there.
|
||||
func packagesAt(pkgs []pkg.Package, coordinates file.Coordinates) []pkg.Package {
|
||||
var out []pkg.Package
|
||||
for _, p := range pkgs {
|
||||
if locatedAt(p, coordinates) {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// locatedAt reports whether one of p's locations is at the given coordinates.
|
||||
func locatedAt(p pkg.Package, coordinates file.Coordinates) bool {
|
||||
for _, l := range p.Locations.ToUnorderedSlice() {
|
||||
if l.Coordinates == coordinates {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func recordArchiveUnknown(builder sbomsync.Builder, coordinates file.Coordinates, reason string) {
|
||||
appendUnknowns(builder, ArchiveCatalogerTaskName, []unknown.CoordinateError{{Coordinates: coordinates, Reason: errors.New(reason)}})
|
||||
}
|
||||
|
||||
// runArchiveTasks runs each task against the archive's contents.
|
||||
//
|
||||
// Failures are recorded as RunTask records them at the top level: errors carrying coordinates keep
|
||||
// them, anything else is attributed to the archive. A failure never fails the archive or the scan.
|
||||
func runArchiveTasks(ctx context.Context, tasks []Task, resolver file.Resolver, archiveCoordinates file.Coordinates, builder sbomsync.Builder) {
|
||||
for _, t := range tasks {
|
||||
err := runSubTaskSafely(ctx, t, resolver, builder)
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
log.WithFields("task", t.Name(), "error", err).Trace("archive sub-cataloger task reported an error")
|
||||
|
||||
unknowns, remaining := unknown.ExtractCoordinateErrors(err)
|
||||
if remaining != nil {
|
||||
unknowns = append(unknowns, unknown.CoordinateError{Coordinates: archiveCoordinates, Reason: remaining})
|
||||
}
|
||||
appendUnknowns(builder, t.Name(), unknowns)
|
||||
}
|
||||
}
|
||||
|
||||
// runSubTaskSafely is runTaskSafely without the task name in a panic's error, since appendUnknowns
|
||||
// already prefixes it.
|
||||
func runSubTaskSafely(ctx context.Context, t Task, resolver file.Resolver, builder sbomsync.Builder) (err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.WithFields("task", t.Name()).Debugf("archive sub-cataloger task panic stack:\n%s", debug.Stack())
|
||||
err = fmt.Errorf("panic: %v", r)
|
||||
}
|
||||
}()
|
||||
return t.Execute(ctx, resolver, builder)
|
||||
}
|
||||
|
||||
// archiveCandidate is one file the walk will try to open.
|
||||
type archiveCandidate struct {
|
||||
location file.Location
|
||||
|
||||
// sniffedAsArchive is true when the file's content types it as an archive. Such a file that cannot
|
||||
// be extracted is a broken archive and is reported; a file that is a candidate only by name is
|
||||
// passed over silently.
|
||||
sniffedAsArchive bool
|
||||
}
|
||||
|
||||
// zipArchiveGlobs find files named like zip archives whose content did not sniff as one, such as a
|
||||
// Spring Boot executable jar: a launcher script with a zip appended. The name only decides what gets
|
||||
// opened; the end-of-central-directory record decides whether it is an archive.
|
||||
//
|
||||
// Globs rather than a MIME lookup, because asking for every executable, script and unrecognized file
|
||||
// returned thousands of locations that were all discarded.
|
||||
var zipArchiveGlobs = []string{
|
||||
"**/*.jar", "**/*.war", "**/*.ear", "**/*.par", "**/*.sar", "**/*.nar", "**/*.kar",
|
||||
"**/*.hpi", "**/*.jpi", "**/*.far", "**/*.rar", "**/*.zip", "**/*.apk", "**/*.aar",
|
||||
"**/*.egg", "**/*.whl", "**/*.lpkg", "**/*.zap", "**/*.exe",
|
||||
}
|
||||
|
||||
var archiveMIMETypes = slices.Sorted(slices.Values(mimetype.ArchiveMIMETypeSet.List()))
|
||||
|
||||
// discoverArchives returns every file worth opening: those whose content is an archive, plus those
|
||||
// named like a zip whose tail may still hold one.
|
||||
func (c *archiveCataloger) discoverArchives(resolver file.Resolver) []archiveCandidate {
|
||||
archives, err := resolver.FilesByMIMEType(archiveMIMETypes...)
|
||||
if err != nil {
|
||||
log.WithFields("error", err).Debug("unable to list archive files for recursive cataloging")
|
||||
return nil
|
||||
}
|
||||
|
||||
var candidates []archiveCandidate
|
||||
seen := map[file.Coordinates]int{}
|
||||
for _, loc := range archives {
|
||||
seen[loc.Coordinates] = len(candidates)
|
||||
candidates = append(candidates, archiveCandidate{location: loc, sniffedAsArchive: true})
|
||||
}
|
||||
|
||||
namedLikeZips, err := resolver.FilesByGlob(zipArchiveGlobs...)
|
||||
if err != nil {
|
||||
log.WithFields("error", err).Debug("unable to list files that may carry an appended archive")
|
||||
return candidates
|
||||
}
|
||||
for _, loc := range namedLikeZips {
|
||||
if i, dup := seen[loc.Coordinates]; dup {
|
||||
// a MIME hit is the bare real path; the name it is reached by (a link such as
|
||||
// lib/dep.jar -> ../store/3f9a1c) is what name-based catalogers like java key off
|
||||
if prev := candidates[i].location; prev.AccessPath == "" || prev.AccessPath == prev.FullNestedPath() {
|
||||
candidates[i].location = loc
|
||||
}
|
||||
continue
|
||||
}
|
||||
seen[loc.Coordinates] = len(candidates)
|
||||
candidates = append(candidates, archiveCandidate{location: loc})
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
// mergeArchiveResults copies everything found in an archive into its parent's SBOM.
|
||||
func mergeArchiveResults(found *sbom.SBOM, builder sbomsync.Builder) {
|
||||
if pkgs := found.Artifacts.Packages.Sorted(); len(pkgs) > 0 {
|
||||
builder.AddPackages(pkgs...)
|
||||
}
|
||||
if accessor, ok := builder.(sbomsync.Accessor); ok {
|
||||
accessor.WriteToSBOM(func(s *sbom.SBOM) {
|
||||
mergeFileArtifacts(&s.Artifacts, &found.Artifacts)
|
||||
})
|
||||
}
|
||||
if len(found.Relationships) > 0 {
|
||||
builder.AddRelationships(found.Relationships...)
|
||||
}
|
||||
}
|
||||
|
||||
// NewArchiveContainsTask relates each archive to every file the SBOM describes directly within it. It must
|
||||
// run after every other task, so the files within archives are those the scan root's configuration
|
||||
// selects (file selection, unknowns, package evidence) and no others.
|
||||
func NewArchiveContainsTask() Task {
|
||||
fn := func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error {
|
||||
var rels []artifact.Relationship
|
||||
builder.(sbomsync.Accessor).ReadFromSBOM(func(s *sbom.SBOM) {
|
||||
rels = archiveContains(s.AllCoordinates())
|
||||
})
|
||||
builder.AddRelationships(rels...)
|
||||
return nil
|
||||
}
|
||||
return NewTask("archive-contains", fn)
|
||||
}
|
||||
|
||||
// archiveContains relates the archive holding each file within one to that file, and each such archive to
|
||||
// the archive holding it, so an archive the SBOM does not otherwise describe still links what is within it.
|
||||
func archiveContains(coordinates []file.Coordinates) []artifact.Relationship {
|
||||
related := file.NewCoordinateSet(coordinates...)
|
||||
var rels []artifact.Relationship
|
||||
for _, c := range coordinates {
|
||||
for {
|
||||
container, ok := archive.ContainingArchive(c)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
rels = append(rels, artifact.Relationship{From: container, To: c, Type: artifact.ContainsRelationship})
|
||||
if related.Contains(container) {
|
||||
break
|
||||
}
|
||||
related.Add(container)
|
||||
c = container
|
||||
}
|
||||
}
|
||||
return rels
|
||||
}
|
||||
|
||||
func mergeFileArtifacts(dst, src *sbom.Artifacts) {
|
||||
mergeInto(&dst.FileMetadata, src.FileMetadata)
|
||||
mergeInto(&dst.FileDigests, src.FileDigests)
|
||||
mergeInto(&dst.FileContents, src.FileContents)
|
||||
mergeInto(&dst.FileLicenses, src.FileLicenses)
|
||||
mergeInto(&dst.Executables, src.Executables)
|
||||
if len(src.Unknowns) > 0 && dst.Unknowns == nil {
|
||||
dst.Unknowns = map[file.Coordinates][]string{}
|
||||
}
|
||||
for coordinates, reasons := range src.Unknowns {
|
||||
dst.Unknowns[coordinates] = append(dst.Unknowns[coordinates], reasons...)
|
||||
}
|
||||
}
|
||||
|
||||
func mergeInto[K comparable, V any](dst *map[K]V, src map[K]V) {
|
||||
if len(src) == 0 {
|
||||
return
|
||||
}
|
||||
if *dst == nil {
|
||||
*dst = make(map[K]V, len(src))
|
||||
}
|
||||
maps.Copy(*dst, src)
|
||||
}
|
||||
@@ -0,0 +1,957 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/wagoodman/go-partybus"
|
||||
"github.com/wagoodman/go-progress"
|
||||
|
||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/mimetype"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/internal/testutils"
|
||||
"github.com/anchore/syft/internal/unknown"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/event"
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
// dirTestResolver is a minimal file.Resolver over a directory. FilesByMIMEType sniffs each file's
|
||||
// content as a real resolver does; FileContentsByLocation opens the file, which is random-access.
|
||||
type dirTestResolver struct {
|
||||
file.Resolver
|
||||
dir string
|
||||
fsid string
|
||||
}
|
||||
|
||||
// streamingDirResolver is dirTestResolver handing out plain streams, so an archive's own bytes must be
|
||||
// held in memory or written to disk rather than read in place.
|
||||
type streamingDirResolver struct {
|
||||
dirTestResolver
|
||||
}
|
||||
|
||||
func (d streamingDirResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) {
|
||||
f, err := d.dirTestResolver.FileContentsByLocation(loc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{f, f}, nil
|
||||
}
|
||||
|
||||
// panickingDirResolver is dirTestResolver whose reads of one file panic, standing in for a decoder
|
||||
// that panics on hostile bytes.
|
||||
type panickingDirResolver struct {
|
||||
dirTestResolver
|
||||
path string
|
||||
}
|
||||
|
||||
func (d panickingDirResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) {
|
||||
if loc.RealPath == d.path {
|
||||
return io.NopCloser(panicReader{}), nil
|
||||
}
|
||||
return d.dirTestResolver.FileContentsByLocation(loc)
|
||||
}
|
||||
|
||||
type panicReader struct{}
|
||||
|
||||
func (panicReader) Read([]byte) (int, error) { panic("decoder blew up") }
|
||||
|
||||
func Test_archiveCataloger_aPanicCostsOnlyThatArchive(t *testing.T) {
|
||||
rootDir := t.TempDir()
|
||||
for _, name := range []string{"bad.zip", "good.zip"} {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600))
|
||||
}
|
||||
|
||||
var seen []string
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, nil))
|
||||
|
||||
resolver := panickingDirResolver{dirTestResolver: dirTestResolver{dir: rootDir}, path: "/bad.zip"}
|
||||
err := tsk.Execute(context.Background(), resolver, sbomsync.NewBuilder(newTestSBOM()))
|
||||
|
||||
// a coordinate error becomes an unknown; a plain error (or an escaped panic) would fail the scan
|
||||
coordErrs, remaining := unknown.ExtractCoordinateErrors(err)
|
||||
require.NoError(t, remaining)
|
||||
require.Len(t, coordErrs, 1)
|
||||
assert.Equal(t, "/bad.zip", coordErrs[0].Coordinates.RealPath)
|
||||
assert.EqualError(t, coordErrs[0].Reason, "panic while cataloging archive: decoder blew up", "no stack trace in the SBOM-bound reason")
|
||||
assert.Equal(t, []string{"/good.zip"}, seen, "the sibling is still cataloged")
|
||||
}
|
||||
|
||||
func newTestTask(t *testing.T, cfg cataloging.ArchiveSearchConfig, subPipeline ...Task) Task {
|
||||
t.Helper()
|
||||
tsk := NewArchiveCatalogerTask(cfg, cataloging.UnknownsConfig{}, subPipeline, nil, nil)
|
||||
require.NotNil(t, tsk)
|
||||
return tsk
|
||||
}
|
||||
|
||||
// newLimitedTestTask builds the task with the limiter's own limits, where zero forbids the resource.
|
||||
func newLimitedTestTask(t *testing.T, maxDepth int, limits archive.Limits, subPipeline ...Task) Task {
|
||||
t.Helper()
|
||||
tsk := newArchiveCatalogerTask(maxDepth, limits, false, subPipeline, nil, nil)
|
||||
require.NotNil(t, tsk)
|
||||
return tsk
|
||||
}
|
||||
|
||||
func newTestSBOM() *sbom.SBOM {
|
||||
return &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}}
|
||||
}
|
||||
|
||||
func countingTask(ran *int) Task {
|
||||
return NewTask("count-runs", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
*ran++
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// capturingTask records the virtual path of every archive the sub-pipeline is run against and how many
|
||||
// files its resolver holds.
|
||||
func capturingTask(t *testing.T, seen *[]string, fileCounts *map[string]int) Task {
|
||||
return NewTask("capture", func(ctx context.Context, r file.Resolver, _ sbomsync.Builder) error {
|
||||
trav := archive.TraversalFromContext(ctx)
|
||||
require.NotNil(t, trav)
|
||||
*seen = append(*seen, archive.VirtualPath(trav.Location))
|
||||
if fileCounts != nil {
|
||||
locs, err := r.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
(*fileCounts)[archive.VirtualPath(trav.Location)] = len(locs)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_truncationStillCatalogs(t *testing.T) {
|
||||
// reaching a limit is a truncation, not a failure: the sub-pipeline still runs over what was stored
|
||||
big := bytes.Repeat([]byte("x"), 4096)
|
||||
outerZip := testutils.Zip(t, map[string][]byte{
|
||||
"a/small.txt": []byte("small"),
|
||||
"b/big.txt": big,
|
||||
})
|
||||
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
// nothing is held in memory, so each entry is charged to disk: small.txt lands, big.txt is refused
|
||||
limits := archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: 6000}
|
||||
|
||||
var seen []string
|
||||
fileCounts := map[string]int{}
|
||||
tsk := newLimitedTestTask(t, 1, limits, capturingTask(t, &seen, &fileCounts))
|
||||
|
||||
s := newTestSBOM()
|
||||
err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))
|
||||
|
||||
require.NoError(t, err, "a truncated extraction must not fail the scan")
|
||||
assert.Equal(t, []string{"/outer.zip"}, seen, "the sub-pipeline must run over the partial contents")
|
||||
assert.Equal(t, 1, fileCounts["/outer.zip"], "only the entry stored before the limit is visible")
|
||||
|
||||
reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
|
||||
require.True(t, ok, "the truncation must be recorded against the archive; got %v", s.Artifacts.Unknowns)
|
||||
require.Len(t, reasons, 1)
|
||||
assert.Contains(t, reasons[0], "archive cataloged from part of its contents")
|
||||
assert.Contains(t, reasons[0], ArchiveCatalogerTaskName, "the unknown names the task that recorded it")
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_aDamagedEntryCostsOnlyThatEntry(t *testing.T) {
|
||||
rootDir := t.TempDir()
|
||||
// entry data overwritten but headers and end-of-central-directory intact: detected as an archive,
|
||||
// and data.bin fails when it is read. ok.txt comes after it and is intact.
|
||||
corrupt := testutils.Zip(t, map[string][]byte{"data.bin": incompressible(4096), "ok.txt": []byte("ok")})
|
||||
require.Greater(t, len(corrupt), 1024)
|
||||
for i := 100; i < 600; i++ {
|
||||
corrupt[i] ^= 0xff
|
||||
}
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "corrupt.zip"), corrupt, 0o600))
|
||||
|
||||
var seen []string
|
||||
fileCounts := map[string]int{}
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, &fileCounts))
|
||||
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
|
||||
|
||||
assert.Equal(t, []string{"/corrupt.zip"}, seen, "the archive is still cataloged")
|
||||
assert.Equal(t, 1, fileCounts["/corrupt.zip"], "from the entries that could be read")
|
||||
reasons := s.Artifacts.Unknowns[file.NewCoordinates("/corrupt.zip", "")]
|
||||
require.Len(t, reasons, 1)
|
||||
assert.Contains(t, reasons[0], `entry "data.bin" could not be read`)
|
||||
}
|
||||
|
||||
func Test_NewArchiveCatalogerTask_gating(t *testing.T) {
|
||||
someTask := NewTask("noop", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error { return nil })
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg cataloging.ArchiveSearchConfig
|
||||
subPipeline []Task
|
||||
wantTask bool
|
||||
}{
|
||||
{
|
||||
name: "disabled at depth 0",
|
||||
cfg: cataloging.DefaultArchiveSearchConfig(),
|
||||
subPipeline: []Task{someTask},
|
||||
},
|
||||
{
|
||||
name: "no sub-pipeline",
|
||||
cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2),
|
||||
subPipeline: nil,
|
||||
},
|
||||
{
|
||||
name: "enabled",
|
||||
cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2),
|
||||
subPipeline: []Task{someTask},
|
||||
wantTask: true,
|
||||
},
|
||||
{
|
||||
name: "enabled at negative depth",
|
||||
cfg: cataloging.DefaultArchiveSearchConfig().WithMaxDepth(-1),
|
||||
subPipeline: []Task{someTask},
|
||||
wantTask: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := NewArchiveCatalogerTask(tt.cfg, cataloging.UnknownsConfig{}, tt.subPipeline, nil, nil)
|
||||
if tt.wantTask {
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, ArchiveCatalogerTaskName, got.Name())
|
||||
return
|
||||
}
|
||||
assert.Nil(t, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_depthStopsAtTheCeiling(t *testing.T) {
|
||||
// "unbounded" must still end, or a zip quine or recursive bomb runs forever, and a configured depth
|
||||
// past the ceiling must too, or the SBOM holds chains no decoder accepts back
|
||||
chain := testutils.Zip(t, map[string][]byte{"deep.txt": []byte("deep")})
|
||||
for range archive.MaxDepth + 4 {
|
||||
chain = testutils.Zip(t, map[string][]byte{"n.zip": chain})
|
||||
}
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), chain, 0o600))
|
||||
|
||||
for _, depth := range []int{-1, archive.MaxDepth + 2} {
|
||||
t.Run(fmt.Sprint(depth), func(t *testing.T) {
|
||||
var seen []string
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth), capturingTask(t, &seen, nil))
|
||||
_ = tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM()))
|
||||
|
||||
assert.Len(t, seen, archive.MaxDepth)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_anArchiveIdenticalToAnAncestorIsNotEntered(t *testing.T) {
|
||||
// a zip quine contains itself. Building one is impractical, so the walk is started as if it were
|
||||
// already inside an archive with these exact bytes, which is the state a quine reaches one level down
|
||||
outerZip := testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")})
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
var ran int
|
||||
c := newTestCataloger(-1, archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), countingTask(&ran))
|
||||
sum := sha1.Sum(outerZip)
|
||||
c.ancestors = []file.Digest{{Algorithm: "sha1", Value: hex.EncodeToString(sum[:])}}
|
||||
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(s)))
|
||||
|
||||
assert.Zero(t, ran, "the sub-pipeline must not run inside a repeat of an ancestor")
|
||||
reasons := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
|
||||
require.Len(t, reasons, 1)
|
||||
assert.Contains(t, reasons[0], "identical to an archive that contains it")
|
||||
assert.Len(t, c.ancestors, 1, "the walk leaves the ancestor stack as it found it")
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_stopsWhenTheContextIsCancelled(t *testing.T) {
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), testutils.Zip(t, map[string][]byte{"a.txt": []byte("a")}), 0o600))
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
var ran int
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), countingTask(&ran))
|
||||
err := tsk.Execute(ctx, dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM()))
|
||||
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.Zero(t, ran)
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_decompressionBudgetIsSharedByATreeAndFreshPerTopLevelArchive(t *testing.T) {
|
||||
// a recursive bomb fits the limits one level at a time, so what bounds it is the total decompressed
|
||||
// under one top-level archive
|
||||
origRatio, origFloor := archiveTreeRatio, archiveTreeFloorBytes
|
||||
archiveTreeRatio, archiveTreeFloorBytes = 0, 100_000
|
||||
t.Cleanup(func() { archiveTreeRatio, archiveTreeFloorBytes = origRatio, origFloor })
|
||||
|
||||
zeros := make([]byte, 60_000)
|
||||
nested := testutils.Zip(t, map[string][]byte{"zeros.bin": zeros})
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "bomb.zip"), testutils.Zip(t, map[string][]byte{"a.zip": nested, "b.zip": nested}), 0o600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "sibling.zip"), testutils.Zip(t, map[string][]byte{"zeros.bin": zeros}), 0o600))
|
||||
|
||||
var seen []string
|
||||
fileCounts := map[string]int{}
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), capturingTask(t, &seen, &fileCounts))
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
|
||||
|
||||
assert.Equal(t, 1, fileCounts["/bomb.zip:a.zip"], "the first nested archive fits the tree's budget")
|
||||
assert.Equal(t, 0, fileCounts["/bomb.zip:b.zip"], "the second does not: the budget is shared, not per archive")
|
||||
reasons := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/bomb.zip", FullPath: "/bomb.zip!/b.zip"}]
|
||||
require.Len(t, reasons, 1)
|
||||
assert.Contains(t, reasons[0], "possible decompression bomb")
|
||||
assert.Equal(t, 1, fileCounts["/sibling.zip"], "another top-level archive starts with a fresh budget")
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_noSelfContainsEdge(t *testing.T) {
|
||||
// a jar java cannot identify records an unknown at its own coordinates; that must not turn into
|
||||
// "x.jar CONTAINS x.jar", which a graph walker would loop on
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "x.jar"), testutils.Zip(t, map[string][]byte{"readme.txt": []byte("no manifest")}), 0o600))
|
||||
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), javaTask(), fileMetadataTask())
|
||||
s := newTestSBOM()
|
||||
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
|
||||
_ = tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s))
|
||||
withArchiveContains(t, s)
|
||||
|
||||
require.Contains(t, s.Artifacts.Unknowns, file.NewCoordinates("/x.jar", ""), "the fixture must reproduce the unknown")
|
||||
var contains int
|
||||
for _, rel := range s.Relationships {
|
||||
if rel.Type != artifact.ContainsRelationship {
|
||||
continue
|
||||
}
|
||||
contains++
|
||||
assert.NotEqual(t, rel.From.ID(), rel.To.ID(), "self-referential CONTAINS edge: %+v", rel)
|
||||
}
|
||||
assert.Positive(t, contains, "the archive still contains its files")
|
||||
}
|
||||
|
||||
func Test_NewArchiveContainsTask(t *testing.T) {
|
||||
// a file within an archive is contained only when the SBOM describes it, as a file at the scan root is
|
||||
// listed only then; an executable no package owns is recorded whatever the file selection
|
||||
in := func(fullPath string) file.Coordinates {
|
||||
return file.Coordinates{RealPath: "/outer.tar", FileSystemID: "layer", FullPath: fullPath}
|
||||
}
|
||||
outer := file.NewCoordinates("/outer.tar", "layer")
|
||||
jar := in("/outer.tar!/lib/app.jar")
|
||||
manifest := in("/outer.tar!/lib/app.jar!/META-INF/MANIFEST.MF")
|
||||
unknownExe := in("/outer.tar!/bin/setup.exe")
|
||||
|
||||
s := newTestSBOM()
|
||||
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{manifest: {}, file.NewCoordinates("/loose.txt", "layer"): {}}
|
||||
s.Artifacts.Executables = map[file.Coordinates]file.Executable{in("/outer.tar!/bin/tool"): {}}
|
||||
s.Artifacts.Unknowns = map[file.Coordinates][]string{unknownExe: {"no package identified in executable file"}}
|
||||
|
||||
withArchiveContains(t, s)
|
||||
|
||||
type edge struct{ from, to file.Coordinates }
|
||||
var edges []edge
|
||||
for _, r := range s.Relationships {
|
||||
assert.Equal(t, artifact.ContainsRelationship, r.Type)
|
||||
edges = append(edges, edge{r.From.(file.Coordinates), r.To.(file.Coordinates)})
|
||||
}
|
||||
assert.ElementsMatch(t, []edge{
|
||||
{jar, manifest},
|
||||
{outer, jar}, // the jar is not described itself, but is on the way to what is
|
||||
{outer, unknownExe},
|
||||
}, edges)
|
||||
}
|
||||
|
||||
func Test_NewArchiveCatalogerTask_dropsItselfFromSubPipeline(t *testing.T) {
|
||||
// a copy of itself in the sub-pipeline would process every nesting level twice
|
||||
cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)
|
||||
|
||||
innerZip := testutils.Zip(t, map[string][]byte{"leaf.txt": []byte("leaf")})
|
||||
outerZip := testutils.Zip(t, map[string][]byte{"nested/inner.zip": innerZip})
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
var ran int
|
||||
countTask := countingTask(&ran)
|
||||
|
||||
poisoned := []Task{countTask, NewArchiveCatalogerTask(cfg, cataloging.UnknownsConfig{}, []Task{countTask}, nil, nil)}
|
||||
require.NotNil(t, poisoned[1])
|
||||
|
||||
tsk := newTestTask(t, cfg, poisoned...)
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.Equal(t, 2, ran, "two archives, one sub-pipeline task each")
|
||||
|
||||
t.Run("a sub-pipeline of nothing but itself yields no task", func(t *testing.T) {
|
||||
only := []Task{NewArchiveCatalogerTask(cfg, cataloging.UnknownsConfig{}, []Task{countTask}, nil, nil)}
|
||||
assert.Nil(t, NewArchiveCatalogerTask(cfg, cataloging.UnknownsConfig{}, only, nil, nil))
|
||||
})
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_nestedArchivesKeepTheLayerFileSystemID(t *testing.T) {
|
||||
// extracted files keep the layer digest as their FileSystemID at every level; the nesting chain
|
||||
// rides on the traversal's VirtualPath and the coordinate's FullPath
|
||||
innerZip := testutils.Zip(t, map[string][]byte{"leaf.txt": []byte("leaf")})
|
||||
outerZip := testutils.Zip(t, map[string][]byte{"nested/inner.zip": innerZip})
|
||||
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
const layerDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
|
||||
var chains, fsids []string
|
||||
captureTask := NewTask("capture-fsid", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
trav := archive.TraversalFromContext(ctx)
|
||||
require.NotNil(t, trav)
|
||||
chains = append(chains, archive.VirtualPath(trav.Location))
|
||||
fsids = append(fsids, trav.Location.FileSystemID)
|
||||
return nil
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2), captureTask)
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir, fsid: layerDigest}, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.Equal(t, []string{layerDigest, layerDigest}, fsids)
|
||||
assert.ElementsMatch(t, []string{"/outer.zip", "/outer.zip:nested/inner.zip"}, chains)
|
||||
}
|
||||
|
||||
// twoLayerResolver reports the same archive path under two filesystem ids with different content, as an
|
||||
// all-layers image scan does for an archive rewritten in a later layer.
|
||||
type twoLayerResolver struct {
|
||||
file.Resolver
|
||||
path string
|
||||
contentBy map[string][]byte
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_sameArchivePathInTwoLayersDoesNotCollide(t *testing.T) {
|
||||
resolver := twoLayerResolver{
|
||||
path: "/app/bundle.zip",
|
||||
contentBy: map[string][]byte{
|
||||
"layer-one": testutils.Zip(t, map[string][]byte{"lib/config.json": []byte(`{"v":1}`)}),
|
||||
"layer-two": testutils.Zip(t, map[string][]byte{"lib/config.json": []byte(`{"v":2}`)}),
|
||||
},
|
||||
}
|
||||
|
||||
var chains []string
|
||||
fileTask := NewTask("record-files", func(ctx context.Context, r file.Resolver, builder sbomsync.Builder) error {
|
||||
trav := archive.TraversalFromContext(ctx)
|
||||
require.NotNil(t, trav)
|
||||
chains = append(chains, trav.Location.FileSystemID)
|
||||
|
||||
locs, err := r.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
metadata := map[file.Coordinates]file.Metadata{}
|
||||
for _, loc := range locs {
|
||||
metadata[loc.Coordinates] = file.Metadata{Path: loc.RealPath}
|
||||
}
|
||||
builder.(sbomsync.Accessor).WriteToSBOM(func(s *sbom.SBOM) {
|
||||
s.Artifacts.FileMetadata = metadata
|
||||
})
|
||||
return nil
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), fileTask)
|
||||
|
||||
s := &sbom.SBOM{Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(),
|
||||
FileMetadata: map[file.Coordinates]file.Metadata{},
|
||||
}}
|
||||
require.NoError(t, tsk.Execute(context.Background(), resolver, sbomsync.NewBuilder(s)))
|
||||
|
||||
assert.ElementsMatch(t, []string{"layer-one", "layer-two"}, chains)
|
||||
// the tables are keyed by Coordinates, so a collision is a silently missing entry
|
||||
assert.Len(t, s.Artifacts.FileMetadata, 2)
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) {
|
||||
// runSubPipeline bypasses RunTask, where the coordinate-error to unknown conversion lives
|
||||
outerZip := testutils.Zip(t, map[string][]byte{"lib/broken.json": []byte("{")})
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1)
|
||||
|
||||
t.Run("a coordinate error keeps the coordinates the cataloger named", func(t *testing.T) {
|
||||
failing := NewTask("failing-cataloger", func(_ context.Context, r file.Resolver, _ sbomsync.Builder) error {
|
||||
locs, err := r.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, locs)
|
||||
var errs error
|
||||
for _, loc := range locs {
|
||||
errs = unknown.Append(errs, loc, errors.New("unable to parse"))
|
||||
}
|
||||
return errs
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cfg, failing)
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)),
|
||||
"a cataloger failing inside an archive must not fail the archive or the scan")
|
||||
|
||||
require.Len(t, s.Artifacts.Unknowns, 1)
|
||||
for coords, reasons := range s.Artifacts.Unknowns {
|
||||
assert.Equal(t, "/outer.zip!/lib/broken.json", coords.FullNestedPath(), "the inner coordinates must survive the merge")
|
||||
assert.Equal(t, "/outer.zip", coords.RealPath, "and must be addressable per archive")
|
||||
assert.Equal(t, []string{"failing-cataloger: unable to parse"}, reasons)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an error with no coordinates is attributed to the archive", func(t *testing.T) {
|
||||
failing := NewTask("bare-error-cataloger", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
return errors.New("cataloger blew up")
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cfg, failing)
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
|
||||
|
||||
require.Len(t, s.Artifacts.Unknowns, 1)
|
||||
reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
|
||||
require.True(t, ok, "an error with no location of its own belongs to the containing archive")
|
||||
assert.Equal(t, []string{"bare-error-cataloger: cataloger blew up"}, reasons)
|
||||
})
|
||||
|
||||
t.Run("a panicking cataloger is recovered and recorded", func(t *testing.T) {
|
||||
panicking := NewTask("panicking-cataloger", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
panic("nope")
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cfg, panicking)
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)),
|
||||
"a panic inside an archive must not fail the scan")
|
||||
|
||||
reasons, ok := s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")]
|
||||
require.True(t, ok)
|
||||
require.Len(t, reasons, 1)
|
||||
assert.Equal(t, "panicking-cataloger: panic: nope", reasons[0])
|
||||
})
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_subPipelineUnknownsAreAddedNotReplaced(t *testing.T) {
|
||||
outerZip := testutils.Zip(t, map[string][]byte{"lib/thing.json": []byte("{}")})
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
preexisting := file.Coordinates{RealPath: "/outer.zip", FullPath: "/outer.zip!/lib/thing.json"}
|
||||
writer := NewTask("scratch-writer", func(_ context.Context, _ file.Resolver, builder sbomsync.Builder) error {
|
||||
builder.(sbomsync.Accessor).WriteToSBOM(func(s *sbom.SBOM) {
|
||||
if s.Artifacts.Unknowns == nil {
|
||||
s.Artifacts.Unknowns = map[file.Coordinates][]string{}
|
||||
}
|
||||
s.Artifacts.Unknowns[preexisting] = append(s.Artifacts.Unknowns[preexisting], "recorded by the cataloger itself")
|
||||
})
|
||||
return nil
|
||||
})
|
||||
failing := NewTask("failing-cataloger", func(_ context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
return errors.New("boom")
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), writer, failing)
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(s)))
|
||||
|
||||
assert.Equal(t, []string{"recorded by the cataloger itself"}, s.Artifacts.Unknowns[preexisting])
|
||||
assert.Equal(t, []string{"failing-cataloger: boom"}, s.Artifacts.Unknowns[file.NewCoordinates("/outer.zip", "")])
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_limitsFallWhenAnArchiveIsReleased(t *testing.T) {
|
||||
entry := bytes.Repeat([]byte("z"), 400)
|
||||
entries := map[string][]byte{}
|
||||
for i := range 5 {
|
||||
entries[fmt.Sprintf("e%d.bin", i)] = entry
|
||||
}
|
||||
|
||||
rootDir := t.TempDir()
|
||||
for _, name := range []string{"one.zip", "two.zip"} {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, name), testutils.Zip(t, entries), 0o600))
|
||||
}
|
||||
|
||||
// room in memory for one archive's entries and their index cost, not for two at once
|
||||
limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: 20000, MaxDiskBytes: 0})
|
||||
|
||||
// sampled while the archive is still held
|
||||
var extracted []int
|
||||
var peakMemory int64
|
||||
sample := NewTask("sample", func(_ context.Context, r file.Resolver, _ sbomsync.Builder) error {
|
||||
locs, err := r.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
extracted = append(extracted, len(locs))
|
||||
if mem, _ := limiter.InUse(); mem > peakMemory {
|
||||
peakMemory = mem
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
c := newTestCataloger(1, limiter, sample)
|
||||
require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.Equal(t, []int{5, 5}, extracted,
|
||||
"both archives must extract in full: the first is released before the second is admitted")
|
||||
assert.Positive(t, peakMemory, "entries within the memory limit are held in memory")
|
||||
|
||||
mem, disk := limiter.InUse()
|
||||
assert.Zero(t, mem, "the memory limit must fall back to nothing once the walk unwinds")
|
||||
assert.Zero(t, disk, "and so must the disk limit")
|
||||
}
|
||||
|
||||
func newTestCataloger(maxDepth int, limiter *archive.Limiter, subPipeline ...Task) *archiveCataloger {
|
||||
return &archiveCataloger{
|
||||
maxDepth: maxDepth,
|
||||
pkgTasks: subPipeline,
|
||||
limiter: limiter,
|
||||
progress: bus.StartCatalogerTask(context.Background(), archiveCatalogerProgressInfo(), -1, ""),
|
||||
}
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_archiveWhoseOwnBytesExceedTheDiskLimitIsSkipped(t *testing.T) {
|
||||
// a streamed archive's own bytes must be placed before it can be opened; when they do not fit on
|
||||
// disk the archive is skipped rather than waited for, and a sibling that fits is cataloged in full
|
||||
oversized := testutils.Zip(t, map[string][]byte{"payload.bin": incompressible(8000)})
|
||||
small := testutils.Zip(t, map[string][]byte{"ok.txt": []byte("ok")})
|
||||
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "oversized.zip"), oversized, 0o600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "small.zip"), small, 0o600))
|
||||
|
||||
// room for the small archive's bytes, its one entry and that entry's index cost
|
||||
diskLimit := int64(len(small)) + 3000
|
||||
require.Greater(t, int64(len(oversized)), diskLimit, "the fixture must actually exceed the limit")
|
||||
|
||||
limits := archive.Limits{MaxMemoryBytes: 0, MaxDiskBytes: diskLimit}
|
||||
|
||||
var seen []string
|
||||
fileCounts := map[string]int{}
|
||||
tsk := newLimitedTestTask(t, 1, limits, capturingTask(t, &seen, &fileCounts))
|
||||
|
||||
s := newTestSBOM()
|
||||
require.NoError(t, tsk.Execute(context.Background(), streamingDirResolver{dirTestResolver{dir: rootDir}}, sbomsync.NewBuilder(s)),
|
||||
"reaching a limit must not fail the scan")
|
||||
|
||||
assert.Equal(t, []string{"/small.zip"}, seen, "the oversized archive is skipped and the sibling is still cataloged")
|
||||
assert.Equal(t, 1, fileCounts["/small.zip"])
|
||||
|
||||
reasons := s.Artifacts.Unknowns[file.NewCoordinates("/oversized.zip", "")]
|
||||
require.Len(t, reasons, 1, "the skip must be recorded against the archive; got %v", s.Artifacts.Unknowns)
|
||||
assert.Contains(t, reasons[0], "archive skipped")
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_discoverArchivesSelectsTarFamily(t *testing.T) {
|
||||
marker := map[string][]byte{"nested/marker.txt": []byte("hello")}
|
||||
|
||||
scanDir := t.TempDir()
|
||||
for name, body := range map[string][]byte{
|
||||
"bundle.tar.gz": testutils.TarGz(t, marker),
|
||||
"bundle.tgz": testutils.TarGz(t, marker),
|
||||
"bundle.tar": testutils.Tar(t, marker),
|
||||
"bundle.zip": testutils.Zip(t, marker),
|
||||
"notes.txt": []byte("not an archive at all"),
|
||||
} {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), body, 0o600))
|
||||
}
|
||||
|
||||
var seen []string
|
||||
fileCounts := map[string]int{}
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), capturingTask(t, &seen, &fileCounts))
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.ElementsMatch(t, []string{"/bundle.tar.gz", "/bundle.tgz", "/bundle.tar", "/bundle.zip"}, seen,
|
||||
"every tar-family archive must be reached, and the plain text file must not be")
|
||||
for _, archivePath := range seen {
|
||||
assert.Equal(t, 1, fileCounts[archivePath], "%s must have been extracted", archivePath)
|
||||
}
|
||||
|
||||
t.Run("the MIME types the tar family is sniffed as are in the set discovery asks for", func(t *testing.T) {
|
||||
for _, mt := range []string{"application/x-tar", "application/gzip", "application/x-gtar", "application/zip"} {
|
||||
assert.True(t, mimetype.ArchiveMIMETypeSet.Has(mt), "%s must be a candidate archive type", mt)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type recordingPublisher struct {
|
||||
events []partybus.Event
|
||||
}
|
||||
|
||||
func Test_archiveCatalogerTask_publishesOneProgressRowPerCataloger(t *testing.T) {
|
||||
// a consumer keying rows by ID (as the syft CLI does) replaces a republished row and never sees it
|
||||
// complete, so each cataloger keeps to the one row it already owns
|
||||
inner := testutils.Zip(t, map[string][]byte{"inner/file.txt": []byte("hello")})
|
||||
scanDir := t.TempDir()
|
||||
for _, name := range []string{"one.zip", "two.zip"} {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(scanDir, name), inner, 0o600))
|
||||
}
|
||||
|
||||
publisher := &recordingPublisher{}
|
||||
bus.Set(publisher)
|
||||
t.Cleanup(func() { bus.Set(nil) })
|
||||
|
||||
// the archive row's stage as each archive's sub-pipeline starts: the previous archive is done by then,
|
||||
// so the row must read the walk's own count rather than anything a nested cataloger left behind
|
||||
var rows []*monitor.TaskProgress
|
||||
var stageAtEntry []string
|
||||
noisy := NewTask("noisy", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
stageAtEntry = append(stageAtEntry, publisher.events[0].Value.(progress.StagedProgressable).Stage())
|
||||
p := bus.StartCatalogerTask(ctx, monitor.GenericTask{ID: "noisy"}, 3, "")
|
||||
rows = append(rows, p)
|
||||
p.AtomicStage.Set("working on something")
|
||||
p.Add(3)
|
||||
p.SetCompleted()
|
||||
return nil
|
||||
})
|
||||
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), noisy)
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: scanDir}, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
var started []string
|
||||
for _, e := range publisher.events {
|
||||
if e.Type != event.CatalogerTaskStarted {
|
||||
continue
|
||||
}
|
||||
info, ok := e.Source.(monitor.GenericTask)
|
||||
require.True(t, ok, "expected a generic task as the event source")
|
||||
started = append(started, info.ID)
|
||||
}
|
||||
|
||||
assert.Equal(t, []string{ArchiveCatalogerTaskName, "noisy"}, started,
|
||||
"each cataloger publishes one row, however many archives the walk enters")
|
||||
|
||||
require.Len(t, rows, 2, "the sub-pipeline ran once per archive")
|
||||
assert.Same(t, rows[0], rows[1], "both runs must report into the one row that cataloger owns")
|
||||
assert.Equal(t, int64(6), rows[0].Current(), "what every run found is counted onto that row")
|
||||
|
||||
require.Len(t, publisher.events, 2)
|
||||
prog, ok := publisher.events[0].Value.(progress.StagedProgressable)
|
||||
require.True(t, ok)
|
||||
assert.ErrorIs(t, prog.Error(), progress.ErrCompleted, "the row must be completed, or a waiting consumer never finishes")
|
||||
assert.Equal(t, int64(2), prog.Current(), "the archive row counts every archive the walk entered")
|
||||
assert.Equal(t, "2 archives", prog.Stage(), "and says so once the walk is done")
|
||||
|
||||
// one.zip sorts before two.zip, and the resolver walks the scan directory in lexical order
|
||||
assert.Equal(t, []string{"", "1 archives (/one.zip)"}, stageAtEntry,
|
||||
"the row must take its stage back from the sub-pipeline once an archive is done")
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_reportsPeakUsageAndTheSlowestArchive(t *testing.T) {
|
||||
// self time excludes nested archives, or the outer archive would be named every scan
|
||||
innerJar := testutils.Zip(t, map[string][]byte{"README.txt": []byte("inner")})
|
||||
outerZip := testutils.Zip(t, map[string][]byte{"lib/inner.jar": innerJar})
|
||||
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "outer.zip"), outerZip, 0o600))
|
||||
|
||||
const innerWork = 40 * time.Millisecond
|
||||
slow := NewTask("slow-inside-the-inner-archive", func(ctx context.Context, _ file.Resolver, _ sbomsync.Builder) error {
|
||||
if trav := archive.TraversalFromContext(ctx); trav != nil && strings.HasSuffix(archive.VirtualPath(trav.Location), "inner.jar") {
|
||||
time.Sleep(innerWork)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
limiter := archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
|
||||
c := newTestCataloger(3, limiter, slow)
|
||||
require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.Equal(t, "/outer.zip!/lib/inner.jar", c.slowestPath, "the slowest archive is named by its full chain")
|
||||
assert.GreaterOrEqual(t, c.slowest, innerWork)
|
||||
assert.Equal(t, int64(2), c.progress.Current())
|
||||
|
||||
// the peaks outlive the walk that set them
|
||||
peakMemory, _ := limiter.Peak()
|
||||
assert.Positive(t, peakMemory, "entries held in memory must show on the peak")
|
||||
memory, disk := limiter.InUse()
|
||||
assert.Zero(t, memory)
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_slowestIsUnsetWhenNoArchiveIsCataloged(t *testing.T) {
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "notes.txt"), []byte("not an archive"), 0o600))
|
||||
|
||||
noop := NewTask("noop", func(context.Context, file.Resolver, sbomsync.Builder) error { return nil })
|
||||
c := newTestCataloger(2, archive.NewLimiter(archive.Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}), noop)
|
||||
require.NoError(t, c.catalog(context.Background(), dirTestResolver{dir: rootDir}, 0, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.Empty(t, c.slowestPath)
|
||||
assert.Zero(t, c.progress.Current())
|
||||
c.logStats() // must not panic with nothing to report
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_aMisnamedNonArchiveIsNotAnError(t *testing.T) {
|
||||
// the name says archive, the content does not: it is never opened
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "notes.zip"),
|
||||
[]byte("this is prose, not an archive, whatever the extension claims\n"), 0o600))
|
||||
|
||||
var ran int
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1), countingTask(&ran))
|
||||
|
||||
err := tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM()))
|
||||
|
||||
assert.NoError(t, err, "a file that is not an archive is not a failure to report")
|
||||
assert.Zero(t, ran, "and nothing is cataloged inside it")
|
||||
}
|
||||
|
||||
func Test_archiveCataloger_catalogsAZipBehindALauncherScript(t *testing.T) {
|
||||
// the Spring Boot executable jar shape: content sniffing types it as a shell script, so it reaches
|
||||
// the walk only by name and qualifies only by its end-of-central-directory record
|
||||
inner := testutils.Zip(t, map[string][]byte{"nested/marker.txt": []byte("found me")})
|
||||
prefixed := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"),
|
||||
testutils.Zip(t, map[string][]byte{"BOOT-INF/lib/inner.jar": inner})...)
|
||||
|
||||
rootDir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(rootDir, "app.jar"), prefixed, 0o600))
|
||||
|
||||
var seen []string
|
||||
tsk := newTestTask(t, cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3), capturingTask(t, &seen, nil))
|
||||
require.NoError(t, tsk.Execute(context.Background(), dirTestResolver{dir: rootDir}, sbomsync.NewBuilder(newTestSBOM())))
|
||||
|
||||
assert.Equal(t, []string{"/app.jar", "/app.jar:BOOT-INF/lib/inner.jar"}, seen,
|
||||
"the stub must not hide the archive, and the walk must carry on into what it holds")
|
||||
}
|
||||
|
||||
func (d dirTestResolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
|
||||
return filesByMIMEType(d.dir, d.fsid, types...)
|
||||
}
|
||||
|
||||
func filesByMIMEType(dir, fsid string, types ...string) ([]file.Location, error) {
|
||||
wanted := strset.New(types...)
|
||||
var locs []file.Location
|
||||
err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil || entry.IsDir() {
|
||||
return err
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mimeType := stereoscopeFile.MIMEType(f)
|
||||
if closeErr := f.Close(); closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if !wanted.Has(mimeType) {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(dir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
coords := file.Coordinates{RealPath: "/" + filepath.ToSlash(rel), FileSystemID: fsid}
|
||||
locs = append(locs, file.NewLocationFromCoordinates(coords))
|
||||
return nil
|
||||
})
|
||||
return locs, err
|
||||
}
|
||||
|
||||
func (d dirTestResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) {
|
||||
return os.Open(filepath.Join(d.dir, filepath.FromSlash(loc.RealPath)))
|
||||
}
|
||||
|
||||
func (d dirTestResolver) FileMetadataByLocation(loc file.Location) (file.Metadata, error) {
|
||||
info, err := os.Stat(filepath.Join(d.dir, filepath.FromSlash(loc.RealPath)))
|
||||
return file.Metadata{FileInfo: info}, err
|
||||
}
|
||||
|
||||
// matchesAnyGlob matches the resolver-relative path with a leading slash, the form a real resolver
|
||||
// globs over. No patterns matches everything.
|
||||
func matchesAnyGlob(root, path string, patterns []string) bool {
|
||||
if len(patterns) == 0 {
|
||||
return true
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
rel = "/" + filepath.ToSlash(rel)
|
||||
for _, pattern := range patterns {
|
||||
if ok, err := doublestar.Match(pattern, rel); err == nil && ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (r twoLayerResolver) FilesByGlob(_ ...string) ([]file.Location, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (d dirTestResolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
|
||||
var locs []file.Location
|
||||
err := filepath.WalkDir(d.dir, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil || entry.IsDir() {
|
||||
return err
|
||||
}
|
||||
if !matchesAnyGlob(d.dir, path, patterns) {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(d.dir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
coords := file.Coordinates{RealPath: "/" + filepath.ToSlash(rel), FileSystemID: d.fsid}
|
||||
locs = append(locs, file.NewLocationFromCoordinates(coords))
|
||||
return nil
|
||||
})
|
||||
return locs, err
|
||||
}
|
||||
|
||||
func (r twoLayerResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) {
|
||||
var locs []file.Location
|
||||
for _, fsid := range []string{"layer-one", "layer-two"} {
|
||||
locs = append(locs, file.NewLocationFromCoordinates(file.NewCoordinates(r.path, fsid)))
|
||||
}
|
||||
return locs, nil
|
||||
}
|
||||
|
||||
func (r twoLayerResolver) FileContentsByLocation(loc file.Location) (io.ReadCloser, error) {
|
||||
body, ok := r.contentBy[loc.FileSystemID]
|
||||
if !ok {
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(body)), nil
|
||||
}
|
||||
|
||||
// FileMetadataByLocation reports no metadata, so the archive falls back to the minimum budget.
|
||||
func (r twoLayerResolver) FileMetadataByLocation(file.Location) (file.Metadata, error) {
|
||||
return file.Metadata{}, os.ErrNotExist
|
||||
}
|
||||
|
||||
// incompressible returns n bytes deflate cannot shrink, so an archive built from them has a
|
||||
// predictable size to set a limit against.
|
||||
func incompressible(n int) []byte {
|
||||
b := make([]byte, n)
|
||||
x := uint32(12345)
|
||||
for i := range b {
|
||||
x = x*1664525 + 1013904223
|
||||
b[i] = byte(x >> 24)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (p *recordingPublisher) Publish(e partybus.Event) {
|
||||
p.events = append(p.events, e)
|
||||
}
|
||||
@@ -47,7 +47,7 @@ func newFileDigestCatalogerTask(selection file.Selection, hashers []crypto.Hash,
|
||||
result, err := filedigest.NewCataloger(hashers).Catalog(ctx, resolver, coordinates...)
|
||||
|
||||
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
|
||||
sbom.Artifacts.FileDigests = result
|
||||
mergeInto(&sbom.Artifacts.FileDigests, result)
|
||||
})
|
||||
|
||||
return err
|
||||
@@ -78,7 +78,7 @@ func newFileMetadataCatalogerTask(selection file.Selection, tags ...string) Task
|
||||
result, err := filemetadata.NewCataloger().Catalog(ctx, resolver, coordinates...)
|
||||
|
||||
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
|
||||
sbom.Artifacts.FileMetadata = result
|
||||
mergeInto(&sbom.Artifacts.FileMetadata, result)
|
||||
})
|
||||
|
||||
return err
|
||||
@@ -104,7 +104,7 @@ func newFileContentCatalogerTask(cfg filecontent.Config, tags ...string) Task {
|
||||
result, err := filecontent.NewCataloger(cfg).Catalog(ctx, resolver)
|
||||
|
||||
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
|
||||
sbom.Artifacts.FileContents = result
|
||||
mergeInto(&sbom.Artifacts.FileContents, result)
|
||||
})
|
||||
|
||||
return err
|
||||
@@ -130,7 +130,7 @@ func newExecutableCatalogerTask(selection file.Selection, cfg executable.Config,
|
||||
result, err := executable.NewCataloger(cfg).CatalogCtx(ctx, resolver)
|
||||
|
||||
accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
|
||||
sbom.Artifacts.Executables = result
|
||||
mergeInto(&sbom.Artifacts.Executables, result)
|
||||
})
|
||||
|
||||
return err
|
||||
|
||||
@@ -40,19 +40,13 @@ func Test_coordinatesForSelection(t *testing.T) {
|
||||
Relationships: []artifact.Relationship{
|
||||
{
|
||||
From: pkg.Package{},
|
||||
To: file.Coordinates{
|
||||
RealPath: "path",
|
||||
FileSystemID: "fs",
|
||||
},
|
||||
To: file.NewCoordinates("path", "fs"),
|
||||
Type: artifact.ContainsRelationship,
|
||||
},
|
||||
},
|
||||
},
|
||||
files: []file.Coordinates{
|
||||
{
|
||||
RealPath: "path",
|
||||
FileSystemID: "fs",
|
||||
},
|
||||
file.NewCoordinates("path", "fs"),
|
||||
},
|
||||
ok: true,
|
||||
},
|
||||
@@ -63,10 +57,7 @@ func Test_coordinatesForSelection(t *testing.T) {
|
||||
Relationships: []artifact.Relationship{
|
||||
{
|
||||
From: pkg.Package{},
|
||||
To: file.Coordinates{
|
||||
RealPath: "path",
|
||||
FileSystemID: "fs",
|
||||
},
|
||||
To: file.NewCoordinates("path", "fs"),
|
||||
// wrong type
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
@@ -82,10 +73,7 @@ func Test_coordinatesForSelection(t *testing.T) {
|
||||
Relationships: []artifact.Relationship{
|
||||
{
|
||||
From: file.Coordinates{}, // wrong type
|
||||
To: file.Coordinates{
|
||||
RealPath: "path",
|
||||
FileSystemID: "fs",
|
||||
},
|
||||
To: file.NewCoordinates("path", "fs"),
|
||||
Type: artifact.ContainsRelationship,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -3,9 +3,11 @@ package task
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/relationship"
|
||||
@@ -48,7 +50,7 @@ func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string
|
||||
HideOnSuccess: true,
|
||||
}
|
||||
|
||||
t := bus.StartCatalogerTask(info, -1, "")
|
||||
t := bus.StartCatalogerTask(ctx, info, -1, "")
|
||||
|
||||
pkgs, relationships, err := c.Catalog(ctx, resolver)
|
||||
|
||||
@@ -167,7 +169,7 @@ func applyComplianceRules(p *pkg.Package, cfg cataloging.ComplianceConfig) (bool
|
||||
loc := "unknown"
|
||||
locs := p.Locations.ToSlice()
|
||||
if len(locs) > 0 {
|
||||
loc = locs[0].Path()
|
||||
loc = locs[0].FullNestedPath()
|
||||
}
|
||||
switch action {
|
||||
case cataloging.ComplianceActionDrop:
|
||||
@@ -236,10 +238,10 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
|
||||
|
||||
locations := map[artifact.ID]file.Location{}
|
||||
|
||||
for _, path := range fileOwner.OwnedFiles() {
|
||||
pathRefs, err := resolver.FilesByPath(path)
|
||||
for _, ownedPath := range fileOwner.OwnedFiles() {
|
||||
pathRefs, err := resolver.FilesByPath(ownedPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to find path for path=%q: %w", path, err)
|
||||
return nil, fmt.Errorf("unable to find path for path=%q: %w", ownedPath, err)
|
||||
}
|
||||
|
||||
if len(pathRefs) == 0 {
|
||||
@@ -250,8 +252,13 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
|
||||
}
|
||||
|
||||
for _, ref := range pathRefs {
|
||||
// an archive looks a relative path up from its root, but an owned path may be relative to
|
||||
// anything (a python RECORD is relative to site-packages), so only an absolute one owns a file in one
|
||||
if archive.ContainerPath(ref.Coordinates) != "" && !path.IsAbs(ownedPath) {
|
||||
continue
|
||||
}
|
||||
if oldRef, ok := locations[ref.ID()]; ok {
|
||||
log.Debugf("found path duplicate of %s", oldRef.RealPath)
|
||||
log.Debugf("found path duplicate of %s", oldRef.FullNestedPath())
|
||||
}
|
||||
locations[ref.ID()] = ref
|
||||
}
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/testutils"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/cpe"
|
||||
@@ -64,8 +68,8 @@ func TestApplyCompliance(t *testing.T) {
|
||||
p2 := pkg.Package{Name: "", Version: "1.0"} // missing name
|
||||
p3 := pkg.Package{Name: "pkg-3", Version: ""} // missing version
|
||||
p4 := pkg.Package{Name: "pkg-4", Version: ""} // missing version
|
||||
c1 := file.Coordinates{RealPath: "/coords/1"}
|
||||
c2 := file.Coordinates{RealPath: "/coords/2"}
|
||||
c1 := file.NewCoordinates("/coords/1", "")
|
||||
c2 := file.NewCoordinates("/coords/2", "")
|
||||
|
||||
for _, p := range []*pkg.Package{&p1, &p2, &p3, &p4} {
|
||||
p.SetID()
|
||||
@@ -281,3 +285,28 @@ func TestApplyComplianceRules_DropAndStub(t *testing.T) {
|
||||
assert.False(t, isCompliant)
|
||||
assert.Equal(t, cataloging.UnknownStubValue, p.Version)
|
||||
}
|
||||
|
||||
func Test_packageFileOwnershipRelationships_exactPathsWithinAnArchive(t *testing.T) {
|
||||
// a wheel's RECORD is relative to site-packages: looked up from the wheel's root it finds the file, but
|
||||
// only an exact path owns one, as at the scan root
|
||||
data := testutils.Zip(t, map[string]string{"pip/__init__.py": "", "bin/tool": ""})
|
||||
whl, err := archive.Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("/pip.whl", ""),
|
||||
archive.NewLimiter(archive.Limits{MaxMemoryBytes: 1 << 20}), archive.NewExclusions(nil))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(whl.Cleanup)
|
||||
|
||||
found, err := whl.FilesByPath("pip/__init__.py")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, found, 1, "a relative path still finds the file")
|
||||
|
||||
p := pkg.Package{Name: "pip", Metadata: pkg.PythonPackage{Files: []pkg.PythonFileRecord{{Path: "pip/__init__.py"}, {Path: "/bin/tool"}}}}
|
||||
p.SetID()
|
||||
rels, err := packageFileOwnershipRelationships(p, whl)
|
||||
require.NoError(t, err)
|
||||
|
||||
var owned []string
|
||||
for _, r := range rels {
|
||||
owned = append(owned, r.To.(file.Coordinates).FullNestedPath())
|
||||
}
|
||||
assert.Equal(t, []string{"/pip.whl!/bin/tool"}, owned)
|
||||
}
|
||||
|
||||
@@ -40,10 +40,11 @@ func packagesToRemove(accessor sbomsync.Accessor) []artifact.ID {
|
||||
}
|
||||
case "":
|
||||
if isPrimaryEvidence {
|
||||
if exists := filterDuplicates[getKey(p, l)]; exists {
|
||||
k := getKey(p, l)
|
||||
if exists := filterDuplicates[k]; exists {
|
||||
break
|
||||
}
|
||||
filterDuplicates[getKey(p, l)] = true
|
||||
filterDuplicates[k] = true
|
||||
noPrimary = false
|
||||
break
|
||||
}
|
||||
@@ -59,5 +60,5 @@ func packagesToRemove(accessor sbomsync.Accessor) []artifact.ID {
|
||||
}
|
||||
|
||||
func getKey(pkg pkg.Package, loc file.Location) string {
|
||||
return fmt.Sprintf("%s-%s-%s-%s", pkg.Name, pkg.Version, loc.RealPath, loc.AccessPath)
|
||||
return fmt.Sprintf("%s-%s-%s-%s", pkg.Name, pkg.Version, loc.FullNestedPath(), loc.AccessPath)
|
||||
}
|
||||
|
||||
@@ -2,8 +2,11 @@ package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
@@ -36,11 +39,20 @@ func (c unknownsLabelerTask) finalize(resolver file.Resolver, s *sbom.SBOM) {
|
||||
hasPackageReference := coordinateReferenceLookup(resolver, s)
|
||||
|
||||
if c.RemoveWhenPackagesDefined {
|
||||
for coords := range s.Artifacts.Unknowns {
|
||||
for coords, reasons := range s.Artifacts.Unknowns {
|
||||
if !hasPackageReference(coords) {
|
||||
continue
|
||||
}
|
||||
delete(s.Artifacts.Unknowns, coords)
|
||||
// what the archive cataloger reports is about the archive's contents (left out, cut short),
|
||||
// which a package describing the archive does not account for
|
||||
kept := slices.DeleteFunc(reasons, func(r string) bool {
|
||||
return !strings.HasPrefix(r, formatUnknown("", ArchiveCatalogerTaskName))
|
||||
})
|
||||
if len(kept) == 0 {
|
||||
delete(s.Artifacts.Unknowns, coords)
|
||||
continue
|
||||
}
|
||||
s.Artifacts.Unknowns[coords] = kept
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,7 +71,7 @@ func (c unknownsLabelerTask) finalize(resolver file.Resolver, s *sbom.SBOM) {
|
||||
if c.IncludeUnexpandedArchives {
|
||||
ctx := context.Background()
|
||||
for coords := range s.Artifacts.FileMetadata {
|
||||
format, _, notArchiveErr := intFile.IdentifyArchive(ctx, coords.RealPath, nil)
|
||||
format, _, notArchiveErr := intFile.IdentifyArchive(ctx, coords.FullNestedPath(), nil)
|
||||
if format != nil && notArchiveErr == nil && !hasPackageReference(coords) {
|
||||
s.Artifacts.Unknowns[coords] = append(s.Artifacts.Unknowns[coords], "archive not cataloged")
|
||||
}
|
||||
@@ -71,6 +83,7 @@ func formatUnknown(err string, task ...string) string {
|
||||
return strings.Join(task, "/") + ": " + err
|
||||
}
|
||||
|
||||
//nolint:gocognit
|
||||
func coordinateReferenceLookup(resolver file.Resolver, s *sbom.SBOM) func(coords file.Coordinates) bool {
|
||||
allPackageCoords := file.NewCoordinateSet()
|
||||
|
||||
@@ -83,7 +96,28 @@ func coordinateReferenceLookup(resolver file.Resolver, s *sbom.SBOM) func(coords
|
||||
// relationships for these owned files may be disabled, but we always want to include them
|
||||
for p := range s.Artifacts.Packages.Enumerate() {
|
||||
if f, ok := p.Metadata.(pkg.FileOwner); ok {
|
||||
// owned paths are within the containers the package was found in
|
||||
var archives []file.Coordinates
|
||||
atRoot := false
|
||||
for _, container := range archive.OwnerContainers(p) {
|
||||
if container.FullPath == "" {
|
||||
atRoot = true
|
||||
} else {
|
||||
archives = append(archives, container)
|
||||
}
|
||||
}
|
||||
for _, ownedFilePath := range f.OwnedFiles() {
|
||||
// an archive's contents are not in this resolver to follow links in, and only an absolute owned
|
||||
// path names a file in one (see packageFileOwnershipRelationships)
|
||||
if path.IsAbs(ownedFilePath) {
|
||||
for _, container := range archives {
|
||||
allPackageCoords.Add(archive.OwnedCoordinates(container, ownedFilePath))
|
||||
}
|
||||
}
|
||||
if !atRoot {
|
||||
continue
|
||||
}
|
||||
|
||||
// resolve these owned files, as they may have symlinks
|
||||
// but coordinates we will test against are always absolute paths
|
||||
locations, err := resolver.FilesByPath(ownedFilePath)
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
func Test_unknownsLabeler_archiveReasonsSurvivePackages(t *testing.T) {
|
||||
jar := file.NewCoordinates("/app.jar", "")
|
||||
p := pkg.Package{Name: "app", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(jar))}
|
||||
p.SetID()
|
||||
s := &sbom.SBOM{Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(p),
|
||||
Unknowns: map[file.Coordinates][]string{jar: {
|
||||
formatUnknown("archive cataloged from part of its contents: too big", ArchiveCatalogerTaskName),
|
||||
formatUnknown("unable to read manifest", "java-archive-cataloger"),
|
||||
}},
|
||||
}}
|
||||
|
||||
unknownsLabelerTask{cataloging.UnknownsConfig{RemoveWhenPackagesDefined: true}}.finalize(file.NewMockResolverForPaths(), s)
|
||||
|
||||
assert.Equal(t, []string{formatUnknown("archive cataloged from part of its contents: too big", ArchiveCatalogerTaskName)}, s.Artifacts.Unknowns[jar],
|
||||
"a package describing the archive explains a parse failure, not contents left out")
|
||||
}
|
||||
|
||||
func Test_unknownsLabeler_archiveContentsAreReportedByFullPath(t *testing.T) {
|
||||
// a package describing an archive does not account for the files within it
|
||||
war := file.NewCoordinates("/app.war", "layer")
|
||||
in := func(fullPath string) file.Coordinates {
|
||||
return file.Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: fullPath}
|
||||
}
|
||||
nativeLib := in("/app.war!/WEB-INF/lib/jna.jar!/linux-x86-64/libjnidispatch.so")
|
||||
unidentifiedJar := in("/app.war!/WEB-INF/lib/javax.inject-1.jar")
|
||||
p := pkg.Package{Name: "app", Locations: file.NewLocationSet(file.NewLocationFromCoordinates(war))}
|
||||
p.SetID()
|
||||
noPackage := formatUnknown("no package identified in archive", "java-archive-cataloger")
|
||||
|
||||
s := &sbom.SBOM{
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(p),
|
||||
Executables: map[file.Coordinates]file.Executable{nativeLib: {}},
|
||||
Unknowns: map[file.Coordinates][]string{unidentifiedJar: {noPackage}},
|
||||
},
|
||||
Relationships: []artifact.Relationship{{From: war, To: unidentifiedJar, Type: artifact.ContainsRelationship}},
|
||||
}
|
||||
|
||||
unknownsLabelerTask{cataloging.UnknownsConfig{RemoveWhenPackagesDefined: true, IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths(), s)
|
||||
|
||||
assert.Equal(t, map[file.Coordinates][]string{
|
||||
nativeLib: {formatUnknown("no package identified in executable file", unknownsLabelerTaskName)},
|
||||
unidentifiedJar: {noPackage},
|
||||
}, s.Artifacts.Unknowns)
|
||||
}
|
||||
|
||||
func Test_unknownsLabeler_relativeOwnedPathsNameNothingInAnArchive(t *testing.T) {
|
||||
// a python RECORD lists paths relative to site-packages, which resolve nowhere at the scan root either
|
||||
whl := func(fullPath string) file.Coordinates {
|
||||
return file.Coordinates{RealPath: "/pip.whl", FullPath: fullPath}
|
||||
}
|
||||
tool := whl("/pip.whl!/pip/tool")
|
||||
p := pkg.Package{
|
||||
Name: "pip",
|
||||
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(whl("/pip.whl!/pip.dist-info/METADATA"))),
|
||||
Metadata: pkg.PythonPackage{Files: []pkg.PythonFileRecord{{Path: "pip/tool"}}},
|
||||
}
|
||||
p.SetID()
|
||||
s := &sbom.SBOM{Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(p),
|
||||
Executables: map[file.Coordinates]file.Executable{tool: {}},
|
||||
}}
|
||||
|
||||
unknownsLabelerTask{cataloging.UnknownsConfig{IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths(), s)
|
||||
|
||||
assert.NotEmpty(t, s.Artifacts.Unknowns[tool])
|
||||
}
|
||||
|
||||
func Test_unknownsLabeler_ownedFilesAreWithinTheOwnersContainer(t *testing.T) {
|
||||
atRoot := file.NewCoordinates("/usr/bin/foo", "")
|
||||
inArchive := file.Coordinates{RealPath: "/rootfs.tar", FullPath: "/rootfs.tar!/usr/bin/foo"}
|
||||
owner := func(at file.Coordinates) pkg.Package {
|
||||
p := pkg.Package{
|
||||
Name: "owner",
|
||||
Type: pkg.RpmPkg,
|
||||
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(at)),
|
||||
Metadata: pkg.RpmDBEntry{Files: []pkg.RpmFileRecord{{Path: "/usr/bin/foo"}}},
|
||||
}
|
||||
p.SetID()
|
||||
return p
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
owner file.Coordinates
|
||||
referenced file.Coordinates
|
||||
}{
|
||||
{name: "an owner at the scan root", owner: file.NewCoordinates("/var/lib/rpm/Packages", ""), referenced: atRoot},
|
||||
{name: "an owner inside an archive", owner: file.Coordinates{RealPath: "/rootfs.tar", FullPath: "/rootfs.tar!/var/lib/rpm/Packages"}, referenced: inArchive},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := &sbom.SBOM{Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(owner(tt.owner)),
|
||||
Executables: map[file.Coordinates]file.Executable{atRoot: {}, inArchive: {}},
|
||||
}}
|
||||
|
||||
unknownsLabelerTask{cataloging.UnknownsConfig{IncludeExecutablesWithoutPackages: true}}.finalize(file.NewMockResolverForPaths("/usr/bin/foo"), s)
|
||||
|
||||
for _, exe := range []file.Coordinates{atRoot, inArchive} {
|
||||
assert.Equal(t, exe != tt.referenced, len(s.Artifacts.Unknowns[exe]) > 0, exe.FullNestedPath())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package testutils
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Zip builds a zip of the entries in sorted name order: archive limits bite as entries are read, so
|
||||
// order decides which land before a truncation.
|
||||
func Zip[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for _, name := range sortedNames(entries) {
|
||||
w, err := zw.Create(name)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte(entries[name]))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, zw.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// Tar builds a tar of the entries in sorted name order.
|
||||
func Tar[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
writeTar(t, &buf, entries)
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// TarGz builds a gzipped tar of the entries in sorted name order.
|
||||
func TarGz[T ~string | ~[]byte](t testing.TB, entries map[string]T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
writeTar(t, gw, entries)
|
||||
require.NoError(t, gw.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// Jar is the smallest jar the java cataloger identifies, a manifest naming the artifact, plus extra.
|
||||
func Jar(t testing.TB, title, version string, extra map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
entries := map[string][]byte{"META-INF/MANIFEST.MF": Manifest(title, version)}
|
||||
for name, body := range extra {
|
||||
entries[name] = body
|
||||
}
|
||||
return Zip(t, entries)
|
||||
}
|
||||
|
||||
// Manifest is a jar manifest naming the artifact.
|
||||
func Manifest(title, version string) []byte {
|
||||
return fmt.Appendf(nil, "Manifest-Version: 1.0\nImplementation-Title: %s\nImplementation-Version: %s\n", title, version)
|
||||
}
|
||||
|
||||
func writeTar[T ~string | ~[]byte](t testing.TB, w io.Writer, entries map[string]T) {
|
||||
t.Helper()
|
||||
tw := tar.NewWriter(w)
|
||||
for _, name := range sortedNames(entries) {
|
||||
body := []byte(entries[name])
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg}))
|
||||
_, err := tw.Write(body)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, tw.Close())
|
||||
}
|
||||
|
||||
func sortedNames[T any](entries map[string]T) []string {
|
||||
names := make([]string, 0, len(entries))
|
||||
for name := range entries {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
@@ -23,9 +23,9 @@ var _ error = (*CoordinateError)(nil)
|
||||
|
||||
func (u *CoordinateError) Error() string {
|
||||
if u.Coordinates.FileSystemID == "" {
|
||||
return fmt.Sprintf("%s: %v", u.Coordinates.RealPath, u.Reason)
|
||||
return fmt.Sprintf("%s: %v", u.Coordinates.FullNestedPath(), u.Reason)
|
||||
}
|
||||
return fmt.Sprintf("%s (%s): %v", u.Coordinates.RealPath, u.Coordinates.FileSystemID, u.Reason)
|
||||
return fmt.Sprintf("%s (%s): %v", u.Coordinates.FullNestedPath(), u.Coordinates.FileSystemID, u.Reason)
|
||||
}
|
||||
|
||||
// New returns a new CoordinateError unless the reason is a CoordinateError itself, in which case
|
||||
|
||||
@@ -169,9 +169,7 @@ func Test_Join(t *testing.T) {
|
||||
}
|
||||
|
||||
func Test_flatten(t *testing.T) {
|
||||
coords := file.Coordinates{
|
||||
RealPath: "real/path",
|
||||
}
|
||||
coords := file.NewCoordinates("real/path", "")
|
||||
e1 := fmt.Errorf("e1")
|
||||
e2 := fmt.Errorf("e2")
|
||||
c1 := New(coords, fmt.Errorf("c1"))
|
||||
|
||||
@@ -19,6 +19,8 @@ func ProcessPathErrors(err error) error {
|
||||
if pathErrorRegex.MatchString(errText) {
|
||||
foundPath := pathErrorRegex.ReplaceAllString(err.Error(), "$1")
|
||||
if foundPath != "" {
|
||||
// a path in an error text is taken as is: nesting is never inferred from "!/" in it, since a real
|
||||
// path can hold one too. A caller with the location in hand should use New instead
|
||||
return New(file.NewLocation(foundPath), err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,28 +17,29 @@ func Test_ProcessPathErrors(t *testing.T) {
|
||||
{
|
||||
errorText: `prefix path="/var/lib/thing" suffix`,
|
||||
expected: &CoordinateError{
|
||||
Coordinates: file.Coordinates{
|
||||
RealPath: "/var/lib/thing",
|
||||
},
|
||||
Reason: fmt.Errorf(`prefix path="/var/lib/thing" suffix`),
|
||||
Coordinates: file.NewCoordinates("/var/lib/thing", ""),
|
||||
Reason: fmt.Errorf(`prefix path="/var/lib/thing" suffix`),
|
||||
},
|
||||
},
|
||||
{
|
||||
errorText: `prefix path="/var/lib/thing"`,
|
||||
expected: &CoordinateError{
|
||||
Coordinates: file.Coordinates{
|
||||
RealPath: "/var/lib/thing",
|
||||
},
|
||||
Reason: fmt.Errorf(`prefix path="/var/lib/thing"`),
|
||||
Coordinates: file.NewCoordinates("/var/lib/thing", ""),
|
||||
Reason: fmt.Errorf(`prefix path="/var/lib/thing"`),
|
||||
},
|
||||
},
|
||||
{
|
||||
errorText: `path="/var/lib/thing" suffix`,
|
||||
expected: &CoordinateError{
|
||||
Coordinates: file.Coordinates{
|
||||
RealPath: "/var/lib/thing",
|
||||
},
|
||||
Reason: fmt.Errorf(`path="/var/lib/thing" suffix`),
|
||||
Coordinates: file.NewCoordinates("/var/lib/thing", ""),
|
||||
Reason: fmt.Errorf(`path="/var/lib/thing" suffix`),
|
||||
},
|
||||
},
|
||||
{
|
||||
errorText: `path="/data/Hello!/world.txt"`,
|
||||
expected: &CoordinateError{
|
||||
Coordinates: file.NewCoordinates("/data/Hello!/world.txt", ""),
|
||||
Reason: fmt.Errorf(`path="/data/Hello!/world.txt"`),
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "anchore.io/schema/syft/json/16.1.11/document",
|
||||
"$id": "anchore.io/schema/syft/json/16.1.12/document",
|
||||
"$ref": "#/$defs/Document",
|
||||
"$defs": {
|
||||
"AlpmDbEntry": {
|
||||
@@ -716,11 +716,15 @@
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string",
|
||||
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)."
|
||||
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem."
|
||||
},
|
||||
"layerID": {
|
||||
"type": "string",
|
||||
"description": "FileSystemID is an ID representing and entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank."
|
||||
"description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in."
|
||||
},
|
||||
"fullPath": {
|
||||
"type": "string",
|
||||
"description": "FullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it is blank, so coordinates outside archives are unchanged by the field (in JSON, equality and ID)."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
@@ -2544,15 +2548,19 @@
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string",
|
||||
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed)."
|
||||
"description": "RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem."
|
||||
},
|
||||
"layerID": {
|
||||
"type": "string",
|
||||
"description": "FileSystemID is an ID representing and entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank."
|
||||
"description": "FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in."
|
||||
},
|
||||
"fullPath": {
|
||||
"type": "string",
|
||||
"description": "FullPath is the full path to the file: RealPath, then for a file within an archive a \"!/\"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. \"/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF\"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it is blank, so coordinates outside archives are unchanged by the field (in JSON, equality and ID)."
|
||||
},
|
||||
"accessPath": {
|
||||
"type": "string",
|
||||
"description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path)"
|
||||
"description": "AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.FullPath)."
|
||||
},
|
||||
"annotations": {
|
||||
"additionalProperties": {
|
||||
|
||||
@@ -0,0 +1,678 @@
|
||||
package syft
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/wagoodman/go-partybus"
|
||||
"github.com/wagoodman/go-progress"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/task"
|
||||
"github.com/anchore/syft/internal/testutils"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/cataloging/filecataloging"
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
"github.com/anchore/syft/syft/event"
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"github.com/anchore/syft/syft/source/directorysource"
|
||||
)
|
||||
|
||||
// Every CreateSBOM call pays to compile the license scanner, so the end-to-end behavior is asserted
|
||||
// over one scan of one fixture directory rather than a scan per claim. The fixture holds every shape
|
||||
// the claims need; the subtests read the one SBOM.
|
||||
//
|
||||
// scan root
|
||||
//
|
||||
// loose.txt an ordinary file, counted by the file catalogers
|
||||
// nested/marker.txt excluded by the root-anchored pattern ./nested/marker.txt
|
||||
// app.zip nested/marker.txt, nested/keep.txt, nested/excluded.txt (**/excluded.txt)
|
||||
// app/data.zip nested/marker.txt, under a directory
|
||||
// outer.zip outer/marker.txt
|
||||
// deeper/inner.zip nested/marker.txt, two levels down
|
||||
// skip/inner-excluded.zip nested/marker.txt, excluded by **/inner-excluded.zip
|
||||
// 日本語/café-🎉/unicode.zip nested/marker.txt, under a non-ASCII path
|
||||
// pkg.rpm a zip holding a marker, excluded by **/*.rpm
|
||||
// KEPT.RPM the same zip, kept: the pattern is lower-case (a distinct base
|
||||
// name, so the two coexist on a case-insensitive filesystem)
|
||||
// app.war WEB-INF/lib/war-lib-2.0.jar
|
||||
// bundle.zip lib/zip-lib-1.2.jar
|
||||
// bundle.tar.gz lib/tgz-lib-1.2.jar
|
||||
var fixtureExclusions = []string{"./nested/marker.txt", "**/excluded.txt", "**/inner-excluded.zip", "**/*.rpm"}
|
||||
|
||||
func writeFixture(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
write := func(name string, data []byte) {
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), data, 0o644))
|
||||
}
|
||||
markerZip := testutils.Zip(t, map[string]string{"nested/marker.txt": "hello"})
|
||||
|
||||
write("loose.txt", []byte("outside"))
|
||||
write("nested/marker.txt", []byte("at the root"))
|
||||
write("app.zip", testutils.Zip(t, map[string]string{
|
||||
"nested/marker.txt": "hello",
|
||||
"nested/keep.txt": "hello",
|
||||
"nested/excluded.txt": "hello",
|
||||
}))
|
||||
write("app/data.zip", markerZip)
|
||||
write("outer.zip", testutils.Zip(t, map[string][]byte{
|
||||
"outer/marker.txt": []byte("hello"),
|
||||
"deeper/inner.zip": markerZip,
|
||||
"skip/inner-excluded.zip": markerZip,
|
||||
"日本語/café-🎉/unicode.zip": markerZip,
|
||||
}))
|
||||
write("pkg.rpm", markerZip)
|
||||
write("KEPT.RPM", markerZip)
|
||||
write("app.war", testutils.Zip(t, map[string][]byte{
|
||||
"WEB-INF/lib/war-lib-2.0.jar": testutils.Jar(t, "war-lib", "2.0", nil),
|
||||
}))
|
||||
write("bundle.zip", testutils.Zip(t, map[string][]byte{
|
||||
"lib/zip-lib-1.2.jar": testutils.Jar(t, "zip-lib", "1.2", nil),
|
||||
}))
|
||||
write("bundle.tar.gz", testutils.TarGz(t, map[string][]byte{
|
||||
"lib/tgz-lib-1.2.jar": testutils.Jar(t, "tgz-lib", "1.2", nil),
|
||||
}))
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestArchiveCataloger(t *testing.T) {
|
||||
scanDir := writeFixture(t)
|
||||
resolvedScanDir, err := filepath.EvalSymlinks(scanDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
resolvedScanDir = filepath.ToSlash(resolvedScanDir)
|
||||
|
||||
recorder := &recordingBus{}
|
||||
bus.Set(recorder)
|
||||
t.Cleanup(func() { bus.Set(nil) })
|
||||
|
||||
var (
|
||||
probeMu sync.Mutex
|
||||
seen []string // every .txt and .zip path any resolver handed a cataloger could see
|
||||
)
|
||||
probe := resolverProbeCataloger{globs: []string{"**/*.txt", "**/*.zip"}, record: func(paths []string) {
|
||||
probeMu.Lock()
|
||||
defer probeMu.Unlock()
|
||||
seen = append(seen, paths...)
|
||||
}}
|
||||
|
||||
// the java catalogers come from the factory, as the CLI builds them: the flag handing recursion
|
||||
// to the archive task is derived onto the factory's config. Both legacy search settings are off,
|
||||
// the cell of the matrix that looks like a bug: with the feature on they no longer gate anything
|
||||
packages := pkgcataloging.DefaultConfig()
|
||||
packages.JavaArchive.IncludeIndexedArchives = false
|
||||
packages.JavaArchive.IncludeUnindexedArchives = false
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")).
|
||||
WithCatalogers(
|
||||
pkgcataloging.NewAlwaysEnabledCatalogerReference(markerCataloger{}),
|
||||
pkgcataloging.NewAlwaysEnabledCatalogerReference(probe),
|
||||
).
|
||||
WithPackagesConfig(packages).
|
||||
WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.AllFilesSelection)).
|
||||
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3))
|
||||
|
||||
s := scanDirWithExclusions(t, scanDir, cfg, fixtureExclusions...)
|
||||
|
||||
t.Run("markers are found at every depth under archive-relative paths", func(t *testing.T) {
|
||||
// the location's RealPath is the outermost archive; FullPath adds the chain of entries within it
|
||||
assert.Equal(t, []string{
|
||||
"KEPT.RPM!/nested/marker.txt",
|
||||
"app.zip!/nested/marker.txt",
|
||||
"app/data.zip!/nested/marker.txt",
|
||||
"outer.zip!/deeper/inner.zip!/nested/marker.txt",
|
||||
"outer.zip!/outer/marker.txt",
|
||||
"outer.zip!/日本語/café-🎉/unicode.zip!/nested/marker.txt",
|
||||
}, packageLocations(s, "marker-pkg"))
|
||||
})
|
||||
|
||||
t.Run("archive contents do not leak into the parent tree", func(t *testing.T) {
|
||||
for _, loc := range packageLocations(s, "marker-pkg") {
|
||||
assert.NotContains(t, loc, ".zip/", "a joined path smears the archive's contents into the parent's tree")
|
||||
assert.NotContains(t, loc, "!//", "a nested path must not be rooted")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unicode entry names survive to output", func(t *testing.T) {
|
||||
for _, loc := range packageLocations(s, "marker-pkg") {
|
||||
assert.True(t, utf8.ValidString(loc))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a CONTAINS relationship links each archive to what was found inside", func(t *testing.T) {
|
||||
var containers []string
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if p.Name != "marker-pkg" {
|
||||
continue
|
||||
}
|
||||
for _, rel := range s.Relationships {
|
||||
if rel.Type != artifact.ContainsRelationship || rel.To.ID() != p.ID() {
|
||||
continue
|
||||
}
|
||||
if coords, ok := rel.From.(file.Coordinates); ok {
|
||||
containers = append(containers, coords.FullNestedPath())
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(containers)
|
||||
assert.Equal(t, []string{
|
||||
"KEPT.RPM",
|
||||
"app.zip",
|
||||
"app/data.zip",
|
||||
"outer.zip",
|
||||
"outer.zip!/deeper/inner.zip",
|
||||
"outer.zip!/日本語/café-🎉/unicode.zip",
|
||||
}, containers, "the archive file, named in its own parent, contains the package")
|
||||
})
|
||||
|
||||
t.Run("each file within an archive is contained by that archive", func(t *testing.T) {
|
||||
containers := fileContainers(s)
|
||||
assert.Equal(t, "app.zip", containers["app.zip!/nested/keep.txt"])
|
||||
assert.Equal(t, "outer.zip!/deeper/inner.zip", containers["outer.zip!/deeper/inner.zip!/nested/marker.txt"])
|
||||
assert.Equal(t, "outer.zip", containers["outer.zip!/deeper/inner.zip"])
|
||||
})
|
||||
|
||||
t.Run("exclusions are the scan's own", func(t *testing.T) {
|
||||
// there is no archive-specific exclusion setting: a scan pattern's shape decides whether it
|
||||
// reaches inside an archive, and an excluded entry is absent from the archive's filesystem
|
||||
// rather than present and skipped
|
||||
markers := packageLocations(s, "marker-pkg")
|
||||
assert.NotContains(t, markers, "nested/marker.txt", "a root-anchored pattern excludes the file at the scan root")
|
||||
assert.Contains(t, markers, "app.zip!/nested/marker.txt", "and says nothing about the same path inside an archive")
|
||||
assert.NotContains(t, markers, "pkg.rpm!/nested/marker.txt", "an archive whose own path is excluded is not extracted")
|
||||
assert.Contains(t, markers, "KEPT.RPM!/nested/marker.txt", "a lower-case pattern does not match an upper-case name")
|
||||
assert.NotContains(t, markers, "outer.zip!/skip/inner-excluded.zip!/nested/marker.txt", "a nested archive is excluded at its own level")
|
||||
|
||||
assert.Contains(t, seen, "app.zip!/nested/keep.txt")
|
||||
assert.NotContains(t, seen, "app.zip!/nested/excluded.txt", "a **/ pattern excludes inside an archive, from the index itself")
|
||||
assert.Contains(t, seen, "outer.zip!/deeper/inner.zip", "the probe ran inside outer.zip")
|
||||
assert.NotContains(t, seen, "outer.zip!/skip/inner-excluded.zip", "the excluded inner archive is absent from outer's filesystem")
|
||||
})
|
||||
|
||||
t.Run("a jar inside an archive is cataloged exactly once, under the virtual path java gives it", func(t *testing.T) {
|
||||
// the failure mode is a duplicate: a jar inside a zip can be reached by the task's walk and by
|
||||
// the java cataloger's own unarchiving. The colon-delimited virtual path appears in published
|
||||
// SBOMs, so it must not change with the mechanism; for a directory source the archive's path
|
||||
// is relative, so the chain has no leading separator
|
||||
assert.Equal(t, map[string][]string{
|
||||
"war-lib": {"app.war:WEB-INF/lib/war-lib-2.0.jar"},
|
||||
"zip-lib": {"bundle.zip:lib/zip-lib-1.2.jar"},
|
||||
"tgz-lib": {"bundle.tar.gz:lib/tgz-lib-1.2.jar"},
|
||||
}, javaVirtualPaths(s), "with both legacy search settings off, every container is still descended")
|
||||
})
|
||||
|
||||
t.Run("no extraction path reaches the SBOM", func(t *testing.T) {
|
||||
// covers syft/archive-content-identity#nested-paths-are-archive-relative: file cataloging is
|
||||
// the one path that enumerates every location rather than searching for them, so a resolver
|
||||
// root that leaked would land in FileMetadata and gain a CONTAINS edge
|
||||
var paths []string
|
||||
for coords := range s.Artifacts.FileMetadata {
|
||||
paths = append(paths, coords.RealPath)
|
||||
}
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
for _, loc := range p.Locations.ToSlice() {
|
||||
paths = append(paths, loc.RealPath, loc.AccessPath)
|
||||
}
|
||||
}
|
||||
for _, rel := range s.Relationships {
|
||||
for _, end := range []artifact.Identifiable{rel.From, rel.To} {
|
||||
if coords, ok := end.(file.Coordinates); ok {
|
||||
paths = append(paths, coords.RealPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
require.NotEmpty(t, paths)
|
||||
for _, p := range paths {
|
||||
assert.NotContains(t, p, "archive-spill-", "the spill file reached the SBOM at %q", p)
|
||||
if filepath.IsAbs(p) {
|
||||
// the directory resolver has always reported the scan root absolute: it is the path the
|
||||
// user asked for. Anything else absolute is a filesystem this feature invented
|
||||
assert.Equal(t, resolvedScanDir, p, "only the scanned directory itself may be reported absolute")
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("file metadata covers every entry of every archive, and nothing else", func(t *testing.T) {
|
||||
// an exact set is also the reproducibility check: anything of a per-run temp directory reaching
|
||||
// output would fail it
|
||||
var got []string
|
||||
for coords := range s.Artifacts.FileMetadata {
|
||||
if archive.ContainerPath(coords) == "" {
|
||||
continue
|
||||
}
|
||||
got = append(got, coords.FullNestedPath())
|
||||
}
|
||||
sort.Strings(got)
|
||||
assert.Equal(t, []string{
|
||||
"KEPT.RPM!/nested/marker.txt",
|
||||
"app.war!/WEB-INF/lib/war-lib-2.0.jar",
|
||||
"app.war!/WEB-INF/lib/war-lib-2.0.jar!/META-INF/MANIFEST.MF",
|
||||
"app.zip!/nested/keep.txt",
|
||||
"app.zip!/nested/marker.txt",
|
||||
"app/data.zip!/nested/marker.txt",
|
||||
"bundle.tar.gz!/lib/tgz-lib-1.2.jar",
|
||||
"bundle.tar.gz!/lib/tgz-lib-1.2.jar!/META-INF/MANIFEST.MF",
|
||||
"bundle.zip!/lib/zip-lib-1.2.jar",
|
||||
"bundle.zip!/lib/zip-lib-1.2.jar!/META-INF/MANIFEST.MF",
|
||||
"outer.zip!/deeper/inner.zip",
|
||||
"outer.zip!/deeper/inner.zip!/nested/marker.txt",
|
||||
"outer.zip!/outer/marker.txt",
|
||||
"outer.zip!/日本語/café-🎉/unicode.zip",
|
||||
"outer.zip!/日本語/café-🎉/unicode.zip!/nested/marker.txt",
|
||||
}, got)
|
||||
})
|
||||
|
||||
t.Run("file catalogers publish one progress row counting archive contents", func(t *testing.T) {
|
||||
// the file catalogers run once over the scan root and again over every archive; each must
|
||||
// report into one row carrying the total of every run, or the count runs past the total and
|
||||
// renders as an overshot bar that reads as complete mid-walk
|
||||
rows := map[string][]*progress.Manual{}
|
||||
for _, e := range recorder.events {
|
||||
if e.Type != event.CatalogerTaskStarted {
|
||||
continue
|
||||
}
|
||||
info, ok := e.Source.(monitor.GenericTask)
|
||||
require.True(t, ok)
|
||||
tp, ok := e.Value.(*monitor.TaskProgress)
|
||||
require.True(t, ok)
|
||||
rows[info.Title.Default] = append(rows[info.Title.Default], tp.Manual)
|
||||
}
|
||||
for _, title := range []string{"File digests", "File metadata"} {
|
||||
published := rows[title]
|
||||
require.Len(t, published, 1, "%s must publish exactly one row for the whole scan", title)
|
||||
row := published[0]
|
||||
assert.Greater(t, row.Current(), int64(len(s.Artifacts.FileMetadata))/2,
|
||||
"%s must count the files inside the archives, got %d", title, row.Current())
|
||||
if row.Size() >= 0 {
|
||||
assert.Equal(t, row.Current(), row.Size(), "%s total must cover every run that reported into it", title)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestArchiveCataloger_featureOff is the one comparison that needs a second scan: with the task
|
||||
// disabled, java's own unarchiving reaches the jars its legacy settings admit, under the same virtual
|
||||
// path the task gives them.
|
||||
func TestArchiveCataloger_featureOff(t *testing.T) {
|
||||
scanDir := writeFixture(t)
|
||||
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java"))
|
||||
|
||||
s := scanDirWithExclusions(t, scanDir, cfg, fixtureExclusions...)
|
||||
|
||||
// indexed archives (zip, war) are searched by default and unindexed ones (tar.gz) are not, so
|
||||
// what a user gets without the feature depends on settings the feature makes irrelevant
|
||||
assert.Equal(t, map[string][]string{
|
||||
"war-lib": {"app.war:WEB-INF/lib/war-lib-2.0.jar"},
|
||||
"zip-lib": {"bundle.zip:lib/zip-lib-1.2.jar"},
|
||||
}, javaVirtualPaths(s), "java's own recursion finds each once, under the virtual path the task also reports")
|
||||
|
||||
t.Run("the top-level archive search booleans reach java", func(t *testing.T) {
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")).
|
||||
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithIncludeUnindexedArchives(true))
|
||||
|
||||
s := scanDirWithExclusions(t, scanDir, cfg, fixtureExclusions...)
|
||||
|
||||
assert.Contains(t, javaVirtualPaths(s), "tgz-lib")
|
||||
assert.False(t, cfg.Packages.JavaArchive.IncludeUnindexedArchives, "the caller's config is not written to")
|
||||
})
|
||||
}
|
||||
|
||||
func TestArchiveCataloger_jarReachedThroughALinkIsIdentified(t *testing.T) {
|
||||
// a content-addressed layout (pnpm, nix, bazel): the jar is stored under a hash and only the link
|
||||
// to it is named like a jar
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gw)
|
||||
jar := testutils.Jar(t, "json-simple", "1.1.1", nil)
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{Name: "store/3f9a1c", Mode: 0o644, Size: int64(len(jar))}))
|
||||
_, err := tw.Write(jar)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{Name: "app/lib/json-simple-1.1.1.jar", Typeflag: tar.TypeSymlink, Linkname: "../../store/3f9a1c"}))
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "bundle.tar.gz"), buf.Bytes(), 0o644))
|
||||
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")).
|
||||
WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.AllFilesSelection)).
|
||||
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2))
|
||||
s := scanDirWithExclusions(t, dir, cfg)
|
||||
|
||||
assert.Equal(t, []string{"bundle.tar.gz!/store/3f9a1c"}, packageLocations(s, "json-simple"))
|
||||
|
||||
// the jar's own files are addressed by its real path, the same one its package location names
|
||||
manifest := file.Coordinates{RealPath: "bundle.tar.gz", FullPath: "bundle.tar.gz!/store/3f9a1c!/META-INF/MANIFEST.MF"}
|
||||
assert.Contains(t, s.Artifacts.FileMetadata, manifest)
|
||||
}
|
||||
|
||||
func TestArchiveCataloger_defaultFileSelectionCoversRootJars(t *testing.T) {
|
||||
// with the default owned-by-package selection, a jar at the root is described by the archive task,
|
||||
// so the root file catalogers must run after it or the jar gets no file entries at all
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "app.jar"), testutils.Jar(t, "app", "1.0", nil), 0o644))
|
||||
|
||||
for name, depth := range map[string]int{"feature off": 0, "feature on": 1} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")).
|
||||
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth))
|
||||
s := scanDirWithExclusions(t, dir, cfg)
|
||||
|
||||
require.Equal(t, []string{"app.jar"}, packageLocations(s, "app"))
|
||||
jar := file.NewCoordinates("app.jar", "")
|
||||
assert.Contains(t, s.Artifacts.FileMetadata, jar)
|
||||
assert.Contains(t, s.Artifacts.FileDigests, jar)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveCataloger_defaultFileSelectionCoversNestedJars(t *testing.T) {
|
||||
// a nested jar's package is found inside the jar but located in the archive holding it, so that
|
||||
// archive's file catalogers must run after it, as the root's do
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "outer.zip"), testutils.Zip(t, map[string][]byte{
|
||||
"lib/inner.jar": testutils.Jar(t, "inner", "1.0", nil),
|
||||
"readme.txt": []byte("hello"),
|
||||
"empty.zip": testutils.Zip(t, map[string]string{"a.txt": "hello"}),
|
||||
}), 0o644))
|
||||
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")).
|
||||
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2))
|
||||
s := scanDirWithExclusions(t, dir, cfg)
|
||||
|
||||
require.Equal(t, []string{"outer.zip!/lib/inner.jar"}, packageLocations(s, "inner"))
|
||||
jar := file.Coordinates{RealPath: "outer.zip", FullPath: "outer.zip!/lib/inner.jar"}
|
||||
assert.Contains(t, s.Artifacts.FileMetadata, jar)
|
||||
assert.Contains(t, s.Artifacts.FileDigests, jar)
|
||||
assert.Equal(t, "outer.zip", fileContainers(s)["outer.zip!/lib/inner.jar"])
|
||||
|
||||
// as at the root, a file no package owns is not described, nor is an archive in which nothing was found
|
||||
var described []string
|
||||
for _, c := range s.AllCoordinates() {
|
||||
described = append(described, c.FullNestedPath())
|
||||
}
|
||||
assert.NotContains(t, described, "outer.zip!/readme.txt")
|
||||
assert.NotContains(t, described, "outer.zip!/empty.zip")
|
||||
assert.NotContains(t, described, "outer.zip!/empty.zip!/a.txt")
|
||||
}
|
||||
|
||||
func TestArchiveCataloger_archiveUnknowns(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
plain := testutils.Zip(t, map[string]string{"readme.txt": "hi"})
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "outer.zip"), testutils.Zip(t, map[string][]byte{
|
||||
"lib/inner.jar": testutils.Jar(t, "inner", "1.0", nil),
|
||||
"lib/plain.zip": plain,
|
||||
}), 0o644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "plain.zip"), plain, 0o644))
|
||||
|
||||
scan := func(depth int) map[string][]string {
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")).
|
||||
WithUnknownsConfig(cataloging.DefaultUnknownsConfig()).
|
||||
WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.AllFilesSelection)).
|
||||
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth))
|
||||
out := map[string][]string{}
|
||||
for c, reasons := range scanDirWithExclusions(t, dir, cfg).Artifacts.Unknowns {
|
||||
out[c.FullNestedPath()] = reasons
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
t.Run("an expanded archive is not reported as not cataloged", func(t *testing.T) {
|
||||
assert.Empty(t, scan(3))
|
||||
})
|
||||
|
||||
t.Run("an archive past the depth limit is reported once, by the archive cataloger", func(t *testing.T) {
|
||||
unknowns := scan(1)
|
||||
assert.NotContains(t, unknowns, "outer.zip")
|
||||
for _, nested := range []string{"outer.zip!/lib/inner.jar", "outer.zip!/lib/plain.zip"} {
|
||||
require.Len(t, unknowns[nested], 1, nested)
|
||||
assert.Contains(t, unknowns[nested][0], "depth limit was reached")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an archive is reported as not cataloged when nesting is off", func(t *testing.T) {
|
||||
assert.Equal(t, []string{"archive not cataloged"}, scan(0)["plain.zip"])
|
||||
})
|
||||
}
|
||||
|
||||
func TestArchiveCataloger_unsupportedArchiveFormatsAreReported(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
var page bytes.Buffer
|
||||
gw := gzip.NewWriter(&page)
|
||||
_, err := gw.Write([]byte("a man page"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gw.Close())
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "outer.zip"), testutils.Zip(t, map[string][]byte{
|
||||
"legacy.7z": []byte("7z\xbc\xaf\x27\x1c\x00\x04"),
|
||||
"page.1.gz": page.Bytes(),
|
||||
}), 0o644))
|
||||
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")).
|
||||
WithUnknownsConfig(cataloging.DefaultUnknownsConfig()).
|
||||
WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.AllFilesSelection)).
|
||||
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3))
|
||||
unknowns := map[string][]string{}
|
||||
for c, reasons := range scanDirWithExclusions(t, dir, cfg).Artifacts.Unknowns {
|
||||
unknowns[c.FullNestedPath()] = reasons
|
||||
}
|
||||
|
||||
assert.Equal(t, map[string][]string{
|
||||
"outer.zip!/legacy.7z": {"archive-cataloger: archive not cataloged: archive format is not supported: application/x-7z-compressed"},
|
||||
}, unknowns, "a bare compressed file holds no files, so it is not an archive left uncataloged")
|
||||
}
|
||||
|
||||
func TestArchiveCataloger_javaParentLinkageMatchesTheLegacyPath(t *testing.T) {
|
||||
// a consumer walking java DEPENDENCY_OF edges (the only structure SPDX and CycloneDX keep) must see
|
||||
// the same war-to-jar linkage whether or not the archive task does the recursion
|
||||
dir := t.TempDir()
|
||||
war := testutils.Jar(t, "app", "1.0", map[string][]byte{
|
||||
"WEB-INF/lib/lib-2.0.jar": testutils.Jar(t, "lib", "2.0", nil),
|
||||
})
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "app.war"), war, 0o644))
|
||||
|
||||
for name, depth := range map[string]int{"feature off": 0, "feature on": 2} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")).
|
||||
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(depth))
|
||||
s := scanDirWithExclusions(t, dir, cfg)
|
||||
|
||||
var edges []string
|
||||
for _, rel := range s.Relationships {
|
||||
from, fromOK := rel.From.(pkg.Package)
|
||||
to, toOK := rel.To.(pkg.Package)
|
||||
if fromOK && toOK && rel.Type == artifact.DependencyOfRelationship {
|
||||
edges = append(edges, from.Name+" -> "+to.Name)
|
||||
}
|
||||
}
|
||||
assert.Equal(t, []string{"lib -> app"}, edges)
|
||||
|
||||
var parent string
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if m, ok := p.Metadata.(pkg.JavaArchive); ok && p.Name == "lib" && m.Parent != nil {
|
||||
parent = m.Parent.Name
|
||||
}
|
||||
}
|
||||
assert.Equal(t, "app", parent)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveCataloger_descriptorRecordsArchiveConfig(t *testing.T) {
|
||||
// a consumer must be able to tell from the SBOM alone how deep archive contents were searched
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithCatalogerSelection(cataloging.NewSelectionRequest().WithDefaults("java")).
|
||||
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(3))
|
||||
s := scanDirWithExclusions(t, t.TempDir(), cfg)
|
||||
|
||||
trail, ok := s.Descriptor.Configuration.(configurationAuditTrail)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, 3, trail.Archive.MaxDepth)
|
||||
assert.Contains(t, trail.Catalogers.Used, task.ArchiveCatalogerTaskName)
|
||||
}
|
||||
|
||||
func TestArchiveCataloger_javaArchiveDepthIsAConfigError(t *testing.T) {
|
||||
// depth and limits are not java settings; setting them on java's deprecated copy must fail fast
|
||||
// rather than be silently ignored
|
||||
for name, set := range map[string]func(*cataloging.ArchiveSearchConfig){
|
||||
"max depth": func(c *cataloging.ArchiveSearchConfig) { c.MaxDepth = 3 },
|
||||
"max memory bytes": func(c *cataloging.ArchiveSearchConfig) { c.MaxMemoryBytes = 1 },
|
||||
"max disk bytes": func(c *cataloging.ArchiveSearchConfig) { c.MaxDiskBytes = 1 },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
supplied := java.DefaultArchiveCatalogerConfig()
|
||||
supplied.ArchiveSearchConfig = cataloging.ArchiveSearchConfig{}
|
||||
set(&supplied.ArchiveSearchConfig)
|
||||
cfg := DefaultCreateSBOMConfig().WithPackagesConfig(pkgcataloging.Config{JavaArchive: supplied})
|
||||
|
||||
src, err := directorysource.New(directorysource.Config{Path: t.TempDir()})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = src.Close() })
|
||||
|
||||
_, err = cfg.Create(context.Background(), src)
|
||||
require.ErrorContains(t, err, "set them with CreateSBOMConfig.Archive")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// packageLocations lists the full path of every location of every package with the name.
|
||||
func packageLocations(s *sbom.SBOM, name string) []string {
|
||||
var out []string
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if p.Name != name {
|
||||
continue
|
||||
}
|
||||
for _, loc := range p.Locations.ToSlice() {
|
||||
out = append(out, loc.FullNestedPath())
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// javaVirtualPaths maps each java package name to the virtual paths it was reported under; a name
|
||||
// with two entries was cataloged twice.
|
||||
func javaVirtualPaths(s *sbom.SBOM) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for _, p := range s.Artifacts.Packages.Sorted() {
|
||||
if metadata, ok := p.Metadata.(pkg.JavaArchive); ok {
|
||||
out[p.Name] = append(out[p.Name], metadata.VirtualPath)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// markerCataloger is a stub package cataloger emitting one package per "marker.txt" it finds, so the
|
||||
// end-to-end tests assert behavior without depending on any real cataloger's fixture format.
|
||||
type markerCataloger struct{}
|
||||
|
||||
func (markerCataloger) Name() string { return "marker-cataloger" }
|
||||
|
||||
func (markerCataloger) Catalog(_ context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
locations, err := resolver.FilesByGlob("**/marker.txt")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var pkgs []pkg.Package
|
||||
for _, loc := range locations {
|
||||
p := pkg.Package{
|
||||
Name: "marker-pkg",
|
||||
Version: "1.0.0",
|
||||
Type: pkg.BinaryPkg,
|
||||
Locations: file.NewLocationSet(loc),
|
||||
}
|
||||
p.SetID()
|
||||
pkgs = append(pkgs, p)
|
||||
}
|
||||
return pkgs, nil, nil
|
||||
}
|
||||
|
||||
// resolverProbeCataloger reports what the resolver it was handed can see, which is how "absent from
|
||||
// the index" is told from "present and skipped" within a scan.
|
||||
type resolverProbeCataloger struct {
|
||||
globs []string
|
||||
record func([]string)
|
||||
}
|
||||
|
||||
func (resolverProbeCataloger) Name() string { return "resolver-probe-cataloger" }
|
||||
|
||||
func (c resolverProbeCataloger) Catalog(_ context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
locations, err := resolver.FilesByGlob(c.globs...)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var paths []string
|
||||
for _, loc := range locations {
|
||||
paths = append(paths, loc.FullNestedPath())
|
||||
}
|
||||
c.record(paths)
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
// recordingBus captures the progress rows a scan publishes.
|
||||
type recordingBus struct {
|
||||
mu sync.Mutex
|
||||
events []partybus.Event
|
||||
}
|
||||
|
||||
func (p *recordingBus) Publish(e partybus.Event) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.events = append(p.events, e)
|
||||
}
|
||||
|
||||
// fileContainers maps the full path of each file an archive CONTAINS to the archive's full path.
|
||||
func fileContainers(s *sbom.SBOM) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, r := range s.Relationships {
|
||||
from, fromFile := r.From.(file.Coordinates)
|
||||
to, toFile := r.To.(file.Coordinates)
|
||||
if r.Type == artifact.ContainsRelationship && fromFile && toFile {
|
||||
out[to.FullNestedPath()] = from.FullNestedPath()
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// scanDirWithExclusions scans with the given exclusion patterns configured on the source, where
|
||||
// exclusions live; archive cataloging has no exclusion setting of its own.
|
||||
func scanDirWithExclusions(t *testing.T, dir string, cfg *CreateSBOMConfig, exclude ...string) *sbom.SBOM {
|
||||
t.Helper()
|
||||
src, err := directorysource.New(directorysource.Config{
|
||||
Path: dir,
|
||||
Exclude: source.ExcludeConfig{Paths: exclude},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = src.Close() })
|
||||
|
||||
s, err := cfg.Create(context.Background(), src)
|
||||
require.NoError(t, err)
|
||||
return s
|
||||
}
|
||||
@@ -1,17 +1,49 @@
|
||||
package cataloging
|
||||
|
||||
import "github.com/anchore/stereoscope/pkg/file"
|
||||
|
||||
const (
|
||||
// DefaultArchiveMaxDepth is 0: nested archives are not cataloged unless asked for.
|
||||
DefaultArchiveMaxDepth = 0
|
||||
|
||||
// DefaultArchiveMaxMemoryBytes bounds the archive content held in memory at once. It decides only
|
||||
// whether an archive costs a write: content that does not fit goes to disk and is cataloged anyway.
|
||||
DefaultArchiveMaxMemoryBytes = 2 * file.GB
|
||||
|
||||
// DefaultArchiveMaxDiskBytes bounds the archive content on disk at once. Content that does not fit
|
||||
// has nowhere to go, so the archive holding it is skipped or truncated. Set high as a backstop
|
||||
// against a pathological archive filling the disk.
|
||||
DefaultArchiveMaxDiskBytes = 100 * file.GB
|
||||
)
|
||||
|
||||
type ArchiveSearchConfig struct {
|
||||
// IncludeIndexedArchives indicates whether to search within indexed archive files (e.g., .zip).
|
||||
IncludeIndexedArchives bool `yaml:"include-indexed-archives" json:"include-indexed-archives" mapstructure:"include-indexed-archives"`
|
||||
|
||||
// IncludeUnindexedArchives indicates whether to search within unindexed archive files (e.g., .tar*).
|
||||
IncludeUnindexedArchives bool `yaml:"include-unindexed-archives" json:"include-unindexed-archives" mapstructure:"include-unindexed-archives"`
|
||||
|
||||
// MaxDepth is how many levels of nested archives to catalog: 0 disables it, and no setting goes deeper than 16 levels
|
||||
// (negative goes that deep).
|
||||
MaxDepth int `yaml:"max-depth" json:"max-depth" mapstructure:"max-depth"`
|
||||
|
||||
// MaxMemoryBytes bounds the archive content held in memory at once; content that does not fit is
|
||||
// written to disk. Zero holds nothing in memory, negative is unbounded.
|
||||
MaxMemoryBytes int64 `yaml:"max-memory-bytes" json:"max-memory-bytes" mapstructure:"max-memory-bytes"`
|
||||
|
||||
// MaxDiskBytes bounds the archive content written to disk at once; an archive whose content does
|
||||
// not fit is skipped or truncated. Zero writes nothing to disk, negative is unbounded. MaxMemoryBytes
|
||||
// and MaxDiskBytes cannot both be zero while MaxDepth is not.
|
||||
MaxDiskBytes int64 `yaml:"max-disk-bytes" json:"max-disk-bytes" mapstructure:"max-disk-bytes"`
|
||||
}
|
||||
|
||||
func DefaultArchiveSearchConfig() ArchiveSearchConfig {
|
||||
return ArchiveSearchConfig{
|
||||
IncludeIndexedArchives: true,
|
||||
IncludeUnindexedArchives: false,
|
||||
MaxDepth: DefaultArchiveMaxDepth,
|
||||
MaxMemoryBytes: DefaultArchiveMaxMemoryBytes,
|
||||
MaxDiskBytes: DefaultArchiveMaxDiskBytes,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,3 +56,18 @@ func (c ArchiveSearchConfig) WithIncludeUnindexedArchives(include bool) ArchiveS
|
||||
c.IncludeUnindexedArchives = include
|
||||
return c
|
||||
}
|
||||
|
||||
func (c ArchiveSearchConfig) WithMaxDepth(depth int) ArchiveSearchConfig {
|
||||
c.MaxDepth = depth
|
||||
return c
|
||||
}
|
||||
|
||||
func (c ArchiveSearchConfig) WithMaxMemoryBytes(size int64) ArchiveSearchConfig {
|
||||
c.MaxMemoryBytes = size
|
||||
return c
|
||||
}
|
||||
|
||||
func (c ArchiveSearchConfig) WithMaxDiskBytes(size int64) ArchiveSearchConfig {
|
||||
c.MaxDiskBytes = size
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package cataloging
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestDefaultArchiveSearchConfig(t *testing.T) {
|
||||
cfg := DefaultArchiveSearchConfig()
|
||||
|
||||
assert.True(t, cfg.IncludeIndexedArchives)
|
||||
assert.False(t, cfg.IncludeUnindexedArchives)
|
||||
assert.Equal(t, DefaultArchiveMaxDepth, cfg.MaxDepth)
|
||||
assert.Equal(t, int64(DefaultArchiveMaxMemoryBytes), cfg.MaxMemoryBytes)
|
||||
assert.Equal(t, int64(DefaultArchiveMaxDiskBytes), cfg.MaxDiskBytes)
|
||||
}
|
||||
|
||||
func TestArchiveSearchConfig_WithMethods(t *testing.T) {
|
||||
cfg := DefaultArchiveSearchConfig()
|
||||
|
||||
cfg = cfg.WithMaxDepth(5)
|
||||
assert.Equal(t, 5, cfg.MaxDepth)
|
||||
|
||||
cfg = cfg.WithMaxMemoryBytes(1024)
|
||||
assert.Equal(t, int64(1024), cfg.MaxMemoryBytes)
|
||||
|
||||
cfg = cfg.WithMaxDiskBytes(2048)
|
||||
assert.Equal(t, int64(2048), cfg.MaxDiskBytes)
|
||||
|
||||
cfg = cfg.WithIncludeIndexedArchives(false)
|
||||
assert.False(t, cfg.IncludeIndexedArchives)
|
||||
|
||||
cfg = cfg.WithIncludeUnindexedArchives(true)
|
||||
assert.True(t, cfg.IncludeUnindexedArchives)
|
||||
}
|
||||
@@ -16,6 +16,7 @@ type configurationAuditTrail struct {
|
||||
DataGeneration cataloging.DataGenerationConfig `json:"data-generation" yaml:"data-generation" mapstructure:"data-generation"`
|
||||
Packages pkgcataloging.Config `json:"packages" yaml:"packages" mapstructure:"packages"`
|
||||
Files filecataloging.Config `json:"files" yaml:"files" mapstructure:"files"`
|
||||
Archive cataloging.ArchiveSearchConfig `json:"archive" yaml:"archive" mapstructure:"archive"`
|
||||
Licenses cataloging.LicenseConfig `json:"licenses" yaml:"licenses" mapstructure:"licenses"`
|
||||
Catalogers catalogerManifest `json:"catalogers" yaml:"catalogers" mapstructure:"catalogers"`
|
||||
ExtraConfigs any `json:"extra,omitempty" yaml:"extra" mapstructure:"extra"`
|
||||
|
||||
+27
-8
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/go-sync"
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/licenses"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
@@ -38,7 +39,7 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (
|
||||
|
||||
srcMetadata := src.Describe()
|
||||
|
||||
taskGroups, audit, err := cfg.makeTaskGroups(srcMetadata)
|
||||
taskGroups, audit, err := cfg.makeTaskGroups(srcMetadata, sourceExclusions(src))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -57,8 +58,9 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (
|
||||
Search: cfg.Search,
|
||||
Relationships: cfg.Relationships,
|
||||
DataGeneration: cfg.DataGeneration,
|
||||
Packages: cfg.Packages,
|
||||
Packages: cfg.packagesWithArchiveSearch(),
|
||||
Files: cfg.Files,
|
||||
Archive: cfg.Archive,
|
||||
Licenses: cfg.Licenses,
|
||||
Catalogers: *audit,
|
||||
ExtraConfigs: cfg.ToolConfiguration,
|
||||
@@ -88,8 +90,8 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (
|
||||
}
|
||||
}
|
||||
|
||||
catalogingProgress := monitorCatalogingTask(src.ID(), taskGroups)
|
||||
packageCatalogingProgress := monitorPackageCatalogingTask()
|
||||
catalogingProgress := monitorCatalogingTask(ctx, src.ID(), taskGroups)
|
||||
packageCatalogingProgress := monitorPackageCatalogingTask(ctx)
|
||||
|
||||
builder := sbomsync.NewBuilder(&s, monitorPackageCount(packageCatalogingProgress))
|
||||
for i := range taskGroups {
|
||||
@@ -112,6 +114,14 @@ func setupContext(ctx context.Context, cfg *CreateSBOMConfig) (context.Context,
|
||||
// configure parallel executors
|
||||
ctx = setContextExecutors(ctx, cfg)
|
||||
|
||||
// one progress row per cataloger for the whole scan, however many archives it is run against
|
||||
ctx = bus.WithCatalogerTaskRegistry(ctx)
|
||||
|
||||
// the archive cataloger task recurses into jars too, so the java cataloger must not unarchive them itself
|
||||
if cfg.Archive.MaxDepth != 0 {
|
||||
ctx = archive.WithNestedCataloging(ctx)
|
||||
}
|
||||
|
||||
// configure temp dir factory for catalogers (if not already set)
|
||||
if tmpdir.FromContext(ctx) == nil {
|
||||
ctx, _ = tmpdir.Root(ctx, "syft-cataloger")
|
||||
@@ -173,7 +183,7 @@ func monitorPackageCount(prog *monitor.TaskProgress) func(s *sbom.SBOM) {
|
||||
}
|
||||
}
|
||||
|
||||
func monitorPackageCatalogingTask() *monitor.TaskProgress {
|
||||
func monitorPackageCatalogingTask(ctx context.Context) *monitor.TaskProgress {
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "Packages",
|
||||
@@ -183,10 +193,10 @@ func monitorPackageCatalogingTask() *monitor.TaskProgress {
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
|
||||
return bus.StartCatalogerTask(info, -1, "")
|
||||
return bus.StartCatalogerTask(ctx, info, -1, "")
|
||||
}
|
||||
|
||||
func monitorCatalogingTask(srcID artifact.ID, tasks [][]task.Task) *monitor.TaskProgress {
|
||||
func monitorCatalogingTask(ctx context.Context, srcID artifact.ID, tasks [][]task.Task) *monitor.TaskProgress {
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "Catalog contents",
|
||||
@@ -203,7 +213,7 @@ func monitorCatalogingTask(srcID artifact.ID, tasks [][]task.Task) *monitor.Task
|
||||
length += int64(len(tg))
|
||||
}
|
||||
|
||||
return bus.StartCatalogerTask(info, length, "")
|
||||
return bus.StartCatalogerTask(ctx, info, length, "")
|
||||
}
|
||||
|
||||
func formatTaskNames(tasks []task.Task) []string {
|
||||
@@ -218,3 +228,12 @@ func formatTaskNames(tasks []task.Task) []string {
|
||||
sort.Strings(list)
|
||||
return list
|
||||
}
|
||||
|
||||
// sourceExclusions returns the exclusion patterns the source was configured with, so the same
|
||||
// exclusions apply inside the archives found in it.
|
||||
func sourceExclusions(src source.Source) []string {
|
||||
if excluder, ok := src.(source.PathExcluder); ok {
|
||||
return excluder.ExcludedPaths()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ type CreateSBOMConfig struct {
|
||||
Packages pkgcataloging.Config
|
||||
Licenses cataloging.LicenseConfig
|
||||
Files filecataloging.Config
|
||||
Archive cataloging.ArchiveSearchConfig
|
||||
Parallelism int
|
||||
CatalogerSelection cataloging.SelectionRequest
|
||||
|
||||
@@ -49,6 +50,7 @@ func DefaultCreateSBOMConfig() *CreateSBOMConfig {
|
||||
Packages: pkgcataloging.DefaultConfig(),
|
||||
Licenses: cataloging.DefaultLicenseConfig(),
|
||||
Files: filecataloging.DefaultConfig(),
|
||||
Archive: cataloging.DefaultArchiveSearchConfig(),
|
||||
Parallelism: 0, // use default: run in parallel based on number of CPUs
|
||||
packageTaskFactories: task.DefaultPackageTaskFactories(),
|
||||
|
||||
@@ -144,6 +146,13 @@ func (c *CreateSBOMConfig) WithFilesConfig(cfg filecataloging.Config) *CreateSBO
|
||||
return c
|
||||
}
|
||||
|
||||
// WithArchiveConfig sets how archives are cataloged: how deep to recurse into nested archives (MaxDepth
|
||||
// 0, the default, does not recurse) and how much extracted content the scan may hold at once.
|
||||
func (c *CreateSBOMConfig) WithArchiveConfig(cfg cataloging.ArchiveSearchConfig) *CreateSBOMConfig {
|
||||
c.Archive = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
// WithoutFiles allows for disabling file cataloging altogether.
|
||||
func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig {
|
||||
c.Files = filecataloging.Config{
|
||||
@@ -183,14 +192,15 @@ func (c *CreateSBOMConfig) WithCatalogers(catalogerRefs ...pkgcataloging.Catalog
|
||||
// groups, where each task in a group can be run concurrently, while tasks in different groups must be run serially.
|
||||
// The final set of task groups is returned along with a cataloger manifest that describes the catalogers that were
|
||||
// selected and the tokens that were sensitive to this selection (both for adding and removing from the final set).
|
||||
func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task, *catalogerManifest, error) {
|
||||
//
|
||||
// exclusions are the source's exclusion patterns, which the archive cataloger applies inside each archive.
|
||||
func (c *CreateSBOMConfig) makeTaskGroups(src source.Description, exclusions []string) ([][]task.Task, *catalogerManifest, error) {
|
||||
var taskGroups [][]task.Task
|
||||
|
||||
// generate package and file tasks based on the configuration
|
||||
environmentTasks := c.environmentTasks()
|
||||
scopeTasks := c.scopeTasks()
|
||||
relationshipsTasks := c.relationshipTasks(src)
|
||||
unknownTasks := c.unknownsTasks()
|
||||
osFeatureDetectionTasks := c.osFeatureDetectionTasks()
|
||||
|
||||
pkgTasks, fileTasks, selectionEvidence, err := c.selectTasks(src)
|
||||
@@ -198,11 +208,29 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// the same package and file catalogers run against the contents of every archive, after the scan
|
||||
// root and before relationship and unknowns post-processing
|
||||
archiveTask := task.NewArchiveCatalogerTask(c.Archive, c.Unknowns, pkgTasks, fileTasks, exclusions)
|
||||
|
||||
// the archive cataloger reports the archives it does not expand, which the labeler can only guess at
|
||||
unknowns := c.Unknowns
|
||||
if archiveTask != nil {
|
||||
unknowns.IncludeUnexpandedArchives = false
|
||||
}
|
||||
unknownTasks := unknownsTasks(unknowns)
|
||||
|
||||
// combine the user-provided and configured tasks
|
||||
if c.Files.Selection == file.FilesOwnedByPackageSelection {
|
||||
switch {
|
||||
case c.Files.Selection == file.FilesOwnedByPackageSelection && archiveTask != nil:
|
||||
// the archive task describes archives at the root too (java leaves them to it), so the root file
|
||||
// catalogers must run after it to see those packages
|
||||
taskGroups = append(taskGroups, pkgTasks, []task.Task{archiveTask}, fileTasks)
|
||||
case c.Files.Selection == file.FilesOwnedByPackageSelection:
|
||||
// special case: we need the package info when we are cataloging files owned by packages
|
||||
taskGroups = append(taskGroups, pkgTasks, fileTasks)
|
||||
} else {
|
||||
case archiveTask != nil:
|
||||
taskGroups = append(taskGroups, append(pkgTasks, fileTasks...), []task.Task{archiveTask})
|
||||
default:
|
||||
taskGroups = append(taskGroups, append(pkgTasks, fileTasks...))
|
||||
}
|
||||
|
||||
@@ -226,6 +254,11 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task
|
||||
taskGroups = append(taskGroups, osFeatureDetectionTasks)
|
||||
}
|
||||
|
||||
// archives are related to the files within them only once every task has decided which files are described
|
||||
if archiveTask != nil {
|
||||
taskGroups = append(taskGroups, []task.Task{task.NewArchiveContainsTask()})
|
||||
}
|
||||
|
||||
// identifying the environment (i.e. the linux release) must be done first as this is required for package cataloging
|
||||
taskGroups = append(
|
||||
[][]task.Task{
|
||||
@@ -237,6 +270,9 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task
|
||||
var allTasks []task.Task
|
||||
allTasks = append(allTasks, pkgTasks...)
|
||||
allTasks = append(allTasks, fileTasks...)
|
||||
if archiveTask != nil {
|
||||
allTasks = append(allTasks, archiveTask)
|
||||
}
|
||||
|
||||
return taskGroups, &catalogerManifest{
|
||||
Requested: selectionEvidence.Request,
|
||||
@@ -260,7 +296,7 @@ func (c *CreateSBOMConfig) selectTasks(src source.Description) ([]task.Task, []t
|
||||
SearchConfig: c.Search,
|
||||
RelationshipsConfig: c.Relationships,
|
||||
DataGenerationConfig: c.DataGeneration,
|
||||
PackagesConfig: c.Packages,
|
||||
PackagesConfig: c.packagesWithArchiveSearch(),
|
||||
LicenseConfig: c.Licenses,
|
||||
ComplianceConfig: c.Compliance,
|
||||
FilesConfig: c.Files,
|
||||
@@ -440,10 +476,10 @@ func (c *CreateSBOMConfig) environmentTasks() []task.Task {
|
||||
|
||||
// unknownsTasks returns a set of tasks that perform any necessary post-processing
|
||||
// to identify SBOM elements as unknowns
|
||||
func (c *CreateSBOMConfig) unknownsTasks() []task.Task {
|
||||
func unknownsTasks(cfg cataloging.UnknownsConfig) []task.Task {
|
||||
var tasks []task.Task
|
||||
|
||||
if t := task.NewUnknownsLabelerTask(c.Unknowns); t != nil {
|
||||
if t := task.NewUnknownsLabelerTask(cfg); t != nil {
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
|
||||
@@ -461,12 +497,36 @@ func (c *CreateSBOMConfig) osFeatureDetectionTasks() []task.Task {
|
||||
return tasks
|
||||
}
|
||||
|
||||
// packagesWithArchiveSearch returns a copy of the packages config with the top-level archive search
|
||||
// booleans applied to java, which is still the only cataloger that reads them.
|
||||
//
|
||||
// ponytail: a top-level boolean wins only when it differs from the default, so callers who set just
|
||||
// java's deprecated copy see no change. Delete this along with that field.
|
||||
func (c *CreateSBOMConfig) packagesWithArchiveSearch() pkgcataloging.Config {
|
||||
pkgs := c.Packages
|
||||
def := cataloging.DefaultArchiveSearchConfig()
|
||||
java := &pkgs.JavaArchive.ArchiveSearchConfig
|
||||
if c.Archive.IncludeIndexedArchives != def.IncludeIndexedArchives {
|
||||
java.IncludeIndexedArchives = c.Archive.IncludeIndexedArchives
|
||||
}
|
||||
if c.Archive.IncludeUnindexedArchives != def.IncludeUnindexedArchives {
|
||||
java.IncludeUnindexedArchives = c.Archive.IncludeUnindexedArchives
|
||||
}
|
||||
return pkgs
|
||||
}
|
||||
|
||||
func (c *CreateSBOMConfig) validate() error {
|
||||
if c.Relationships.ExcludeBinaryPackagesWithFileOwnershipOverlap {
|
||||
if !c.Relationships.PackageFileOwnershipOverlap {
|
||||
return fmt.Errorf("invalid configuration: to exclude binary packages based on file ownership overlap relationships, cataloging file ownership overlap relationships must be enabled")
|
||||
}
|
||||
}
|
||||
if j := c.Packages.JavaArchive.ArchiveSearchConfig; j.MaxDepth != 0 || j.MaxMemoryBytes != 0 || j.MaxDiskBytes != 0 {
|
||||
return fmt.Errorf("invalid configuration: nested archive depth and limits are not java cataloger settings, set them with CreateSBOMConfig.Archive")
|
||||
}
|
||||
if a := c.Archive; a.MaxDepth != 0 && a.MaxMemoryBytes == 0 && a.MaxDiskBytes == 0 {
|
||||
return fmt.Errorf("invalid configuration: nested archive cataloging is enabled but its memory and disk limits are both 0, leaving nowhere to extract archives")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/anchore/syft/syft/cataloging/pkgcataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/java"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
@@ -329,7 +330,7 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) {
|
||||
require.NotEmpty(t, tt.wantTaskNames)
|
||||
|
||||
// test the subject
|
||||
gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src)
|
||||
gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src, nil)
|
||||
tt.wantErr(t, err)
|
||||
if err != nil {
|
||||
return
|
||||
@@ -554,6 +555,31 @@ func TestCreateSBOMConfig_validate(t *testing.T) {
|
||||
),
|
||||
wantErr: assert.Error,
|
||||
},
|
||||
{
|
||||
name: "nested archives enabled with no memory or disk",
|
||||
cfg: DefaultCreateSBOMConfig().WithArchiveConfig(
|
||||
cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1).WithMaxMemoryBytes(0).WithMaxDiskBytes(0),
|
||||
),
|
||||
wantErr: assert.Error,
|
||||
},
|
||||
{
|
||||
name: "nested archives enabled with memory only",
|
||||
cfg: DefaultCreateSBOMConfig().WithArchiveConfig(
|
||||
cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1).WithMaxDiskBytes(0),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "nested archives enabled with disk only",
|
||||
cfg: DefaultCreateSBOMConfig().WithArchiveConfig(
|
||||
cataloging.DefaultArchiveSearchConfig().WithMaxDepth(1).WithMaxMemoryBytes(0),
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "nested archives disabled with no memory or disk",
|
||||
cfg: DefaultCreateSBOMConfig().WithArchiveConfig(
|
||||
cataloging.DefaultArchiveSearchConfig().WithMaxMemoryBytes(0).WithMaxDiskBytes(0),
|
||||
),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -564,3 +590,64 @@ func TestCreateSBOMConfig_validate(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateSBOMConfig_archiveTaskGroup(t *testing.T) {
|
||||
src := source.Description{Metadata: source.DirectoryMetadata{}}
|
||||
|
||||
archiveTaskIn := func(groups [][]task.Task) bool {
|
||||
for _, group := range groups {
|
||||
for _, tsk := range group {
|
||||
if tsk.Name() == task.ArchiveCatalogerTaskName {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
t.Run("no archive task by default", func(t *testing.T) {
|
||||
// the feature is off by default, and off must mean no task at all rather than a registered
|
||||
// task that does nothing: a registered task is reported as a selected cataloger
|
||||
groups, manifest, err := DefaultCreateSBOMConfig().makeTaskGroups(src, nil)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, archiveTaskIn(groups))
|
||||
assert.NotContains(t, manifest.Used, task.ArchiveCatalogerTaskName)
|
||||
})
|
||||
|
||||
t.Run("archive task is present at negative depth", func(t *testing.T) {
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(-1))
|
||||
groups, _, err := cfg.makeTaskGroups(src, nil)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, archiveTaskIn(groups))
|
||||
})
|
||||
|
||||
t.Run("archive task is present when enabled", func(t *testing.T) {
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2))
|
||||
groups, manifest, err := cfg.makeTaskGroups(src, nil)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, archiveTaskIn(groups))
|
||||
assert.Contains(t, manifest.Used, task.ArchiveCatalogerTaskName, "the manifest must say archive contents were searched")
|
||||
})
|
||||
|
||||
t.Run("enabling the feature does not mutate the caller's package config", func(t *testing.T) {
|
||||
// the java side learns the archive task is running from the context, not from its config: a
|
||||
// consumer who handed us a package config gets it back exactly as supplied.
|
||||
supplied := java.DefaultArchiveCatalogerConfig().WithUseNetwork(true)
|
||||
|
||||
cfg := DefaultCreateSBOMConfig().
|
||||
WithArchiveConfig(cataloging.DefaultArchiveSearchConfig().WithMaxDepth(2)).
|
||||
WithPackagesConfig(pkgcataloging.Config{JavaArchive: supplied})
|
||||
|
||||
_, _, err := cfg.makeTaskGroups(src, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, supplied, cfg.Packages.JavaArchive,
|
||||
"the caller's java config must come back byte-identical")
|
||||
assert.True(t, cfg.Packages.JavaArchive.UseNetwork,
|
||||
"a caller-supplied field must survive untouched")
|
||||
assert.Zero(t, cfg.Packages.JavaArchive.MaxDepth,
|
||||
"java's own squash-inlined depth must not be written to either")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -64,11 +64,11 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prog := catalogingProgress(int64(len(locs)))
|
||||
prog := catalogingProgress(ctx, int64(len(locs)))
|
||||
|
||||
results := make(map[file.Coordinates]file.Executable)
|
||||
errs := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(locs), func(loc file.Location) (*file.Executable, error) {
|
||||
prog.AtomicStage.Set(loc.Path())
|
||||
prog.AtomicStage.Set(loc.FullNestedPath())
|
||||
|
||||
exec, err := processExecutableLocation(loc, resolver)
|
||||
if err != nil {
|
||||
@@ -93,15 +93,15 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map
|
||||
func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec *file.Executable, err error) {
|
||||
reader, err := resolver.FileContentsByLocation(loc)
|
||||
if err != nil {
|
||||
log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get file contents")
|
||||
log.WithFields("error", err, "path", loc.FullNestedPath()).Debug("unable to get file contents")
|
||||
return nil, fmt.Errorf("unable to get file contents: %w", err)
|
||||
}
|
||||
defer internal.CloseAndLogError(reader, loc.RealPath)
|
||||
defer internal.CloseAndLogError(reader, loc.FullNestedPath())
|
||||
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
// a malformed binary must not take down the whole SBOM; the caller records this as an unknown
|
||||
log.WithFields("path", loc.RealPath, "panic", r).Debug("recovered from panic while reading executable")
|
||||
log.WithFields("path", loc.FullNestedPath(), "panic", r).Debug("recovered from panic while reading executable")
|
||||
log.Tracef("executable panic stack:\n%s", debug.Stack())
|
||||
exec, err = nil, fmt.Errorf("recovered from panic while reading executable: %v", r)
|
||||
}
|
||||
@@ -109,14 +109,14 @@ func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec
|
||||
|
||||
uReader, err := unionreader.GetUnionReader(reader)
|
||||
if err != nil {
|
||||
log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get union reader")
|
||||
log.WithFields("error", err, "path", loc.FullNestedPath()).Debug("unable to get union reader")
|
||||
return nil, fmt.Errorf("unable to get union reader: %w", err)
|
||||
}
|
||||
|
||||
return processExecutable(loc, uReader)
|
||||
}
|
||||
|
||||
func catalogingProgress(locations int64) *monitor.TaskProgress {
|
||||
func catalogingProgress(ctx context.Context, locations int64) *monitor.TaskProgress {
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "Executables",
|
||||
@@ -124,7 +124,7 @@ func catalogingProgress(locations int64) *monitor.TaskProgress {
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
|
||||
return bus.StartCatalogerTask(info, locations, "")
|
||||
return bus.StartCatalogerTask(ctx, info, locations, "")
|
||||
}
|
||||
|
||||
func filterByGlobs(locs []file.Location, globs []string) ([]file.Location, error) {
|
||||
@@ -146,7 +146,7 @@ func filterByGlobs(locs []file.Location, globs []string) ([]file.Location, error
|
||||
|
||||
func locationMatchesGlob(loc file.Location, globs []string) (bool, error) {
|
||||
for _, glob := range globs {
|
||||
for _, path := range []string{loc.RealPath, loc.AccessPath} {
|
||||
for _, path := range []string{loc.LocalPath(), loc.AccessPath} {
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
@@ -169,13 +169,13 @@ func processExecutable(loc file.Location, reader unionreader.UnionReader) (*file
|
||||
|
||||
format, err := findExecutableFormat(reader)
|
||||
if err != nil {
|
||||
log.Debugf("unable to determine executable kind for %v: %v", loc.RealPath, err)
|
||||
log.Debugf("unable to determine executable kind for %v: %v", loc.FullNestedPath(), err)
|
||||
return nil, fmt.Errorf("unable to determine executable kind: %w", err)
|
||||
}
|
||||
|
||||
if format == "" {
|
||||
// this is not an "unknown", so just log -- this binary does not have parseable data in it
|
||||
log.Debugf("unable to determine executable format for %q", loc.RealPath)
|
||||
log.Debugf("unable to determine executable format for %q", loc.FullNestedPath())
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -184,17 +184,17 @@ func processExecutable(loc file.Location, reader unionreader.UnionReader) (*file
|
||||
switch format {
|
||||
case file.ELF:
|
||||
if err = findELFFeatures(&data, reader); err != nil {
|
||||
log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine ELF features")
|
||||
log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine ELF features")
|
||||
err = fmt.Errorf("unable to determine ELF features: %w", err)
|
||||
}
|
||||
case file.PE:
|
||||
if err = findPEFeatures(&data, reader); err != nil {
|
||||
log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine PE features")
|
||||
log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine PE features")
|
||||
err = fmt.Errorf("unable to determine PE features: %w", err)
|
||||
}
|
||||
case file.MachO:
|
||||
if err = findMachoFeatures(&data, reader); err != nil {
|
||||
log.WithFields("error", err, "path", loc.RealPath).Trace("unable to determine Macho features")
|
||||
log.WithFields("error", err, "path", loc.FullNestedPath()).Trace("unable to determine Macho features")
|
||||
err = fmt.Errorf("unable to determine Macho features: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ func NewCataloger(cfg Config) *Cataloger {
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Cataloger) Catalog(_ context.Context, resolver file.Resolver) (map[file.Coordinates]string, error) {
|
||||
func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) (map[file.Coordinates]string, error) {
|
||||
results := make(map[file.Coordinates]string)
|
||||
var locations []file.Location
|
||||
var errs error
|
||||
@@ -54,10 +54,10 @@ func (i *Cataloger) Catalog(_ context.Context, resolver file.Resolver) (map[file
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prog := catalogingProgress(int64(len(locations)))
|
||||
prog := catalogingProgress(ctx, int64(len(locations)))
|
||||
|
||||
for _, location := range locations {
|
||||
prog.AtomicStage.Set(location.Path())
|
||||
prog.AtomicStage.Set(location.FullNestedPath())
|
||||
|
||||
metadata, err := resolver.FileMetadataByLocation(location)
|
||||
if err != nil {
|
||||
@@ -98,12 +98,12 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer internal.CloseAndLogError(contentReader, location.AccessPath)
|
||||
defer internal.CloseAndLogError(contentReader, location.FullNestedPath())
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
encoder := base64.NewEncoder(base64.StdEncoding, buf)
|
||||
if _, err = io.Copy(encoder, contentReader); err != nil {
|
||||
return "", internal.ErrPath{Context: "content-cataloger", Path: location.RealPath, Err: err}
|
||||
return "", internal.ErrPath{Context: "content-cataloger", Path: location.FullNestedPath(), Err: err}
|
||||
}
|
||||
// note: it's important to close the reader before reading from the buffer since closing will flush the remaining bytes
|
||||
if err := encoder.Close(); err != nil {
|
||||
@@ -113,7 +113,7 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
func catalogingProgress(locations int64) *monitor.TaskProgress {
|
||||
func catalogingProgress(ctx context.Context, locations int64) *monitor.TaskProgress {
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "File contents",
|
||||
@@ -121,5 +121,5 @@ func catalogingProgress(locations int64) *monitor.TaskProgress {
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
|
||||
return bus.StartCatalogerTask(info, locations, "")
|
||||
return bus.StartCatalogerTask(ctx, info, locations, "")
|
||||
}
|
||||
|
||||
@@ -42,15 +42,27 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
|
||||
locations = intCataloger.AllRegularFiles(ctx, resolver)
|
||||
} else {
|
||||
for _, c := range coordinates {
|
||||
locs, err := resolver.FilesByPath(c.RealPath)
|
||||
locs, err := resolver.FilesByPath(c.FullNestedPath())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to get file locations for path %q: %w", c.RealPath, err)
|
||||
return nil, fmt.Errorf("unable to get file locations for path %q: %w", c.FullNestedPath(), err)
|
||||
}
|
||||
locations = append(locations, locs...)
|
||||
}
|
||||
}
|
||||
|
||||
prog := catalogingProgress(int64(len(locations)))
|
||||
// symlinks resolve to the same file, so drop repeated coordinates to avoid hashing the same contents more than once
|
||||
seen := make(map[file.Coordinates]struct{}, len(locations))
|
||||
unique := locations[:0]
|
||||
for _, l := range locations {
|
||||
if _, ok := seen[l.Coordinates]; ok {
|
||||
continue
|
||||
}
|
||||
seen[l.Coordinates] = struct{}{}
|
||||
unique = append(unique, l)
|
||||
}
|
||||
locations = unique
|
||||
|
||||
prog := catalogingProgress(ctx, int64(len(locations)))
|
||||
|
||||
err := sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(locations), func(location file.Location) ([]file.Digest, error) {
|
||||
result, err := i.catalogLocation(ctx, resolver, location)
|
||||
@@ -59,10 +71,10 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
prog.AtomicStage.Set(location.Path())
|
||||
prog.AtomicStage.Set(location.FullNestedPath())
|
||||
|
||||
if internal.IsErrPathPermission(err) {
|
||||
log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
|
||||
log.Debugf("file digests cataloger skipping %q: %+v", location.FullNestedPath(), err)
|
||||
return nil, unknown.New(location, err)
|
||||
}
|
||||
|
||||
@@ -103,12 +115,12 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer internal.CloseAndLogError(contentReader, location.AccessPath)
|
||||
defer internal.CloseAndLogError(contentReader, location.FullNestedPath())
|
||||
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
// the caller records this as an unknown for the location instead of failing the whole SBOM
|
||||
log.WithFields("path", location.RealPath, "panic", r).Debug("recovered from panic while digesting file")
|
||||
log.WithFields("path", location.FullNestedPath(), "panic", r).Debug("recovered from panic while digesting file")
|
||||
log.Tracef("file digest panic stack:\n%s", debug.Stack())
|
||||
digests, err = nil, fmt.Errorf("recovered from panic while digesting file: %v", r)
|
||||
}
|
||||
@@ -116,13 +128,13 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver,
|
||||
|
||||
digests, err = intFile.NewDigestsFromFile(ctx, contentReader, i.hashes)
|
||||
if err != nil {
|
||||
return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err}
|
||||
return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.FullNestedPath(), Err: err}
|
||||
}
|
||||
|
||||
return digests, nil
|
||||
}
|
||||
|
||||
func catalogingProgress(locations int64) *monitor.TaskProgress {
|
||||
func catalogingProgress(ctx context.Context, locations int64) *monitor.TaskProgress {
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "File digests",
|
||||
@@ -130,5 +142,5 @@ func catalogingProgress(locations int64) *monitor.TaskProgress {
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
|
||||
return bus.StartCatalogerTask(info, locations, "")
|
||||
return bus.StartCatalogerTask(ctx, info, locations, "")
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -207,6 +208,45 @@ func TestFileDigestCataloger_GivenCoordinates(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
// countingResolver records how many times each coordinate's contents are opened
|
||||
type countingResolver struct {
|
||||
file.Resolver
|
||||
lock sync.Mutex
|
||||
counts map[file.Coordinates]int
|
||||
}
|
||||
|
||||
func (r *countingResolver) FileContentsByLocation(l file.Location) (io.ReadCloser, error) {
|
||||
r.lock.Lock()
|
||||
r.counts[l.Coordinates]++
|
||||
r.lock.Unlock()
|
||||
return r.Resolver.FileContentsByLocation(l)
|
||||
}
|
||||
|
||||
func TestDigestsCataloger_hashesEachCoordinateOnce(t *testing.T) {
|
||||
testImage := "image-file-type-mix"
|
||||
|
||||
testutils.SkipWithoutLinuxContainers(t)
|
||||
img := imagetest.GetFixtureImage(t, "docker-archive", testImage)
|
||||
|
||||
src := stereoscopesource.New(img, stereoscopesource.ImageConfig{
|
||||
Reference: testImage,
|
||||
})
|
||||
|
||||
resolver, err := src.FileResolver(source.SquashedScope)
|
||||
require.NoError(t, err)
|
||||
|
||||
// /symlink-1 and /file-1.txt both resolve to the same coordinates
|
||||
counter := &countingResolver{Resolver: resolver, counts: make(map[file.Coordinates]int)}
|
||||
|
||||
actual, err := NewCataloger([]crypto.Hash{crypto.MD5}).Catalog(context.Background(), counter)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, actual)
|
||||
|
||||
for c, n := range counter.counts {
|
||||
assert.Equal(t, 1, n, "contents of %+v opened %d times", c, n)
|
||||
}
|
||||
}
|
||||
|
||||
type panicReader struct{}
|
||||
|
||||
func (panicReader) Read([]byte) (int, error) { panic("boom") }
|
||||
|
||||
@@ -34,7 +34,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for _, c := range coordinates {
|
||||
locs, err := resolver.FilesByPath(c.RealPath)
|
||||
locs, err := resolver.FilesByPath(c.FullNestedPath())
|
||||
if err != nil {
|
||||
errs = unknown.Append(errs, c, err)
|
||||
continue
|
||||
@@ -53,9 +53,9 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
|
||||
}()
|
||||
}
|
||||
|
||||
prog := catalogingProgress(-1)
|
||||
prog := catalogingProgress(ctx, -1)
|
||||
for location := range locations {
|
||||
prog.AtomicStage.Set(location.Path())
|
||||
prog.AtomicStage.Set(location.FullNestedPath())
|
||||
|
||||
metadata, err := resolver.FileMetadataByLocation(location)
|
||||
if err != nil {
|
||||
@@ -76,7 +76,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
|
||||
return results, errs
|
||||
}
|
||||
|
||||
func catalogingProgress(locations int64) *monitor.TaskProgress {
|
||||
func catalogingProgress(ctx context.Context, locations int64) *monitor.TaskProgress {
|
||||
info := monitor.GenericTask{
|
||||
Title: monitor.Title{
|
||||
Default: "File metadata",
|
||||
@@ -84,5 +84,5 @@ func catalogingProgress(locations int64) *monitor.TaskProgress {
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
|
||||
return bus.StartCatalogerTask(info, locations, "")
|
||||
return bus.StartCatalogerTask(ctx, info, locations, "")
|
||||
}
|
||||
|
||||
@@ -12,16 +12,16 @@ func AllRegularFiles(ctx context.Context, resolver file.Resolver) (locations []f
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
for location := range resolver.AllLocations(ctx) {
|
||||
resolvedLocations, err := resolver.FilesByPath(location.RealPath)
|
||||
resolvedLocations, err := resolver.FilesByPath(location.LocalPath())
|
||||
if err != nil {
|
||||
log.Debugf("unable to resolve %+v: %+v", location, err)
|
||||
log.Debugf("unable to resolve %q: %+v", location.FullNestedPath(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, resolvedLocation := range resolvedLocations {
|
||||
metadata, err := resolver.FileMetadataByLocation(resolvedLocation)
|
||||
if err != nil {
|
||||
log.Debugf("unable to get metadata for %+v: %+v", location, err)
|
||||
log.Debugf("unable to get metadata for %q: %+v", resolvedLocation.FullNestedPath(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -78,6 +78,9 @@ func (s CoordinateSet) ToSlice(sorters ...func(a, b Coordinates) int) []Coordina
|
||||
if !sorted {
|
||||
sort.SliceStable(coordinates, func(i, j int) bool {
|
||||
if coordinates[i].FileSystemID == coordinates[j].FileSystemID {
|
||||
if coordinates[i].RealPath == coordinates[j].RealPath {
|
||||
return coordinates[i].FullNestedPath() < coordinates[j].FullNestedPath()
|
||||
}
|
||||
return coordinates[i].RealPath < coordinates[j].RealPath
|
||||
}
|
||||
return coordinates[i].FileSystemID < coordinates[j].FileSystemID
|
||||
|
||||
@@ -2,18 +2,46 @@ package file
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
)
|
||||
|
||||
const ArchivePathSeparator = "!/"
|
||||
|
||||
// Coordinates contains the minimal information needed to describe how to find a file within any possible source object (e.g. image and directory sources)
|
||||
type Coordinates struct {
|
||||
// RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed).
|
||||
// RealPath is the canonical absolute form of the path accessed (all symbolic links have been followed and relative path components like '.' and '..' have been removed). For a file found within an archive, it is the real path of the outermost archive on the scanned filesystem.
|
||||
RealPath string `json:"path" cyclonedx:"path"`
|
||||
|
||||
// FileSystemID is an ID representing and entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank.
|
||||
// FileSystemID is an ID representing an entire filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank. A file inside an archive carries the FileSystemID of the filesystem the archive was found in.
|
||||
FileSystemID string `json:"layerID,omitempty" cyclonedx:"layerID"`
|
||||
|
||||
// FullPath is the full path to the file: RealPath, then for a file within an archive a "!/"-delimited chain of entry paths through any nested archives, ending with the file's own entry path (e.g. "/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF"). Each segment is a real path (links resolved) relative to the archive before it. For a file not within an archive it is blank, so coordinates outside archives are unchanged by the field (in JSON, equality and ID).
|
||||
FullPath string `json:"fullPath,omitempty" cyclonedx:"fullPath"`
|
||||
}
|
||||
|
||||
// FullNestedPath is FullPath, or RealPath where FullPath is blank.
|
||||
func (c Coordinates) FullNestedPath() string {
|
||||
if c.FullPath == "" {
|
||||
return c.RealPath
|
||||
}
|
||||
return c.FullPath
|
||||
}
|
||||
|
||||
// LocalPath is the rooted path within the resolver holding the file: the last entry path in FullPath, or
|
||||
// RealPath for a file not within an archive. RealPath is cut off before looking for a separator, so a
|
||||
// "!/" in a real path is never taken for an archive boundary.
|
||||
func (c Coordinates) LocalPath() string {
|
||||
entries, ok := strings.CutPrefix(c.FullPath, c.RealPath+ArchivePathSeparator)
|
||||
if !ok {
|
||||
return c.RealPath
|
||||
}
|
||||
if i := strings.LastIndex(entries, ArchivePathSeparator); i >= 0 {
|
||||
entries = entries[i+len(ArchivePathSeparator):]
|
||||
}
|
||||
return "/" + entries
|
||||
}
|
||||
|
||||
func NewCoordinates(realPath, fsID string) Coordinates {
|
||||
@@ -40,9 +68,22 @@ func (c Coordinates) String() string {
|
||||
if c.FileSystemID != "" {
|
||||
str += fmt.Sprintf(" Layer=%q", c.FileSystemID)
|
||||
}
|
||||
|
||||
if c.FullNestedPath() != c.RealPath {
|
||||
str += fmt.Sprintf(" FullPath=%q", c.FullNestedPath())
|
||||
}
|
||||
return fmt.Sprintf("Location<%s>", str)
|
||||
}
|
||||
|
||||
func (c Coordinates) GetCoordinates() Coordinates {
|
||||
return c
|
||||
}
|
||||
|
||||
// HashInclude keeps a FullPath that adds nothing to RealPath out of the artifact ID hash (see
|
||||
// artifact.IDByHash), so coordinates outside archives keep the IDs they had before the field existed.
|
||||
func (c Coordinates) HashInclude(field string, _ any) (bool, error) {
|
||||
if field == "FullPath" && c.FullNestedPath() == c.RealPath {
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
+12
-6
@@ -33,7 +33,7 @@ type LocationData struct {
|
||||
// note: it is IMPORTANT to ignore anything but the coordinates for a Location when considering the ID (hash value)
|
||||
// since the coordinates are the minimally correct ID for a location (symlinks should not come into play)
|
||||
|
||||
// AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path)
|
||||
// AccessPath is the path used to retrieve file contents (which may or may not have hardlinks / symlinks in the path). For a file within an archive, it is a full path (see Coordinates.FullPath).
|
||||
AccessPath string `hash:"ignore" json:"accessPath"`
|
||||
|
||||
// ref is the stereoscope file reference relative to the stereoscope.FileCatalog that has more information about this location.
|
||||
@@ -110,12 +110,12 @@ func NewVirtualLocation(realPath, accessPath string) Location {
|
||||
}}
|
||||
}
|
||||
|
||||
// NewLocationFromCoordinates creates a new location for the given Coordinates.
|
||||
// NewLocationFromCoordinates creates a new location for the given Coordinates, accessed by their full path.
|
||||
func NewLocationFromCoordinates(coordinates Coordinates) Location {
|
||||
return Location{
|
||||
LocationData: LocationData{
|
||||
Coordinates: coordinates,
|
||||
AccessPath: coordinates.RealPath,
|
||||
AccessPath: coordinates.FullNestedPath(),
|
||||
},
|
||||
LocationMetadata: LocationMetadata{
|
||||
Annotations: map[string]string{},
|
||||
@@ -205,11 +205,13 @@ func NewVirtualLocationFromDirectory(responsePath, responseAccessPath string, re
|
||||
}
|
||||
}
|
||||
|
||||
// Path is the path this location was accessed by, or its full path (see Coordinates.FullNestedPath). For a
|
||||
// file within an archive both are full paths, so use LocalPath for path arithmetic against the resolver.
|
||||
func (l Location) Path() string {
|
||||
if l.AccessPath != "" {
|
||||
return l.AccessPath
|
||||
}
|
||||
return l.RealPath
|
||||
return l.FullNestedPath()
|
||||
}
|
||||
|
||||
func (l Location) String() string {
|
||||
@@ -220,10 +222,14 @@ func (l Location) String() string {
|
||||
|
||||
str += fmt.Sprintf("RealPath=%q", l.RealPath)
|
||||
|
||||
if l.AccessPath != "" && l.AccessPath != l.RealPath {
|
||||
if l.AccessPath != "" {
|
||||
str += fmt.Sprintf(" AccessPath=%q", l.AccessPath)
|
||||
}
|
||||
|
||||
if l.FullNestedPath() != l.RealPath {
|
||||
str += fmt.Sprintf(" FullPath=%q", l.FullNestedPath())
|
||||
}
|
||||
|
||||
if l.FileSystemID != "" {
|
||||
str += fmt.Sprintf(" Layer=%q", l.FileSystemID)
|
||||
}
|
||||
@@ -231,7 +237,7 @@ func (l Location) String() string {
|
||||
}
|
||||
|
||||
func (l Location) Equals(other Location) bool {
|
||||
return l.RealPath == other.RealPath &&
|
||||
return l.FullNestedPath() == other.FullNestedPath() &&
|
||||
l.AccessPath == other.AccessPath &&
|
||||
l.FileSystemID == other.FileSystemID
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"sort"
|
||||
|
||||
"github.com/gohugoio/hashstructure"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
)
|
||||
@@ -112,8 +113,13 @@ func (s *LocationSet) Empty() bool {
|
||||
}
|
||||
|
||||
func (s LocationSet) Hash() (uint64, error) {
|
||||
// access paths and filesystem IDs are not considered when hashing a location set, only the real paths
|
||||
return hashstructure.Hash(s.CoordinateSet().Paths(), &hashstructure.HashOptions{
|
||||
// access paths and filesystem IDs are not considered when hashing a location set, only the full paths
|
||||
// (unlike CoordinateSet.Paths, which ownership-by-overlap needs as real paths only)
|
||||
paths := strset.New()
|
||||
for _, c := range s.CoordinateSet().ToUnorderedSlice() {
|
||||
paths.Add(c.FullNestedPath())
|
||||
}
|
||||
return hashstructure.Hash(paths.List(), &hashstructure.HashOptions{
|
||||
ZeroNil: true,
|
||||
SlicesAsSets: true,
|
||||
})
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/gohugoio/hashstructure"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
@@ -255,6 +256,10 @@ func TestLocationSet_Hash(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
inA := NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FullPath: "/app.war!/a.jar!/package.json"})
|
||||
inB := NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FullPath: "/app.war!/b.jar!/package.json"})
|
||||
etcFull := Location{LocationData: LocationData{Coordinates: Coordinates{RealPath: "/etc/hosts", FileSystemID: "a", FullPath: "/etc/hosts"}}}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
setA LocationSet
|
||||
@@ -297,6 +302,18 @@ func TestLocationSet_Hash(t *testing.T) {
|
||||
setB: NewLocationSet(binB, etcB),
|
||||
want: assert.Equal,
|
||||
},
|
||||
{
|
||||
name: "same outer archive but different nested entries have different hashes",
|
||||
setA: NewLocationSet(inA),
|
||||
setB: NewLocationSet(inB),
|
||||
want: assert.NotEqual,
|
||||
},
|
||||
{
|
||||
name: "a full path equal to the real path has the same hash as a blank one",
|
||||
setA: NewLocationSet(etcA),
|
||||
setB: NewLocationSet(etcFull),
|
||||
want: assert.Equal,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -308,3 +325,13 @@ func TestLocationSet_Hash(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// IDs outside archives must not change from when only real paths were hashed
|
||||
func TestLocationSet_Hash_unchangedOutsideArchives(t *testing.T) {
|
||||
want, err := hashstructure.Hash([]string{"/etc/hosts", "/bin"}, &hashstructure.HashOptions{ZeroNil: true, SlicesAsSets: true})
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := NewLocationSet(NewLocation("/etc/hosts"), NewVirtualLocation("/bin", "/usr/bin")).Hash()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, got)
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/file"
|
||||
)
|
||||
@@ -48,3 +50,73 @@ func TestLocation_ID(t *testing.T) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestLocation_fullPathIsIdentity(t *testing.T) {
|
||||
a := NewLocationFromCoordinates(Coordinates{RealPath: "/a.jar", FullPath: "/a.jar!/META-INF/MANIFEST.MF"})
|
||||
b := NewLocationFromCoordinates(Coordinates{RealPath: "/b.jar", FullPath: "/b.jar!/META-INF/MANIFEST.MF"})
|
||||
|
||||
assert.False(t, a.Equals(b))
|
||||
assert.NotEqual(t, a.String(), b.String())
|
||||
assert.Contains(t, a.String(), `RealPath="/a.jar"`)
|
||||
}
|
||||
|
||||
func TestLocation_JSON(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
location Location
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "a file outside an archive writes no fullPath",
|
||||
location: NewVirtualLocationFromDirectory("/etc/os-release", "/etc/link", file.Reference{}).WithAnnotation("k", "v"),
|
||||
expected: `{"path":"/etc/os-release","accessPath":"/etc/link","annotations":{"k":"v"}}`,
|
||||
},
|
||||
{
|
||||
name: "a file within an archive writes its full path",
|
||||
location: NewLocationFromCoordinates(Coordinates{RealPath: "/app.war", FileSystemID: "layer", FullPath: "/app.war!/META-INF/MANIFEST.MF"}),
|
||||
expected: `{"path":"/app.war","layerID":"layer","fullPath":"/app.war!/META-INF/MANIFEST.MF","accessPath":"/app.war!/META-INF/MANIFEST.MF"}`,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
data, err := json.Marshal(tt.location)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, tt.expected, string(data))
|
||||
|
||||
var decoded Location
|
||||
require.NoError(t, json.Unmarshal(data, &decoded))
|
||||
assert.Equal(t, tt.location.Coordinates, decoded.Coordinates, "decoding fills FullPath back in")
|
||||
assert.Equal(t, tt.location.AccessPath, decoded.AccessPath)
|
||||
|
||||
data, err = json.Marshal(tt.location.Coordinates)
|
||||
require.NoError(t, err)
|
||||
var coordinates Coordinates
|
||||
require.NoError(t, json.Unmarshal(data, &coordinates))
|
||||
assert.Equal(t, tt.location.Coordinates, coordinates)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinates_ID_unchangedByTopLevelFullRealPath(t *testing.T) {
|
||||
assert.Equal(t, Coordinates{RealPath: "/a"}.ID(), NewCoordinates("/a", "").ID())
|
||||
assert.NotEqual(t, NewCoordinates("/a", "").ID(), Coordinates{RealPath: "/a", FullPath: "/a!/b"}.ID())
|
||||
}
|
||||
|
||||
func TestCoordinates_LocalPath(t *testing.T) {
|
||||
assert.Equal(t, "/a.jar", NewCoordinates("/a.jar", "").LocalPath())
|
||||
assert.Equal(t, "/a.jar", Coordinates{RealPath: "/a.jar"}.LocalPath())
|
||||
assert.Equal(t, "/META-INF/MANIFEST.MF", Coordinates{RealPath: "/a.war", FullPath: "/a.war!/lib/b.jar!/META-INF/MANIFEST.MF"}.LocalPath())
|
||||
assert.Equal(t, "/data/Hello!/world.txt", NewCoordinates("/data/Hello!/world.txt", "").LocalPath(), "a separator in a real path is not an archive boundary")
|
||||
assert.Equal(t, "/data/Hello!/world.txt", Coordinates{RealPath: "/data/Hello!/world.txt", FullPath: "/data/Hello!/world.txt"}.LocalPath())
|
||||
assert.Equal(t, "/x.txt", Coordinates{RealPath: "/data/Hello!/a.zip", FullPath: "/data/Hello!/a.zip!/x.txt"}.LocalPath())
|
||||
assert.Equal(t, "/x.txt", Coordinates{RealPath: "/data/Hello!/a.zip", FullPath: "/data/Hello!/a.zip!/b.jar!/x.txt"}.LocalPath())
|
||||
}
|
||||
|
||||
func TestLocation_Path(t *testing.T) {
|
||||
nested := Coordinates{RealPath: "/a.war", FullPath: "/a.war!/lib/b.jar!/META-INF/MANIFEST.MF"}
|
||||
assert.Equal(t, "/a.jar", NewLocation("/a.jar").Path())
|
||||
assert.Equal(t, "/link", NewVirtualLocation("/a.jar", "/link").Path())
|
||||
assert.Equal(t, nested.FullPath, NewLocationFromCoordinates(nested).Path())
|
||||
assert.Equal(t, nested.FullPath, Location{LocationData: LocationData{Coordinates: nested}}.Path(), "a nested location without an access path is still full")
|
||||
assert.Equal(t, nested.FullPath, NewVirtualLocationFromCoordinates(nested, nested.FullPath).Path())
|
||||
}
|
||||
|
||||
+12
-2
@@ -83,7 +83,12 @@ func LocationSorter(layers []string) func(a, b Location) int { //nolint:gocognit
|
||||
return strings.Compare(a.AccessPath, b.AccessPath)
|
||||
}
|
||||
|
||||
return strings.Compare(a.RealPath, b.RealPath)
|
||||
if a.RealPath != b.RealPath {
|
||||
return strings.Compare(a.RealPath, b.RealPath)
|
||||
}
|
||||
|
||||
// the same path in different archives is a different file
|
||||
return strings.Compare(a.FullNestedPath(), b.FullNestedPath())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,6 +123,11 @@ func CoordinatesSorter(layers []string) func(a, b Coordinates) int {
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Compare(a.RealPath, b.RealPath)
|
||||
if a.RealPath != b.RealPath {
|
||||
return strings.Compare(a.RealPath, b.RealPath)
|
||||
}
|
||||
|
||||
// the same path in different archives is a different file
|
||||
return strings.Compare(a.FullNestedPath(), b.FullNestedPath())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/anchore/syft/internal/evidence"
|
||||
)
|
||||
@@ -555,3 +556,24 @@ func TestLocationAndCoordinatesSorters(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSorters_fullPathBreaksTies(t *testing.T) {
|
||||
// the same path in two archives must sort the same way whatever order the inputs arrive in, or
|
||||
// output built from map iteration is not reproducible
|
||||
a := Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"}
|
||||
b := Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/META-INF/MANIFEST.MF"}
|
||||
want := []string{a.FullNestedPath(), b.FullNestedPath()}
|
||||
|
||||
for _, in := range [][]Coordinates{{a, b}, {b, a}} {
|
||||
coords := slices.Clone(in)
|
||||
slices.SortFunc(coords, CoordinatesSorter(nil))
|
||||
assert.Equal(t, want, []string{coords[0].FullNestedPath(), coords[1].FullNestedPath()}, "CoordinatesSorter")
|
||||
|
||||
locs := []Location{NewLocationFromCoordinates(in[0]), NewLocationFromCoordinates(in[1])}
|
||||
slices.SortFunc(locs, LocationSorter(nil))
|
||||
assert.Equal(t, want, []string{locs[0].FullNestedPath(), locs[1].FullNestedPath()}, "LocationSorter")
|
||||
|
||||
got := NewCoordinateSet(in...).ToSlice()
|
||||
assert.Equal(t, want, []string{got[0].FullNestedPath(), got[1].FullNestedPath()}, "CoordinateSet.ToSlice")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,9 +60,9 @@ func NewMockResolverForPathsWithMetadata(metadata map[Coordinates]Metadata) *Moc
|
||||
l := NewLocationFromCoordinates(c)
|
||||
locations = append(locations, l)
|
||||
mimeTypeIndex[m.MIMEType] = append(mimeTypeIndex[m.MIMEType], l)
|
||||
ext := path.Ext(l.RealPath)
|
||||
ext := path.Ext(l.FullNestedPath())
|
||||
extension[ext] = append(extension[ext], l)
|
||||
bn := path.Base(l.RealPath)
|
||||
bn := path.Base(l.FullNestedPath())
|
||||
basename[bn] = append(basename[bn], l)
|
||||
}
|
||||
|
||||
@@ -97,9 +97,9 @@ func NewMockResolverForMediaTypes(mediaTypes map[string][]Location) *MockResolve
|
||||
mediaTypeIndex[mediaType] = append(mediaTypeIndex[mediaType], locs...)
|
||||
for _, l := range locs {
|
||||
locations = append(locations, l)
|
||||
ext := path.Ext(l.RealPath)
|
||||
ext := path.Ext(l.FullNestedPath())
|
||||
extension[ext] = append(extension[ext], l)
|
||||
bn := path.Base(l.RealPath)
|
||||
bn := path.Base(l.FullNestedPath())
|
||||
basename[bn] = append(basename[bn], l)
|
||||
}
|
||||
}
|
||||
@@ -121,7 +121,7 @@ func (r MockResolver) ImageReference() string {
|
||||
// HasPath indicates if the given path exists in the underlying source.
|
||||
func (r MockResolver) HasPath(path string) bool {
|
||||
for _, l := range r.locations {
|
||||
if l.RealPath == path {
|
||||
if l.FullNestedPath() == path {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -142,7 +142,7 @@ func (r MockResolver) FileContentsByLocation(location Location) (io.ReadCloser,
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("no file for location: %v", location)
|
||||
return nil, fmt.Errorf("no file for location: %q", location.FullNestedPath())
|
||||
}
|
||||
|
||||
// FilesByPath returns all Locations that match the given paths.
|
||||
@@ -150,7 +150,7 @@ func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) {
|
||||
var results []Location
|
||||
for _, p := range paths {
|
||||
for _, location := range r.locations {
|
||||
if p == location.RealPath {
|
||||
if p == location.LocalPath() {
|
||||
results = append(results, NewLocation(p))
|
||||
}
|
||||
}
|
||||
@@ -164,7 +164,7 @@ func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) {
|
||||
var results []Location
|
||||
for _, pattern := range patterns {
|
||||
for _, location := range r.locations {
|
||||
matches, err := doublestar.Match(pattern, location.RealPath)
|
||||
matches, err := doublestar.Match(pattern, location.LocalPath())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cpe"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
formatInternal "github.com/anchore/syft/syft/format/internal"
|
||||
"github.com/anchore/syft/syft/format/internal/cyclonedxutil/helpers"
|
||||
"github.com/anchore/syft/syft/linux"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
@@ -42,7 +43,8 @@ func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM {
|
||||
cdxBOM.SerialNumber = uuid.New().URN()
|
||||
cdxBOM.Metadata = toBomDescriptor(s.Descriptor.Name, s.Descriptor.Version, s.Source)
|
||||
|
||||
coordinates, locationSorter := getCoordinates(s)
|
||||
locationSorter, coordinateSorter := formatInternal.GetLocationSorters(s)
|
||||
top, entries := formatInternal.ArchiveEntries(s.AllCoordinates(), coordinateSorter)
|
||||
|
||||
// Packages
|
||||
packages := s.Artifacts.Packages.Sorted()
|
||||
@@ -52,40 +54,7 @@ func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM {
|
||||
}
|
||||
components = append(components, toOSComponent(s.Artifacts.LinuxDistribution)...)
|
||||
|
||||
artifacts := s.Artifacts
|
||||
|
||||
for _, coordinate := range coordinates {
|
||||
var metadata *file.Metadata
|
||||
// File Info
|
||||
fileMetadata, exists := artifacts.FileMetadata[coordinate]
|
||||
// no file metadata then don't include in SBOM
|
||||
// the syft config allows for sometimes only capturing files owned by packages
|
||||
// so there can be a map miss here where we have less metadata than all coordinates
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
if fileMetadata.Type == stfile.TypeDirectory ||
|
||||
fileMetadata.Type == stfile.TypeSocket ||
|
||||
fileMetadata.Type == stfile.TypeSymLink {
|
||||
// skip dir, symlinks and sockets for the final bom
|
||||
continue
|
||||
}
|
||||
metadata = &fileMetadata
|
||||
|
||||
// Digests
|
||||
var digests []file.Digest
|
||||
if digestsForLocation, exists := artifacts.FileDigests[coordinate]; exists {
|
||||
digests = digestsForLocation
|
||||
}
|
||||
|
||||
cdxHashes := digestsToHashes(digests)
|
||||
components = append(components, cyclonedx.Component{
|
||||
BOMRef: string(coordinate.ID()),
|
||||
Type: cyclonedx.ComponentTypeFile,
|
||||
Name: metadata.Path,
|
||||
Hashes: &cdxHashes,
|
||||
})
|
||||
}
|
||||
components = append(components, toFileComponents(s.Artifacts, entries, top)...)
|
||||
cdxBOM.Components = &components
|
||||
|
||||
dependencies := toDependencies(s.Relationships)
|
||||
@@ -96,19 +65,41 @@ func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM {
|
||||
return cdxBOM
|
||||
}
|
||||
|
||||
func getCoordinates(s sbom.SBOM) ([]file.Coordinates, func(a, b file.Location) int) {
|
||||
var layers []string
|
||||
if m, ok := s.Source.Metadata.(source.ImageMetadata); ok {
|
||||
for _, l := range m.Layers {
|
||||
layers = append(layers, l.Digest)
|
||||
// toFileComponents describes each of coordinates that is a regular file with metadata, named by its path
|
||||
// (within its archive, for a file within one), with the files within it nested in its component.
|
||||
func toFileComponents(artifacts sbom.Artifacts, entries map[file.Coordinates][]file.Coordinates, coordinates []file.Coordinates) []cyclonedx.Component {
|
||||
var components []cyclonedx.Component
|
||||
for _, coordinate := range coordinates {
|
||||
nested := toFileComponents(artifacts, entries, entries[coordinate])
|
||||
// no file metadata then don't include in SBOM
|
||||
// the syft config allows for sometimes only capturing files owned by packages
|
||||
// so there can be a map miss here where we have less metadata than all coordinates
|
||||
metadata, exists := artifacts.FileMetadata[coordinate]
|
||||
if !isFileComponent(metadata, exists) && len(nested) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
cdxHashes := digestsToHashes(artifacts.FileDigests[coordinate])
|
||||
component := cyclonedx.Component{
|
||||
BOMRef: string(coordinate.ID()),
|
||||
Type: cyclonedx.ComponentTypeFile,
|
||||
Name: coordinate.LocalPath(),
|
||||
Hashes: &cdxHashes,
|
||||
}
|
||||
if len(nested) > 0 {
|
||||
component.Components = &nested
|
||||
}
|
||||
components = append(components, component)
|
||||
}
|
||||
return components
|
||||
}
|
||||
|
||||
coordSorter := file.CoordinatesSorter(layers)
|
||||
coordinates := s.AllCoordinates()
|
||||
|
||||
slices.SortFunc(coordinates, coordSorter)
|
||||
return coordinates, file.LocationSorter(layers)
|
||||
// isFileComponent skips dirs, symlinks and sockets for the final bom
|
||||
func isFileComponent(metadata file.Metadata, exists bool) bool {
|
||||
return exists &&
|
||||
metadata.Type != stfile.TypeDirectory &&
|
||||
metadata.Type != stfile.TypeSocket &&
|
||||
metadata.Type != stfile.TypeSymLink
|
||||
}
|
||||
|
||||
func digestsToHashes(digests []file.Digest) []cyclonedx.Hash {
|
||||
|
||||
@@ -608,3 +608,65 @@ func Test_toOSBomRef(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_nestedArchiveFiles(t *testing.T) {
|
||||
top := file.NewCoordinates("/etc/os-release", "")
|
||||
inA := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF"}
|
||||
inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt"}
|
||||
inWar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/index.html"}
|
||||
dir := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/c.jar!/META-INF"}
|
||||
sha := []file.Digest{{Algorithm: "sha256", Value: "abc"}}
|
||||
s := sbom.SBOM{Artifacts: sbom.Artifacts{
|
||||
FileMetadata: map[file.Coordinates]file.Metadata{
|
||||
// a directory resolver's metadata path is the path on the host, not the one scanned
|
||||
top: {Path: "/host/scan/etc/os-release"},
|
||||
inA: {},
|
||||
inB: {},
|
||||
inWar: {},
|
||||
dir: {Type: stfile.TypeDirectory},
|
||||
},
|
||||
FileDigests: map[file.Coordinates][]file.Digest{inA: sha},
|
||||
}}
|
||||
|
||||
bom := ToFormatModel(s)
|
||||
|
||||
type node struct {
|
||||
Name string
|
||||
Children []node
|
||||
}
|
||||
var tree func(cs *[]cyclonedx.Component) []node
|
||||
tree = func(cs *[]cyclonedx.Component) []node {
|
||||
if cs == nil {
|
||||
return nil
|
||||
}
|
||||
var out []node
|
||||
for _, c := range *cs {
|
||||
require.Equal(t, cyclonedx.ComponentTypeFile, c.Type)
|
||||
out = append(out, node{Name: c.Name, Children: tree(c.Components)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
// each archive holding a file is a component, synthesized when not otherwise described; an archive
|
||||
// holding only skipped files is left out
|
||||
assert.Equal(t, []node{
|
||||
{Name: "/app.war", Children: []node{
|
||||
{Name: "/WEB-INF/lib/a.jar", Children: []node{{Name: "/META-INF/MANIFEST.MF"}}},
|
||||
{Name: "/WEB-INF/lib/b.jar", Children: []node{{Name: "/odd:na%me.txt"}}},
|
||||
{Name: "/index.html"},
|
||||
}},
|
||||
{Name: "/etc/os-release"},
|
||||
}, tree(bom.Components))
|
||||
|
||||
decoded, err := helpers.ToSyftModel(bom)
|
||||
require.NoError(t, err)
|
||||
|
||||
war := file.NewCoordinates("/app.war", "")
|
||||
a := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar"}
|
||||
b := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar"}
|
||||
var got []file.Coordinates
|
||||
for c := range decoded.Artifacts.FileMetadata {
|
||||
got = append(got, c)
|
||||
}
|
||||
assert.ElementsMatch(t, []file.Coordinates{top, war, a, b, inA, inB, inWar}, got)
|
||||
assert.Equal(t, map[file.Coordinates][]file.Digest{inA: sha}, decoded.Artifacts.FileDigests)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -53,7 +54,9 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document {
|
||||
rels := relationship.NewIndex(s.Relationships...)
|
||||
packages, otherLicenses := toPackages(rels, s.Artifacts.Packages, s)
|
||||
|
||||
allRelationships := toRelationships(rels.All())
|
||||
sourceFile := toSourceFile(s)
|
||||
files, archiveRelationships := toFiles(s, sourceFile)
|
||||
allRelationships := append(toRelationships(rels.All()), archiveRelationships...)
|
||||
|
||||
// for valid SPDX we need a document describes relationship
|
||||
describesID := spdx.ElementID("DOCUMENT")
|
||||
@@ -62,8 +65,7 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document {
|
||||
if rootPackage != nil {
|
||||
describesID = rootPackage.PackageSPDXIdentifier
|
||||
|
||||
// add all relationships from the document root to all other packages
|
||||
allRelationships = append(allRelationships, toRootRelationships(rootPackage, packages)...)
|
||||
allRelationships = append(allRelationships, toRootRelationships(rootPackage, sourceFile, packages, files, allRelationships)...)
|
||||
|
||||
// append the root package
|
||||
packages = append(packages, rootPackage)
|
||||
@@ -155,25 +157,64 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document {
|
||||
CreatorComment: "",
|
||||
},
|
||||
Packages: packages,
|
||||
Files: toFiles(s),
|
||||
Files: files,
|
||||
Relationships: allRelationships,
|
||||
OtherLicenses: convertOtherLicense(otherLicenses),
|
||||
}
|
||||
}
|
||||
|
||||
func toRootRelationships(rootPackage *spdx.Package, packages []*spdx.Package) (out []*spdx.Relationship) {
|
||||
for _, p := range packages {
|
||||
// toRootRelationships relates the root package to every package, and to each file nothing else CONTAINS: what is
|
||||
// within an archive is reached only through the archive's file, and an owned file through its package. When the
|
||||
// source is itself a file, the root package DESCRIBES it, and a scanned directory CONTAINS those files instead. A
|
||||
// package containing files must have analyzed them, so the root package then gets a verification code over them.
|
||||
func toRootRelationships(rootPackage *spdx.Package, sourceFile *file.Coordinates, packages []*spdx.Package, files []*spdx.File, relationships []*spdx.Relationship) (out []*spdx.Relationship) {
|
||||
contained := map[spdx.ElementID]struct{}{}
|
||||
for _, r := range relationships {
|
||||
if r.Relationship == string(helpers.ContainsRelationship) {
|
||||
contained[r.RefB.ElementRefID] = struct{}{}
|
||||
}
|
||||
}
|
||||
relate := func(a spdx.ElementID, typ helpers.RelationshipType, b spdx.ElementID) {
|
||||
out = append(out, &spdx.Relationship{
|
||||
RefA: spdx.DocElementID{
|
||||
ElementRefID: rootPackage.PackageSPDXIdentifier,
|
||||
},
|
||||
Relationship: string(helpers.ContainsRelationship),
|
||||
RefB: spdx.DocElementID{
|
||||
ElementRefID: p.PackageSPDXIdentifier,
|
||||
},
|
||||
RefA: spdx.DocElementID{ElementRefID: a},
|
||||
Relationship: string(typ),
|
||||
RefB: spdx.DocElementID{ElementRefID: b},
|
||||
})
|
||||
}
|
||||
return
|
||||
|
||||
fileContainer := rootPackage.PackageSPDXIdentifier
|
||||
if sourceFile != nil {
|
||||
id := toSPDXID(*sourceFile)
|
||||
relate(rootPackage.PackageSPDXIdentifier, helpers.DescribesRelationship, id)
|
||||
contained[id] = struct{}{}
|
||||
if isScanRoot(*sourceFile) {
|
||||
fileContainer = id
|
||||
}
|
||||
}
|
||||
|
||||
for _, p := range packages {
|
||||
relate(rootPackage.PackageSPDXIdentifier, helpers.ContainsRelationship, p.PackageSPDXIdentifier)
|
||||
}
|
||||
var sha1s []string
|
||||
for _, f := range files {
|
||||
if _, has := contained[f.FileSPDXIdentifier]; has {
|
||||
continue
|
||||
}
|
||||
relate(fileContainer, helpers.ContainsRelationship, f.FileSPDXIdentifier)
|
||||
if fileContainer != rootPackage.PackageSPDXIdentifier {
|
||||
continue
|
||||
}
|
||||
for _, c := range f.Checksums {
|
||||
if c.Algorithm == spdx.SHA1 {
|
||||
sha1s = append(sha1s, c.Value)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rootPackage.PackageVerificationCode = toVerificationCode(sha1s)
|
||||
rootPackage.FilesAnalyzed = rootPackage.PackageVerificationCode != nil
|
||||
return out
|
||||
}
|
||||
|
||||
//nolint:funlen
|
||||
@@ -355,7 +396,7 @@ func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID {
|
||||
return spdx.ElementID(helpers.SanitizeElementID(id))
|
||||
}
|
||||
p := ""
|
||||
parts := strings.Split(it.RealPath, "/")
|
||||
parts := strings.Split(it.FullNestedPath(), "/")
|
||||
for i := len(parts); i > 0; i-- {
|
||||
part := parts[i-1]
|
||||
if len(part) == 0 {
|
||||
@@ -663,8 +704,11 @@ func toRelationships(relationships []artifact.Relationship) (result []*spdx.Rela
|
||||
continue
|
||||
}
|
||||
|
||||
// FIXME: we are only currently including Package -> * relationships
|
||||
if _, ok := r.From.(pkg.Package); !ok {
|
||||
// FIXME: we are only currently including Package -> * relationships, plus an archive containing a
|
||||
// package found inside it, which is the only structure nested archive cataloging records for
|
||||
// non-java packages. Every file-side From is in the document: toFiles covers every coordinate a
|
||||
// relationship names, and itself relates each archive to the files within it.
|
||||
if _, ok := r.From.(pkg.Package); !ok && !isFileContainsPackage(r) {
|
||||
log.Debugf("skipping non-package relationship: %+v", r)
|
||||
continue
|
||||
}
|
||||
@@ -683,6 +727,12 @@ func toRelationships(relationships []artifact.Relationship) (result []*spdx.Rela
|
||||
return result
|
||||
}
|
||||
|
||||
func isFileContainsPackage(r artifact.Relationship) bool {
|
||||
_, fromFile := r.From.(file.Coordinates)
|
||||
_, toPkg := r.To.(pkg.Package)
|
||||
return fromFile && toPkg && r.Type == artifact.ContainsRelationship
|
||||
}
|
||||
|
||||
func lookupRelationship(ty artifact.RelationshipType) (bool, helpers.RelationshipType, string) {
|
||||
switch ty {
|
||||
case artifact.ContainsRelationship:
|
||||
@@ -697,60 +747,60 @@ func lookupRelationship(ty artifact.RelationshipType) (bool, helpers.Relationshi
|
||||
return false, "", ""
|
||||
}
|
||||
|
||||
func toFiles(s sbom.SBOM) (results []*spdx.File) {
|
||||
artifacts := s.Artifacts
|
||||
const layerIDCommentPrefix = "layerID: "
|
||||
|
||||
// toFiles describes every file, including each archive holding one. A file within an archive is named by
|
||||
// its path in that archive, and the archive CONTAINS it.
|
||||
func toFiles(s sbom.SBOM, sourceFile *file.Coordinates) (results []*spdx.File, archiveRelationships []*spdx.Relationship) {
|
||||
_, coordinateSorter := formatInternal.GetLocationSorters(s)
|
||||
top, entries := formatInternal.ArchiveEntries(s.AllCoordinates(), coordinateSorter)
|
||||
|
||||
coordinates := s.AllCoordinates()
|
||||
slices.SortFunc(coordinates, coordinateSorter)
|
||||
|
||||
for _, c := range coordinates {
|
||||
var metadata *file.Metadata
|
||||
if metadataForLocation, exists := artifacts.FileMetadata[c]; exists {
|
||||
metadata = &metadataForLocation
|
||||
// a file holding entries is an archive, whatever its content starts with (such as a launch script)
|
||||
describe := func(c file.Coordinates, name string) *spdx.File {
|
||||
f := toFile(s.Artifacts, c, name)
|
||||
if len(entries[c]) > 0 {
|
||||
f.FileTypes = withArchiveType(f.FileTypes)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
var digests []file.Digest
|
||||
if digestsForLocation, exists := artifacts.FileDigests[c]; exists {
|
||||
digests = digestsForLocation
|
||||
var addEntries func(archive file.Coordinates)
|
||||
addEntries = func(archive file.Coordinates) {
|
||||
for _, c := range entries[archive] {
|
||||
archiveRelationships = append(archiveRelationships, &spdx.Relationship{
|
||||
RefA: spdx.DocElementID{ElementRefID: toSPDXID(archive)},
|
||||
Relationship: string(helpers.ContainsRelationship),
|
||||
RefB: spdx.DocElementID{ElementRefID: toSPDXID(c)},
|
||||
})
|
||||
results = append(results, describe(c, strings.TrimPrefix(c.LocalPath(), "/")))
|
||||
addEntries(c)
|
||||
}
|
||||
|
||||
// if we don't have any metadata or digests for this location
|
||||
// then the file is most likely a symlink or non-regular file
|
||||
// for now we include a 0 sha1 digest as requested by the spdx spec
|
||||
// TODO: update location code in core SBOM so that we can map complex links
|
||||
// back to their real file digest location.
|
||||
if len(digests) == 0 {
|
||||
digests = append(digests, file.Digest{Algorithm: "sha1", Value: "0000000000000000000000000000000000000000"})
|
||||
}
|
||||
|
||||
// TODO: add file classifications (?) and content as a snippet
|
||||
|
||||
var comment string
|
||||
if c.FileSystemID != "" {
|
||||
comment = fmt.Sprintf("layerID: %s", c.FileSystemID)
|
||||
}
|
||||
|
||||
relativePath, err := convertAbsoluteToRelative(c.RealPath)
|
||||
}
|
||||
for _, c := range top {
|
||||
name, err := convertAbsoluteToRelative(c.RealPath)
|
||||
if err != nil {
|
||||
log.Debugf("unable to convert relative path '%s' to absolute path: %s", c.RealPath, err)
|
||||
relativePath = c.RealPath
|
||||
name = c.RealPath
|
||||
}
|
||||
|
||||
results = append(results, &spdx.File{
|
||||
FileSPDXIdentifier: toSPDXID(c),
|
||||
FileComment: comment,
|
||||
// required, no attempt made to determine license information
|
||||
LicenseConcluded: noAssertion,
|
||||
FileCopyrightText: noAssertion,
|
||||
Checksums: toFileChecksums(digests),
|
||||
FileName: relativePath,
|
||||
FileTypes: toFileTypes(metadata),
|
||||
LicenseInfoInFiles: []string{ // required in SPDX 2.2
|
||||
helpers.NOASSERTION,
|
||||
},
|
||||
})
|
||||
var archive *source.FileMetadata
|
||||
if isScanRoot(c) {
|
||||
// the scanned root is the root package, unless it is the file the source is
|
||||
if sourceFile == nil || c != *sourceFile {
|
||||
continue
|
||||
}
|
||||
name = "."
|
||||
if m, ok := s.Source.Metadata.(source.FileMetadata); ok {
|
||||
archive = &m
|
||||
name = filepath.Base(m.Path)
|
||||
}
|
||||
}
|
||||
f := describe(c, name)
|
||||
if archive != nil {
|
||||
// an extracted archive is the archive file, not the directory it was extracted to
|
||||
f.FileTypes = withArchiveType(toFileTypes(&file.Metadata{MIMEType: archive.MIMEType}))
|
||||
}
|
||||
results = append(results, f)
|
||||
addEntries(c)
|
||||
}
|
||||
|
||||
// sort by real path then virtual path to ensure the result is stable across multiple runs
|
||||
@@ -760,7 +810,90 @@ func toFiles(s sbom.SBOM) (results []*spdx.File) {
|
||||
}
|
||||
return results[i].FileName < results[j].FileName
|
||||
})
|
||||
return results
|
||||
return results, archiveRelationships
|
||||
}
|
||||
|
||||
// toSourceFile finds the file the scanned source is: a directory, or an extracted archive, is the scanned root,
|
||||
// while a single file is indexed beside its parent directory, which was not scanned.
|
||||
func toSourceFile(s sbom.SBOM) *file.Coordinates {
|
||||
var top []file.Coordinates
|
||||
for _, c := range s.AllCoordinates() {
|
||||
if c.FullNestedPath() == c.RealPath {
|
||||
top = append(top, c)
|
||||
}
|
||||
}
|
||||
match := isScanRoot
|
||||
switch m := s.Source.Metadata.(type) {
|
||||
case source.DirectoryMetadata:
|
||||
case source.FileMetadata:
|
||||
name := filepath.Base(m.Path)
|
||||
single := func(c file.Coordinates) bool { return strings.TrimPrefix(c.RealPath, "/") == name }
|
||||
if slices.ContainsFunc(top, single) {
|
||||
match = single
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
i := slices.IndexFunc(top, match)
|
||||
if i < 0 {
|
||||
return nil
|
||||
}
|
||||
return &top[i]
|
||||
}
|
||||
|
||||
// withArchiveType adds the archive file type, in place of other
|
||||
func withArchiveType(types []string) []string {
|
||||
if slices.Contains(types, string(helpers.ArchiveFileType)) {
|
||||
return types
|
||||
}
|
||||
types = slices.DeleteFunc(types, func(t string) bool { return t == string(helpers.OtherFileType) })
|
||||
return append(types, string(helpers.ArchiveFileType))
|
||||
}
|
||||
|
||||
func isScanRoot(c file.Coordinates) bool {
|
||||
return c.RealPath == "" || c.RealPath == "/"
|
||||
}
|
||||
|
||||
func toFile(artifacts sbom.Artifacts, c file.Coordinates, name string) *spdx.File {
|
||||
var metadata *file.Metadata
|
||||
if metadataForLocation, exists := artifacts.FileMetadata[c]; exists {
|
||||
metadata = &metadataForLocation
|
||||
}
|
||||
|
||||
var digests []file.Digest
|
||||
if digestsForLocation, exists := artifacts.FileDigests[c]; exists {
|
||||
digests = digestsForLocation
|
||||
}
|
||||
|
||||
// if we don't have any metadata or digests for this location
|
||||
// then the file is most likely a symlink or non-regular file
|
||||
// for now we include a 0 sha1 digest as requested by the spdx spec
|
||||
// TODO: update location code in core SBOM so that we can map complex links
|
||||
// back to their real file digest location.
|
||||
if len(digests) == 0 {
|
||||
digests = append(digests, file.Digest{Algorithm: "sha1", Value: "0000000000000000000000000000000000000000"})
|
||||
}
|
||||
|
||||
// TODO: add file classifications (?) and content as a snippet
|
||||
|
||||
var comment string
|
||||
if c.FileSystemID != "" {
|
||||
comment = layerIDCommentPrefix + c.FileSystemID
|
||||
}
|
||||
|
||||
return &spdx.File{
|
||||
FileSPDXIdentifier: toSPDXID(c),
|
||||
FileComment: comment,
|
||||
// required, no attempt made to determine license information
|
||||
LicenseConcluded: noAssertion,
|
||||
FileCopyrightText: noAssertion,
|
||||
Checksums: toFileChecksums(digests),
|
||||
FileName: name,
|
||||
FileTypes: toFileTypes(metadata),
|
||||
LicenseInfoInFiles: []string{ // required in SPDX 2.2
|
||||
helpers.NOASSERTION,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func toFileChecksums(digests []file.Digest) (checksums []spdx.Checksum) {
|
||||
@@ -836,7 +969,7 @@ func newPackageVerificationCode(rels *relationship.Index, p pkg.Package, sbom sb
|
||||
// if a sha1 for a file is provided, then the validator will fail if the package does not have
|
||||
// a package verification code
|
||||
coordinates := rels.Coordinates(p, artifact.ContainsRelationship)
|
||||
var digests []file.Digest
|
||||
var sha1s []string
|
||||
for _, c := range coordinates {
|
||||
digest := sbom.Artifacts.FileDigests[c]
|
||||
if len(digest) == 0 {
|
||||
@@ -850,27 +983,23 @@ func newPackageVerificationCode(rels *relationship.Index, p pkg.Package, sbom sb
|
||||
break
|
||||
}
|
||||
}
|
||||
digests = append(digests, d)
|
||||
sha1s = append(sha1s, d.Value)
|
||||
}
|
||||
return toVerificationCode(sha1s)
|
||||
}
|
||||
|
||||
if len(digests) == 0 {
|
||||
func toVerificationCode(sha1s []string) *spdx.PackageVerificationCode {
|
||||
if len(sha1s) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// sort templist in ascending order by SHA1 value
|
||||
sort.SliceStable(digests, func(i, j int) bool {
|
||||
return digests[i].Value < digests[j].Value
|
||||
})
|
||||
sort.Strings(sha1s)
|
||||
|
||||
// filelist = templist with "/n"s removed. /* ordered sequence of SHA1 values with no separators
|
||||
var b strings.Builder
|
||||
for _, digest := range digests {
|
||||
b.WriteString(digest.Value)
|
||||
}
|
||||
|
||||
//nolint:gosec
|
||||
hasher := sha1.New()
|
||||
_, _ = hasher.Write([]byte(b.String()))
|
||||
_, _ = hasher.Write([]byte(strings.Join(sha1s, "")))
|
||||
return &spdx.PackageVerificationCode{
|
||||
// 7.9.1: Package Verification Code Value
|
||||
// Cardinality: mandatory, one
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
stfile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/internal/relationship"
|
||||
"github.com/anchore/syft/internal/sourcemetadata"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
@@ -556,7 +558,7 @@ func Test_toFiles(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in sbom.SBOM
|
||||
want spdx.File
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "File paths are converted to relative in final SPDX collection",
|
||||
@@ -584,16 +586,57 @@ func Test_toFiles(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
want: spdx.File{
|
||||
FileName: "some/path",
|
||||
want: []string{"some/path"},
|
||||
},
|
||||
{
|
||||
name: "the scanned root is the root package, not a file",
|
||||
in: sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{
|
||||
{RealPath: ""}: {Type: stfile.TypeDirectory},
|
||||
{RealPath: "/etc"}: {Type: stfile.TypeDirectory},
|
||||
}}},
|
||||
want: []string{"etc"},
|
||||
},
|
||||
{
|
||||
name: "a scanned directory is a file",
|
||||
in: sbom.SBOM{
|
||||
Source: source.Description{Metadata: source.DirectoryMetadata{Path: "/dir"}},
|
||||
Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{
|
||||
{RealPath: ""}: {Type: stfile.TypeDirectory},
|
||||
{RealPath: "/etc"}: {Type: stfile.TypeDirectory},
|
||||
}},
|
||||
},
|
||||
want: []string{".", "etc"},
|
||||
},
|
||||
{
|
||||
name: "a scanned file is indexed beside its parent directory, which was not scanned",
|
||||
in: sbom.SBOM{
|
||||
Source: source.Description{Metadata: source.FileMetadata{Path: "/dir/go.mod"}},
|
||||
Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{
|
||||
{RealPath: ""}: {Type: stfile.TypeDirectory},
|
||||
{RealPath: "/go.mod"}: {Type: stfile.TypeRegular},
|
||||
}},
|
||||
},
|
||||
want: []string{"go.mod"},
|
||||
},
|
||||
{
|
||||
name: "the absolute root is the root package, not a file",
|
||||
in: sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{
|
||||
{RealPath: "/"}: {Type: stfile.TypeDirectory},
|
||||
{RealPath: "/etc"}: {Type: stfile.TypeDirectory},
|
||||
}}},
|
||||
want: []string{"etc"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
files := toFiles(test.in)
|
||||
got := files[0]
|
||||
assert.Equal(t, test.want.FileName, got.FileName)
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
files, _ := toFiles(test.in, toSourceFile(test.in))
|
||||
var names []string
|
||||
for _, f := range files {
|
||||
names = append(names, f.FileName)
|
||||
}
|
||||
assert.Equal(t, test.want, names)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -762,10 +805,7 @@ func Test_fileIDsForPackage(t *testing.T) {
|
||||
Name: "bogus",
|
||||
}
|
||||
|
||||
c := file.Coordinates{
|
||||
RealPath: "/path",
|
||||
FileSystemID: "nowhere",
|
||||
}
|
||||
c := file.NewCoordinates("/path", "nowhere")
|
||||
|
||||
docElementId := func(identifiable artifact.Identifiable) spdx.DocElementID {
|
||||
return spdx.DocElementID{
|
||||
@@ -824,7 +864,7 @@ func Test_fileIDsForPackage(t *testing.T) {
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
name: "ignore file-to-package",
|
||||
name: "include an archive file containing a package",
|
||||
relationships: []artifact.Relationship{
|
||||
{
|
||||
From: c,
|
||||
@@ -832,6 +872,23 @@ func Test_fileIDsForPackage(t *testing.T) {
|
||||
Type: artifact.ContainsRelationship,
|
||||
},
|
||||
},
|
||||
expected: []*spdx.Relationship{
|
||||
{
|
||||
Relationship: "CONTAINS",
|
||||
RefA: docElementId(c),
|
||||
RefB: docElementId(p),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ignore other file-to-package",
|
||||
relationships: []artifact.Relationship{
|
||||
{
|
||||
From: c,
|
||||
To: p,
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
},
|
||||
expected: nil,
|
||||
},
|
||||
{
|
||||
@@ -1010,17 +1067,13 @@ func Test_toSPDXID(t *testing.T) {
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "short filename",
|
||||
it: file.Coordinates{
|
||||
RealPath: "/short/path/file.txt",
|
||||
},
|
||||
name: "short filename",
|
||||
it: file.NewCoordinates("/short/path/file.txt", ""),
|
||||
expected: "File-short-path-file.txt",
|
||||
},
|
||||
{
|
||||
name: "long filename",
|
||||
it: file.Coordinates{
|
||||
RealPath: "/some/long/path/with/a/lot/of-text/that-contains-a/file.txt",
|
||||
},
|
||||
name: "long filename",
|
||||
it: file.NewCoordinates("/some/long/path/with/a/lot/of-text/that-contains-a/file.txt", ""),
|
||||
expected: "File-...a-lot-of-text-that-contains-a-file.txt",
|
||||
},
|
||||
{
|
||||
@@ -1146,6 +1199,287 @@ func Test_otherLicenses(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func Test_nestedArchiveFiles(t *testing.T) {
|
||||
top := file.NewCoordinates("/etc/os-release", "sha256:layer")
|
||||
inA := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", FileSystemID: "sha256:layer"}
|
||||
inB := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar!/odd:na%me.txt", FileSystemID: "sha256:layer"}
|
||||
owned := file.NewCoordinates("/usr/bin/tool", "sha256:layer")
|
||||
a := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar", FileSystemID: "sha256:layer"}
|
||||
war := file.NewCoordinates("/app.war", "sha256:layer")
|
||||
b := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/b.jar", FileSystemID: "sha256:layer"}
|
||||
topPkg := pkg.Package{Name: "top"}
|
||||
topPkg.SetID()
|
||||
nestedPkg := pkg.Package{Name: "nested"}
|
||||
nestedPkg.SetID()
|
||||
|
||||
sources := []source.Description{
|
||||
{Name: "img", Metadata: source.ImageMetadata{UserInput: "img:latest", ManifestDigest: "sha256:abc"}},
|
||||
{Name: "dir", Metadata: source.DirectoryMetadata{Path: "/dir"}},
|
||||
}
|
||||
for _, src := range sources {
|
||||
t.Run(src.Name, func(t *testing.T) {
|
||||
s := sbom.SBOM{
|
||||
Source: src,
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(topPkg, nestedPkg),
|
||||
FileMetadata: map[file.Coordinates]file.Metadata{top: {}, inA: {}, inB: {}, owned: {}},
|
||||
},
|
||||
Relationships: []artifact.Relationship{
|
||||
{From: topPkg, To: owned, Type: artifact.ContainsRelationship},
|
||||
{From: a, To: nestedPkg, Type: artifact.ContainsRelationship},
|
||||
},
|
||||
}
|
||||
|
||||
doc := ToFormatModel(s)
|
||||
|
||||
// each archive holding a described file is described too, named by its path in its own container
|
||||
names := map[spdx.ElementID]string{}
|
||||
for _, f := range doc.Files {
|
||||
names[f.FileSPDXIdentifier] = f.FileName
|
||||
assert.Equal(t, "layerID: sha256:layer", f.FileComment)
|
||||
}
|
||||
assert.Equal(t, map[spdx.ElementID]string{
|
||||
toSPDXID(top): "etc/os-release",
|
||||
toSPDXID(owned): "usr/bin/tool",
|
||||
toSPDXID(war): "app.war",
|
||||
toSPDXID(a): "WEB-INF/lib/a.jar",
|
||||
toSPDXID(b): "WEB-INF/lib/b.jar",
|
||||
toSPDXID(inA): "META-INF/MANIFEST.MF",
|
||||
toSPDXID(inB): "odd:na%me.txt",
|
||||
}, names)
|
||||
|
||||
var described []spdx.ElementID
|
||||
var contains [][2]spdx.ElementID
|
||||
for _, r := range doc.Relationships {
|
||||
switch r.Relationship {
|
||||
case string(helpers.DescribesRelationship):
|
||||
described = append(described, r.RefB.ElementRefID)
|
||||
case string(helpers.ContainsRelationship):
|
||||
contains = append(contains, [2]spdx.ElementID{r.RefA.ElementRefID, r.RefB.ElementRefID})
|
||||
}
|
||||
}
|
||||
rootPkg := doc.Packages[len(doc.Packages)-1]
|
||||
root := rootPkg.PackageSPDXIdentifier
|
||||
assert.Equal(t, []spdx.ElementID{root}, described)
|
||||
|
||||
// the root contains every package, but only the files nothing else does: not what is within an archive,
|
||||
// nor an owned file
|
||||
assert.ElementsMatch(t, [][2]spdx.ElementID{
|
||||
{root, toSPDXID(topPkg)},
|
||||
{root, toSPDXID(nestedPkg)},
|
||||
{root, toSPDXID(top)},
|
||||
{root, toSPDXID(war)},
|
||||
{toSPDXID(topPkg), toSPDXID(owned)},
|
||||
{toSPDXID(a), toSPDXID(nestedPkg)},
|
||||
{toSPDXID(war), toSPDXID(a)},
|
||||
{toSPDXID(war), toSPDXID(b)},
|
||||
{toSPDXID(a), toSPDXID(inA)},
|
||||
{toSPDXID(b), toSPDXID(inB)},
|
||||
}, contains)
|
||||
|
||||
// a package containing files must have analyzed them
|
||||
assert.True(t, rootPkg.FilesAnalyzed)
|
||||
assert.NotNil(t, rootPkg.PackageVerificationCode)
|
||||
|
||||
// other tools may instead have the document describe the top-level files directly
|
||||
describesFiles := ToFormatModel(s)
|
||||
for _, r := range describesFiles.Relationships {
|
||||
if r.RefA.ElementRefID == root && (r.RefB.ElementRefID == toSPDXID(top) || r.RefB.ElementRefID == toSPDXID(war)) {
|
||||
r.RefA.ElementRefID = "DOCUMENT"
|
||||
r.Relationship = string(helpers.DescribesRelationship)
|
||||
}
|
||||
}
|
||||
|
||||
for _, d := range []struct {
|
||||
name string
|
||||
doc *spdx.Document
|
||||
}{
|
||||
{name: "root contains files", doc: doc},
|
||||
{name: "document describes files", doc: describesFiles},
|
||||
} {
|
||||
t.Run(d.name, func(t *testing.T) {
|
||||
decoded, err := ToSyftModel(d.doc)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, src.Name, decoded.Source.Name)
|
||||
assert.IsType(t, src.Metadata, decoded.Source.Metadata)
|
||||
|
||||
// decoded paths are relative, as SPDX names are
|
||||
relative := func(c file.Coordinates) file.Coordinates {
|
||||
c.RealPath = strings.TrimPrefix(c.RealPath, "/")
|
||||
c.FullPath = strings.TrimPrefix(c.FullPath, "/")
|
||||
return c
|
||||
}
|
||||
var got []file.Coordinates
|
||||
for c := range decoded.Artifacts.FileMetadata {
|
||||
got = append(got, c)
|
||||
}
|
||||
assert.ElementsMatch(t, []file.Coordinates{relative(top), relative(owned), relative(war), relative(a), relative(b), relative(inA), relative(inB)}, got)
|
||||
|
||||
var gotContains []artifact.Relationship
|
||||
for _, r := range decoded.Relationships {
|
||||
_, fromFile := r.From.(file.Coordinates)
|
||||
_, toFile := r.To.(file.Coordinates)
|
||||
if fromFile && toFile {
|
||||
gotContains = append(gotContains, r)
|
||||
}
|
||||
}
|
||||
assert.ElementsMatch(t, []artifact.Relationship{
|
||||
{From: relative(war), To: relative(a), Type: artifact.ContainsRelationship},
|
||||
{From: relative(war), To: relative(b), Type: artifact.ContainsRelationship},
|
||||
{From: relative(a), To: relative(inA), Type: artifact.ContainsRelationship},
|
||||
{From: relative(b), To: relative(inB), Type: artifact.ContainsRelationship},
|
||||
}, gotContains)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_sourceFile(t *testing.T) {
|
||||
root := file.NewCoordinates("", "")
|
||||
war := file.NewCoordinates("/app.war", "")
|
||||
jar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/a.jar"}
|
||||
etc := file.NewCoordinates("/etc/os-release", "")
|
||||
p := pkg.Package{Name: "p"}
|
||||
p.SetID()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
source source.Description
|
||||
files []file.Coordinates
|
||||
// relationships as "from type to", with the root package as "root"
|
||||
want []string
|
||||
wantAnalyzed bool
|
||||
// the file types of the file the root package describes
|
||||
wantTypes []string
|
||||
// decoded files, and the file relationships among them, as "from type to"
|
||||
wantFiles []string
|
||||
wantContains []string
|
||||
}{
|
||||
{
|
||||
name: "a scanned directory is described, and contains what is at its root",
|
||||
source: source.Description{Name: "dir", Metadata: source.DirectoryMetadata{Path: "/dir"}},
|
||||
files: []file.Coordinates{root, etc, war, jar},
|
||||
want: []string{
|
||||
"root DESCRIBES .",
|
||||
"root CONTAINS p",
|
||||
". CONTAINS etc/os-release",
|
||||
". CONTAINS app.war",
|
||||
"app.war CONTAINS a.jar",
|
||||
},
|
||||
// the scanned root decodes back to the scanned root, and holds nothing as an archive would
|
||||
wantFiles: []string{"", "etc/os-release", "app.war", "app.war!/a.jar"},
|
||||
wantContains: []string{"app.war contains app.war!/a.jar"},
|
||||
},
|
||||
{
|
||||
name: "an extracted archive is described, and contains what is at its root",
|
||||
source: source.Description{Name: "t.tar", Metadata: source.FileMetadata{Path: "/dir/t.tar"}},
|
||||
files: []file.Coordinates{root, etc},
|
||||
want: []string{
|
||||
"root DESCRIBES t.tar",
|
||||
"root CONTAINS p",
|
||||
"t.tar CONTAINS etc/os-release",
|
||||
},
|
||||
// the scanned root is the directory the archive was extracted to, but describes the archive
|
||||
wantTypes: []string{"ARCHIVE"},
|
||||
// the described archive is the scanned root, holding what is at the root
|
||||
wantFiles: []string{"", "etc/os-release"},
|
||||
},
|
||||
{
|
||||
name: "an extracted archive takes its file types from its MIME type",
|
||||
source: source.Description{Name: "t.zip", Metadata: source.FileMetadata{Path: "/dir/t.zip", MIMEType: "application/zip"}},
|
||||
files: []file.Coordinates{root, etc},
|
||||
want: []string{
|
||||
"root DESCRIBES t.zip",
|
||||
"root CONTAINS p",
|
||||
"t.zip CONTAINS etc/os-release",
|
||||
},
|
||||
wantTypes: []string{"APPLICATION", "ARCHIVE"},
|
||||
wantFiles: []string{"", "etc/os-release"},
|
||||
},
|
||||
{
|
||||
name: "a scanned file is described, but not its parent directory",
|
||||
source: source.Description{Name: "app.war", Metadata: source.FileMetadata{Path: "/dir/app.war"}},
|
||||
files: []file.Coordinates{root, war, jar},
|
||||
want: []string{
|
||||
"root DESCRIBES app.war",
|
||||
"root CONTAINS p",
|
||||
"app.war CONTAINS a.jar",
|
||||
},
|
||||
// an archive read in place is the scanned root too, so decodes as if it had been extracted
|
||||
wantFiles: []string{"", "a.jar"},
|
||||
},
|
||||
{
|
||||
name: "without the scanned root, the root package contains what is at its root",
|
||||
source: source.Description{Name: "dir", Metadata: source.DirectoryMetadata{Path: "/dir"}},
|
||||
files: []file.Coordinates{etc},
|
||||
want: []string{
|
||||
"root CONTAINS p",
|
||||
"root CONTAINS etc/os-release",
|
||||
},
|
||||
wantAnalyzed: true,
|
||||
wantFiles: []string{"etc/os-release"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
metadata := map[file.Coordinates]file.Metadata{}
|
||||
for _, c := range tt.files {
|
||||
metadata[c] = file.Metadata{}
|
||||
}
|
||||
s := sbom.SBOM{
|
||||
Source: tt.source,
|
||||
Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(p), FileMetadata: metadata},
|
||||
}
|
||||
|
||||
doc := ToFormatModel(s)
|
||||
|
||||
rootPkg := doc.Packages[len(doc.Packages)-1]
|
||||
names := map[spdx.ElementID]string{rootPkg.PackageSPDXIdentifier: "root", toSPDXID(p): "p"}
|
||||
for _, f := range doc.Files {
|
||||
names[f.FileSPDXIdentifier] = f.FileName
|
||||
}
|
||||
var got []string
|
||||
for _, r := range doc.Relationships {
|
||||
if r.RefA.ElementRefID == "DOCUMENT" {
|
||||
continue
|
||||
}
|
||||
got = append(got, names[r.RefA.ElementRefID]+" "+r.Relationship+" "+names[r.RefB.ElementRefID])
|
||||
}
|
||||
assert.ElementsMatch(t, tt.want, got)
|
||||
assert.Equal(t, tt.wantAnalyzed, rootPkg.FilesAnalyzed)
|
||||
if tt.wantTypes != nil {
|
||||
for _, r := range doc.Relationships {
|
||||
if r.RefA.ElementRefID == rootPkg.PackageSPDXIdentifier && r.Relationship == string(helpers.DescribesRelationship) {
|
||||
i := slices.IndexFunc(doc.Files, func(f *spdx.File) bool { return f.FileSPDXIdentifier == r.RefB.ElementRefID })
|
||||
require.GreaterOrEqual(t, i, 0)
|
||||
assert.Equal(t, tt.wantTypes, doc.Files[i].FileTypes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
decoded, err := ToSyftModel(doc)
|
||||
require.NoError(t, err)
|
||||
|
||||
var gotFiles []string
|
||||
for c := range decoded.Artifacts.FileMetadata {
|
||||
gotFiles = append(gotFiles, c.FullNestedPath())
|
||||
}
|
||||
assert.ElementsMatch(t, tt.wantFiles, gotFiles)
|
||||
|
||||
var gotContains []string
|
||||
for _, r := range decoded.Relationships {
|
||||
if from, ok := r.From.(file.Coordinates); ok {
|
||||
to, _ := r.To.(file.Coordinates)
|
||||
gotContains = append(gotContains, from.FullNestedPath()+" "+string(r.Type)+" "+to.FullNestedPath())
|
||||
}
|
||||
}
|
||||
assert.ElementsMatch(t, tt.wantContains, gotContains)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_toRootPackage_name(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -1191,3 +1525,27 @@ func Test_toRootPackage_name(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_toFiles_archiveType(t *testing.T) {
|
||||
// a file holding entries is an archive, even when its content starts like something else, such as a Spring Boot
|
||||
// executable jar's launch script
|
||||
jar := file.NewCoordinates("/app.jar", "")
|
||||
s := sbom.SBOM{Artifacts: sbom.Artifacts{FileMetadata: map[file.Coordinates]file.Metadata{
|
||||
jar: {MIMEType: "text/x-shellscript"},
|
||||
file.NewCoordinates("/plain.zip", ""): {MIMEType: "application/zip"},
|
||||
{RealPath: "/app.jar", FullPath: "/app.jar!/lib/b.jar!/c.txt"}: {MIMEType: "text/plain"},
|
||||
}}}
|
||||
|
||||
files, _ := toFiles(s, nil)
|
||||
got := map[string][]string{}
|
||||
for _, f := range files {
|
||||
got[f.FileName] = f.FileTypes
|
||||
}
|
||||
assert.Equal(t, map[string][]string{
|
||||
"app.jar": {"TEXT", "ARCHIVE"},
|
||||
"plain.zip": {"APPLICATION", "ARCHIVE"},
|
||||
// added only to hold its entry, with no metadata of its own
|
||||
"lib/b.jar": {"ARCHIVE"},
|
||||
"c.txt": {"TEXT"},
|
||||
}, got)
|
||||
}
|
||||
|
||||
@@ -38,6 +38,9 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) {
|
||||
|
||||
spdxIDMap := make(map[string]any)
|
||||
|
||||
// found before the root package and its relationships are removed
|
||||
sourceFile := findSourceFile(doc)
|
||||
|
||||
s := &sbom.SBOM{
|
||||
Source: extractSource(spdxIDMap, doc),
|
||||
Artifacts: sbom.Artifacts{
|
||||
@@ -50,7 +53,7 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) {
|
||||
|
||||
collectSyftPackages(s, spdxIDMap, doc)
|
||||
|
||||
collectSyftFiles(s, spdxIDMap, doc)
|
||||
collectSyftFiles(s, spdxIDMap, doc, sourceFile)
|
||||
|
||||
s.Relationships = toSyftRelationships(spdxIDMap, doc)
|
||||
|
||||
@@ -379,26 +382,122 @@ func collectSyftPackages(s *sbom.SBOM, spdxIDMap map[string]any, doc *spdx.Docum
|
||||
}
|
||||
}
|
||||
|
||||
func collectSyftFiles(s *sbom.SBOM, spdxIDMap map[string]any, doc *spdx.Document) {
|
||||
// findSourceFile returns the file the root package DESCRIBES when it CONTAINS files: an archive scanned as the source
|
||||
// is the scanned root, so what it contains is at the root rather than within an archive.
|
||||
func findSourceFile(doc *spdx.Document) spdx.ElementID {
|
||||
roots := findRootPackages(doc)
|
||||
if len(roots) != 1 {
|
||||
return ""
|
||||
}
|
||||
var described spdx.ElementID
|
||||
containers := map[spdx.ElementID]bool{}
|
||||
for _, r := range doc.Relationships {
|
||||
switch helpers.RelationshipType(r.Relationship) {
|
||||
case helpers.DescribesRelationship:
|
||||
if r.RefA.ElementRefID == roots[0].PackageSPDXIdentifier {
|
||||
described = r.RefB.ElementRefID
|
||||
}
|
||||
case helpers.ContainsRelationship:
|
||||
containers[r.RefA.ElementRefID] = true
|
||||
}
|
||||
}
|
||||
if !containers[described] {
|
||||
return ""
|
||||
}
|
||||
return described
|
||||
}
|
||||
|
||||
func collectSyftFiles(s *sbom.SBOM, spdxIDMap map[string]any, doc *spdx.Document, sourceFile spdx.ElementID) {
|
||||
files := map[spdx.ElementID]*spdx.File{}
|
||||
for _, p := range doc.Packages {
|
||||
for _, f := range p.Files {
|
||||
l := toSyftLocation(f)
|
||||
spdxIDMap[string(f.FileSPDXIdentifier)] = l
|
||||
files[f.FileSPDXIdentifier] = f
|
||||
}
|
||||
}
|
||||
for _, f := range doc.Files {
|
||||
files[f.FileSPDXIdentifier] = f
|
||||
}
|
||||
|
||||
s.Artifacts.FileMetadata[l.Coordinates] = toFileMetadata(f)
|
||||
s.Artifacts.FileDigests[l.Coordinates] = toFileDigests(f)
|
||||
nested := newNestedFiles[spdx.ElementID]()
|
||||
for id, f := range files {
|
||||
nested.own[id] = toSyftCoordinates(f)
|
||||
}
|
||||
if c, ok := nested.own[sourceFile]; ok {
|
||||
// the scanned root
|
||||
c.RealPath = ""
|
||||
nested.own[sourceFile] = c
|
||||
}
|
||||
for _, r := range doc.Relationships {
|
||||
container, entry := r.RefA.ElementRefID, r.RefB.ElementRefID
|
||||
switch helpers.RelationshipType(r.Relationship) {
|
||||
case helpers.ContainsRelationship:
|
||||
case helpers.ContainedByRelationship:
|
||||
container, entry = entry, container
|
||||
default:
|
||||
continue
|
||||
}
|
||||
// the scanned root holds what is at the root, not within an archive
|
||||
if files[container] != nil && files[entry] != nil && !isScanRoot(nested.own[container]) {
|
||||
nested.container[entry] = container
|
||||
}
|
||||
}
|
||||
|
||||
for _, f := range doc.Files {
|
||||
l := toSyftLocation(f)
|
||||
spdxIDMap[string(f.FileSPDXIdentifier)] = l
|
||||
for id, f := range files {
|
||||
c, ok := nested.coordinates(id)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
l := file.NewLocationFromCoordinates(c)
|
||||
spdxIDMap[string(id)] = l
|
||||
|
||||
s.Artifacts.FileMetadata[l.Coordinates] = toFileMetadata(f)
|
||||
s.Artifacts.FileDigests[l.Coordinates] = toFileDigests(f)
|
||||
}
|
||||
}
|
||||
|
||||
// nestedFiles resolves the coordinates of files within archives, where an archive file CONTAINS a file
|
||||
// named by its path within the archive.
|
||||
type nestedFiles[K comparable] struct {
|
||||
own map[K]file.Coordinates
|
||||
container map[K]K
|
||||
resolved map[K]file.Coordinates
|
||||
refused map[K]bool
|
||||
}
|
||||
|
||||
func newNestedFiles[K comparable]() nestedFiles[K] {
|
||||
return nestedFiles[K]{own: map[K]file.Coordinates{}, container: map[K]K{}, resolved: map[K]file.Coordinates{}, refused: map[K]bool{}}
|
||||
}
|
||||
|
||||
// coordinates reports false for a file within a chain a scan would refuse (a bad name, too deep, too long), and
|
||||
// for everything within it. Such a file is dropped rather than moved, since no scan put it anywhere else.
|
||||
func (n nestedFiles[K]) coordinates(id K) (file.Coordinates, bool) {
|
||||
// walk up to a resolved or outermost file, then resolve back down, so a long chain costs no stack
|
||||
chain := []K{id}
|
||||
for {
|
||||
next, ok := n.container[chain[len(chain)-1]]
|
||||
if _, resolved := n.resolved[chain[len(chain)-1]]; resolved || !ok {
|
||||
break
|
||||
}
|
||||
// a containment cycle ends where it started
|
||||
delete(n.container, chain[len(chain)-1])
|
||||
chain = append(chain, next)
|
||||
}
|
||||
for i := len(chain) - 1; i >= 0; i-- {
|
||||
if _, ok := n.resolved[chain[i]]; ok {
|
||||
continue
|
||||
}
|
||||
c := n.own[chain[i]]
|
||||
if i+1 < len(chain) {
|
||||
container := chain[i+1]
|
||||
nested, ok := internal.NestedCoordinates(n.resolved[container], c.RealPath)
|
||||
n.refused[chain[i]] = !ok || n.refused[container]
|
||||
c = nested
|
||||
}
|
||||
n.resolved[chain[i]] = c
|
||||
}
|
||||
return n.resolved[id], !n.refused[id]
|
||||
}
|
||||
|
||||
func toFileDigests(f *spdx.File) (digests []file.Digest) {
|
||||
for _, digest := range f.Checksums {
|
||||
digests = append(digests, file.Digest{
|
||||
@@ -452,6 +551,10 @@ func collectDocRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out
|
||||
}
|
||||
a := spdxIDMap[string(r.RefA.ElementRefID)]
|
||||
b := spdxIDMap[string(r.RefB.ElementRefID)]
|
||||
if rel, ok := fileContainsRelationship(helpers.RelationshipType(r.Relationship), a, b); ok {
|
||||
out = append(out, rel)
|
||||
continue
|
||||
}
|
||||
from, fromOk := a.(pkg.Package)
|
||||
toPackage, toPackageOk := b.(pkg.Package)
|
||||
toLocation, toLocationOk := b.(file.Location)
|
||||
@@ -505,6 +608,31 @@ func collectDocRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out
|
||||
return out
|
||||
}
|
||||
|
||||
// fileContainsRelationship decodes an archive file containing a file or package within it.
|
||||
func fileContainsRelationship(typ helpers.RelationshipType, a, b any) (artifact.Relationship, bool) {
|
||||
switch typ {
|
||||
case helpers.ContainsRelationship:
|
||||
case helpers.ContainedByRelationship:
|
||||
a, b = b, a
|
||||
default:
|
||||
return artifact.Relationship{}, false
|
||||
}
|
||||
container, ok := a.(file.Location)
|
||||
if !ok || isScanRoot(container.Coordinates) {
|
||||
return artifact.Relationship{}, false
|
||||
}
|
||||
var entry artifact.Identifiable
|
||||
switch to := b.(type) {
|
||||
case file.Location:
|
||||
entry = to.Coordinates
|
||||
case pkg.Package:
|
||||
entry = to
|
||||
default:
|
||||
return artifact.Relationship{}, false
|
||||
}
|
||||
return artifact.Relationship{From: container.Coordinates, To: entry, Type: artifact.ContainsRelationship}, true
|
||||
}
|
||||
|
||||
// collectPackageFileRelationships add relationships for direct files
|
||||
func collectPackageFileRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out []artifact.Relationship) {
|
||||
for _, p := range doc.Packages {
|
||||
@@ -530,23 +658,22 @@ func collectPackageFileRelationships(spdxIDMap map[string]any, doc *spdx.Documen
|
||||
}
|
||||
|
||||
func toSyftCoordinates(f *spdx.File) file.Coordinates {
|
||||
const layerIDPrefix = "layerID: "
|
||||
var fileSystemID string
|
||||
if strings.HasPrefix(f.FileComment, layerIDPrefix) {
|
||||
fileSystemID = strings.TrimPrefix(f.FileComment, layerIDPrefix)
|
||||
}
|
||||
if strings.HasPrefix(string(f.FileSPDXIdentifier), layerIDPrefix) {
|
||||
fileSystemID = strings.TrimPrefix(string(f.FileSPDXIdentifier), layerIDPrefix)
|
||||
}
|
||||
return file.Coordinates{
|
||||
RealPath: f.FileName,
|
||||
FileSystemID: fileSystemID,
|
||||
}
|
||||
return coordinatesFromFile(f.FileName, f.FileComment, string(f.FileSPDXIdentifier))
|
||||
}
|
||||
|
||||
func toSyftLocation(f *spdx.File) file.Location {
|
||||
l := file.NewVirtualLocationFromCoordinates(toSyftCoordinates(f), f.FileName)
|
||||
return l
|
||||
func coordinatesFromFile(name, comment, id string) file.Coordinates {
|
||||
// the scanned root
|
||||
if name == "." {
|
||||
name = ""
|
||||
}
|
||||
c := file.NewCoordinates(name, "")
|
||||
if v, ok := strings.CutPrefix(comment, layerIDCommentPrefix); ok {
|
||||
c.FileSystemID = v
|
||||
}
|
||||
if v, ok := strings.CutPrefix(id, layerIDCommentPrefix); ok {
|
||||
c.FileSystemID = v
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func requireAndTrimPrefix(val any, prefix string) string {
|
||||
|
||||
@@ -2,6 +2,7 @@ package spdxhelpers
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/packageurl-go"
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
@@ -795,3 +797,24 @@ func Test_skipsPackagesWithGeneratedFromRelationship(t *testing.T) {
|
||||
assert.NotNil(t, s.Artifacts.Packages.Package("1"))
|
||||
assert.Nil(t, s.Artifacts.Packages.Package("1-src"))
|
||||
}
|
||||
|
||||
func Test_nestedFiles_longChainIsCapped(t *testing.T) {
|
||||
// a document can claim any chain; it decodes no deeper than a scan goes, without recursing per level, and
|
||||
// what is past that is dropped
|
||||
n := newNestedFiles[int]()
|
||||
const length = 100_000
|
||||
for i := range length {
|
||||
n.own[i] = file.NewCoordinates("/a.jar", "")
|
||||
if i > 0 {
|
||||
n.container[i] = i - 1
|
||||
}
|
||||
}
|
||||
|
||||
for i := length - 1; i >= 0; i-- {
|
||||
c, ok := n.coordinates(i)
|
||||
assert.Equal(t, i <= archive.MaxDepth, ok, "%d", i)
|
||||
if ok {
|
||||
assert.Equal(t, "/a.jar"+strings.Repeat("!/a.jar", i), c.FullNestedPath())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +31,9 @@ func ToSyftModelV3(doc *spdx.Document) (*sbom.SBOM, error) {
|
||||
|
||||
spdxMap := ptrMap[any]{}
|
||||
|
||||
// found before the root package and its relationships are removed
|
||||
sourceFile := v3findSourceFile(doc)
|
||||
|
||||
s := &sbom.SBOM{
|
||||
Source: v3extractSource(spdxMap, doc),
|
||||
Artifacts: sbom.Artifacts{
|
||||
@@ -45,7 +48,7 @@ func ToSyftModelV3(doc *spdx.Document) (*sbom.SBOM, error) {
|
||||
|
||||
v3collectSyftPackages(s, spdxMap, relationships, doc)
|
||||
|
||||
v3collectSyftFiles(s, spdxMap, doc)
|
||||
v3collectSyftFiles(s, spdxMap, doc, sourceFile)
|
||||
|
||||
s.Relationships = v3toSyftRelationships(spdxMap, doc)
|
||||
|
||||
@@ -282,21 +285,87 @@ func v3collectSyftPackages(s *sbom.SBOM, spdxMap ptrMap[any], relationships ptrM
|
||||
}
|
||||
}
|
||||
|
||||
func v3collectSyftFiles(s *sbom.SBOM, spdxMap ptrMap[any], doc *spdx.Document) {
|
||||
found := ptrMap[struct{}]{}
|
||||
// v3findSourceFile returns the file the root package DESCRIBES when it CONTAINS files: an archive scanned as the
|
||||
// source is the scanned root, so what it contains is at the root rather than within an archive.
|
||||
func v3findSourceFile(doc *spdx.Document) spdx.AnyFile {
|
||||
roots := v3findRootPackages(doc)
|
||||
if len(roots) != 1 {
|
||||
return nil
|
||||
}
|
||||
var described spdx.AnyFile
|
||||
containers := ptrMap[bool]{}
|
||||
for _, r := range doc.Elements.Relationships() {
|
||||
from := r.GetFrom()
|
||||
if from == nil {
|
||||
continue
|
||||
}
|
||||
switch r.GetType() {
|
||||
case spdx.RelationshipType_Describes:
|
||||
if from == spdx.AnyElement(roots[0]) {
|
||||
for _, to := range r.GetTo() {
|
||||
if f, ok := to.(spdx.AnyFile); ok {
|
||||
described = f
|
||||
}
|
||||
}
|
||||
}
|
||||
case spdx.RelationshipType_Contains:
|
||||
containers.Set(from, true)
|
||||
}
|
||||
}
|
||||
if described == nil || !containers.Get(described) {
|
||||
return nil
|
||||
}
|
||||
return described
|
||||
}
|
||||
|
||||
func v3collectSyftFiles(s *sbom.SBOM, spdxMap ptrMap[any], doc *spdx.Document, sourceFile spdx.AnyFile) {
|
||||
nested := newNestedFiles[reflect.Value]()
|
||||
var files []spdx.AnyFile
|
||||
for _, elementList := range []spdx.ElementList{doc.Elements, doc.RootElements} {
|
||||
for _, f := range elementList.Files() {
|
||||
if found.Has(f) {
|
||||
if _, found := nested.own[ptrTo(f)]; found {
|
||||
continue
|
||||
}
|
||||
found.Set(f, struct{}{})
|
||||
l := v3toSyftLocation(f)
|
||||
spdxMap.Set(f, l)
|
||||
|
||||
s.Artifacts.FileMetadata[l.Coordinates] = v3toFileMetadata(f)
|
||||
s.Artifacts.FileDigests[l.Coordinates] = v3toFileDigests(f)
|
||||
nested.own[ptrTo(f)] = v3toSyftCoordinates(f)
|
||||
files = append(files, f)
|
||||
}
|
||||
}
|
||||
if sourceFile != nil {
|
||||
if c, ok := nested.own[ptrTo(sourceFile)]; ok {
|
||||
// the scanned root
|
||||
c.RealPath = ""
|
||||
nested.own[ptrTo(sourceFile)] = c
|
||||
}
|
||||
}
|
||||
for _, r := range doc.Elements.Relationships() {
|
||||
container, ok := r.GetFrom().(spdx.AnyFile)
|
||||
if !ok || r.GetType() != spdx.RelationshipType_Contains {
|
||||
continue
|
||||
}
|
||||
for _, to := range r.GetTo() {
|
||||
entry, ok := to.(spdx.AnyFile)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
_, knownContainer := nested.own[ptrTo(container)]
|
||||
_, knownEntry := nested.own[ptrTo(entry)]
|
||||
if knownContainer && knownEntry && !isScanRoot(nested.own[ptrTo(container)]) {
|
||||
nested.container[ptrTo(entry)] = ptrTo(container)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
c, ok := nested.coordinates(ptrTo(f))
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
l := file.NewLocationFromCoordinates(c)
|
||||
spdxMap.Set(f, l)
|
||||
|
||||
s.Artifacts.FileMetadata[l.Coordinates] = v3toFileMetadata(f)
|
||||
s.Artifacts.FileDigests[l.Coordinates] = v3toFileDigests(f)
|
||||
}
|
||||
}
|
||||
|
||||
func v3toFileDigests(f spdx.AnyFile) (digests []file.Digest) {
|
||||
@@ -341,6 +410,12 @@ func v3collectDocRelationships(spdxMap ptrMap[any], doc *spdx.Document) (out []a
|
||||
|
||||
for _, to := range r.GetTo() {
|
||||
b := spdxMap.Get(to)
|
||||
if r.GetType() == spdx.RelationshipType_Contains {
|
||||
if rel, ok := fileContainsRelationship(helpers.ContainsRelationship, a, b); ok {
|
||||
out = append(out, rel)
|
||||
continue
|
||||
}
|
||||
}
|
||||
from, fromOk := a.(pkg.Package)
|
||||
toPackage, toPackageOk := b.(pkg.Package)
|
||||
toLocation, toLocationOk := b.(file.Location)
|
||||
@@ -393,23 +468,7 @@ func v3collectDocRelationships(spdxMap ptrMap[any], doc *spdx.Document) (out []a
|
||||
}
|
||||
|
||||
func v3toSyftCoordinates(f spdx.AnyFile) file.Coordinates {
|
||||
const layerIDPrefix = "layerID: "
|
||||
var fileSystemID string
|
||||
if strings.HasPrefix(f.GetComment(), layerIDPrefix) {
|
||||
fileSystemID = strings.TrimPrefix(f.GetComment(), layerIDPrefix)
|
||||
}
|
||||
if strings.HasPrefix(f.GetID(), layerIDPrefix) {
|
||||
fileSystemID = strings.TrimPrefix(f.GetID(), layerIDPrefix)
|
||||
}
|
||||
return file.Coordinates{
|
||||
RealPath: f.GetName(),
|
||||
FileSystemID: fileSystemID,
|
||||
}
|
||||
}
|
||||
|
||||
func v3toSyftLocation(f spdx.AnyFile) file.Location {
|
||||
l := file.NewVirtualLocationFromCoordinates(v3toSyftCoordinates(f), f.GetName())
|
||||
return l
|
||||
return coordinatesFromFile(f.GetName(), f.GetComment(), f.GetID())
|
||||
}
|
||||
|
||||
func v3findQualifierValue(purl packageurl.PackageURL, qualifier string) string {
|
||||
|
||||
@@ -193,14 +193,14 @@ func TestToSyftModelV3_documentConversion(t *testing.T) {
|
||||
assert.Equal(t, "BSD-3-Clause AND GPL-2.0+", pkg2Concluded[0].Value)
|
||||
|
||||
// Verify files
|
||||
coords1 := file.Coordinates{RealPath: "/src/main.go"}
|
||||
coords1 := file.NewCoordinates("/src/main.go", "")
|
||||
digests, ok := result.Artifacts.FileDigests[coords1]
|
||||
require.True(t, ok, "file1 digests not found")
|
||||
require.Len(t, digests, 1)
|
||||
assert.Equal(t, "sha256", digests[0].Algorithm)
|
||||
assert.Equal(t, "abc123", digests[0].Value)
|
||||
|
||||
coords2 := file.Coordinates{RealPath: "/src/util.go"}
|
||||
coords2 := file.NewCoordinates("/src/util.go", "")
|
||||
_, ok = result.Artifacts.FileMetadata[coords2]
|
||||
assert.True(t, ok, "file2 metadata not found")
|
||||
|
||||
@@ -434,3 +434,44 @@ func Test_v3parseSPDXLicenses(t *testing.T) {
|
||||
require.Len(t, declared, 1)
|
||||
assert.Equal(t, "Apache-2.0 AND BSD-3-Clause", declared[0].Value)
|
||||
}
|
||||
|
||||
func TestToSyftModelV3_nestedArchiveFiles(t *testing.T) {
|
||||
war := &spdx.File{ID: "war", Name: "/app.war", Comment: "layerID: sha256:layer"}
|
||||
jar := &spdx.File{ID: "jar", Name: "WEB-INF/lib/a.jar"}
|
||||
manifest := &spdx.File{ID: "manifest", Name: "META-INF/MANIFEST.MF"}
|
||||
p := &spdx.Package{ID: "pkg", Name: "a"}
|
||||
warContainsJar := &spdx.Relationship{From: war, To: spdx.ElementList{jar}, Type: spdx.RelationshipType_Contains}
|
||||
jarContains := &spdx.Relationship{From: jar, To: spdx.ElementList{manifest, p}, Type: spdx.RelationshipType_Contains}
|
||||
|
||||
doc := &spdx.Document{SpdxDocument: spdx.SpdxDocument{
|
||||
Elements: spdx.ElementList{war, jar, manifest, p, warContainsJar, jarContains},
|
||||
}}
|
||||
|
||||
s, err := ToSyftModelV3(doc)
|
||||
require.NoError(t, err)
|
||||
|
||||
wantWar := file.NewCoordinates("/app.war", "sha256:layer")
|
||||
wantJar := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar", FileSystemID: "sha256:layer"}
|
||||
wantManifest := file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF", FileSystemID: "sha256:layer"}
|
||||
var got []file.Coordinates
|
||||
for c := range s.Artifacts.FileMetadata {
|
||||
got = append(got, c)
|
||||
}
|
||||
assert.ElementsMatch(t, []file.Coordinates{wantWar, wantJar, wantManifest}, got)
|
||||
|
||||
var fileRels []string
|
||||
for _, r := range s.Relationships {
|
||||
if from, ok := r.From.(file.Coordinates); ok {
|
||||
to := "package"
|
||||
if c, ok := r.To.(file.Coordinates); ok {
|
||||
to = c.FullNestedPath()
|
||||
}
|
||||
fileRels = append(fileRels, from.FullNestedPath()+" "+string(r.Type)+" "+to)
|
||||
}
|
||||
}
|
||||
assert.ElementsMatch(t, []string{
|
||||
"/app.war contains /app.war!/WEB-INF/lib/a.jar",
|
||||
"/app.war!/WEB-INF/lib/a.jar contains /app.war!/WEB-INF/lib/a.jar!/META-INF/MANIFEST.MF",
|
||||
"/app.war!/WEB-INF/lib/a.jar contains package",
|
||||
}, fileRels)
|
||||
}
|
||||
|
||||
@@ -114,6 +114,45 @@ func TestCycloneDxImageEncoder(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
func TestCycloneDxSourcePackageEncoder(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
testutil.AssertEncoderAgainstGoldenSnapshot(t,
|
||||
testutil.EncoderSnapshotTestConfig{
|
||||
Subject: testutil.SourcePackagesInput(t, dir),
|
||||
Format: getEncoder(t),
|
||||
UpdateSnapshot: *updateSnapshot,
|
||||
PersistRedactionsInSnapshot: true,
|
||||
IsJSON: true,
|
||||
Redactor: redactor(dir),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func TestCycloneDxSourcePackageEncoder_SpecVersions(t *testing.T) {
|
||||
s := testutil.SourcePackagesInput(t, t.TempDir())
|
||||
tests := []struct {
|
||||
version string
|
||||
want bool
|
||||
}{
|
||||
{version: "1.4", want: false},
|
||||
{version: "1.5", want: false},
|
||||
{version: "1.6", want: true},
|
||||
{version: cyclonedxutil.DefaultVersion, want: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.version, func(t *testing.T) {
|
||||
enc, err := NewFormatEncoderWithConfig(EncoderConfig{Version: tt.version})
|
||||
require.NoError(t, err)
|
||||
|
||||
var buf bytes.Buffer
|
||||
require.NoError(t, enc.Encode(&buf, s))
|
||||
|
||||
// the source-distribution reference type only exists from spec 1.6
|
||||
assert.Equal(t, tt.want, strings.Contains(buf.String(), `"source-distribution"`))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func redactor(values ...string) testutil.Redactor {
|
||||
return testutil.NewRedactions().
|
||||
WithValuesRedacted(values...).
|
||||
|
||||
+275
@@ -0,0 +1,275 @@
|
||||
{
|
||||
"$schema": "http://cyclonedx.org/schema/bom-1.7.schema.json",
|
||||
"bomFormat": "CycloneDX",
|
||||
"specVersion": "1.7",
|
||||
"serialNumber": "urn:uuid:redacted",
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"timestamp": "timestamp:redacted",
|
||||
"tools": {
|
||||
"components": [
|
||||
{
|
||||
"type": "application",
|
||||
"author": "anchore",
|
||||
"name": "syft",
|
||||
"version": "v0.42.0-bogus"
|
||||
}
|
||||
]
|
||||
},
|
||||
"component": {
|
||||
"bom-ref": "redacted",
|
||||
"type": "file",
|
||||
"name": "some/path"
|
||||
}
|
||||
},
|
||||
"components": [
|
||||
{
|
||||
"bom-ref": "pkg:deb/debian/attr@1%3A2.4.47-2%2Bb1?arch=amd64&distro=debian-12&package-id=5cc0e8979d7f5115&upstream=attr%401%3A2.4.47-2",
|
||||
"type": "library",
|
||||
"name": "attr",
|
||||
"version": "1:2.4.47-2+b1",
|
||||
"purl": "pkg:deb/debian/attr@1%3A2.4.47-2%2Bb1?arch=amd64&distro=debian-12&upstream=attr%401%3A2.4.47-2",
|
||||
"externalReferences": [
|
||||
{
|
||||
"url": "pkg:deb/debian/attr@1%3A2.4.47-2?arch=source&distro=debian-12",
|
||||
"type": "source-distribution"
|
||||
}
|
||||
],
|
||||
"properties": [
|
||||
{
|
||||
"name": "syft:package:foundBy",
|
||||
"value": "dpkg-db-cataloger"
|
||||
},
|
||||
{
|
||||
"name": "syft:package:type",
|
||||
"value": "deb"
|
||||
},
|
||||
{
|
||||
"name": "syft:package:metadataType",
|
||||
"value": "dpkg-db-entry"
|
||||
},
|
||||
{
|
||||
"name": "syft:location:0:path",
|
||||
"value": "/var/lib/dpkg/status"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:installedSize",
|
||||
"value": "0"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:source",
|
||||
"value": "attr"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:sourceVersion",
|
||||
"value": "1:2.4.47-2"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"bom-ref": "pkg:rpm/centos/bind-export-libs@9.11.13-3.el8?arch=x86_64&distro=centos-8&epoch=32&package-id=6008fca05ca7b418&upstream=bind-9.11.13-3.el8.src.rpm",
|
||||
"type": "library",
|
||||
"name": "bind-export-libs",
|
||||
"version": "32:9.11.13-3.el8",
|
||||
"purl": "pkg:rpm/centos/bind-export-libs@9.11.13-3.el8?arch=x86_64&distro=centos-8&epoch=32&upstream=bind-9.11.13-3.el8.src.rpm",
|
||||
"externalReferences": [
|
||||
{
|
||||
"url": "pkg:rpm/centos/bind@9.11.13-3.el8?arch=src&distro=centos-8&epoch=32",
|
||||
"type": "source-distribution"
|
||||
}
|
||||
],
|
||||
"properties": [
|
||||
{
|
||||
"name": "syft:package:foundBy",
|
||||
"value": "rpm-db-cataloger"
|
||||
},
|
||||
{
|
||||
"name": "syft:package:type",
|
||||
"value": "rpm"
|
||||
},
|
||||
{
|
||||
"name": "syft:package:metadataType",
|
||||
"value": "rpm-db-entry"
|
||||
},
|
||||
{
|
||||
"name": "syft:location:0:path",
|
||||
"value": "/var/lib/rpm/Packages"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:epoch",
|
||||
"value": "32"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:release",
|
||||
"value": "3.el8"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:size",
|
||||
"value": "0"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:sourceRpm",
|
||||
"value": "bind-9.11.13-3.el8.src.rpm"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"bom-ref": "pkg:apk/alpine/busybox@1.35.0-r17?arch=x86_64&distro=alpine-3.16.3&package-id=aa74af1190de1c40",
|
||||
"type": "library",
|
||||
"name": "busybox",
|
||||
"version": "1.35.0-r17",
|
||||
"purl": "pkg:apk/alpine/busybox@1.35.0-r17?arch=x86_64&distro=alpine-3.16.3",
|
||||
"properties": [
|
||||
{
|
||||
"name": "syft:package:foundBy",
|
||||
"value": "apk-db-cataloger"
|
||||
},
|
||||
{
|
||||
"name": "syft:package:type",
|
||||
"value": "apk"
|
||||
},
|
||||
{
|
||||
"name": "syft:package:metadataType",
|
||||
"value": "apk-db-entry"
|
||||
},
|
||||
{
|
||||
"name": "syft:location:0:path",
|
||||
"value": "/lib/apk/db/installed"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:installedSize",
|
||||
"value": "0"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:originPackage",
|
||||
"value": "busybox"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:size",
|
||||
"value": "0"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"bom-ref": "pkg:alpm/arch/gcc-libs@13.2.1-3?arch=x86_64&distro=arch-rolling&package-id=58a2fcf749a9a3cc&upstream=gcc",
|
||||
"type": "library",
|
||||
"name": "gcc-libs",
|
||||
"version": "13.2.1-3",
|
||||
"purl": "pkg:alpm/arch/gcc-libs@13.2.1-3?arch=x86_64&distro=arch-rolling&upstream=gcc",
|
||||
"properties": [
|
||||
{
|
||||
"name": "syft:package:foundBy",
|
||||
"value": "alpm-db-cataloger"
|
||||
},
|
||||
{
|
||||
"name": "syft:package:type",
|
||||
"value": "alpm"
|
||||
},
|
||||
{
|
||||
"name": "syft:package:metadataType",
|
||||
"value": "alpm-db-entry"
|
||||
},
|
||||
{
|
||||
"name": "syft:location:0:path",
|
||||
"value": "/var/lib/pacman/local/gcc-libs-13.2.1-3/desc"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:architecture",
|
||||
"value": "x86_64"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:basepackage",
|
||||
"value": "gcc"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:package",
|
||||
"value": "gcc-libs"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:size",
|
||||
"value": "0"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:version",
|
||||
"value": "13.2.1-3"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"bom-ref": "pkg:apk/alpine/libc-utils@0.7.2-r3?arch=x86_64&distro=alpine-3.16.3&package-id=16a8c7b19fde6f4f&upstream=libc-dev",
|
||||
"type": "library",
|
||||
"name": "libc-utils",
|
||||
"version": "0.7.2-r3",
|
||||
"purl": "pkg:apk/alpine/libc-utils@0.7.2-r3?arch=x86_64&distro=alpine-3.16.3&upstream=libc-dev",
|
||||
"properties": [
|
||||
{
|
||||
"name": "syft:package:foundBy",
|
||||
"value": "apk-db-cataloger"
|
||||
},
|
||||
{
|
||||
"name": "syft:package:type",
|
||||
"value": "apk"
|
||||
},
|
||||
{
|
||||
"name": "syft:package:metadataType",
|
||||
"value": "apk-db-entry"
|
||||
},
|
||||
{
|
||||
"name": "syft:location:0:path",
|
||||
"value": "/lib/apk/db/installed"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:installedSize",
|
||||
"value": "0"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:originPackage",
|
||||
"value": "libc-dev"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:size",
|
||||
"value": "0"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"bom-ref": "pkg:deb/debian/libpam-runtime@1.5.2-6%2Bdeb12u1?arch=all&distro=debian-12&package-id=7b4ddc97df57aec4&upstream=pam",
|
||||
"type": "library",
|
||||
"name": "libpam-runtime",
|
||||
"version": "1.5.2-6+deb12u1",
|
||||
"purl": "pkg:deb/debian/libpam-runtime@1.5.2-6%2Bdeb12u1?arch=all&distro=debian-12&upstream=pam",
|
||||
"externalReferences": [
|
||||
{
|
||||
"url": "pkg:deb/debian/pam@1.5.2-6%2Bdeb12u1?arch=source&distro=debian-12",
|
||||
"type": "source-distribution"
|
||||
}
|
||||
],
|
||||
"properties": [
|
||||
{
|
||||
"name": "syft:package:foundBy",
|
||||
"value": "dpkg-db-cataloger"
|
||||
},
|
||||
{
|
||||
"name": "syft:package:type",
|
||||
"value": "deb"
|
||||
},
|
||||
{
|
||||
"name": "syft:package:metadataType",
|
||||
"value": "dpkg-db-entry"
|
||||
},
|
||||
{
|
||||
"name": "syft:location:0:path",
|
||||
"value": "/var/lib/dpkg/status"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:installedSize",
|
||||
"value": "0"
|
||||
},
|
||||
{
|
||||
"name": "syft:metadata:source",
|
||||
"value": "pam"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -87,6 +87,20 @@ func TestCycloneDxImageEncoder(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
func TestCycloneDxSourcePackageEncoder(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
testutil.AssertEncoderAgainstGoldenSnapshot(t,
|
||||
testutil.EncoderSnapshotTestConfig{
|
||||
Subject: testutil.SourcePackagesInput(t, dir),
|
||||
Format: getEncoder(t),
|
||||
UpdateSnapshot: *updateSnapshot,
|
||||
PersistRedactionsInSnapshot: true,
|
||||
IsJSON: false,
|
||||
Redactor: redactor(dir),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func redactor(values ...string) testutil.Redactor {
|
||||
return testutil.NewRedactions().
|
||||
WithValuesRedacted(values...).
|
||||
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<bom xmlns="http://cyclonedx.org/schema/bom/1.7" serialNumber="redacted" version="1">
|
||||
<metadata>
|
||||
<timestamp>redacted</timestamp>
|
||||
<tools>
|
||||
<components>
|
||||
<component type="application">
|
||||
<author>anchore</author>
|
||||
<name>syft</name>
|
||||
<version>v0.42.0-bogus</version>
|
||||
</component>
|
||||
</components>
|
||||
</tools>
|
||||
<component bom-ref="redacted" type="file">
|
||||
<name>some/path</name>
|
||||
</component>
|
||||
</metadata>
|
||||
<components>
|
||||
<component bom-ref="pkg:deb/debian/attr@1%3A2.4.47-2%2Bb1?arch=amd64&distro=debian-12&package-id=5cc0e8979d7f5115&upstream=attr%401%3A2.4.47-2" type="library">
|
||||
<name>attr</name>
|
||||
<version>1:2.4.47-2+b1</version>
|
||||
<purl>pkg:deb/debian/attr@1%3A2.4.47-2%2Bb1?arch=amd64&distro=debian-12&upstream=attr%401%3A2.4.47-2</purl>
|
||||
<externalReferences>
|
||||
<reference type="source-distribution">
|
||||
<url>pkg:deb/debian/attr@1%3A2.4.47-2?arch=source&distro=debian-12</url>
|
||||
</reference>
|
||||
</externalReferences>
|
||||
<properties>
|
||||
<property name="syft:package:foundBy">dpkg-db-cataloger</property>
|
||||
<property name="syft:package:type">deb</property>
|
||||
<property name="syft:package:metadataType">dpkg-db-entry</property>
|
||||
<property name="syft:location:0:path">/var/lib/dpkg/status</property>
|
||||
<property name="syft:metadata:installedSize">0</property>
|
||||
<property name="syft:metadata:source">attr</property>
|
||||
<property name="syft:metadata:sourceVersion">1:2.4.47-2</property>
|
||||
</properties>
|
||||
</component>
|
||||
<component bom-ref="pkg:rpm/centos/bind-export-libs@9.11.13-3.el8?arch=x86_64&distro=centos-8&epoch=32&package-id=6008fca05ca7b418&upstream=bind-9.11.13-3.el8.src.rpm" type="library">
|
||||
<name>bind-export-libs</name>
|
||||
<version>32:9.11.13-3.el8</version>
|
||||
<purl>pkg:rpm/centos/bind-export-libs@9.11.13-3.el8?arch=x86_64&distro=centos-8&epoch=32&upstream=bind-9.11.13-3.el8.src.rpm</purl>
|
||||
<externalReferences>
|
||||
<reference type="source-distribution">
|
||||
<url>pkg:rpm/centos/bind@9.11.13-3.el8?arch=src&distro=centos-8&epoch=32</url>
|
||||
</reference>
|
||||
</externalReferences>
|
||||
<properties>
|
||||
<property name="syft:package:foundBy">rpm-db-cataloger</property>
|
||||
<property name="syft:package:type">rpm</property>
|
||||
<property name="syft:package:metadataType">rpm-db-entry</property>
|
||||
<property name="syft:location:0:path">/var/lib/rpm/Packages</property>
|
||||
<property name="syft:metadata:epoch">32</property>
|
||||
<property name="syft:metadata:release">3.el8</property>
|
||||
<property name="syft:metadata:size">0</property>
|
||||
<property name="syft:metadata:sourceRpm">bind-9.11.13-3.el8.src.rpm</property>
|
||||
</properties>
|
||||
</component>
|
||||
<component bom-ref="pkg:apk/alpine/busybox@1.35.0-r17?arch=x86_64&distro=alpine-3.16.3&package-id=aa74af1190de1c40" type="library">
|
||||
<name>busybox</name>
|
||||
<version>1.35.0-r17</version>
|
||||
<purl>pkg:apk/alpine/busybox@1.35.0-r17?arch=x86_64&distro=alpine-3.16.3</purl>
|
||||
<properties>
|
||||
<property name="syft:package:foundBy">apk-db-cataloger</property>
|
||||
<property name="syft:package:type">apk</property>
|
||||
<property name="syft:package:metadataType">apk-db-entry</property>
|
||||
<property name="syft:location:0:path">/lib/apk/db/installed</property>
|
||||
<property name="syft:metadata:installedSize">0</property>
|
||||
<property name="syft:metadata:originPackage">busybox</property>
|
||||
<property name="syft:metadata:size">0</property>
|
||||
</properties>
|
||||
</component>
|
||||
<component bom-ref="pkg:alpm/arch/gcc-libs@13.2.1-3?arch=x86_64&distro=arch-rolling&package-id=58a2fcf749a9a3cc&upstream=gcc" type="library">
|
||||
<name>gcc-libs</name>
|
||||
<version>13.2.1-3</version>
|
||||
<purl>pkg:alpm/arch/gcc-libs@13.2.1-3?arch=x86_64&distro=arch-rolling&upstream=gcc</purl>
|
||||
<properties>
|
||||
<property name="syft:package:foundBy">alpm-db-cataloger</property>
|
||||
<property name="syft:package:type">alpm</property>
|
||||
<property name="syft:package:metadataType">alpm-db-entry</property>
|
||||
<property name="syft:location:0:path">/var/lib/pacman/local/gcc-libs-13.2.1-3/desc</property>
|
||||
<property name="syft:metadata:architecture">x86_64</property>
|
||||
<property name="syft:metadata:basepackage">gcc</property>
|
||||
<property name="syft:metadata:package">gcc-libs</property>
|
||||
<property name="syft:metadata:size">0</property>
|
||||
<property name="syft:metadata:version">13.2.1-3</property>
|
||||
</properties>
|
||||
</component>
|
||||
<component bom-ref="pkg:apk/alpine/libc-utils@0.7.2-r3?arch=x86_64&distro=alpine-3.16.3&package-id=16a8c7b19fde6f4f&upstream=libc-dev" type="library">
|
||||
<name>libc-utils</name>
|
||||
<version>0.7.2-r3</version>
|
||||
<purl>pkg:apk/alpine/libc-utils@0.7.2-r3?arch=x86_64&distro=alpine-3.16.3&upstream=libc-dev</purl>
|
||||
<properties>
|
||||
<property name="syft:package:foundBy">apk-db-cataloger</property>
|
||||
<property name="syft:package:type">apk</property>
|
||||
<property name="syft:package:metadataType">apk-db-entry</property>
|
||||
<property name="syft:location:0:path">/lib/apk/db/installed</property>
|
||||
<property name="syft:metadata:installedSize">0</property>
|
||||
<property name="syft:metadata:originPackage">libc-dev</property>
|
||||
<property name="syft:metadata:size">0</property>
|
||||
</properties>
|
||||
</component>
|
||||
<component bom-ref="pkg:deb/debian/libpam-runtime@1.5.2-6%2Bdeb12u1?arch=all&distro=debian-12&package-id=7b4ddc97df57aec4&upstream=pam" type="library">
|
||||
<name>libpam-runtime</name>
|
||||
<version>1.5.2-6+deb12u1</version>
|
||||
<purl>pkg:deb/debian/libpam-runtime@1.5.2-6%2Bdeb12u1?arch=all&distro=debian-12&upstream=pam</purl>
|
||||
<externalReferences>
|
||||
<reference type="source-distribution">
|
||||
<url>pkg:deb/debian/pam@1.5.2-6%2Bdeb12u1?arch=source&distro=debian-12</url>
|
||||
</reference>
|
||||
</externalReferences>
|
||||
<properties>
|
||||
<property name="syft:package:foundBy">dpkg-db-cataloger</property>
|
||||
<property name="syft:package:type">deb</property>
|
||||
<property name="syft:package:metadataType">dpkg-db-entry</property>
|
||||
<property name="syft:location:0:path">/var/lib/dpkg/status</property>
|
||||
<property name="syft:metadata:installedSize">0</property>
|
||||
<property name="syft:metadata:source">pam</property>
|
||||
</properties>
|
||||
</component>
|
||||
</components>
|
||||
</bom>
|
||||
@@ -113,11 +113,7 @@ func toPath(s source.Description, p pkg.Package) string {
|
||||
locations := p.Locations.ToSlice()
|
||||
if len(locations) > 0 {
|
||||
location := locations[0]
|
||||
packagePath := location.RealPath
|
||||
if location.AccessPath != "" {
|
||||
packagePath = location.AccessPath
|
||||
}
|
||||
packagePath = strings.TrimPrefix(packagePath, "/")
|
||||
packagePath := strings.TrimPrefix(location.Path(), "/")
|
||||
switch metadata := s.Metadata.(type) {
|
||||
case source.ImageMetadata:
|
||||
image := strings.ReplaceAll(metadata.UserInput, ":/", "//")
|
||||
|
||||
@@ -41,30 +41,21 @@ func sbomFixture() sbom.SBOM {
|
||||
Name: "pkg-1",
|
||||
Version: "1.0.1",
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocationFromCoordinates(file.Coordinates{
|
||||
RealPath: "/usr/lib",
|
||||
FileSystemID: "fsid-1",
|
||||
}),
|
||||
file.NewLocationFromCoordinates(file.NewCoordinates("/usr/lib", "fsid-1")),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "pkg-2",
|
||||
Version: "2.0.2",
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocationFromCoordinates(file.Coordinates{
|
||||
RealPath: "/usr/lib",
|
||||
FileSystemID: "fsid-1",
|
||||
}),
|
||||
file.NewLocationFromCoordinates(file.NewCoordinates("/usr/lib", "fsid-1")),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "pkg-3",
|
||||
Version: "3.0.3",
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocationFromCoordinates(file.Coordinates{
|
||||
RealPath: "/etc",
|
||||
FileSystemID: "fsid-1",
|
||||
}),
|
||||
file.NewLocationFromCoordinates(file.NewCoordinates("/etc", "fsid-1")),
|
||||
),
|
||||
},
|
||||
} {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user