mirror of
https://github.com/anchore/syft.git
synced 2026-11-04 16:36:40 +01:00
Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
Keith Zantow
@@ -227,7 +227,6 @@ func getSource(ctx context.Context, opts *options.Catalog, userInput string, sou
|
||||
Paths: opts.Exclusions,
|
||||
}).
|
||||
WithBasePath(opts.Source.BasePath).
|
||||
WithMaxArchiveRecursiveIndexDepth(opts.Source.Directory.MaxArchiveRecursiveIndexDepth).
|
||||
WithSources(sources...).
|
||||
WithDefaultImagePullSource(opts.Source.Image.DefaultPullSource)
|
||||
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
package options
|
||||
|
||||
import (
|
||||
"github.com/anchore/clio"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
)
|
||||
|
||||
type archiveConfig struct {
|
||||
MaxDepth int `yaml:"max-depth" json:"max-depth" mapstructure:"max-depth"`
|
||||
MaxExtractionSizeBytes int64 `yaml:"max-extraction-size-bytes" json:"max-extraction-size-bytes" mapstructure:"max-extraction-size-bytes"`
|
||||
MaxFileCount int `yaml:"max-file-count" json:"max-file-count" mapstructure:"max-file-count"`
|
||||
MaxTotalExtractionBytes int64 `yaml:"max-total-extraction-bytes" json:"max-total-extraction-bytes" mapstructure:"max-total-extraction-bytes"`
|
||||
ExcludeExtensions []string `yaml:"exclude-extensions" json:"exclude-extensions" mapstructure:"exclude-extensions"`
|
||||
}
|
||||
|
||||
var _ interface {
|
||||
clio.FieldDescriber
|
||||
} = (*archiveConfig)(nil)
|
||||
|
||||
func defaultArchiveConfig() archiveConfig {
|
||||
return archiveConfig{
|
||||
MaxDepth: cataloging.DefaultArchiveMaxDepth,
|
||||
MaxExtractionSizeBytes: cataloging.DefaultArchiveMaxExtractionSizeBytes,
|
||||
MaxFileCount: cataloging.DefaultArchiveMaxFileCount,
|
||||
MaxTotalExtractionBytes: cataloging.DefaultArchiveMaxTotalExtractionBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (o *archiveConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
|
||||
descriptions.Add(&o.MaxDepth, `maximum depth of recursive archive extraction (0 = disabled, no recursive extraction)
|
||||
enabling this allows syft to discover packages inside nested archives (e.g., a tar.gz containing Python packages)`)
|
||||
descriptions.Add(&o.MaxExtractionSizeBytes, `maximum total bytes to extract from a single archive before stopping`)
|
||||
descriptions.Add(&o.MaxFileCount, `maximum number of files to extract from a single archive before stopping`)
|
||||
descriptions.Add(&o.MaxTotalExtractionBytes, `maximum total bytes to extract across all archives before stopping`)
|
||||
descriptions.Add(&o.ExcludeExtensions, `archive file extensions to skip during recursive extraction (e.g., [".rpm", ".deb"])`)
|
||||
}
|
||||
@@ -37,7 +37,6 @@ type Catalog struct {
|
||||
Package packageConfig `yaml:"package" json:"package" mapstructure:"package"`
|
||||
License licenseConfig `yaml:"license" json:"license" mapstructure:"license"`
|
||||
File fileConfig `yaml:"file" json:"file" mapstructure:"file"`
|
||||
Archive archiveConfig `yaml:"archive" json:"archive" mapstructure:"archive"`
|
||||
Scope string `yaml:"scope" json:"scope" mapstructure:"scope"`
|
||||
Parallelism int `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"` // the number of catalog workers to run in parallel
|
||||
Relationships relationshipsConfig `yaml:"relationships" json:"relationships" mapstructure:"relationships"`
|
||||
@@ -77,7 +76,6 @@ func DefaultCatalog() Catalog {
|
||||
Scope: source.SquashedScope.String(),
|
||||
Package: defaultPackageConfig(),
|
||||
License: defaultLicenseConfig(),
|
||||
Archive: defaultArchiveConfig(),
|
||||
LinuxKernel: defaultLinuxKernelConfig(),
|
||||
JavaScript: defaultJavaScriptConfig(),
|
||||
Python: defaultPythonConfig(),
|
||||
@@ -104,8 +102,6 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
|
||||
WithPackagesConfig(cfg.ToPackagesConfig()).
|
||||
WithLicenseConfig(cfg.ToLicenseConfig()).
|
||||
WithFilesConfig(cfg.ToFilesConfig()).
|
||||
WithArchiveConfig(cfg.ToArchiveConfig()).
|
||||
WithExclusions(cfg.Exclusions).
|
||||
WithCatalogerSelection(
|
||||
cataloging.NewSelectionRequest().
|
||||
WithDefaults(cfg.DefaultCatalogers...).
|
||||
@@ -113,18 +109,6 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
|
||||
)
|
||||
}
|
||||
|
||||
func (cfg Catalog) ToArchiveConfig() cataloging.ArchiveSearchConfig {
|
||||
return cataloging.ArchiveSearchConfig{
|
||||
IncludeIndexedArchives: cfg.Package.SearchIndexedArchives,
|
||||
IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
|
||||
MaxDepth: cfg.Archive.MaxDepth,
|
||||
MaxExtractionSizeBytes: cfg.Archive.MaxExtractionSizeBytes,
|
||||
MaxFileCount: cfg.Archive.MaxFileCount,
|
||||
MaxTotalExtractionBytes: cfg.Archive.MaxTotalExtractionBytes,
|
||||
ExcludeExtensions: cfg.Archive.ExcludeExtensions,
|
||||
}
|
||||
}
|
||||
|
||||
func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig {
|
||||
return cataloging.SearchConfig{
|
||||
Scope: source.ParseScope(cfg.Scope),
|
||||
|
||||
@@ -14,25 +14,36 @@ import (
|
||||
)
|
||||
|
||||
type sourceConfig struct {
|
||||
Name string `json:"name" yaml:"name" mapstructure:"name"`
|
||||
Version string `json:"version" yaml:"version" mapstructure:"version"`
|
||||
Supplier string `json:"supplier" yaml:"supplier" mapstructure:"supplier"`
|
||||
Source string `json:"source" yaml:"source" mapstructure:"source"`
|
||||
BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths
|
||||
File fileSource `json:"file" yaml:"file" mapstructure:"file"`
|
||||
Directory directorySource `json:"directory" yaml:"directory" mapstructure:"directory"`
|
||||
Image imageSource `json:"image" yaml:"image" mapstructure:"image"`
|
||||
Name string `json:"name" yaml:"name" mapstructure:"name"`
|
||||
Version string `json:"version" yaml:"version" mapstructure:"version"`
|
||||
Supplier string `json:"supplier" yaml:"supplier" mapstructure:"supplier"`
|
||||
Source string `json:"source" yaml:"source" mapstructure:"source"`
|
||||
BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths
|
||||
File fileSource `json:"file" yaml:"file" mapstructure:"file"`
|
||||
Image imageSource `json:"image" yaml:"image" mapstructure:"image"`
|
||||
}
|
||||
|
||||
var _ clio.FieldDescriber = (*sourceConfig)(nil)
|
||||
type fileSource struct {
|
||||
Digests []string `json:"digests" yaml:"digests" mapstructure:"digests"`
|
||||
}
|
||||
|
||||
var _ interface {
|
||||
clio.FieldDescriber
|
||||
} = (*sourceConfig)(nil)
|
||||
|
||||
var _ clio.PostLoader = (*imageSource)(nil)
|
||||
|
||||
func (o *sourceConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
|
||||
descriptions.Add(&o.File.Digests, `the file digest algorithms to use on the scanned file (options: "md5", "sha1", "sha224", "sha256", "sha384", "sha512")`)
|
||||
descriptions.Add(&o.Directory.MaxArchiveRecursiveIndexDepth, `depth to recursively resolve archive parents, no limit if == -1`)
|
||||
descriptions.Add(&o.Image.DefaultPullSource, `allows users to specify which image source should be used to generate the sbom
|
||||
valid values are: registry, docker, podman`)
|
||||
}
|
||||
|
||||
type imageSource struct {
|
||||
DefaultPullSource string `json:"default-pull-source" yaml:"default-pull-source" mapstructure:"default-pull-source"`
|
||||
MaxLayerSize string `json:"max-layer-size" yaml:"max-layer-size" mapstructure:"max-layer-size"`
|
||||
}
|
||||
|
||||
func defaultSourceConfig() sourceConfig {
|
||||
var digests []string
|
||||
for _, alg := range sourceproviders.DefaultConfig().DigestAlgorithms {
|
||||
@@ -45,12 +56,6 @@ func defaultSourceConfig() sourceConfig {
|
||||
}
|
||||
}
|
||||
|
||||
var _ clio.PostLoader = (*fileSource)(nil)
|
||||
|
||||
type fileSource struct {
|
||||
Digests []string `json:"digests" yaml:"digests" mapstructure:"digests"`
|
||||
}
|
||||
|
||||
func (c *fileSource) PostLoad() error {
|
||||
digests := strset.New(c.Digests...).List()
|
||||
sort.Strings(digests)
|
||||
@@ -58,23 +63,6 @@ func (c *fileSource) PostLoad() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ clio.PostLoader = (*directorySource)(nil)
|
||||
|
||||
type directorySource struct {
|
||||
MaxArchiveRecursiveIndexDepth int `json:"max-archive-recursive-index-depth" yaml:"max-archive-recursive-index-depth" mapstructure:"max-archive-recursive-index-depth"`
|
||||
}
|
||||
|
||||
func (c *directorySource) PostLoad() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ clio.PostLoader = (*imageSource)(nil)
|
||||
|
||||
type imageSource struct {
|
||||
DefaultPullSource string `json:"default-pull-source" yaml:"default-pull-source" mapstructure:"default-pull-source"`
|
||||
MaxLayerSize string `json:"max-layer-size" yaml:"max-layer-size" mapstructure:"max-layer-size"`
|
||||
}
|
||||
|
||||
func (c *imageSource) PostLoad() error {
|
||||
if c.MaxLayerSize != "" {
|
||||
perFileReadLimit, err := humanize.ParseBytes(c.MaxLayerSize)
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
# Recursive Archive Cataloging — Design
|
||||
|
||||
Status: **approved, in implementation**
|
||||
Branch: `feat/recursive-archive-combined`
|
||||
|
||||
## Background
|
||||
|
||||
Syft can scan inside Java archives (JAR/WAR/EAR, including shaded JARs) but ignores the
|
||||
contents of generic archives (`.tar`, `.tar.gz`/`.tgz`, `.zip`, `.apk`, `.gem`, `.iso`, …)
|
||||
found in a scanned directory or image. Tracked by issues
|
||||
[#246](https://github.com/anchore/syft/issues/246),
|
||||
[#1379](https://github.com/anchore/syft/issues/1379),
|
||||
[#2407](https://github.com/anchore/syft/issues/2407).
|
||||
|
||||
Two community PRs attempted this and are **superseded** by this design:
|
||||
|
||||
- **#4044** (@Rupikz) — recursion at the `fileresolver` layer.
|
||||
- **#4761** (@toabctl) — a composite resolver + orchestrator wired into `CreateSBOM`.
|
||||
|
||||
Both author's commits are preserved in this branch's history for credit; their
|
||||
approaches are replaced for the reasons below.
|
||||
|
||||
## Why the prior approaches were rejected
|
||||
|
||||
Search in syft *is* an index lookup. `FilesByPath`/`FilesByGlob`
|
||||
(`syft/internal/fileresolver/filetree_resolver.go`) call
|
||||
`SearchContext.SearchByPath(...)` and then `r.Index.Get(ref)`; a file that is not in
|
||||
*that resolver's own* `filetree.Index` causes the lookup to error and the file is
|
||||
**silently skipped**. A composite/wrapped resolver that merges archive contents without
|
||||
merging a real index therefore excludes files from indexing and breaks search. Pushing
|
||||
recursion into the directory resolver also smears archive contents into the parent's
|
||||
tree/chroot and forces a signature change on every `NewFromDirectory` caller.
|
||||
|
||||
## Core principle
|
||||
|
||||
**Each archive is treated as its own fully-indexed, standalone filesystem** — exactly how
|
||||
syft already treats container image layers. Extract the archive to a temp directory and
|
||||
build a resolver with `fileresolver.NewFromDirectory(tempDir, …)`, which produces a
|
||||
complete `Tree` + `Index` + `SearchContext`. No resolver merging, no shared index.
|
||||
|
||||
## Design decisions (locked)
|
||||
|
||||
1. **Detection is content-based** via `internal/file.IdentifyArchive` (catches `.tgz`,
|
||||
renamed archives, etc.).
|
||||
2. **A new `archive-cataloger` Task** (`internal/task/`) drives
|
||||
detect → extract → run the cataloger sub-pipeline → recurse. It is added to the
|
||||
package-cataloger task group, **before** relationship/unknowns tasks.
|
||||
3. **Provenance is expressed only via file-level `CONTAINS` relationships** in the first
|
||||
iteration: `From` = the archive file's `Coordinates`, `To` = each nested file/package.
|
||||
No changes to `Location`/`Coordinates`/metadata schema.
|
||||
4. **`Coordinates` is never modified** — it must remain a comparable two-string struct
|
||||
(used as `map[file.Coordinates]…` keys throughout `sbom.Artifacts` and as the
|
||||
`LocationData` key in `LocationSet`).
|
||||
5. **Node identity uses a per-archive `FileSystemID`** (an *existing* `Coordinates`
|
||||
field — no struct change). The chain rule is:
|
||||
|
||||
```
|
||||
childFSID = parentFSID == "" ? archiveAccessPath
|
||||
: parentFSID + "/" + archiveAccessPath
|
||||
```
|
||||
|
||||
- Uses the archive Location's **AccessPath**.
|
||||
- For non-image sources the root FSID is empty, so the chain starts with the relative
|
||||
archive path (e.g. `some/path/to.zip/some/nested/path.tar.gz`). Image sources start
|
||||
from the layer digest (e.g. `<layerDigest>/some/path/to.zip/…`).
|
||||
- The `FiletreeResolver` carries a `fileSystemID` and stamps it onto every returned
|
||||
`Location`, the same pattern image-layer resolvers already use. This keeps
|
||||
same-named files in different archives from colliding in the coordinate-keyed tables,
|
||||
with no post-processing in catalogers.
|
||||
6. **Java is left as-is in phase 1.** The generic archive cataloger recurses only into
|
||||
**non-JAR** types; detection excludes the formats the Java cataloger owns
|
||||
(`.jar/.war/.ear/.par/.sar/.nar/.jpi/.hpi/.kar/.far/.lpkg/.rar/.zap`). Unifying JARs
|
||||
into the generic mechanism is deferred.
|
||||
7. **Disabled by default.** Opt-in via `--archive-max-depth` / `SYFT_ARCHIVE_MAX_DEPTH`
|
||||
(default `0`), plus the extraction safety limits (max extraction size, file count,
|
||||
total bytes) carried over from the prior work.
|
||||
|
||||
## Reused from the prior PRs
|
||||
|
||||
- `internal/archive/extractor.go` — archive extractor + zip-bomb / size-limit guards.
|
||||
- `syft/cataloging/archive_search.go` — the `ArchiveSearchConfig` (depth + limits).
|
||||
- `syft/pkg/cataloger/java/parse_pom_xml.go` — skip `META-INF/maven/*/pom.xml` phantom
|
||||
packages.
|
||||
|
||||
## Implementation phases
|
||||
|
||||
| Phase | Work |
|
||||
|------|------|
|
||||
| 0 | Remove PR-4044's resolver recursion and PR-4761's composite/orchestrator; restore the 2-arg `NewFromDirectory`. Keep the items listed under "Reused" above. |
|
||||
| 1 | `FiletreeResolver.fileSystemID` + stamp on all Location-returning call sites; add `NewFromDirectoryWithFS(root, base, fileSystemID, …)` (existing `NewFromDirectory` delegates with `""`). |
|
||||
| 2 | `extractToResolver(...)` — extract to a temp dir and build a standalone indexed resolver with the computed `childFSID`. |
|
||||
| 3 | The `archive-cataloger` Task — detect (non-JAR) archives, run the package/file cataloger sub-pipeline into a scratch builder, merge into the shared builder, emit `CONTAINS` edges, and recurse to `MaxDepth`. |
|
||||
| 4 | Wire the task into `makeTaskGroups`; add `--archive-max-depth` (default 0) and limits. |
|
||||
| 5 | Tests: FSID stamping, extractor limits/zip-slip, nested-fixture FSID/relationship/dedup assertions, CLI integration. |
|
||||
@@ -126,7 +126,6 @@ require (
|
||||
github.com/ProtonMail/go-crypto v1.4.0 // indirect
|
||||
github.com/STARRY-S/zip v0.2.3 // indirect
|
||||
github.com/agext/levenshtein v1.2.1 // indirect
|
||||
github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51 // indirect
|
||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||
github.com/anchore/go-struct-converter v0.1.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
@@ -205,7 +204,6 @@ require (
|
||||
github.com/go-restruct/restruct v1.2.0-alpha // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||
@@ -251,7 +249,6 @@ require (
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/nwaples/rardecode v1.1.3 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
|
||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
|
||||
github.com/olekukonko/errors v1.2.0 // indirect
|
||||
|
||||
@@ -120,8 +120,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51 h1:yhk+P8lF3ZiROjmaVRao9WGTRo4b/wYjoKEiAHWrKwc=
|
||||
github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51/go.mod h1:nwuGSd7aZp0rtYt79YggCGafz1RYsclE7pi3fhLwvuw=
|
||||
github.com/anchore/bubbly v0.2.0 h1:VE6nLlWsYV+bTOqqCzS46NirrVmvHah3UQDtFcjzgHU=
|
||||
github.com/anchore/bubbly v0.2.0/go.mod h1:zRg1mfRCCe4vq1t9Ibu24YurErmnp9LAVCbpnpsb1pQ=
|
||||
github.com/anchore/clio v0.1.0 h1:lrlSgD38V0T6TLjOG5Kn3miEXCQyYr6ZlwTSnW1WIJg=
|
||||
@@ -495,8 +493,6 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
@@ -761,8 +757,6 @@ github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 h1:kpt9ZfKcm+
|
||||
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1/go.mod h1:qgCw4bBKZX8qMgGeEZzGFVT3notl42dBjNqO2jut0M0=
|
||||
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE=
|
||||
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8=
|
||||
github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc=
|
||||
github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
|
||||
github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A=
|
||||
github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
|
||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc=
|
||||
|
||||
@@ -1,524 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
)
|
||||
|
||||
// TestIntegration_TarGzContainingPythonPackage tests the scenario where a tar.gz
|
||||
// archive contains Python package metadata files that would normally be found by
|
||||
// the Python cataloger. This verifies the orchestrator makes those files visible.
|
||||
func TestIntegration_TarGzContainingPythonPackage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a tar.gz containing Python dist-info metadata
|
||||
tarPath := filepath.Join(dir, "python-packages.tar.gz")
|
||||
createTarGzWithFiles(t, tarPath, map[string]string{
|
||||
"lib/python3.9/site-packages/requests-2.28.0.dist-info/METADATA": `Metadata-Version: 2.1
|
||||
Name: requests
|
||||
Version: 2.28.0
|
||||
Summary: Python HTTP for Humans.
|
||||
`,
|
||||
"lib/python3.9/site-packages/requests-2.28.0.dist-info/RECORD": "requests/__init__.py,sha256=abc,1234",
|
||||
"lib/python3.9/site-packages/requests/__init__.py": "# requests library",
|
||||
})
|
||||
tarContent, err := os.ReadFile(tarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Base resolver sees the tar.gz as a single opaque file
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/app/python-packages.tar.gz": string(tarContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeUnindexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Extract the archive
|
||||
count := orch.DiscoverAndExtract(ctx, 0)
|
||||
require.Equal(t, 1, count, "should extract the tar.gz")
|
||||
|
||||
// The Python METADATA file should now be visible through the composite resolver
|
||||
resolver := orch.Resolver()
|
||||
locs, err := resolver.FilesByPath("/lib/python3.9/site-packages/requests-2.28.0.dist-info/METADATA")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1, "METADATA file should be visible through the composite resolver")
|
||||
|
||||
// Verify the access path shows it came from the archive
|
||||
assert.Contains(t, locs[0].AccessPath, "python-packages.tar.gz:")
|
||||
assert.Contains(t, locs[0].AccessPath, "METADATA")
|
||||
|
||||
// Verify we can read the content
|
||||
reader, err := resolver.FileContentsByLocation(locs[0])
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
|
||||
buf := make([]byte, 1024)
|
||||
n, _ := reader.Read(buf)
|
||||
assert.Contains(t, string(buf[:n]), "Name: requests")
|
||||
}
|
||||
|
||||
// TestIntegration_ZipContainingJar tests a zip archive containing a JAR file,
|
||||
// which itself should be extractable in a second round.
|
||||
func TestIntegration_ZipContainingJar(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an inner JAR (which is just a zip) with a MANIFEST.MF
|
||||
innerJarPath := filepath.Join(dir, "inner.jar")
|
||||
createZipWithFiles(t, innerJarPath, map[string]string{
|
||||
"META-INF/MANIFEST.MF": `Manifest-Version: 1.0
|
||||
Implementation-Title: example-lib
|
||||
Implementation-Version: 1.2.3
|
||||
`,
|
||||
})
|
||||
innerJarContent, err := os.ReadFile(innerJarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create an outer zip containing the JAR and other files
|
||||
outerZipPath := filepath.Join(dir, "app-bundle.zip")
|
||||
createZipWithFiles(t, outerZipPath, map[string]string{
|
||||
"lib/example-lib-1.2.3.jar": string(innerJarContent),
|
||||
"README.md": "# Application Bundle",
|
||||
})
|
||||
outerZipContent, err := os.ReadFile(outerZipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/downloads/app-bundle.zip": string(outerZipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 2
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Round 1: extract outer zip
|
||||
count1 := orch.DiscoverAndExtract(ctx, 0)
|
||||
require.Equal(t, 1, count1, "should extract outer zip")
|
||||
|
||||
// Round 2: extract inner JAR found inside the zip
|
||||
count2 := orch.DiscoverAndExtract(ctx, 1)
|
||||
require.Equal(t, 1, count2, "should extract inner JAR")
|
||||
|
||||
resolver := orch.Resolver()
|
||||
|
||||
// The MANIFEST.MF from inside the JAR should be visible
|
||||
locs, err := resolver.FilesByPath("/META-INF/MANIFEST.MF")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1, "MANIFEST.MF should be visible from nested JAR")
|
||||
|
||||
// The access path should show the full nesting chain
|
||||
accessPath := locs[0].AccessPath
|
||||
assert.Contains(t, accessPath, "app-bundle.zip:")
|
||||
assert.Contains(t, accessPath, "example-lib-1.2.3.jar:")
|
||||
assert.Contains(t, accessPath, "MANIFEST.MF")
|
||||
|
||||
// Verify relationships
|
||||
rels := orch.Relationships()
|
||||
assert.Len(t, rels, 2, "should have relationships for both extracted archives")
|
||||
}
|
||||
|
||||
// TestIntegration_MixedArchiveTypes tests a directory containing both
|
||||
// zip and tar.gz archives that should be extracted in a single round.
|
||||
func TestIntegration_MixedArchiveTypes(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a zip
|
||||
zipPath := createTestZip(t, dir, map[string]string{
|
||||
"from-zip.txt": "zip content",
|
||||
})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a tar.gz
|
||||
tarPath := createTestTarGz(t, dir, map[string]string{
|
||||
"from-tar.txt": "tar content",
|
||||
})
|
||||
tarContent, err := os.ReadFile(tarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/data/archive.zip": string(zipContent),
|
||||
"/data/archive.tar.gz": string(tarContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.IncludeUnindexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
count := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 2, count, "should extract both archives")
|
||||
|
||||
resolver := orch.Resolver()
|
||||
|
||||
// Both files should be visible
|
||||
zipLocs, err := resolver.FilesByPath("/from-zip.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, zipLocs, 1, "file from zip should be visible")
|
||||
|
||||
tarLocs, err := resolver.FilesByPath("/from-tar.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, tarLocs, 1, "file from tar should be visible")
|
||||
}
|
||||
|
||||
// TestIntegration_DepthLimitPreventsDeepExtraction verifies that the
|
||||
// depth limit correctly stops extraction at the configured level.
|
||||
func TestIntegration_DepthLimitPreventsDeepExtraction(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a deeply nested archive: zip -> zip -> zip -> file
|
||||
innermostDir := filepath.Join(dir, "inner")
|
||||
require.NoError(t, os.MkdirAll(innermostDir, 0o755))
|
||||
innermost := createTestZip(t, innermostDir, map[string]string{
|
||||
"secret.txt": "you found me!",
|
||||
})
|
||||
innermostContent, err := os.ReadFile(innermost)
|
||||
require.NoError(t, err)
|
||||
|
||||
middleDir := filepath.Join(dir, "middle")
|
||||
require.NoError(t, os.MkdirAll(middleDir, 0o755))
|
||||
middle := createTestZip(t, middleDir, map[string]string{
|
||||
"innermost.zip": string(innermostContent),
|
||||
})
|
||||
middleContent, err := os.ReadFile(middle)
|
||||
require.NoError(t, err)
|
||||
|
||||
outerDir := filepath.Join(dir, "outer")
|
||||
require.NoError(t, os.MkdirAll(outerDir, 0o755))
|
||||
outerZip := createTestZip(t, outerDir, map[string]string{
|
||||
"middle.zip": string(middleContent),
|
||||
})
|
||||
outerContent, err := os.ReadFile(outerZip)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/nested.zip": string(outerContent),
|
||||
})
|
||||
|
||||
// Only allow depth 1 — should only extract the outer zip
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
count := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 1, count, "should only extract the outer zip")
|
||||
|
||||
// middle.zip should be visible but secret.txt should NOT be
|
||||
resolver := orch.Resolver()
|
||||
|
||||
middleLocs, err := resolver.FilesByPath("/middle.zip")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, middleLocs, 1, "middle.zip should be visible at depth 1")
|
||||
|
||||
secretLocs, err := resolver.FilesByPath("/secret.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, secretLocs, 0, "secret.txt should NOT be visible (depth limit)")
|
||||
}
|
||||
|
||||
// TestIntegration_ExcludeExtensionPreventsExtraction verifies that
|
||||
// excluded extensions are respected.
|
||||
func TestIntegration_ExcludeExtensionPreventsExtraction(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{
|
||||
"data.txt": "from zip",
|
||||
})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/archive.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.ExcludeExtensions = []string{".zip"} // exclude zip files
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
count := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, count, "zip should be excluded by extension")
|
||||
}
|
||||
|
||||
// TestIntegration_MultiLevelMixedFormats tests a 3-level nesting chain with
|
||||
// different archive formats: tar.gz -> zip -> tar.gz -> files.
|
||||
// This is the most realistic cross-format recursive extraction scenario.
|
||||
func TestIntegration_MultiLevelMixedFormats(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Level 3 (deepest): a tar.gz containing application files
|
||||
level3Dir := filepath.Join(dir, "level3")
|
||||
require.NoError(t, os.MkdirAll(level3Dir, 0o755))
|
||||
level3Path := filepath.Join(level3Dir, "app-data.tar.gz")
|
||||
createTarGzWithFiles(t, level3Path, map[string]string{
|
||||
"config.yaml": "app: myapp\nversion: 1.0",
|
||||
"bin/run.sh": "#!/bin/sh\necho hello",
|
||||
})
|
||||
level3Content, err := os.ReadFile(level3Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Level 2: a zip containing the level3 tar.gz and some metadata
|
||||
level2Dir := filepath.Join(dir, "level2")
|
||||
require.NoError(t, os.MkdirAll(level2Dir, 0o755))
|
||||
level2Path := filepath.Join(level2Dir, "package.zip")
|
||||
createZipWithFiles(t, level2Path, map[string]string{
|
||||
"app-data.tar.gz": string(level3Content),
|
||||
"package-info.json": `{"name": "mypackage", "version": "2.0"}`,
|
||||
})
|
||||
level2Content, err := os.ReadFile(level2Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Level 1 (outermost): a tar.gz containing the level2 zip
|
||||
level1Dir := filepath.Join(dir, "level1")
|
||||
require.NoError(t, os.MkdirAll(level1Dir, 0o755))
|
||||
level1Path := filepath.Join(level1Dir, "distribution.tar.gz")
|
||||
createTarGzWithFiles(t, level1Path, map[string]string{
|
||||
"package.zip": string(level2Content),
|
||||
"LICENSE": "MIT License",
|
||||
})
|
||||
level1Content, err := os.ReadFile(level1Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/distribution.tar.gz": string(level1Content),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 3
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.IncludeUnindexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Round 1: extract distribution.tar.gz
|
||||
count1 := orch.DiscoverAndExtract(ctx, 0)
|
||||
require.Equal(t, 1, count1, "round 1: should extract distribution.tar.gz")
|
||||
|
||||
// Round 2: extract package.zip from inside distribution.tar.gz
|
||||
count2 := orch.DiscoverAndExtract(ctx, 1)
|
||||
require.Equal(t, 1, count2, "round 2: should extract package.zip")
|
||||
|
||||
// Round 3: extract app-data.tar.gz from inside package.zip
|
||||
count3 := orch.DiscoverAndExtract(ctx, 2)
|
||||
require.Equal(t, 1, count3, "round 3: should extract app-data.tar.gz")
|
||||
|
||||
resolver := orch.Resolver()
|
||||
|
||||
// Verify files from all levels are accessible:
|
||||
|
||||
// Level 1: LICENSE from distribution.tar.gz
|
||||
locs, err := resolver.FilesByPath("/LICENSE")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1, "LICENSE from level 1 should be visible")
|
||||
assert.Contains(t, locs[0].AccessPath, "distribution.tar.gz:")
|
||||
|
||||
// Level 2: package-info.json from package.zip
|
||||
locs, err = resolver.FilesByPath("/package-info.json")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1, "package-info.json from level 2 should be visible")
|
||||
assert.Contains(t, locs[0].AccessPath, "package.zip:")
|
||||
|
||||
// Level 3: config.yaml from app-data.tar.gz
|
||||
locs, err = resolver.FilesByPath("/config.yaml")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1, "config.yaml from level 3 should be visible")
|
||||
assert.Contains(t, locs[0].AccessPath, "app-data.tar.gz:")
|
||||
|
||||
// Level 3: bin/run.sh from app-data.tar.gz
|
||||
locs, err = resolver.FilesByPath("/bin/run.sh")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1, "bin/run.sh from level 3 should be visible")
|
||||
|
||||
// Verify we can read deeply nested content
|
||||
reader, err := resolver.FileContentsByLocation(locs[0])
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
buf := make([]byte, 1024)
|
||||
n, _ := reader.Read(buf)
|
||||
assert.Contains(t, string(buf[:n]), "echo hello")
|
||||
|
||||
// Verify relationships cover all 3 levels
|
||||
rels := orch.Relationships()
|
||||
assert.Len(t, rels, 3, "should have 3 relationships (one per extracted archive)")
|
||||
}
|
||||
|
||||
// TestIntegration_TarInsideZipInsideTar tests tar -> zip -> tar nesting
|
||||
// (the reverse of the above test), ensuring bidirectional format crossing.
|
||||
func TestIntegration_TarInsideZipInsideTar(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Innermost: a tar.gz with a single file
|
||||
innerTarDir := filepath.Join(dir, "innertar")
|
||||
require.NoError(t, os.MkdirAll(innerTarDir, 0o755))
|
||||
innerTarPath := filepath.Join(innerTarDir, "inner.tar.gz")
|
||||
createTarGzWithFiles(t, innerTarPath, map[string]string{
|
||||
"deeply-nested.txt": "found at the bottom",
|
||||
})
|
||||
innerTarContent, err := os.ReadFile(innerTarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Middle: a zip containing the inner tar
|
||||
middleZipDir := filepath.Join(dir, "middlezip")
|
||||
require.NoError(t, os.MkdirAll(middleZipDir, 0o755))
|
||||
middleZipPath := filepath.Join(middleZipDir, "middle.zip")
|
||||
createZipWithFiles(t, middleZipPath, map[string]string{
|
||||
"inner.tar.gz": string(innerTarContent),
|
||||
"middle.txt": "middle level",
|
||||
})
|
||||
middleZipContent, err := os.ReadFile(middleZipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Outermost: a tar.gz containing the middle zip
|
||||
outerTarDir := filepath.Join(dir, "outertar")
|
||||
require.NoError(t, os.MkdirAll(outerTarDir, 0o755))
|
||||
outerTarPath := filepath.Join(outerTarDir, "outer.tar.gz")
|
||||
createTarGzWithFiles(t, outerTarPath, map[string]string{
|
||||
"middle.zip": string(middleZipContent),
|
||||
"outer.txt": "outer level",
|
||||
})
|
||||
outerTarContent, err := os.ReadFile(outerTarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/outer.tar.gz": string(outerTarContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 3
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.IncludeUnindexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Extract all 3 levels
|
||||
count1 := orch.DiscoverAndExtract(ctx, 0)
|
||||
require.Equal(t, 1, count1)
|
||||
count2 := orch.DiscoverAndExtract(ctx, 1)
|
||||
require.Equal(t, 1, count2)
|
||||
count3 := orch.DiscoverAndExtract(ctx, 2)
|
||||
require.Equal(t, 1, count3)
|
||||
|
||||
resolver := orch.Resolver()
|
||||
|
||||
// Check all levels are accessible
|
||||
outerLocs, err := resolver.FilesByPath("/outer.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, outerLocs, 1)
|
||||
|
||||
middleLocs, err := resolver.FilesByPath("/middle.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, middleLocs, 1)
|
||||
|
||||
deepLocs, err := resolver.FilesByPath("/deeply-nested.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, deepLocs, 1)
|
||||
|
||||
// Read the deeply nested content
|
||||
reader, err := resolver.FileContentsByLocation(deepLocs[0])
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
buf := make([]byte, 1024)
|
||||
n, _ := reader.Read(buf)
|
||||
assert.Equal(t, "found at the bottom", string(buf[:n]))
|
||||
|
||||
// The access path should show the full chain: tar -> zip -> tar -> file
|
||||
accessPath := deepLocs[0].AccessPath
|
||||
assert.Contains(t, accessPath, "outer.tar.gz:")
|
||||
assert.Contains(t, accessPath, "middle.zip:")
|
||||
assert.Contains(t, accessPath, "inner.tar.gz:")
|
||||
assert.Contains(t, accessPath, "deeply-nested.txt")
|
||||
}
|
||||
|
||||
// createZipWithFiles creates a zip at the given path with the given files.
|
||||
func createZipWithFiles(t *testing.T, zipPath string, files map[string]string) {
|
||||
t.Helper()
|
||||
f, err := os.Create(zipPath)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
w := zip.NewWriter(f)
|
||||
for name, content := range files {
|
||||
fw, err := w.Create(name)
|
||||
require.NoError(t, err)
|
||||
_, err = fw.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, w.Close())
|
||||
}
|
||||
|
||||
// createTarGzWithFiles creates a tar.gz at the given path with the given files.
|
||||
func createTarGzWithFiles(t *testing.T, tarPath string, files map[string]string) {
|
||||
t.Helper()
|
||||
f, err := os.Create(tarPath)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
gw := gzip.NewWriter(f)
|
||||
tw := tar.NewWriter(gw)
|
||||
|
||||
// sort keys for deterministic output
|
||||
for name, content := range files {
|
||||
// create parent dirs as needed
|
||||
dir := filepath.Dir(name)
|
||||
if dir != "." {
|
||||
hdr := &tar.Header{
|
||||
Typeflag: tar.TypeDir,
|
||||
Name: dir + "/",
|
||||
Mode: 0o755,
|
||||
}
|
||||
require.NoError(t, tw.WriteHeader(hdr))
|
||||
}
|
||||
|
||||
hdr := &tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(content)),
|
||||
}
|
||||
require.NoError(t, tw.WriteHeader(hdr))
|
||||
_, err = tw.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
}
|
||||
|
||||
@@ -1,362 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// ResolverFactory creates a file.Resolver for a directory path.
|
||||
// This is injectable to avoid direct dependency on syft/internal/fileresolver.
|
||||
type ResolverFactory func(root string) (file.Resolver, error)
|
||||
|
||||
// Orchestrator manages iterative archive extraction and cataloging. It discovers
|
||||
// archives in the file resolver, extracts them, builds child resolvers for their
|
||||
// contents, and tracks the relationships between archives and their contents.
|
||||
type Orchestrator struct {
|
||||
composite *CompositeResolver
|
||||
extractors []Extractor
|
||||
config cataloging.ArchiveSearchConfig
|
||||
tmpDir string
|
||||
resolverFactory ResolverFactory
|
||||
|
||||
// tracking state
|
||||
processed map[processedKey]bool // archives we've already considered for extraction
|
||||
totalBytesWritten int64
|
||||
archiveNodes []*archiveNode // for relationship tracking
|
||||
cleanups []func()
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// archiveNode tracks an extracted archive for relationship generation.
|
||||
type archiveNode struct {
|
||||
location file.Location // where the archive is in the resolver
|
||||
fsID string // the filesystem ID assigned to this archive's contents
|
||||
depth int
|
||||
}
|
||||
|
||||
// processedKey identifies an archive location uniquely. We key the processed
|
||||
// map on a struct (not a "<fsID>:<realPath>" string) because both fields can
|
||||
// legally contain ':' — fsIDs are emitted as "archive:<hex>" and POSIX paths
|
||||
// permit colons — so a string key would have an ambiguous boundary.
|
||||
type processedKey struct {
|
||||
fsID string
|
||||
realPath string
|
||||
}
|
||||
|
||||
// NewOrchestrator creates an Orchestrator that manages archive extraction.
|
||||
// The resolverFactory is used to build file.Resolvers for extracted archive directories.
|
||||
func NewOrchestrator(baseResolver file.Resolver, cfg cataloging.ArchiveSearchConfig, tmpDir string, resolverFactory ResolverFactory) *Orchestrator {
|
||||
return &Orchestrator{
|
||||
composite: NewCompositeResolver(baseResolver),
|
||||
extractors: DefaultExtractors(),
|
||||
config: cfg,
|
||||
tmpDir: tmpDir,
|
||||
resolverFactory: resolverFactory,
|
||||
processed: make(map[processedKey]bool),
|
||||
}
|
||||
}
|
||||
|
||||
// Resolver returns the composite resolver that includes all extracted archive contents.
|
||||
func (o *Orchestrator) Resolver() *CompositeResolver {
|
||||
return o.composite
|
||||
}
|
||||
|
||||
// DiscoverAndExtract finds archives in the current resolver view, extracts them,
|
||||
// and adds their contents as child resolvers. Returns the number of new archives extracted.
|
||||
// The depth parameter indicates the current nesting depth (0 = scanning base filesystem).
|
||||
func (o *Orchestrator) DiscoverAndExtract(ctx context.Context, depth int) int {
|
||||
if depth >= o.config.MaxDepth {
|
||||
return 0
|
||||
}
|
||||
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
|
||||
if o.config.MaxTotalExtractionBytes > 0 && o.totalBytesWritten >= o.config.MaxTotalExtractionBytes {
|
||||
log.Debug("archive orchestrator: total extraction size limit reached, stopping")
|
||||
return 0
|
||||
}
|
||||
|
||||
// build the list of extractors actually permitted by config — every
|
||||
// non-archive location in the resolver is going to pay a peek read against
|
||||
// these, so excluding disabled types up-front avoids reading files we
|
||||
// would only end up rejecting after detection.
|
||||
enabled := o.enabledExtractors()
|
||||
if len(enabled) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
newArchives := 0
|
||||
|
||||
// scan all locations in the current resolver for archives
|
||||
for loc := range o.composite.AllLocations(ctx) {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
// skip already-processed archives
|
||||
key := processedKey{fsID: loc.FileSystemID, realPath: loc.RealPath}
|
||||
if o.processed[key] {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip excluded extensions
|
||||
if IsExcludedExtension(loc.RealPath, o.config.ExcludeExtensions) {
|
||||
continue
|
||||
}
|
||||
|
||||
// check if this location is an archive we can extract — peek a small
|
||||
// buffer rather than materializing the full file content, which would
|
||||
// be a full disk-to-disk copy of every non-archive file in the resolver.
|
||||
ext := o.detectFormat(ctx, loc, enabled)
|
||||
if ext == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// mark as processed before extraction to avoid re-processing
|
||||
o.processed[key] = true
|
||||
|
||||
// extract the archive
|
||||
extracted, err := o.extractArchive(ctx, ext, loc, depth+1)
|
||||
if err != nil {
|
||||
log.WithFields("archive", loc.Path(), "error", err).Debug("failed to extract archive")
|
||||
continue
|
||||
}
|
||||
if extracted {
|
||||
newArchives++
|
||||
}
|
||||
}
|
||||
|
||||
return newArchives
|
||||
}
|
||||
|
||||
// enabledExtractors returns the subset of o.extractors whose corresponding
|
||||
// config gate is on (IncludeIndexedArchives for ZipExtractor,
|
||||
// IncludeUnindexedArchives for TarExtractor).
|
||||
func (o *Orchestrator) enabledExtractors() []Extractor {
|
||||
var out []Extractor
|
||||
for _, ext := range o.extractors {
|
||||
switch ext.(type) {
|
||||
case *ZipExtractor:
|
||||
if o.config.IncludeIndexedArchives {
|
||||
out = append(out, ext)
|
||||
}
|
||||
case *TarExtractor:
|
||||
if o.config.IncludeUnindexedArchives {
|
||||
out = append(out, ext)
|
||||
}
|
||||
default:
|
||||
// unknown extractor types: include by default so plugin extractors
|
||||
// added in the future aren't silently disabled
|
||||
out = append(out, ext)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// extractArchive extracts a single archive and registers its contents as a child resolver.
|
||||
func (o *Orchestrator) extractArchive(ctx context.Context, ext Extractor, loc file.Location, depth int) (bool, error) {
|
||||
// create a temp dir for this archive's contents
|
||||
extractDir, err := os.MkdirTemp(o.tmpDir, fmt.Sprintf("archive-depth%d-*", depth))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("unable to create temp dir: %w", err)
|
||||
}
|
||||
o.cleanups = append(o.cleanups, func() {
|
||||
os.RemoveAll(extractDir)
|
||||
})
|
||||
|
||||
// extractors need a file path, so materialize the archive contents to a temp file
|
||||
tmpPath, err := o.copyArchiveToTempFile(loc)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
// calculate remaining budget
|
||||
limits := ExtractionLimits{
|
||||
MaxExtractionSizeBytes: o.config.MaxExtractionSizeBytes,
|
||||
MaxFileCount: o.config.MaxFileCount,
|
||||
}
|
||||
if o.config.MaxTotalExtractionBytes > 0 {
|
||||
remaining := o.config.MaxTotalExtractionBytes - o.totalBytesWritten
|
||||
if remaining <= 0 {
|
||||
return false, fmt.Errorf("total extraction budget exhausted")
|
||||
}
|
||||
if limits.MaxExtractionSizeBytes == 0 || remaining < limits.MaxExtractionSizeBytes {
|
||||
limits.MaxExtractionSizeBytes = remaining
|
||||
}
|
||||
}
|
||||
|
||||
result, err := ext.Extract(ctx, tmpPath, extractDir, limits)
|
||||
if err != nil {
|
||||
log.WithFields("archive", loc.Path(), "error", err).Debug("archive extraction stopped")
|
||||
// partial extraction is still useful, continue if we got some files
|
||||
if result.FilesExtracted == 0 {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
o.totalBytesWritten += result.BytesWritten
|
||||
|
||||
if result.FilesExtracted == 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// build a directory resolver for the extracted contents
|
||||
childResolver, resolverErr := o.resolverFactory(extractDir)
|
||||
if resolverErr != nil {
|
||||
return false, fmt.Errorf("unable to build resolver for extracted archive: %w", resolverErr)
|
||||
}
|
||||
|
||||
fsID := o.composite.AddChild(childResolver, loc, depth)
|
||||
|
||||
// track for relationship generation
|
||||
o.archiveNodes = append(o.archiveNodes, &archiveNode{
|
||||
location: loc,
|
||||
fsID: fsID,
|
||||
depth: depth,
|
||||
})
|
||||
|
||||
log.WithFields(
|
||||
"archive", loc.Path(),
|
||||
"depth", depth,
|
||||
"files", result.FilesExtracted,
|
||||
"bytes", result.BytesWritten,
|
||||
).Debug("extracted archive contents")
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// formatProbeBytes is the size of the buffer peeked from each candidate file
|
||||
// to decide whether it's an extractable archive.
|
||||
//
|
||||
// Single-step formats only need their magic bytes (ZIP: 4, gzip/bzip2/xz/zstd:
|
||||
// 2-6, tar: a 512-byte first-block header). The size that matters is for
|
||||
// compressed-tar without a recognized extension: there mholt/archives must
|
||||
// gzip-decode our peek buffer and feed the result to tar.NewReader.Next(),
|
||||
// which needs a full 512-byte tar header out the back. 4 KiB of compressed
|
||||
// input gives gzip a comfortable margin and is still a single filesystem
|
||||
// block read.
|
||||
const formatProbeBytes = 4096
|
||||
|
||||
// detectFormat returns the first enabled extractor that can handle loc by
|
||||
// peeking a small buffer of its contents (no full materialization). Returns
|
||||
// nil when no extractor matches or when the file can't be opened. The caller
|
||||
// is responsible for narrowing extractors to those permitted by config so we
|
||||
// don't pay peek I/O for archive types that would only be rejected later.
|
||||
func (o *Orchestrator) detectFormat(ctx context.Context, loc file.Location, extractors []Extractor) Extractor {
|
||||
reader, err := o.composite.FileContentsByLocation(loc)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer internal.CloseAndLogError(reader, loc.RealPath)
|
||||
|
||||
var peek [formatProbeBytes]byte
|
||||
n, _ := io.ReadFull(reader, peek[:])
|
||||
if n == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, ext := range extractors {
|
||||
if ext.CanExtract(ctx, loc.RealPath, bytes.NewReader(peek[:n])) {
|
||||
return ext
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyArchiveToTempFile reads the archive contents from the resolver and writes them
|
||||
// to a freshly-created temp file under o.tmpDir. The returned path must be removed
|
||||
// by the caller (typically via defer).
|
||||
func (o *Orchestrator) copyArchiveToTempFile(loc file.Location) (string, error) {
|
||||
reader, err := o.composite.FileContentsByLocation(loc)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to get archive contents: %w", err)
|
||||
}
|
||||
defer internal.CloseAndLogError(reader, loc.RealPath)
|
||||
|
||||
tmpFile, err := os.CreateTemp(o.tmpDir, "archive-src-*")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to create temp file: %w", err)
|
||||
}
|
||||
_, copyErr := safeIOCopy(tmpFile, reader)
|
||||
tmpFile.Close()
|
||||
if copyErr != nil {
|
||||
os.Remove(tmpFile.Name())
|
||||
return "", fmt.Errorf("unable to copy archive to temp: %w", copyErr)
|
||||
}
|
||||
return tmpFile.Name(), nil
|
||||
}
|
||||
|
||||
// Relationships returns the archive containment relationships discovered during extraction.
|
||||
func (o *Orchestrator) Relationships() []artifact.Relationship {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
|
||||
var rels []artifact.Relationship
|
||||
for _, node := range o.archiveNodes {
|
||||
// Create a "contains" relationship: the archive contains its extracted content
|
||||
// The archive file (identified by its coordinates) contains the child filesystem
|
||||
rels = append(rels, artifact.Relationship{
|
||||
From: node.location.Coordinates,
|
||||
To: file.Coordinates{RealPath: "/", FileSystemID: node.fsID},
|
||||
Type: artifact.ContainsRelationship,
|
||||
})
|
||||
}
|
||||
return rels
|
||||
}
|
||||
|
||||
// Cleanup removes all temporary directories created during extraction.
|
||||
func (o *Orchestrator) Cleanup() {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
for _, cleanup := range o.cleanups {
|
||||
cleanup()
|
||||
}
|
||||
o.cleanups = nil
|
||||
}
|
||||
|
||||
// safeIOCopy copies with a size limit to prevent decompression bombs.
|
||||
func safeIOCopy(dst *os.File, src interface{ Read([]byte) (int, error) }) (int64, error) {
|
||||
const maxCopySize = 2 * 1024 * 1024 * 1024 // 2GB
|
||||
return limitedCopy(dst, src, maxCopySize)
|
||||
}
|
||||
|
||||
func limitedCopy(dst *os.File, src interface{ Read([]byte) (int, error) }, limit int64) (int64, error) {
|
||||
n, err := copyN(dst, src, limit)
|
||||
if n >= limit {
|
||||
return n, fmt.Errorf("copy size limit reached (%d bytes)", limit)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func copyN(dst *os.File, src interface{ Read([]byte) (int, error) }, limit int64) (int64, error) {
|
||||
buf := make([]byte, 32*1024)
|
||||
var written int64
|
||||
for written < limit {
|
||||
nr, readErr := src.Read(buf)
|
||||
if nr > 0 {
|
||||
nw, writeErr := dst.Write(buf[:nr])
|
||||
written += int64(nw)
|
||||
if writeErr != nil {
|
||||
return written, writeErr
|
||||
}
|
||||
}
|
||||
if readErr != nil {
|
||||
if readErr == io.EOF {
|
||||
return written, nil
|
||||
}
|
||||
return written, readErr
|
||||
}
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
@@ -1,537 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// testResolverFactory creates a simple mock resolver factory for testing.
|
||||
func testResolverFactory(root string) (file.Resolver, error) {
|
||||
// Walk the directory and create a mock resolver from its contents
|
||||
files := make(map[string]string)
|
||||
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info.IsDir() {
|
||||
return err
|
||||
}
|
||||
relPath, _ := filepath.Rel(root, path)
|
||||
content, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
files["/"+relPath] = string(content)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newMockResolver(files), nil
|
||||
}
|
||||
|
||||
func TestOrchestrator_DiscoverAndExtract_ZipArchive(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a zip file in the "source" directory
|
||||
zipFiles := map[string]string{
|
||||
"inner.txt": "hello from inside the zip",
|
||||
"lib/a.so": "library content",
|
||||
}
|
||||
zipPath := createTestZip(t, dir, zipFiles)
|
||||
|
||||
// Create a parent resolver that sees the zip file
|
||||
parentFiles := map[string]string{
|
||||
"/test.zip": "placeholder", // the resolver sees the file
|
||||
}
|
||||
parent := newMockResolver(parentFiles)
|
||||
|
||||
// Override FileContentsByLocation to return actual zip content
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
parent.contents["/test.zip"] = string(zipContent)
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 1, newCount)
|
||||
|
||||
// the composite resolver should now have the child files
|
||||
resolver := orch.Resolver()
|
||||
assert.Equal(t, 1, resolver.ChildCount())
|
||||
}
|
||||
|
||||
func TestOrchestrator_DiscoverAndExtract_TarGzArchive(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
tarFiles := map[string]string{
|
||||
"inner.txt": "hello from tar",
|
||||
}
|
||||
tarPath := createTestTarGz(t, dir, tarFiles)
|
||||
|
||||
tarContent, err := os.ReadFile(tarPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.tar.gz": string(tarContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeUnindexedArchives = true // tar needs this enabled
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 1, newCount)
|
||||
assert.Equal(t, 1, orch.Resolver().ChildCount())
|
||||
}
|
||||
|
||||
func TestOrchestrator_DiscoverAndExtract_DepthLimit(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/file.txt": "just a file",
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// at depth 1 (== MaxDepth), should extract nothing
|
||||
newCount := orch.DiscoverAndExtract(ctx, 1)
|
||||
assert.Equal(t, 0, newCount)
|
||||
}
|
||||
|
||||
func TestOrchestrator_DiscoverAndExtract_NoArchives(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/file.txt": "plain text",
|
||||
"/data.csv": "a,b,c",
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 3
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, newCount)
|
||||
assert.Equal(t, 0, orch.Resolver().ChildCount())
|
||||
}
|
||||
|
||||
func TestOrchestrator_DiscoverAndExtract_ExcludeExtension(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.ExcludeExtensions = []string{".zip"}
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, newCount)
|
||||
}
|
||||
|
||||
func TestOrchestrator_DiscoverAndExtract_TotalSizeLimit(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.MaxTotalExtractionBytes = 1 // 1 byte total limit - will be exceeded
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// should still extract but hit the limit
|
||||
_ = orch.DiscoverAndExtract(ctx, 0)
|
||||
// second call should be blocked by total limit
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, newCount)
|
||||
}
|
||||
|
||||
func TestOrchestrator_Relationships(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
orch.DiscoverAndExtract(ctx, 0)
|
||||
|
||||
rels := orch.Relationships()
|
||||
assert.NotEmpty(t, rels)
|
||||
|
||||
// should have a contains relationship for the archive
|
||||
for _, rel := range rels {
|
||||
assert.Equal(t, "contains", string(rel.Type))
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrchestrator_Cleanup(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
|
||||
orch.DiscoverAndExtract(ctx, 0)
|
||||
|
||||
// verify temp dirs exist
|
||||
entries, err := os.ReadDir(tmpDir)
|
||||
require.NoError(t, err)
|
||||
initialCount := len(entries)
|
||||
assert.True(t, initialCount > 0, "expected temp dirs to be created")
|
||||
|
||||
orch.Cleanup()
|
||||
|
||||
// verify temp dirs are cleaned up
|
||||
entries, err = os.ReadDir(tmpDir)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, len(entries) < initialCount, "expected temp dirs to be cleaned up")
|
||||
}
|
||||
|
||||
func TestOrchestrator_DoesNotReprocess(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 2
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// first call extracts the archive
|
||||
first := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 1, first)
|
||||
|
||||
// second call should not re-extract the same archive
|
||||
second := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, second)
|
||||
assert.Equal(t, 1, orch.Resolver().ChildCount())
|
||||
}
|
||||
|
||||
func TestOrchestrator_NestedArchives_ZipInsideZip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create inner zip
|
||||
innerZipPath := createTestZip(t, dir, map[string]string{
|
||||
"deep.txt": "deeply nested content",
|
||||
})
|
||||
innerZipContent, err := os.ReadFile(innerZipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create outer zip containing the inner zip
|
||||
outerDir := filepath.Join(dir, "outer")
|
||||
require.NoError(t, os.MkdirAll(outerDir, 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(outerDir, "inner.zip"), innerZipContent, 0o644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(outerDir, "readme.txt"), []byte("readme"), 0o644))
|
||||
outerZipPath := createTestZip(t, outerDir, map[string]string{
|
||||
"inner.zip": string(innerZipContent),
|
||||
"readme.txt": "readme",
|
||||
})
|
||||
outerZipContent, err := os.ReadFile(outerZipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/outer.zip": string(outerZipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 3
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Round 1: extract outer.zip
|
||||
count1 := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 1, count1, "should extract outer.zip")
|
||||
assert.Equal(t, 1, orch.Resolver().ChildCount())
|
||||
|
||||
// Round 2: extract inner.zip found inside outer.zip
|
||||
count2 := orch.DiscoverAndExtract(ctx, 1)
|
||||
assert.Equal(t, 1, count2, "should extract inner.zip from within outer.zip")
|
||||
assert.Equal(t, 2, orch.Resolver().ChildCount())
|
||||
|
||||
// Round 3: no more archives to extract
|
||||
count3 := orch.DiscoverAndExtract(ctx, 2)
|
||||
assert.Equal(t, 0, count3, "no more archives to extract")
|
||||
}
|
||||
|
||||
func TestOrchestrator_ContextCancellation(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel immediately
|
||||
|
||||
// should handle gracefully without hanging
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.True(t, newCount >= 0) // may be 0 or 1 depending on timing
|
||||
}
|
||||
|
||||
func TestOrchestrator_MultipleArchivesSameLevel(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zip1Path := createTestZip(t, dir, map[string]string{"file1.txt": "from zip1"})
|
||||
zip1Content, err := os.ReadFile(zip1Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Need a different dir to avoid overwriting test.zip
|
||||
dir2 := t.TempDir()
|
||||
zip2Path := createTestZip(t, dir2, map[string]string{"file2.txt": "from zip2"})
|
||||
zip2Content, err := os.ReadFile(zip2Path)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/archive1.zip": string(zip1Content),
|
||||
"/archive2.zip": string(zip2Content),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 2, newCount, "should extract both archives")
|
||||
assert.Equal(t, 2, orch.Resolver().ChildCount())
|
||||
}
|
||||
|
||||
func TestOrchestrator_CorruptArchiveSkipped(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// A file with .zip extension but invalid content
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/corrupt.zip": "this is not a valid zip file",
|
||||
"/normal.txt": "just text",
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Should not panic or error out; corrupt archives should be skipped
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, newCount, "corrupt archive should be skipped")
|
||||
}
|
||||
|
||||
func TestOrchestrator_EmptyArchive(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create an empty zip
|
||||
emptyZipPath := createTestZip(t, dir, map[string]string{})
|
||||
emptyZipContent, err := os.ReadFile(emptyZipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/empty.zip": string(emptyZipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, newCount, "empty archive should not count as extracted")
|
||||
}
|
||||
|
||||
func TestOrchestrator_PerArchiveFileLimitApplied(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a zip with many files
|
||||
files := map[string]string{}
|
||||
for i := 0; i < 10; i++ {
|
||||
files[filepath.Join("dir", string(rune('a'+i))+".txt")] = "content"
|
||||
}
|
||||
zipPath := createTestZip(t, dir, files)
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/big.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
cfg.MaxFileCount = 3 // only allow 3 files to be extracted
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
// Should partially extract but still register the archive
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
// Extraction will fail after 3 files, but partial extraction counts
|
||||
assert.True(t, newCount >= 0)
|
||||
}
|
||||
|
||||
func TestOrchestrator_RelationshipsHaveArchiveCoordinates(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"file.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/archive.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = true
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
orch.DiscoverAndExtract(ctx, 0)
|
||||
|
||||
rels := orch.Relationships()
|
||||
require.Len(t, rels, 1)
|
||||
|
||||
// verify the relationship structure
|
||||
rel := rels[0]
|
||||
assert.Equal(t, "contains", string(rel.Type))
|
||||
|
||||
// From should be the archive's coordinates
|
||||
fromCoords, ok := rel.From.(file.Coordinates)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "/archive.zip", fromCoords.RealPath)
|
||||
}
|
||||
|
||||
func TestOrchestrator_ZipDisabledByConfig(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tmpDir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
zipPath := createTestZip(t, dir, map[string]string{"inner.txt": "data"})
|
||||
zipContent, err := os.ReadFile(zipPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/test.zip": string(zipContent),
|
||||
})
|
||||
|
||||
cfg := cataloging.DefaultArchiveSearchConfig()
|
||||
cfg.MaxDepth = 1
|
||||
cfg.IncludeIndexedArchives = false // disable zip extraction
|
||||
|
||||
orch := NewOrchestrator(parent, cfg, tmpDir, testResolverFactory)
|
||||
defer orch.Cleanup()
|
||||
|
||||
newCount := orch.DiscoverAndExtract(ctx, 0)
|
||||
assert.Equal(t, 0, newCount)
|
||||
}
|
||||
@@ -1,274 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
const (
|
||||
// ArchivePathSeparator is used to encode nested archive paths in AccessPath,
|
||||
// matching the convention established by the Java cataloger.
|
||||
ArchivePathSeparator = ":"
|
||||
)
|
||||
|
||||
// childResolver holds a resolver for an extracted archive along with metadata about the archive.
|
||||
type childResolver struct {
|
||||
resolver file.Resolver
|
||||
archiveLocation file.Location // where the archive file lives in the parent resolver
|
||||
fsID string // unique filesystem ID for this archive
|
||||
depth int // nesting depth (0 = base filesystem)
|
||||
}
|
||||
|
||||
// CompositeResolver implements file.Resolver by compositing a parent resolver with
|
||||
// child resolvers created from extracted archive contents. Files from child resolvers
|
||||
// appear transparently alongside files from the parent.
|
||||
type CompositeResolver struct {
|
||||
parent file.Resolver
|
||||
children []*childResolver
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
var _ file.Resolver = (*CompositeResolver)(nil)
|
||||
|
||||
// NewCompositeResolver creates a new CompositeResolver wrapping the given parent resolver.
|
||||
func NewCompositeResolver(parent file.Resolver) *CompositeResolver {
|
||||
return &CompositeResolver{
|
||||
parent: parent,
|
||||
}
|
||||
}
|
||||
|
||||
// AddChild registers a child resolver for an extracted archive.
|
||||
func (r *CompositeResolver) AddChild(resolver file.Resolver, archiveLocation file.Location, depth int) string {
|
||||
fsID := generateArchiveFSID(archiveLocation)
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.children = append(r.children, &childResolver{
|
||||
resolver: resolver,
|
||||
archiveLocation: archiveLocation,
|
||||
fsID: fsID,
|
||||
depth: depth,
|
||||
})
|
||||
return fsID
|
||||
}
|
||||
|
||||
// ChildCount returns the number of child resolvers registered.
|
||||
func (r *CompositeResolver) ChildCount() int {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return len(r.children)
|
||||
}
|
||||
|
||||
// HasPath checks if the path exists in the parent or any child resolver.
|
||||
func (r *CompositeResolver) HasPath(path string) bool {
|
||||
if r.parent.HasPath(path) {
|
||||
return true
|
||||
}
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
if child.resolver.HasPath(path) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FilesByPath returns locations from the parent and all child resolvers.
|
||||
func (r *CompositeResolver) FilesByPath(paths ...string) ([]file.Location, error) {
|
||||
var results []file.Location
|
||||
|
||||
parentLocs, err := r.parent.FilesByPath(paths...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, parentLocs...)
|
||||
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
childLocs, err := child.resolver.FilesByPath(paths...)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, loc := range childLocs {
|
||||
results = append(results, r.transformLocation(loc, child))
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// FilesByGlob returns locations from the parent and all child resolvers.
|
||||
func (r *CompositeResolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
|
||||
var results []file.Location
|
||||
|
||||
parentLocs, err := r.parent.FilesByGlob(patterns...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, parentLocs...)
|
||||
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
childLocs, err := child.resolver.FilesByGlob(patterns...)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, loc := range childLocs {
|
||||
results = append(results, r.transformLocation(loc, child))
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// FilesByMIMEType returns locations from the parent and all child resolvers.
|
||||
func (r *CompositeResolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
|
||||
var results []file.Location
|
||||
|
||||
parentLocs, err := r.parent.FilesByMIMEType(types...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, parentLocs...)
|
||||
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
childLocs, err := child.resolver.FilesByMIMEType(types...)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, loc := range childLocs {
|
||||
results = append(results, r.transformLocation(loc, child))
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// RelativeFileByPath looks up a file relative to a given location.
|
||||
func (r *CompositeResolver) RelativeFileByPath(location file.Location, path string) *file.Location {
|
||||
// try parent first
|
||||
if loc := r.parent.RelativeFileByPath(location, path); loc != nil {
|
||||
return loc
|
||||
}
|
||||
|
||||
// try children - match on the archive's FSID
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
if location.FileSystemID == child.fsID {
|
||||
if loc := child.resolver.RelativeFileByPath(location, path); loc != nil {
|
||||
transformed := r.transformLocation(*loc, child)
|
||||
return &transformed
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FileContentsByLocation routes to the appropriate resolver based on FileSystemID.
|
||||
func (r *CompositeResolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
|
||||
// check if this location belongs to a child resolver
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
if location.FileSystemID == child.fsID {
|
||||
// look up the file by path in the child resolver to get a location with
|
||||
// a valid internal reference (needed by directory resolvers that use
|
||||
// stereoscope file references for content lookup)
|
||||
childLocs, err := child.resolver.FilesByPath(location.RealPath)
|
||||
if err != nil || len(childLocs) == 0 {
|
||||
// fallback: try with a constructed location
|
||||
childLoc := file.NewLocation(location.RealPath)
|
||||
return child.resolver.FileContentsByLocation(childLoc)
|
||||
}
|
||||
return child.resolver.FileContentsByLocation(childLocs[0])
|
||||
}
|
||||
}
|
||||
|
||||
// fall through to parent
|
||||
return r.parent.FileContentsByLocation(location)
|
||||
}
|
||||
|
||||
// AllLocations returns locations from the parent and all child resolvers.
|
||||
func (r *CompositeResolver) AllLocations(ctx context.Context) <-chan file.Location {
|
||||
results := make(chan file.Location)
|
||||
go func() {
|
||||
defer close(results)
|
||||
|
||||
// parent locations
|
||||
for loc := range r.parent.AllLocations(ctx) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case results <- loc:
|
||||
}
|
||||
}
|
||||
|
||||
// child locations
|
||||
r.mu.RLock()
|
||||
children := make([]*childResolver, len(r.children))
|
||||
copy(children, r.children)
|
||||
r.mu.RUnlock()
|
||||
|
||||
for _, child := range children {
|
||||
for loc := range child.resolver.AllLocations(ctx) {
|
||||
transformed := r.transformLocation(loc, child)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case results <- transformed:
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return results
|
||||
}
|
||||
|
||||
// FileMetadataByLocation routes to the appropriate resolver based on FileSystemID.
|
||||
func (r *CompositeResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, child := range r.children {
|
||||
if location.FileSystemID == child.fsID {
|
||||
childLocs, err := child.resolver.FilesByPath(location.RealPath)
|
||||
if err != nil || len(childLocs) == 0 {
|
||||
childLoc := file.NewLocation(location.RealPath)
|
||||
return child.resolver.FileMetadataByLocation(childLoc)
|
||||
}
|
||||
return child.resolver.FileMetadataByLocation(childLocs[0])
|
||||
}
|
||||
}
|
||||
return r.parent.FileMetadataByLocation(location)
|
||||
}
|
||||
|
||||
// transformLocation transforms a location from a child resolver to include the archive context.
|
||||
func (r *CompositeResolver) transformLocation(loc file.Location, child *childResolver) file.Location {
|
||||
archivePath := child.archiveLocation.Path()
|
||||
return file.Location{
|
||||
LocationData: file.LocationData{
|
||||
Coordinates: file.Coordinates{
|
||||
RealPath: loc.RealPath,
|
||||
FileSystemID: child.fsID,
|
||||
},
|
||||
AccessPath: archivePath + ArchivePathSeparator + loc.Path(),
|
||||
},
|
||||
LocationMetadata: loc.LocationMetadata,
|
||||
}
|
||||
}
|
||||
|
||||
// generateArchiveFSID creates a deterministic filesystem ID for an archive based on its location.
|
||||
func generateArchiveFSID(archiveLocation file.Location) string {
|
||||
h := sha256.New()
|
||||
h.Write([]byte(archiveLocation.RealPath))
|
||||
h.Write([]byte(archiveLocation.FileSystemID))
|
||||
return fmt.Sprintf("archive:%x", h.Sum(nil)[:12])
|
||||
}
|
||||
@@ -1,584 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// mockResolver implements file.Resolver for testing.
|
||||
type mockResolver struct {
|
||||
locations map[string]file.Location // path -> location
|
||||
contents map[string]string // realPath -> content
|
||||
}
|
||||
|
||||
func newMockResolver(files map[string]string) *mockResolver {
|
||||
r := &mockResolver{
|
||||
locations: make(map[string]file.Location),
|
||||
contents: files,
|
||||
}
|
||||
for path := range files {
|
||||
r.locations[path] = file.NewLocation(path)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (m *mockResolver) HasPath(path string) bool {
|
||||
_, ok := m.locations[path]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (m *mockResolver) FilesByPath(paths ...string) ([]file.Location, error) {
|
||||
var results []file.Location
|
||||
for _, p := range paths {
|
||||
if loc, ok := m.locations[p]; ok {
|
||||
results = append(results, loc)
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (m *mockResolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
|
||||
var results []file.Location
|
||||
for _, pattern := range patterns {
|
||||
for path, loc := range m.locations {
|
||||
matched, _ := filepath.Match(pattern, path)
|
||||
if matched {
|
||||
results = append(results, loc)
|
||||
}
|
||||
// also try with the basename for simple glob patterns
|
||||
if !matched {
|
||||
matched, _ = filepath.Match(pattern, filepath.Base(path))
|
||||
if matched {
|
||||
results = append(results, loc)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (m *mockResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockResolver) RelativeFileByPath(_ file.Location, path string) *file.Location {
|
||||
if loc, ok := m.locations[path]; ok {
|
||||
return &loc
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockResolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
|
||||
if content, ok := m.contents[location.RealPath]; ok {
|
||||
return io.NopCloser(strings.NewReader(content)), nil
|
||||
}
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
|
||||
func (m *mockResolver) AllLocations(ctx context.Context) <-chan file.Location {
|
||||
ch := make(chan file.Location)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
for _, loc := range m.locations {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case ch <- loc:
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
func (m *mockResolver) FileMetadataByLocation(_ file.Location) (file.Metadata, error) {
|
||||
return file.Metadata{}, nil
|
||||
}
|
||||
|
||||
// strictMockResolver mimics real directory resolvers (fileresolver.Directory) which
|
||||
// require that FileContentsByLocation receives a location object that was originally
|
||||
// returned by FilesByPath/FilesByGlob — not a freshly constructed file.NewLocation().
|
||||
// Real directory resolvers need the internal stereoscope file.Reference for index lookup.
|
||||
// We simulate this by annotating returned locations and only accepting those back.
|
||||
type strictMockResolver struct {
|
||||
mockResolver
|
||||
resolvedAnnotation string // annotation key that marks locations as "resolved"
|
||||
}
|
||||
|
||||
func newStrictMockResolver(files map[string]string) *strictMockResolver {
|
||||
return &strictMockResolver{
|
||||
mockResolver: *newMockResolver(files),
|
||||
resolvedAnnotation: "strict-resolved",
|
||||
}
|
||||
}
|
||||
|
||||
func (s *strictMockResolver) annotate(loc file.Location) file.Location {
|
||||
return loc.WithAnnotation(s.resolvedAnnotation, "true")
|
||||
}
|
||||
|
||||
func (s *strictMockResolver) isResolved(loc file.Location) bool {
|
||||
return loc.Annotations[s.resolvedAnnotation] == "true"
|
||||
}
|
||||
|
||||
func (s *strictMockResolver) FilesByPath(paths ...string) ([]file.Location, error) {
|
||||
locs, err := s.mockResolver.FilesByPath(paths...)
|
||||
for i := range locs {
|
||||
locs[i] = s.annotate(locs[i])
|
||||
}
|
||||
return locs, err
|
||||
}
|
||||
|
||||
func (s *strictMockResolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
|
||||
locs, err := s.mockResolver.FilesByGlob(patterns...)
|
||||
for i := range locs {
|
||||
locs[i] = s.annotate(locs[i])
|
||||
}
|
||||
return locs, err
|
||||
}
|
||||
|
||||
func (s *strictMockResolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
|
||||
if !s.isResolved(location) {
|
||||
return nil, fmt.Errorf("location %q was not resolved via FilesByPath/FilesByGlob first (simulating real directory resolver behavior)", location.RealPath)
|
||||
}
|
||||
return s.mockResolver.FileContentsByLocation(location)
|
||||
}
|
||||
|
||||
func (s *strictMockResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
|
||||
if !s.isResolved(location) {
|
||||
return file.Metadata{}, fmt.Errorf("location %q was not resolved via FilesByPath/FilesByGlob first", location.RealPath)
|
||||
}
|
||||
return s.mockResolver.FileMetadataByLocation(location)
|
||||
}
|
||||
|
||||
// TestCompositeResolver_FileContentsByLocation_ChildRequiresLookup tests that
|
||||
// FileContentsByLocation works when the child resolver requires locations to be
|
||||
// resolved via FilesByPath first (as real directory resolvers do). This catches
|
||||
// the bug where we constructed a bare file.NewLocation() without a valid internal
|
||||
// reference, which fails on real fileresolver.Directory implementations.
|
||||
func TestCompositeResolver_FileContentsByLocation_ChildRequiresLookup(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{})
|
||||
child := newStrictMockResolver(map[string]string{
|
||||
"/lib/data.txt": "important data",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
archiveLoc := file.NewLocation("/archive.zip")
|
||||
fsID := composite.AddChild(child, archiveLoc, 1)
|
||||
|
||||
// Simulate what a cataloger does: find the file, then read its content.
|
||||
// The location returned has the archive's fsID, not the child's internal ref.
|
||||
locs, err := composite.FilesByPath("/lib/data.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, fsID, locs[0].FileSystemID)
|
||||
|
||||
// Now read content - this must work even though the location was transformed
|
||||
reader, err := composite.FileContentsByLocation(locs[0])
|
||||
require.NoError(t, err, "FileContentsByLocation should resolve the file in the child resolver via FilesByPath before reading")
|
||||
defer reader.Close()
|
||||
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "important data", string(content))
|
||||
}
|
||||
|
||||
// TestCompositeResolver_FileMetadataByLocation_ChildRequiresLookup is the same
|
||||
// test as above but for FileMetadataByLocation.
|
||||
func TestCompositeResolver_FileMetadataByLocation_ChildRequiresLookup(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{})
|
||||
child := newStrictMockResolver(map[string]string{
|
||||
"/lib/data.txt": "important data",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
archiveLoc := file.NewLocation("/archive.zip")
|
||||
fsID := composite.AddChild(child, archiveLoc, 1)
|
||||
|
||||
locs, err := composite.FilesByPath("/lib/data.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, fsID, locs[0].FileSystemID)
|
||||
|
||||
// FileMetadataByLocation must also resolve via FilesByPath first
|
||||
_, err = composite.FileMetadataByLocation(locs[0])
|
||||
require.NoError(t, err, "FileMetadataByLocation should resolve the file in the child resolver via FilesByPath before reading metadata")
|
||||
}
|
||||
|
||||
func TestCompositeResolver_FilesByPath_ParentOnly(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/parent.txt": "parent content",
|
||||
})
|
||||
composite := NewCompositeResolver(parent)
|
||||
|
||||
locs, err := composite.FilesByPath("/parent.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, "/parent.txt", locs[0].RealPath)
|
||||
}
|
||||
|
||||
func TestCompositeResolver_FilesByPath_WithChild(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/parent.txt": "parent content",
|
||||
})
|
||||
child := newMockResolver(map[string]string{
|
||||
"/child.txt": "child content",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
archiveLoc := file.NewLocation("/archive.tar.gz")
|
||||
composite.AddChild(child, archiveLoc, 1)
|
||||
|
||||
locs, err := composite.FilesByPath("/child.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Contains(t, locs[0].AccessPath, "archive.tar.gz:")
|
||||
assert.Contains(t, locs[0].AccessPath, "child.txt")
|
||||
}
|
||||
|
||||
func TestCompositeResolver_FileContentsByLocation_Parent(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/parent.txt": "parent content",
|
||||
})
|
||||
composite := NewCompositeResolver(parent)
|
||||
|
||||
locs, err := composite.FilesByPath("/parent.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
|
||||
reader, err := composite.FileContentsByLocation(locs[0])
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "parent content", string(content))
|
||||
}
|
||||
|
||||
func TestCompositeResolver_FileContentsByLocation_Child(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{})
|
||||
child := newMockResolver(map[string]string{
|
||||
"/child.txt": "child content",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
archiveLoc := file.NewLocation("/archive.zip")
|
||||
fsID := composite.AddChild(child, archiveLoc, 1)
|
||||
|
||||
// construct a location as returned by FilesByPath
|
||||
loc := file.Location{
|
||||
LocationData: file.LocationData{
|
||||
Coordinates: file.Coordinates{
|
||||
RealPath: "/child.txt",
|
||||
FileSystemID: fsID,
|
||||
},
|
||||
AccessPath: "/archive.zip:/child.txt",
|
||||
},
|
||||
}
|
||||
|
||||
reader, err := composite.FileContentsByLocation(loc)
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "child content", string(content))
|
||||
}
|
||||
|
||||
func TestCompositeResolver_HasPath(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/parent.txt": "p",
|
||||
})
|
||||
child := newMockResolver(map[string]string{
|
||||
"/child.txt": "c",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
archiveLoc := file.NewLocation("/archive.zip")
|
||||
composite.AddChild(child, archiveLoc, 1)
|
||||
|
||||
assert.True(t, composite.HasPath("/parent.txt"))
|
||||
assert.True(t, composite.HasPath("/child.txt"))
|
||||
assert.False(t, composite.HasPath("/nonexistent.txt"))
|
||||
}
|
||||
|
||||
func TestCompositeResolver_AllLocations(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/parent.txt": "p",
|
||||
})
|
||||
child := newMockResolver(map[string]string{
|
||||
"/child.txt": "c",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
archiveLoc := file.NewLocation("/archive.zip")
|
||||
composite.AddChild(child, archiveLoc, 1)
|
||||
|
||||
ctx := context.Background()
|
||||
var allLocs []file.Location
|
||||
for loc := range composite.AllLocations(ctx) {
|
||||
allLocs = append(allLocs, loc)
|
||||
}
|
||||
|
||||
assert.Len(t, allLocs, 2)
|
||||
|
||||
hasArchivePrefix := false
|
||||
for _, loc := range allLocs {
|
||||
if strings.Contains(loc.AccessPath, "archive.zip:") {
|
||||
hasArchivePrefix = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, hasArchivePrefix)
|
||||
}
|
||||
|
||||
func TestCompositeResolver_ChildCount(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{})
|
||||
composite := NewCompositeResolver(parent)
|
||||
assert.Equal(t, 0, composite.ChildCount())
|
||||
|
||||
child := newMockResolver(map[string]string{"/a.txt": "a"})
|
||||
archiveLoc := file.NewLocation("/archive.zip")
|
||||
composite.AddChild(child, archiveLoc, 1)
|
||||
assert.Equal(t, 1, composite.ChildCount())
|
||||
}
|
||||
|
||||
func TestCompositeResolver_TransformLocation(t *testing.T) {
|
||||
composite := NewCompositeResolver(nil)
|
||||
|
||||
archiveLoc := file.NewLocation("/path/to/archive.tar.gz")
|
||||
child := &childResolver{
|
||||
archiveLocation: archiveLoc,
|
||||
fsID: "archive:abc123",
|
||||
depth: 1,
|
||||
}
|
||||
|
||||
loc := file.NewLocation("/inner/file.txt")
|
||||
transformed := composite.transformLocation(loc, child)
|
||||
|
||||
assert.Equal(t, "/inner/file.txt", transformed.RealPath)
|
||||
assert.Equal(t, "archive:abc123", transformed.FileSystemID)
|
||||
assert.Equal(t, "/path/to/archive.tar.gz:/inner/file.txt", transformed.AccessPath)
|
||||
}
|
||||
|
||||
func TestGenerateArchiveFSID(t *testing.T) {
|
||||
loc1 := file.NewLocation("/archive1.zip")
|
||||
loc2 := file.NewLocation("/archive2.zip")
|
||||
|
||||
fsID1 := generateArchiveFSID(loc1)
|
||||
fsID2 := generateArchiveFSID(loc2)
|
||||
|
||||
assert.NotEqual(t, fsID1, fsID2)
|
||||
assert.Equal(t, fsID1, generateArchiveFSID(loc1))
|
||||
assert.True(t, strings.HasPrefix(fsID1, "archive:"))
|
||||
}
|
||||
|
||||
func TestCompositeResolver_MultipleChildren(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/base.txt": "base",
|
||||
})
|
||||
child1 := newMockResolver(map[string]string{
|
||||
"/lib.so": "library1",
|
||||
})
|
||||
child2 := newMockResolver(map[string]string{
|
||||
"/lib.so": "library2",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
composite.AddChild(child1, file.NewLocation("/archive1.zip"), 1)
|
||||
composite.AddChild(child2, file.NewLocation("/archive2.zip"), 1)
|
||||
|
||||
assert.Equal(t, 2, composite.ChildCount())
|
||||
|
||||
// searching for /lib.so should find it in both children
|
||||
locs, err := composite.FilesByPath("/lib.so")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, locs, 2)
|
||||
|
||||
// each should have different archive prefixes
|
||||
var accessPaths []string
|
||||
for _, loc := range locs {
|
||||
accessPaths = append(accessPaths, loc.AccessPath)
|
||||
}
|
||||
assert.Contains(t, accessPaths[0], "archive1.zip:")
|
||||
assert.Contains(t, accessPaths[1], "archive2.zip:")
|
||||
}
|
||||
|
||||
func TestCompositeResolver_NestedChildren_ChildInChild(t *testing.T) {
|
||||
// Simulates: base filesystem -> archive1.tar.gz -> archive2.zip -> inner file
|
||||
// In practice the orchestrator adds children at different depths, but the resolver
|
||||
// itself just has a flat list of children. The nesting is encoded in the AccessPath.
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/base.txt": "base",
|
||||
})
|
||||
|
||||
// First level: contents of archive1.tar.gz
|
||||
child1 := newMockResolver(map[string]string{
|
||||
"/inner-archive.zip": "zipdata", // this is an archive inside archive1
|
||||
"/readme.txt": "readme from archive1",
|
||||
})
|
||||
|
||||
// Second level: contents of inner-archive.zip (which was inside archive1.tar.gz)
|
||||
child2 := newMockResolver(map[string]string{
|
||||
"/deep-file.txt": "deeply nested content",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
|
||||
// Add child1 as extracted from archive1.tar.gz (depth 1)
|
||||
archive1Loc := file.NewLocation("/archive1.tar.gz")
|
||||
fsID1 := composite.AddChild(child1, archive1Loc, 1)
|
||||
|
||||
// Add child2 as extracted from inner-archive.zip which was in archive1 (depth 2)
|
||||
// The archive location for child2 refers to the inner-archive.zip as it appears in child1
|
||||
innerArchiveLoc := file.Location{
|
||||
LocationData: file.LocationData{
|
||||
Coordinates: file.Coordinates{
|
||||
RealPath: "/inner-archive.zip",
|
||||
FileSystemID: fsID1,
|
||||
},
|
||||
AccessPath: "/archive1.tar.gz:/inner-archive.zip",
|
||||
},
|
||||
}
|
||||
fsID2 := composite.AddChild(child2, innerArchiveLoc, 2)
|
||||
|
||||
assert.Equal(t, 2, composite.ChildCount())
|
||||
assert.NotEqual(t, fsID1, fsID2)
|
||||
|
||||
// Should find base file
|
||||
locs, err := composite.FilesByPath("/base.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, locs, 1)
|
||||
assert.Equal(t, "/base.txt", locs[0].AccessPath)
|
||||
|
||||
// Should find readme from archive1
|
||||
locs, err = composite.FilesByPath("/readme.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
assert.Equal(t, "/archive1.tar.gz:/readme.txt", locs[0].AccessPath)
|
||||
assert.Equal(t, fsID1, locs[0].FileSystemID)
|
||||
|
||||
// Should find deeply nested file
|
||||
locs, err = composite.FilesByPath("/deep-file.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locs, 1)
|
||||
// The access path shows the full chain: outer-archive:inner-archive:file
|
||||
assert.Equal(t, "/archive1.tar.gz:/inner-archive.zip:/deep-file.txt", locs[0].AccessPath)
|
||||
assert.Equal(t, fsID2, locs[0].FileSystemID)
|
||||
|
||||
// Should be able to read the deeply nested content
|
||||
reader, err := composite.FileContentsByLocation(locs[0])
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "deeply nested content", string(content))
|
||||
}
|
||||
|
||||
func TestCompositeResolver_EmptyChild(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/file.txt": "content",
|
||||
})
|
||||
child := newMockResolver(map[string]string{})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
composite.AddChild(child, file.NewLocation("/empty.zip"), 1)
|
||||
|
||||
// should still find parent files
|
||||
locs, err := composite.FilesByPath("/file.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, locs, 1)
|
||||
|
||||
// no extra locations from empty child
|
||||
ctx := context.Background()
|
||||
var all []file.Location
|
||||
for loc := range composite.AllLocations(ctx) {
|
||||
all = append(all, loc)
|
||||
}
|
||||
assert.Len(t, all, 1)
|
||||
}
|
||||
|
||||
func TestCompositeResolver_SameFileInParentAndChild(t *testing.T) {
|
||||
// File with the same path exists in both parent and child
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/config.yaml": "parent version",
|
||||
})
|
||||
child := newMockResolver(map[string]string{
|
||||
"/config.yaml": "child version",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
fsID := composite.AddChild(child, file.NewLocation("/archive.zip"), 1)
|
||||
|
||||
locs, err := composite.FilesByPath("/config.yaml")
|
||||
require.NoError(t, err)
|
||||
// Should find both versions
|
||||
assert.Len(t, locs, 2)
|
||||
|
||||
// parent version has no archive prefix
|
||||
parentLoc := locs[0]
|
||||
assert.Equal(t, "/config.yaml", parentLoc.AccessPath)
|
||||
assert.Empty(t, parentLoc.FileSystemID)
|
||||
|
||||
// child version has archive prefix
|
||||
childLoc := locs[1]
|
||||
assert.Equal(t, "/archive.zip:/config.yaml", childLoc.AccessPath)
|
||||
assert.Equal(t, fsID, childLoc.FileSystemID)
|
||||
|
||||
// can read both
|
||||
r1, err := composite.FileContentsByLocation(parentLoc)
|
||||
require.NoError(t, err)
|
||||
c1, _ := io.ReadAll(r1)
|
||||
r1.Close()
|
||||
assert.Equal(t, "parent version", string(c1))
|
||||
|
||||
r2, err := composite.FileContentsByLocation(childLoc)
|
||||
require.NoError(t, err)
|
||||
c2, _ := io.ReadAll(r2)
|
||||
r2.Close()
|
||||
assert.Equal(t, "child version", string(c2))
|
||||
}
|
||||
|
||||
func TestCompositeResolver_AllLocations_ContextCancellation(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/a.txt": "a",
|
||||
"/b.txt": "b",
|
||||
"/c.txt": "c",
|
||||
})
|
||||
|
||||
composite := NewCompositeResolver(parent)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel immediately
|
||||
|
||||
var count int
|
||||
for range composite.AllLocations(ctx) {
|
||||
count++
|
||||
}
|
||||
// with cancelled context, we may get 0 or some locations, but shouldn't hang
|
||||
assert.True(t, count <= 3)
|
||||
}
|
||||
|
||||
func TestCompositeResolver_NoChildren_BehavesLikeParent(t *testing.T) {
|
||||
parent := newMockResolver(map[string]string{
|
||||
"/file.txt": "content",
|
||||
})
|
||||
composite := NewCompositeResolver(parent)
|
||||
|
||||
// FilesByPath
|
||||
parentLocs, _ := parent.FilesByPath("/file.txt")
|
||||
compositeLocs, _ := composite.FilesByPath("/file.txt")
|
||||
assert.Equal(t, len(parentLocs), len(compositeLocs))
|
||||
|
||||
// HasPath
|
||||
assert.Equal(t, parent.HasPath("/file.txt"), composite.HasPath("/file.txt"))
|
||||
assert.Equal(t, parent.HasPath("/nope"), composite.HasPath("/nope"))
|
||||
}
|
||||
@@ -17,7 +17,6 @@ type configurationAuditTrail struct {
|
||||
Packages pkgcataloging.Config `json:"packages" yaml:"packages" mapstructure:"packages"`
|
||||
Files filecataloging.Config `json:"files" yaml:"files" mapstructure:"files"`
|
||||
Licenses cataloging.LicenseConfig `json:"licenses" yaml:"licenses" mapstructure:"licenses"`
|
||||
Archive cataloging.ArchiveSearchConfig `json:"archive" yaml:"archive" mapstructure:"archive"`
|
||||
Catalogers catalogerManifest `json:"catalogers" yaml:"catalogers" mapstructure:"catalogers"`
|
||||
ExtraConfigs any `json:"extra,omitempty" yaml:"extra" mapstructure:"extra"`
|
||||
}
|
||||
|
||||
+1
-72
@@ -10,7 +10,6 @@ import (
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/go-sync"
|
||||
intArchive "github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/licenses"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
@@ -20,12 +19,9 @@ import (
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"github.com/anchore/syft/syft/source/directorysource"
|
||||
)
|
||||
|
||||
// CreateSBOM creates a software bill-of-materials from the given source. If the CreateSBOMConfig is nil, then
|
||||
@@ -64,7 +60,6 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (
|
||||
Packages: cfg.Packages,
|
||||
Files: cfg.Files,
|
||||
Licenses: cfg.Licenses,
|
||||
Archive: cfg.Archive,
|
||||
Catalogers: *audit,
|
||||
ExtraConfigs: cfg.ToolConfiguration,
|
||||
},
|
||||
@@ -97,17 +92,9 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (
|
||||
packageCatalogingProgress := monitorPackageCatalogingTask()
|
||||
|
||||
builder := sbomsync.NewBuilder(&s, monitorPackageCount(packageCatalogingProgress))
|
||||
|
||||
// determine the effective resolver: if recursive archive extraction is enabled,
|
||||
// wrap the resolver with the archive orchestrator
|
||||
effectiveResolver, archiveCleanup, archiveRelationships := setupArchiveOrchestration(ctx, resolver, cfg)
|
||||
if archiveCleanup != nil {
|
||||
defer archiveCleanup()
|
||||
}
|
||||
|
||||
for i := range taskGroups {
|
||||
err = sync.Collect(&ctx, cataloging.ExecutorFile, sync.ToSeq(taskGroups[i]), func(t task.Task) (any, error) {
|
||||
return nil, task.RunTask(ctx, t, effectiveResolver, builder, catalogingProgress)
|
||||
return nil, task.RunTask(ctx, t, resolver, builder, catalogingProgress)
|
||||
}, nil)
|
||||
if err != nil {
|
||||
// TODO: tie this to the open progress monitors...
|
||||
@@ -115,70 +102,12 @@ func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (
|
||||
}
|
||||
}
|
||||
|
||||
// add archive containment relationships to the SBOM
|
||||
if len(archiveRelationships) > 0 {
|
||||
builder.AddRelationships(archiveRelationships...)
|
||||
}
|
||||
|
||||
packageCatalogingProgress.SetCompleted()
|
||||
catalogingProgress.SetCompleted()
|
||||
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
// setupArchiveOrchestration wraps the resolver with an archive orchestrator when recursive
|
||||
// archive extraction is enabled (MaxDepth > 0). It performs the iterative extract loop
|
||||
// and returns the effective resolver, a cleanup function, and any archive relationships.
|
||||
func setupArchiveOrchestration(ctx context.Context, baseResolver file.Resolver, cfg *CreateSBOMConfig) (file.Resolver, func(), []artifact.Relationship) {
|
||||
if cfg.Archive.MaxDepth <= 0 {
|
||||
return baseResolver, nil, nil
|
||||
}
|
||||
|
||||
td := tmpdir.FromContext(ctx)
|
||||
if td == nil {
|
||||
log.Warn("archive orchestration requested but no temp dir available")
|
||||
return baseResolver, nil, nil
|
||||
}
|
||||
|
||||
archiveTmpDir, archiveTmpCleanup, err := td.NewChild("archive-extraction") //nolint:gocritic // cleanup is returned to caller, not deferred here
|
||||
if err != nil {
|
||||
log.WithFields("error", err).Warn("unable to create temp dir for archive extraction")
|
||||
return baseResolver, nil, nil
|
||||
}
|
||||
|
||||
// build user-exclusion path filters once and reuse them for every child
|
||||
// resolver, so that --exclude patterns also apply inside extracted archives
|
||||
resolverFactory := func(root string) (file.Resolver, error) {
|
||||
filters, err := directorysource.GetDirectoryExclusionFunctions(root, append([]string(nil), cfg.Exclusions...))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid exclusion patterns for archive child resolver: %w", err)
|
||||
}
|
||||
// depth 0: the archive Orchestrator (PR #4761) drives recursion here, so the
|
||||
// resolver-level recursive archive indexing (PR #4044) is disabled to avoid
|
||||
// double extraction. With depth 0 no temp dir is created, so the cleanup is a no-op.
|
||||
res, _, err := fileresolver.NewFromDirectory(root, "", 0, filters...)
|
||||
return res, err
|
||||
}
|
||||
|
||||
orch := intArchive.NewOrchestrator(baseResolver, cfg.Archive, archiveTmpDir, resolverFactory)
|
||||
|
||||
// iteratively discover and extract archives up to the configured max depth
|
||||
for depth := 0; depth < cfg.Archive.MaxDepth; depth++ {
|
||||
newArchives := orch.DiscoverAndExtract(ctx, depth)
|
||||
if newArchives == 0 {
|
||||
break
|
||||
}
|
||||
log.WithFields("depth", depth+1, "archives", newArchives).Debug("extracted archives for recursive cataloging")
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
orch.Cleanup()
|
||||
archiveTmpCleanup()
|
||||
}
|
||||
|
||||
return orch.Resolver(), cleanup, orch.Relationships()
|
||||
}
|
||||
|
||||
func setupContext(ctx context.Context, cfg *CreateSBOMConfig) (context.Context, error) {
|
||||
// configure parallel executors
|
||||
ctx = setContextExecutors(ctx, cfg)
|
||||
|
||||
@@ -28,16 +28,9 @@ type CreateSBOMConfig struct {
|
||||
Packages pkgcataloging.Config
|
||||
Licenses cataloging.LicenseConfig
|
||||
Files filecataloging.Config
|
||||
Archive cataloging.ArchiveSearchConfig
|
||||
Parallelism int
|
||||
CatalogerSelection cataloging.SelectionRequest
|
||||
|
||||
// Exclusions are user-provided glob patterns that the source resolver already
|
||||
// applies; they are also propagated into child resolvers built during recursive
|
||||
// archive extraction so that a "--exclude=**/foo" pattern keeps working inside
|
||||
// extracted archive contents.
|
||||
Exclusions []string
|
||||
|
||||
// audit what tool is being used to generate the SBOM
|
||||
ToolName string
|
||||
ToolVersion string
|
||||
@@ -56,7 +49,6 @@ func DefaultCreateSBOMConfig() *CreateSBOMConfig {
|
||||
Packages: pkgcataloging.DefaultConfig(),
|
||||
Licenses: cataloging.DefaultLicenseConfig(),
|
||||
Files: filecataloging.DefaultConfig(),
|
||||
Archive: cataloging.DefaultArchiveSearchConfig(),
|
||||
Parallelism: 0, // use default: run in parallel based on number of CPUs
|
||||
packageTaskFactories: task.DefaultPackageTaskFactories(),
|
||||
|
||||
@@ -161,20 +153,6 @@ func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig {
|
||||
return c
|
||||
}
|
||||
|
||||
// WithArchiveConfig allows for defining archive extraction and recursive cataloging parameters.
|
||||
func (c *CreateSBOMConfig) WithArchiveConfig(cfg cataloging.ArchiveSearchConfig) *CreateSBOMConfig {
|
||||
c.Archive = cfg
|
||||
return c
|
||||
}
|
||||
|
||||
// WithExclusions sets the glob patterns that should be propagated to child resolvers
|
||||
// constructed during recursive archive extraction. Patterns must follow the same
|
||||
// "./", "*/", or "**/"-prefix convention as the syft --exclude flag.
|
||||
func (c *CreateSBOMConfig) WithExclusions(exclusions []string) *CreateSBOMConfig {
|
||||
c.Exclusions = exclusions
|
||||
return c
|
||||
}
|
||||
|
||||
// WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag.
|
||||
func (c *CreateSBOMConfig) WithCatalogerSelection(selection cataloging.SelectionRequest) *CreateSBOMConfig {
|
||||
c.CatalogerSelection = selection
|
||||
|
||||
@@ -51,11 +51,6 @@ func (c *GetSourceConfig) WithBasePath(basePath string) *GetSourceConfig {
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *GetSourceConfig) WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth int) *GetSourceConfig {
|
||||
c.SourceProviderConfig = c.SourceProviderConfig.WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth)
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *GetSourceConfig) WithSources(sources ...string) *GetSourceConfig {
|
||||
c.Sources = sources
|
||||
return c
|
||||
|
||||
@@ -3,28 +3,11 @@ package fileresolver
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/archiver/v3"
|
||||
"github.com/anchore/stereoscope/pkg/filetree"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
type archiveAccessPath struct {
|
||||
realPath string
|
||||
accessPath string
|
||||
archiveRealPath string
|
||||
archiveDepth int
|
||||
}
|
||||
|
||||
const (
|
||||
archiveTempPathPattern = "syft-archivePaths-"
|
||||
archiveContentTempPathPattern = "syft-archive-contents-"
|
||||
)
|
||||
|
||||
var ErrSkipPath = errors.New("skip path")
|
||||
|
||||
var _ file.Resolver = (*Directory)(nil)
|
||||
@@ -36,29 +19,13 @@ type Directory struct {
|
||||
indexer *directoryIndexer
|
||||
}
|
||||
|
||||
func NewFromDirectory(root, base string, maxArchiveRecursiveIndexDepth int, pathFilters ...PathIndexVisitor) (*Directory, func() error, error) {
|
||||
var cleanupFn = func() error { return nil }
|
||||
directory, err := newFromDirectoryWithoutIndex(root, base, pathFilters...)
|
||||
func NewFromDirectory(root, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
|
||||
resolver, err := newFromDirectoryWithoutIndex(root, base, pathFilters...)
|
||||
if err != nil {
|
||||
return nil, cleanupFn, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = directory.buildIndex(); err != nil {
|
||||
return nil, cleanupFn, fmt.Errorf("unable to build index: %w", err)
|
||||
}
|
||||
|
||||
if maxArchiveRecursiveIndexDepth != 0 && len(directory.indexer.archivePaths) > 0 {
|
||||
archiveTempDir, err := os.MkdirTemp("", archiveTempPathPattern)
|
||||
if err != nil {
|
||||
return nil, cleanupFn, fmt.Errorf("unable to create tempdir for archive processing: %w", err)
|
||||
}
|
||||
|
||||
return directory, func() error {
|
||||
return os.RemoveAll(archiveTempDir)
|
||||
}, directory.buildArchiveIndex(archiveTempDir, directory.indexer.archivePaths, maxArchiveRecursiveIndexDepth)
|
||||
}
|
||||
|
||||
return directory, cleanupFn, nil
|
||||
return resolver, resolver.buildIndex()
|
||||
}
|
||||
|
||||
func newFromDirectoryWithoutIndex(root, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
|
||||
@@ -98,91 +65,6 @@ func (r *Directory) buildIndex() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Directory) buildArchiveIndex(archiveTempDir string, archives []string, maxArchiveIndexDepth int) error {
|
||||
archivesToIndex := make([]archiveAccessPath, len(archives))
|
||||
for i, archive := range archives {
|
||||
archivesToIndex[i] = archiveAccessPath{
|
||||
realPath: r.Chroot.ToChrootPath(archive),
|
||||
accessPath: r.Chroot.ToChrootPath(archive),
|
||||
archiveRealPath: archive,
|
||||
}
|
||||
}
|
||||
|
||||
loop:
|
||||
for {
|
||||
var currentArchivePath archiveAccessPath
|
||||
switch len(archivesToIndex) {
|
||||
case 0:
|
||||
break loop
|
||||
case 1:
|
||||
currentArchivePath, archivesToIndex = archivesToIndex[0], nil
|
||||
default:
|
||||
currentArchivePath, archivesToIndex = archivesToIndex[0], archivesToIndex[1:]
|
||||
}
|
||||
|
||||
if maxArchiveIndexDepth != -1 && currentArchivePath.archiveDepth >= maxArchiveIndexDepth {
|
||||
continue
|
||||
}
|
||||
|
||||
envelopedUnarchiver, err := archiver.ByExtension(currentArchivePath.archiveRealPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not determine archive envelope type: %w", err)
|
||||
}
|
||||
|
||||
unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver)
|
||||
if !ok {
|
||||
return fmt.Errorf("unarchiver %T is not an archiver", envelopedUnarchiver)
|
||||
}
|
||||
|
||||
archiveDestinationTempDir, err := os.MkdirTemp(archiveTempDir, archiveContentTempPathPattern)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create tempdir for archive processing: %w", err)
|
||||
}
|
||||
|
||||
archiveDestinationPath, err := filepath.EvalSymlinks(archiveDestinationTempDir)
|
||||
if err != nil {
|
||||
var pathErr *os.PathError
|
||||
if errors.As(err, &pathErr) {
|
||||
// we can't index the path, but we shouldn't consider this to be fatal
|
||||
// TODO: known-unknowns
|
||||
log.WithFields("archivePath", archiveDestinationTempDir, "error", err).Trace("unable to evaluate symlink while indexing branch")
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if err = unarchiver.Unarchive(currentArchivePath.archiveRealPath, archiveDestinationPath); err != nil {
|
||||
return fmt.Errorf("unable to unarchive nested file: %w", err)
|
||||
}
|
||||
|
||||
d, err := newFromDirectoryWithoutIndex(archiveDestinationPath, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = d.buildIndex(); err != nil {
|
||||
return fmt.Errorf("unable to build index of archive: %w", err)
|
||||
}
|
||||
|
||||
for _, archive := range d.indexer.archivePaths {
|
||||
archivesToIndex = append(archivesToIndex, archiveAccessPath{
|
||||
realPath: currentArchivePath.realPath,
|
||||
accessPath: strings.Replace(archive, archiveDestinationPath, currentArchivePath.accessPath, 1),
|
||||
archiveRealPath: archive,
|
||||
archiveDepth: currentArchivePath.archiveDepth + 1,
|
||||
})
|
||||
}
|
||||
|
||||
d.realPath = currentArchivePath.realPath
|
||||
d.accessPath = currentArchivePath.accessPath
|
||||
d.archiveRealPath = archiveDestinationPath
|
||||
|
||||
r.archives = append(r.archives, &d.FiletreeResolver)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stringer to represent a directory path data source
|
||||
func (r *Directory) String() string {
|
||||
return fmt.Sprintf("dir:%s", r.path)
|
||||
|
||||
@@ -9,13 +9,13 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/archiver/v3"
|
||||
"github.com/wagoodman/go-progress"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/stereoscope/pkg/filetree"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/internal/windows"
|
||||
"github.com/wagoodman/go-progress"
|
||||
)
|
||||
|
||||
type PathIndexVisitor func(string, string, os.FileInfo, error) error
|
||||
@@ -27,7 +27,6 @@ type directoryIndexer struct {
|
||||
errPaths map[string]error
|
||||
tree filetree.ReadWriter
|
||||
index filetree.Index
|
||||
archivePaths []string
|
||||
}
|
||||
|
||||
func newDirectoryIndexer(path, base string, visitors ...PathIndexVisitor) *directoryIndexer {
|
||||
@@ -145,10 +144,6 @@ func (r *directoryIndexer) indexTree(root string, stager *progress.AtomicStage)
|
||||
func(path string, info os.FileInfo, err error) error {
|
||||
stager.Set(path)
|
||||
|
||||
if isArchive(path, info) {
|
||||
r.archivePaths = append(r.archivePaths, path)
|
||||
}
|
||||
|
||||
newRoot, err := r.indexPath(path, info, err)
|
||||
|
||||
if err != nil {
|
||||
@@ -169,17 +164,6 @@ func (r *directoryIndexer) indexTree(root string, stager *progress.AtomicStage)
|
||||
return roots, nil
|
||||
}
|
||||
|
||||
// isArchive returns true if the path appears to be an archive
|
||||
func isArchive(path string, info os.FileInfo) bool {
|
||||
if info == nil || !info.Mode().IsRegular() {
|
||||
return false
|
||||
}
|
||||
|
||||
envelopedUnarchiver, err := archiver.ByExtension(path)
|
||||
_, ok := envelopedUnarchiver.(archiver.Unarchiver)
|
||||
return err == nil && ok
|
||||
}
|
||||
|
||||
func isRealPath(root string) (bool, error) {
|
||||
rootParent := filepath.Clean(filepath.Dir(root))
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/stereoscope/pkg/filetree"
|
||||
@@ -22,11 +21,6 @@ type FiletreeResolver struct {
|
||||
Index filetree.IndexReader
|
||||
SearchContext filetree.Searcher
|
||||
Opener func(stereoscopeFile.Reference) (io.ReadCloser, error)
|
||||
|
||||
realPath string
|
||||
accessPath string
|
||||
archiveRealPath string
|
||||
archives []*FiletreeResolver
|
||||
}
|
||||
|
||||
func nativeOSFileOpener(ref stereoscopeFile.Reference) (io.ReadCloser, error) {
|
||||
@@ -41,16 +35,7 @@ func nativeOSFileOpener(ref stereoscopeFile.Reference) (io.ReadCloser, error) {
|
||||
}
|
||||
|
||||
func (r *FiletreeResolver) requestPath(userPath string) (string, error) {
|
||||
requestPath, err := r.Chroot.ToNativePath(userPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if r.accessPath != "" && r.archiveRealPath != "" {
|
||||
return strings.Replace(requestPath, r.accessPath, r.archiveRealPath, 1), nil
|
||||
}
|
||||
|
||||
return requestPath, nil
|
||||
return r.Chroot.ToNativePath(userPath)
|
||||
}
|
||||
|
||||
// responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the file resolver.
|
||||
@@ -58,32 +43,13 @@ func (r FiletreeResolver) responsePath(path string) string {
|
||||
return r.Chroot.ToChrootPath(path)
|
||||
}
|
||||
|
||||
func (r FiletreeResolver) responseAccessPath(path string) string {
|
||||
if r.accessPath != "" && r.archiveRealPath != "" {
|
||||
path = strings.Replace(path, r.archiveRealPath, r.accessPath, 1)
|
||||
}
|
||||
|
||||
return r.Chroot.ToChrootPath(path)
|
||||
}
|
||||
|
||||
// HasPath indicates if the given path exists in the underlying source.
|
||||
func (r *FiletreeResolver) HasPath(userPath string) bool {
|
||||
requestPath, err := r.requestPath(userPath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if r.Tree.HasPath(stereoscopeFile.Path(requestPath)) {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, archive := range r.archives {
|
||||
if archive.HasPath(userPath) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
return r.Tree.HasPath(stereoscopeFile.Path(requestPath))
|
||||
}
|
||||
|
||||
// FilesByPath returns all file.References that match the given paths from the file index.
|
||||
@@ -127,35 +93,18 @@ func (r FiletreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err
|
||||
references = append(references,
|
||||
file.NewVirtualLocationFromDirectory(
|
||||
r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root
|
||||
r.responseAccessPath(userStrPath), // the path used to access this file, relative to the resolver root
|
||||
r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root
|
||||
*ref.Reference,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
for _, archive := range r.archives {
|
||||
locations, err := archive.FilesByPath(userPaths...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
references = append(references, locations...)
|
||||
}
|
||||
|
||||
return references, nil
|
||||
}
|
||||
|
||||
func (r FiletreeResolver) requestGlob(pattern string) (string, error) {
|
||||
nativeGlob, err := r.Chroot.ToNativeGlob(pattern)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if r.accessPath != "" && r.archiveRealPath != "" {
|
||||
return strings.Replace(nativeGlob, r.accessPath, r.archiveRealPath, 1), nil
|
||||
}
|
||||
|
||||
return nativeGlob, nil
|
||||
return r.Chroot.ToNativeGlob(pattern)
|
||||
}
|
||||
|
||||
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
|
||||
@@ -187,8 +136,8 @@ func (r FiletreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro
|
||||
}
|
||||
|
||||
loc := file.NewVirtualLocationFromDirectory(
|
||||
r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root
|
||||
r.responseAccessPath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
|
||||
r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root
|
||||
r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
|
||||
*refVia.Reference,
|
||||
)
|
||||
uniqueFileIDs.Add(*refVia.Reference)
|
||||
@@ -196,20 +145,6 @@ func (r FiletreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro
|
||||
}
|
||||
}
|
||||
|
||||
for _, archive := range r.archives {
|
||||
archiveUniqueLocations, err := archive.FilesByGlob(patterns...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, archiveUniqueLocation := range archiveUniqueLocations {
|
||||
if uniqueFileIDs.Contains(archiveUniqueLocation.Reference()) {
|
||||
continue
|
||||
}
|
||||
uniqueLocations = append(uniqueLocations, archiveUniqueLocation)
|
||||
}
|
||||
}
|
||||
|
||||
return uniqueLocations, nil
|
||||
}
|
||||
|
||||
@@ -235,14 +170,6 @@ func (r FiletreeResolver) FileContentsByLocation(location file.Location) (io.Rea
|
||||
}
|
||||
|
||||
entry, err := r.Index.Get(location.Reference())
|
||||
if err != nil {
|
||||
for _, archive := range r.archives {
|
||||
entry, err = archive.Index.Get(location.Reference())
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -263,39 +190,21 @@ func (r *FiletreeResolver) AllLocations(ctx context.Context) <-chan file.Locatio
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case results <- file.NewVirtualLocationFromDirectory(r.responsePath(string(ref.RealPath)), r.responseAccessPath(string(ref.RealPath)), ref):
|
||||
case results <- file.NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), "", ref):
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
for _, archive := range r.archives {
|
||||
for location := range archive.AllLocations(ctx) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case results <- location:
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return results
|
||||
}
|
||||
|
||||
func (r *FiletreeResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
|
||||
entry, err := r.Index.Get(location.Reference())
|
||||
if err == nil {
|
||||
return entry.Metadata, nil
|
||||
if err != nil {
|
||||
return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
|
||||
}
|
||||
|
||||
for _, archive := range r.archives {
|
||||
entry, err = archive.Index.Get(location.Reference())
|
||||
if err == nil {
|
||||
return entry.Metadata, nil
|
||||
}
|
||||
}
|
||||
|
||||
return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
|
||||
return entry.Metadata, nil
|
||||
}
|
||||
|
||||
func (r *FiletreeResolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
|
||||
@@ -315,26 +224,12 @@ func (r *FiletreeResolver) FilesByMIMEType(types ...string) ([]file.Location, er
|
||||
}
|
||||
location := file.NewVirtualLocationFromDirectory(
|
||||
r.responsePath(string(refVia.RealPath)),
|
||||
r.responseAccessPath(string(refVia.RequestPath)),
|
||||
r.responsePath(string(refVia.RequestPath)),
|
||||
*refVia.Reference,
|
||||
)
|
||||
uniqueFileIDs.Add(*refVia.Reference)
|
||||
uniqueLocations = append(uniqueLocations, location)
|
||||
}
|
||||
|
||||
for _, archive := range r.archives {
|
||||
archiveUniqueLocations, err := archive.FilesByMIMEType(types...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, archiveUniqueLocation := range archiveUniqueLocations {
|
||||
if uniqueFileIDs.Contains(archiveUniqueLocation.Reference()) {
|
||||
continue
|
||||
}
|
||||
uniqueLocations = append(uniqueLocations, archiveUniqueLocation)
|
||||
}
|
||||
}
|
||||
|
||||
return uniqueLocations, nil
|
||||
}
|
||||
|
||||
@@ -505,12 +505,9 @@ func TestDirectoryResolver_FilesByPath_request_response(t *testing.T) {
|
||||
require.NoError(t, os.Chdir(testDir))
|
||||
})
|
||||
|
||||
resolver, cleanupFn, err := NewFromDirectory(c.root, c.base, 0)
|
||||
resolver, err := NewFromDirectory(c.root, c.base)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
refs, err := resolver.FilesByPath(c.input)
|
||||
require.NoError(t, err)
|
||||
@@ -560,12 +557,8 @@ func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) {
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory(c.relativeRoot, "", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
resolver, err := NewFromDirectory(c.relativeRoot, "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
refs, err := resolver.FilesByPath(c.input)
|
||||
require.NoError(t, err)
|
||||
@@ -619,12 +612,8 @@ func TestDirectoryResolver_FilesByPath_absoluteRoot(t *testing.T) {
|
||||
absRoot, err := filepath.Abs(c.relativeRoot)
|
||||
require.NoError(t, err)
|
||||
|
||||
resolver, cleanupFn, err := NewFromDirectory(absRoot, "", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
resolver, err := NewFromDirectory(absRoot, "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
refs, err := resolver.FilesByPath(c.input)
|
||||
require.NoError(t, err)
|
||||
@@ -684,12 +673,8 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory(c.root, "", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
resolver, err := NewFromDirectory(c.root, "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
hasPath := resolver.HasPath(c.input)
|
||||
if !c.forcePositiveHasPath {
|
||||
@@ -736,13 +721,8 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata", "", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
resolver, err := NewFromDirectory("./testdata", "")
|
||||
assert.NoError(t, err)
|
||||
refs, err := resolver.FilesByPath(c.input...)
|
||||
assert.NoError(t, err)
|
||||
|
||||
@@ -754,13 +734,8 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata", "", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
resolver, err := NewFromDirectory("./testdata", "")
|
||||
assert.NoError(t, err)
|
||||
refs, err := resolver.FilesByGlob("**/image-symlinks/file*")
|
||||
assert.NoError(t, err)
|
||||
|
||||
@@ -768,26 +743,16 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata/image-symlinks", "", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
resolver, err := NewFromDirectory("./testdata/image-symlinks", "")
|
||||
assert.NoError(t, err)
|
||||
refs, err := resolver.FilesByGlob("**/*.txt")
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, refs, 6)
|
||||
}
|
||||
|
||||
func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata", "", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
resolver, err := NewFromDirectory("./testdata", "")
|
||||
assert.NoError(t, err)
|
||||
refs, err := resolver.FilesByGlob("**/image-symlinks/*1.txt")
|
||||
assert.NoError(t, err)
|
||||
|
||||
@@ -813,12 +778,8 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-simple", "", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
resolver, err := NewFromDirectory("./testdata/symlinks-simple", "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
refs, err := resolver.FilesByPath(test.fixture)
|
||||
require.NoError(t, err)
|
||||
@@ -840,12 +801,8 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {
|
||||
|
||||
func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) {
|
||||
// let's make certain that "dev/place" is not ignored, since it is not "/dev/place"
|
||||
resolver, cleanupFn, err := NewFromDirectory("testdata/system_paths/target", "", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
resolver, err := NewFromDirectory("testdata/system_paths/target", "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
// all paths should be found (non filtering matches a path)
|
||||
locations, err := resolver.FilesByGlob("**/place")
|
||||
@@ -883,13 +840,8 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.fixturePath, func(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory(test.fixturePath, "", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
resolver, err := NewFromDirectory(test.fixturePath, "")
|
||||
assert.NoError(t, err)
|
||||
locations, err := resolver.FilesByMIMEType(test.mimeType)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, test.expectedPaths.Size(), len(locations))
|
||||
@@ -901,12 +853,8 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
|
||||
}
|
||||
|
||||
func Test_IndexingNestedSymLinks(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-simple", "", 0)
|
||||
resolver, err := NewFromDirectory("./testdata/symlinks-simple", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
// check that we can get the real path
|
||||
locations, err := resolver.FilesByPath("./readme")
|
||||
@@ -961,12 +909,8 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
|
||||
return nil
|
||||
}
|
||||
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-simple", "", 0, filterFn)
|
||||
resolver, err := NewFromDirectory("./testdata/symlinks-simple", "", filterFn)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
// the path to the real file is PRUNED from the index, so we should NOT expect a location returned
|
||||
locations, err := resolver.FilesByPath("./readme")
|
||||
@@ -985,12 +929,8 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
|
||||
}
|
||||
|
||||
func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-multiple-roots/root", "", 0)
|
||||
resolver, err := NewFromDirectory("./testdata/symlinks-multiple-roots/root", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
// check that we can get the real path
|
||||
locations, err := resolver.FilesByPath("./readme")
|
||||
@@ -1007,12 +947,8 @@ func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
|
||||
}
|
||||
|
||||
func Test_RootViaSymlink(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinked-root/nested/link-root", "", 0)
|
||||
resolver, err := NewFromDirectory("./testdata/symlinked-root/nested/link-root", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
locations, err := resolver.FilesByPath("./file1.txt")
|
||||
require.NoError(t, err)
|
||||
@@ -1031,14 +967,10 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
|
||||
cwd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
|
||||
resolver, cleanupFn, err := NewFromDirectory(".", "", 0)
|
||||
r, err := NewFromDirectory(".", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
exists, existingPath, err := resolver.Tree.File(stereoscopeFile.Path(filepath.Join(cwd, "testdata/image-simple/file-1.txt")))
|
||||
exists, existingPath, err := r.Tree.File(stereoscopeFile.Path(filepath.Join(cwd, "testdata/image-simple/file-1.txt")))
|
||||
require.True(t, exists)
|
||||
require.NoError(t, err)
|
||||
require.True(t, existingPath.HasReference())
|
||||
@@ -1062,7 +994,8 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
actual, err := resolver.FileContentsByLocation(test.location)
|
||||
|
||||
actual, err := r.FileContentsByLocation(test.location)
|
||||
if test.err {
|
||||
require.Error(t, err)
|
||||
return
|
||||
@@ -1080,12 +1013,8 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
|
||||
|
||||
func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) {
|
||||
test := func(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-loop", "", 0)
|
||||
resolver, err := NewFromDirectory("./testdata/symlinks-loop", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
locations, err := resolver.FilesByGlob("**/file.target")
|
||||
require.NoError(t, err)
|
||||
@@ -1155,12 +1084,8 @@ func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) {
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory(c.root, c.root, 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
resolver, err := NewFromDirectory(c.root, c.root)
|
||||
assert.NoError(t, err)
|
||||
|
||||
refs, err := resolver.FilesByPath(c.input)
|
||||
require.NoError(t, err)
|
||||
@@ -1303,12 +1228,9 @@ func Test_directoryResolver_resolvesLinks(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "", 0)
|
||||
resolver, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
actual := test.runner(resolver)
|
||||
|
||||
@@ -1318,12 +1240,8 @@ func Test_directoryResolver_resolvesLinks(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-prune-indexing", "", 0)
|
||||
resolver, err := NewFromDirectory("./testdata/symlinks-prune-indexing", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
var allRealPaths []stereoscopeFile.Path
|
||||
for l := range resolver.AllLocations(context.Background()) {
|
||||
@@ -1345,12 +1263,8 @@ func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
|
||||
|
||||
func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
|
||||
defer goleak.VerifyNone(t)
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata/system_paths", "", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
resolver, err := NewFromDirectory("./testdata/system_paths", "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
var dirLoc *file.Location
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
@@ -1372,12 +1286,8 @@ func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDirectoryResolver_AllLocations(t *testing.T) {
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "", 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
resolver, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "")
|
||||
assert.NoError(t, err)
|
||||
|
||||
paths := strset.New()
|
||||
for loc := range resolver.AllLocations(context.Background()) {
|
||||
@@ -1409,13 +1319,8 @@ func TestDirectoryResolver_AllLocations(t *testing.T) {
|
||||
|
||||
func TestAllLocationsDoesNotLeakGoRoutine(t *testing.T) {
|
||||
defer goleak.VerifyNone(t)
|
||||
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "", 0)
|
||||
resolver, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
for range resolver.AllLocations(ctx) {
|
||||
break
|
||||
@@ -1648,7 +1553,6 @@ func Test_fileResolver_FileContentsByLocation(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFileResolver_AllLocations_errorOnDirRequest(t *testing.T) {
|
||||
defer goleak.VerifyNone(t)
|
||||
filePath := "./testdata/system_paths/target/home/place"
|
||||
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
|
||||
require.NoError(t, err)
|
||||
@@ -1675,10 +1579,11 @@ func TestFileResolver_AllLocations_errorOnDirRequest(t *testing.T) {
|
||||
reader, err := resolver.FileContentsByLocation(*dirLoc)
|
||||
require.Error(t, err)
|
||||
require.Nil(t, reader)
|
||||
|
||||
goleak.VerifyNone(t)
|
||||
}
|
||||
|
||||
func TestFileResolver_AllLocations(t *testing.T) {
|
||||
defer goleak.VerifyNone(t)
|
||||
// Verify both the parent and the file itself are indexed
|
||||
filePath := "./testdata/system_paths/target/home/place"
|
||||
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
|
||||
@@ -1703,10 +1608,11 @@ func TestFileResolver_AllLocations(t *testing.T) {
|
||||
sort.Strings(pathsList)
|
||||
|
||||
assert.ElementsMatchf(t, expected, pathsList, "expected all paths to be indexed, but found different paths: \n%s", cmp.Diff(expected, paths.List()))
|
||||
|
||||
goleak.VerifyNone(t)
|
||||
}
|
||||
|
||||
func Test_FileResolver_AllLocationsDoesNotLeakGoRoutine(t *testing.T) {
|
||||
defer goleak.VerifyNone(t)
|
||||
filePath := "./testdata/system_paths/target/home/place"
|
||||
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
|
||||
require.NoError(t, err)
|
||||
@@ -1723,4 +1629,6 @@ func Test_FileResolver_AllLocationsDoesNotLeakGoRoutine(t *testing.T) {
|
||||
break
|
||||
}
|
||||
cancel()
|
||||
|
||||
goleak.VerifyNone(t)
|
||||
}
|
||||
|
||||
@@ -273,12 +273,8 @@ func Test_mavenResolverRemote(t *testing.T) {
|
||||
}
|
||||
|
||||
func Test_relativePathParent(t *testing.T) {
|
||||
resolver, cleanupFn, err := fileresolver.NewFromDirectory("testdata/local", "", 0)
|
||||
resolver, err := fileresolver.NewFromDirectory("testdata/local", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resolver)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, cleanupFn())
|
||||
})
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
|
||||
@@ -21,11 +21,10 @@ import (
|
||||
var _ source.Source = (*directorySource)(nil)
|
||||
|
||||
type Config struct {
|
||||
Path string
|
||||
Base string
|
||||
Exclude source.ExcludeConfig
|
||||
Alias source.Alias
|
||||
MaxArchiveRecursiveIndexDepth int
|
||||
Path string
|
||||
Base string
|
||||
Exclude source.ExcludeConfig
|
||||
Alias source.Alias
|
||||
}
|
||||
|
||||
type directorySource struct {
|
||||
@@ -33,7 +32,6 @@ type directorySource struct {
|
||||
config Config
|
||||
resolver file.Resolver
|
||||
mutex *sync.Mutex
|
||||
closer func() error
|
||||
}
|
||||
|
||||
func NewFromPath(path string) (source.Source, error) {
|
||||
@@ -107,12 +105,11 @@ func (s *directorySource) FileResolver(_ source.Scope) (file.Resolver, error) {
|
||||
// this should be the only file resolver that might have overlap with where files are cached
|
||||
exclusionFunctions = append(exclusionFunctions, excludeCachePathVisitors()...)
|
||||
|
||||
res, cleanupFn, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, s.config.MaxArchiveRecursiveIndexDepth, exclusionFunctions...)
|
||||
res, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, exclusionFunctions...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
|
||||
}
|
||||
|
||||
s.closer = cleanupFn
|
||||
s.resolver = res
|
||||
return s.resolver, nil
|
||||
}
|
||||
@@ -122,10 +119,7 @@ func (s *directorySource) Close() error {
|
||||
defer s.mutex.Unlock()
|
||||
|
||||
s.resolver = nil
|
||||
if s.closer == nil {
|
||||
return nil
|
||||
}
|
||||
return s.closer()
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileresolver.PathIndexVisitor, error) {
|
||||
|
||||
@@ -10,22 +10,20 @@ import (
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
func NewSourceProvider(path string, exclude source.ExcludeConfig, alias source.Alias, basePath string, maxArchiveRecursiveIndexDepth int) source.Provider {
|
||||
func NewSourceProvider(path string, exclude source.ExcludeConfig, alias source.Alias, basePath string) source.Provider {
|
||||
return &directorySourceProvider{
|
||||
path: path,
|
||||
basePath: basePath,
|
||||
exclude: exclude,
|
||||
alias: alias,
|
||||
maxArchiveRecursiveIndexDepth: maxArchiveRecursiveIndexDepth,
|
||||
path: path,
|
||||
basePath: basePath,
|
||||
exclude: exclude,
|
||||
alias: alias,
|
||||
}
|
||||
}
|
||||
|
||||
type directorySourceProvider struct {
|
||||
path string
|
||||
basePath string
|
||||
exclude source.ExcludeConfig
|
||||
alias source.Alias
|
||||
maxArchiveRecursiveIndexDepth int
|
||||
path string
|
||||
basePath string
|
||||
exclude source.ExcludeConfig
|
||||
alias source.Alias
|
||||
}
|
||||
|
||||
func (l directorySourceProvider) Name() string {
|
||||
@@ -50,11 +48,10 @@ func (l directorySourceProvider) Provide(_ context.Context) (source.Source, erro
|
||||
|
||||
return New(
|
||||
Config{
|
||||
Path: location,
|
||||
Base: basePath(l.basePath, location),
|
||||
Exclude: l.exclude,
|
||||
Alias: l.alias,
|
||||
MaxArchiveRecursiveIndexDepth: l.maxArchiveRecursiveIndexDepth,
|
||||
Path: location,
|
||||
Base: basePath(l.basePath, location),
|
||||
Exclude: l.exclude,
|
||||
Alias: l.alias,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -27,12 +27,11 @@ import (
|
||||
var _ source.Source = (*fileSource)(nil)
|
||||
|
||||
type Config struct {
|
||||
Path string
|
||||
Exclude source.ExcludeConfig
|
||||
DigestAlgorithms []crypto.Hash
|
||||
Alias source.Alias
|
||||
SkipExtractArchive bool
|
||||
MaxArchiveRecursiveIndexDepth int
|
||||
Path string
|
||||
Exclude source.ExcludeConfig
|
||||
DigestAlgorithms []crypto.Hash
|
||||
Alias source.Alias
|
||||
SkipExtractArchive bool
|
||||
}
|
||||
|
||||
type fileSource struct {
|
||||
@@ -154,19 +153,12 @@ func (s fileSource) FileResolver(_ source.Scope) (file.Resolver, error) {
|
||||
|
||||
if isArchiveAnalysis := fi.IsDir(); isArchiveAnalysis {
|
||||
// this is an analysis of an archive file... we should scan the directory where the archive contents
|
||||
res, cleanupFn, err := fileresolver.NewFromDirectory(s.analysisPath, "", s.config.MaxArchiveRecursiveIndexDepth, exclusionFunctions...)
|
||||
res, err := fileresolver.NewFromDirectory(s.analysisPath, "", exclusionFunctions...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
|
||||
}
|
||||
|
||||
s.resolver = res
|
||||
s.closer = func() error {
|
||||
if err := cleanupFn(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.closer()
|
||||
}
|
||||
return s.resolver, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -11,22 +11,20 @@ import (
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
func NewSourceProvider(path string, exclude source.ExcludeConfig, digestAlgorithms []crypto.Hash, alias source.Alias, maxArchiveRecursiveIndexDepth int) source.Provider {
|
||||
func NewSourceProvider(path string, exclude source.ExcludeConfig, digestAlgorithms []crypto.Hash, alias source.Alias) source.Provider {
|
||||
return &fileSourceProvider{
|
||||
path: path,
|
||||
exclude: exclude,
|
||||
digestAlgorithms: digestAlgorithms,
|
||||
alias: alias,
|
||||
maxArchiveRecursiveIndexDepth: maxArchiveRecursiveIndexDepth,
|
||||
path: path,
|
||||
exclude: exclude,
|
||||
digestAlgorithms: digestAlgorithms,
|
||||
alias: alias,
|
||||
}
|
||||
}
|
||||
|
||||
type fileSourceProvider struct {
|
||||
path string
|
||||
exclude source.ExcludeConfig
|
||||
digestAlgorithms []crypto.Hash
|
||||
alias source.Alias
|
||||
maxArchiveRecursiveIndexDepth int
|
||||
path string
|
||||
exclude source.ExcludeConfig
|
||||
digestAlgorithms []crypto.Hash
|
||||
alias source.Alias
|
||||
}
|
||||
|
||||
func (p fileSourceProvider) Name() string {
|
||||
@@ -51,11 +49,10 @@ func (p fileSourceProvider) Provide(_ context.Context) (source.Source, error) {
|
||||
|
||||
return New(
|
||||
Config{
|
||||
Path: location,
|
||||
Exclude: p.exclude,
|
||||
DigestAlgorithms: p.digestAlgorithms,
|
||||
Alias: p.alias,
|
||||
MaxArchiveRecursiveIndexDepth: p.maxArchiveRecursiveIndexDepth,
|
||||
Path: location,
|
||||
Exclude: p.exclude,
|
||||
DigestAlgorithms: p.digestAlgorithms,
|
||||
Alias: p.alias,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -9,13 +9,12 @@ import (
|
||||
|
||||
// Config is the uber-configuration for all Syft source providers
|
||||
type Config struct {
|
||||
Platform *image.Platform
|
||||
Alias source.Alias
|
||||
RegistryOptions *image.RegistryOptions
|
||||
Exclude source.ExcludeConfig
|
||||
DigestAlgorithms []crypto.Hash
|
||||
BasePath string
|
||||
MaxArchiveRecursiveIndexDepth int
|
||||
Platform *image.Platform
|
||||
Alias source.Alias
|
||||
RegistryOptions *image.RegistryOptions
|
||||
Exclude source.ExcludeConfig
|
||||
DigestAlgorithms []crypto.Hash
|
||||
BasePath string
|
||||
}
|
||||
|
||||
func (c *Config) WithAlias(alias source.Alias) *Config {
|
||||
@@ -48,11 +47,6 @@ func (c *Config) WithBasePath(basePath string) *Config {
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Config) WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth int) *Config {
|
||||
c.MaxArchiveRecursiveIndexDepth = maxArchiveRecursiveIndexDepth
|
||||
return c
|
||||
}
|
||||
|
||||
func DefaultConfig() *Config {
|
||||
return &Config{
|
||||
DigestAlgorithms: []crypto.Hash{
|
||||
|
||||
@@ -36,8 +36,8 @@ func All(userInput string, cfg *Config) []collections.TaggedValue[source.Provide
|
||||
Join(tagProvider(snapsource.NewLocalSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias), SnapTag)).
|
||||
|
||||
// 2. try unspecific, local sources after other local sources last...
|
||||
Join(tagProvider(filesource.NewSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias, cfg.MaxArchiveRecursiveIndexDepth), FileTag)).
|
||||
Join(tagProvider(directorysource.NewSourceProvider(userInput, cfg.Exclude, cfg.Alias, cfg.BasePath, cfg.MaxArchiveRecursiveIndexDepth), DirTag)).
|
||||
Join(tagProvider(filesource.NewSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias), FileTag)).
|
||||
Join(tagProvider(directorysource.NewSourceProvider(userInput, cfg.Exclude, cfg.Alias, cfg.BasePath), DirTag)).
|
||||
|
||||
// 3. try remote sources after everything else...
|
||||
|
||||
|
||||
@@ -10,8 +10,6 @@ import (
|
||||
)
|
||||
|
||||
func TestArchiveScan(t *testing.T) {
|
||||
archive := createArchive(t, "testdata/archive", t.TempDir())
|
||||
nestedArchive := createArchive(t, filepath.Dir(archive), t.TempDir())
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
@@ -25,21 +23,7 @@ func TestArchiveScan(t *testing.T) {
|
||||
"scan",
|
||||
"-o",
|
||||
"json",
|
||||
"file:" + archive,
|
||||
},
|
||||
assertions: []traitAssertion{
|
||||
assertSuccessfulReturnCode,
|
||||
assertJsonReport,
|
||||
assertPackageCount(1),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "scan an nested archive within the temp dir",
|
||||
args: []string{
|
||||
"scan",
|
||||
"-o",
|
||||
"json",
|
||||
"file:" + nestedArchive,
|
||||
"file:" + createArchive(t, "testdata/archive", t.TempDir()),
|
||||
},
|
||||
assertions: []traitAssertion{
|
||||
assertSuccessfulReturnCode,
|
||||
|
||||
Reference in New Issue
Block a user