mirror of
https://github.com/anchore/syft.git
synced 2026-11-04 16:36:40 +01:00
Merge PR #4044: recursive nested-archive scanning at the resolver layer
Adds recursive archive indexing/traversal in the fileresolver and source provider layers. Original work by @Rupikz in anchore/syft#4044. # Conflicts: # cmd/syft/internal/options/source.go # syft/internal/fileresolver/directory.go # syft/internal/fileresolver/filetree_resolver.go # syft/internal/fileresolver/filetree_resolver_test.go # syft/pkg/cataloger/java/internal/maven/resolver_test.go # syft/source/directorysource/directory_source.go # test/cli/archive_test.go
This commit is contained in:
@@ -227,6 +227,7 @@ func getSource(ctx context.Context, opts *options.Catalog, userInput string, sou
|
|||||||
Paths: opts.Exclusions,
|
Paths: opts.Exclusions,
|
||||||
}).
|
}).
|
||||||
WithBasePath(opts.Source.BasePath).
|
WithBasePath(opts.Source.BasePath).
|
||||||
|
WithMaxArchiveRecursiveIndexDepth(opts.Source.Directory.MaxArchiveRecursiveIndexDepth).
|
||||||
WithSources(sources...).
|
WithSources(sources...).
|
||||||
WithDefaultImagePullSource(opts.Source.Image.DefaultPullSource)
|
WithDefaultImagePullSource(opts.Source.Image.DefaultPullSource)
|
||||||
|
|
||||||
|
|||||||
@@ -20,30 +20,19 @@ type sourceConfig struct {
|
|||||||
Source string `json:"source" yaml:"source" mapstructure:"source"`
|
Source string `json:"source" yaml:"source" mapstructure:"source"`
|
||||||
BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths
|
BasePath string `yaml:"base-path" json:"base-path" mapstructure:"base-path"` // specify base path for all file paths
|
||||||
File fileSource `json:"file" yaml:"file" mapstructure:"file"`
|
File fileSource `json:"file" yaml:"file" mapstructure:"file"`
|
||||||
|
Directory directorySource `json:"directory" yaml:"directory" mapstructure:"directory"`
|
||||||
Image imageSource `json:"image" yaml:"image" mapstructure:"image"`
|
Image imageSource `json:"image" yaml:"image" mapstructure:"image"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type fileSource struct {
|
var _ clio.FieldDescriber = (*sourceConfig)(nil)
|
||||||
Digests []string `json:"digests" yaml:"digests" mapstructure:"digests"`
|
|
||||||
}
|
|
||||||
|
|
||||||
var _ interface {
|
|
||||||
clio.FieldDescriber
|
|
||||||
} = (*sourceConfig)(nil)
|
|
||||||
|
|
||||||
var _ clio.PostLoader = (*imageSource)(nil)
|
|
||||||
|
|
||||||
func (o *sourceConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
|
func (o *sourceConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
|
||||||
descriptions.Add(&o.File.Digests, `the file digest algorithms to use on the scanned file (options: "md5", "sha1", "sha224", "sha256", "sha384", "sha512")`)
|
descriptions.Add(&o.File.Digests, `the file digest algorithms to use on the scanned file (options: "md5", "sha1", "sha224", "sha256", "sha384", "sha512")`)
|
||||||
|
descriptions.Add(&o.Directory.MaxArchiveRecursiveIndexDepth, `depth to recursively resolve archive parents, no limit if == -1`)
|
||||||
descriptions.Add(&o.Image.DefaultPullSource, `allows users to specify which image source should be used to generate the sbom
|
descriptions.Add(&o.Image.DefaultPullSource, `allows users to specify which image source should be used to generate the sbom
|
||||||
valid values are: registry, docker, podman`)
|
valid values are: registry, docker, podman`)
|
||||||
}
|
}
|
||||||
|
|
||||||
type imageSource struct {
|
|
||||||
DefaultPullSource string `json:"default-pull-source" yaml:"default-pull-source" mapstructure:"default-pull-source"`
|
|
||||||
MaxLayerSize string `json:"max-layer-size" yaml:"max-layer-size" mapstructure:"max-layer-size"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func defaultSourceConfig() sourceConfig {
|
func defaultSourceConfig() sourceConfig {
|
||||||
var digests []string
|
var digests []string
|
||||||
for _, alg := range sourceproviders.DefaultConfig().DigestAlgorithms {
|
for _, alg := range sourceproviders.DefaultConfig().DigestAlgorithms {
|
||||||
@@ -56,6 +45,12 @@ func defaultSourceConfig() sourceConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var _ clio.PostLoader = (*fileSource)(nil)
|
||||||
|
|
||||||
|
type fileSource struct {
|
||||||
|
Digests []string `json:"digests" yaml:"digests" mapstructure:"digests"`
|
||||||
|
}
|
||||||
|
|
||||||
func (c *fileSource) PostLoad() error {
|
func (c *fileSource) PostLoad() error {
|
||||||
digests := strset.New(c.Digests...).List()
|
digests := strset.New(c.Digests...).List()
|
||||||
sort.Strings(digests)
|
sort.Strings(digests)
|
||||||
@@ -63,6 +58,23 @@ func (c *fileSource) PostLoad() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var _ clio.PostLoader = (*directorySource)(nil)
|
||||||
|
|
||||||
|
type directorySource struct {
|
||||||
|
MaxArchiveRecursiveIndexDepth int `json:"max-archive-recursive-index-depth" yaml:"max-archive-recursive-index-depth" mapstructure:"max-archive-recursive-index-depth"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *directorySource) PostLoad() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ clio.PostLoader = (*imageSource)(nil)
|
||||||
|
|
||||||
|
type imageSource struct {
|
||||||
|
DefaultPullSource string `json:"default-pull-source" yaml:"default-pull-source" mapstructure:"default-pull-source"`
|
||||||
|
MaxLayerSize string `json:"max-layer-size" yaml:"max-layer-size" mapstructure:"max-layer-size"`
|
||||||
|
}
|
||||||
|
|
||||||
func (c *imageSource) PostLoad() error {
|
func (c *imageSource) PostLoad() error {
|
||||||
if c.MaxLayerSize != "" {
|
if c.MaxLayerSize != "" {
|
||||||
perFileReadLimit, err := humanize.ParseBytes(c.MaxLayerSize)
|
perFileReadLimit, err := humanize.ParseBytes(c.MaxLayerSize)
|
||||||
|
|||||||
@@ -126,6 +126,7 @@ require (
|
|||||||
github.com/ProtonMail/go-crypto v1.4.0 // indirect
|
github.com/ProtonMail/go-crypto v1.4.0 // indirect
|
||||||
github.com/STARRY-S/zip v0.2.3 // indirect
|
github.com/STARRY-S/zip v0.2.3 // indirect
|
||||||
github.com/agext/levenshtein v1.2.1 // indirect
|
github.com/agext/levenshtein v1.2.1 // indirect
|
||||||
|
github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51 // indirect
|
||||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||||
github.com/anchore/go-struct-converter v0.1.0 // indirect
|
github.com/anchore/go-struct-converter v0.1.0 // indirect
|
||||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||||
@@ -204,6 +205,7 @@ require (
|
|||||||
github.com/go-restruct/restruct v1.2.0-alpha // indirect
|
github.com/go-restruct/restruct v1.2.0-alpha // indirect
|
||||||
github.com/gogo/protobuf v1.3.2 // indirect
|
github.com/gogo/protobuf v1.3.2 // indirect
|
||||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||||
|
github.com/golang/snappy v0.0.4 // indirect
|
||||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect
|
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect
|
||||||
github.com/google/s2a-go v0.1.9 // indirect
|
github.com/google/s2a-go v0.1.9 // indirect
|
||||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||||
@@ -249,6 +251,7 @@ require (
|
|||||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||||
github.com/muesli/termenv v0.16.0 // indirect
|
github.com/muesli/termenv v0.16.0 // indirect
|
||||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||||
|
github.com/nwaples/rardecode v1.1.3 // indirect
|
||||||
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
|
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
|
||||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
|
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
|
||||||
github.com/olekukonko/errors v1.2.0 // indirect
|
github.com/olekukonko/errors v1.2.0 // indirect
|
||||||
|
|||||||
@@ -120,6 +120,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
|
|||||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||||
|
github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51 h1:yhk+P8lF3ZiROjmaVRao9WGTRo4b/wYjoKEiAHWrKwc=
|
||||||
|
github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51/go.mod h1:nwuGSd7aZp0rtYt79YggCGafz1RYsclE7pi3fhLwvuw=
|
||||||
github.com/anchore/bubbly v0.2.0 h1:VE6nLlWsYV+bTOqqCzS46NirrVmvHah3UQDtFcjzgHU=
|
github.com/anchore/bubbly v0.2.0 h1:VE6nLlWsYV+bTOqqCzS46NirrVmvHah3UQDtFcjzgHU=
|
||||||
github.com/anchore/bubbly v0.2.0/go.mod h1:zRg1mfRCCe4vq1t9Ibu24YurErmnp9LAVCbpnpsb1pQ=
|
github.com/anchore/bubbly v0.2.0/go.mod h1:zRg1mfRCCe4vq1t9Ibu24YurErmnp9LAVCbpnpsb1pQ=
|
||||||
github.com/anchore/clio v0.1.0 h1:lrlSgD38V0T6TLjOG5Kn3miEXCQyYr6ZlwTSnW1WIJg=
|
github.com/anchore/clio v0.1.0 h1:lrlSgD38V0T6TLjOG5Kn3miEXCQyYr6ZlwTSnW1WIJg=
|
||||||
@@ -493,6 +495,8 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
|
|||||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||||
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||||
|
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||||
|
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||||
@@ -757,6 +761,8 @@ github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 h1:kpt9ZfKcm+
|
|||||||
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1/go.mod h1:qgCw4bBKZX8qMgGeEZzGFVT3notl42dBjNqO2jut0M0=
|
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1/go.mod h1:qgCw4bBKZX8qMgGeEZzGFVT3notl42dBjNqO2jut0M0=
|
||||||
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE=
|
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE=
|
||||||
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8=
|
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8=
|
||||||
|
github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc=
|
||||||
|
github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
|
||||||
github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A=
|
github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A=
|
||||||
github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
|
github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
|
||||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc=
|
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc=
|
||||||
|
|||||||
+5
-1
@@ -153,7 +153,11 @@ func setupArchiveOrchestration(ctx context.Context, baseResolver file.Resolver,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("invalid exclusion patterns for archive child resolver: %w", err)
|
return nil, fmt.Errorf("invalid exclusion patterns for archive child resolver: %w", err)
|
||||||
}
|
}
|
||||||
return fileresolver.NewFromDirectory(root, "", filters...)
|
// depth 0: the archive Orchestrator (PR #4761) drives recursion here, so the
|
||||||
|
// resolver-level recursive archive indexing (PR #4044) is disabled to avoid
|
||||||
|
// double extraction. With depth 0 no temp dir is created, so the cleanup is a no-op.
|
||||||
|
res, _, err := fileresolver.NewFromDirectory(root, "", 0, filters...)
|
||||||
|
return res, err
|
||||||
}
|
}
|
||||||
|
|
||||||
orch := intArchive.NewOrchestrator(baseResolver, cfg.Archive, archiveTmpDir, resolverFactory)
|
orch := intArchive.NewOrchestrator(baseResolver, cfg.Archive, archiveTmpDir, resolverFactory)
|
||||||
|
|||||||
@@ -51,6 +51,11 @@ func (c *GetSourceConfig) WithBasePath(basePath string) *GetSourceConfig {
|
|||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *GetSourceConfig) WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth int) *GetSourceConfig {
|
||||||
|
c.SourceProviderConfig = c.SourceProviderConfig.WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
func (c *GetSourceConfig) WithSources(sources ...string) *GetSourceConfig {
|
func (c *GetSourceConfig) WithSources(sources ...string) *GetSourceConfig {
|
||||||
c.Sources = sources
|
c.Sources = sources
|
||||||
return c
|
return c
|
||||||
|
|||||||
@@ -3,11 +3,28 @@ package fileresolver
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/anchore/archiver/v3"
|
||||||
"github.com/anchore/stereoscope/pkg/filetree"
|
"github.com/anchore/stereoscope/pkg/filetree"
|
||||||
|
"github.com/anchore/syft/internal/log"
|
||||||
"github.com/anchore/syft/syft/file"
|
"github.com/anchore/syft/syft/file"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type archiveAccessPath struct {
|
||||||
|
realPath string
|
||||||
|
accessPath string
|
||||||
|
archiveRealPath string
|
||||||
|
archiveDepth int
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
archiveTempPathPattern = "syft-archivePaths-"
|
||||||
|
archiveContentTempPathPattern = "syft-archive-contents-"
|
||||||
|
)
|
||||||
|
|
||||||
var ErrSkipPath = errors.New("skip path")
|
var ErrSkipPath = errors.New("skip path")
|
||||||
|
|
||||||
var _ file.Resolver = (*Directory)(nil)
|
var _ file.Resolver = (*Directory)(nil)
|
||||||
@@ -19,13 +36,29 @@ type Directory struct {
|
|||||||
indexer *directoryIndexer
|
indexer *directoryIndexer
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewFromDirectory(root, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
|
func NewFromDirectory(root, base string, maxArchiveRecursiveIndexDepth int, pathFilters ...PathIndexVisitor) (*Directory, func() error, error) {
|
||||||
resolver, err := newFromDirectoryWithoutIndex(root, base, pathFilters...)
|
var cleanupFn = func() error { return nil }
|
||||||
|
directory, err := newFromDirectoryWithoutIndex(root, base, pathFilters...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, cleanupFn, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return resolver, resolver.buildIndex()
|
if err = directory.buildIndex(); err != nil {
|
||||||
|
return nil, cleanupFn, fmt.Errorf("unable to build index: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if maxArchiveRecursiveIndexDepth != 0 && len(directory.indexer.archivePaths) > 0 {
|
||||||
|
archiveTempDir, err := os.MkdirTemp("", archiveTempPathPattern)
|
||||||
|
if err != nil {
|
||||||
|
return nil, cleanupFn, fmt.Errorf("unable to create tempdir for archive processing: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return directory, func() error {
|
||||||
|
return os.RemoveAll(archiveTempDir)
|
||||||
|
}, directory.buildArchiveIndex(archiveTempDir, directory.indexer.archivePaths, maxArchiveRecursiveIndexDepth)
|
||||||
|
}
|
||||||
|
|
||||||
|
return directory, cleanupFn, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newFromDirectoryWithoutIndex(root, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
|
func newFromDirectoryWithoutIndex(root, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
|
||||||
@@ -65,6 +98,91 @@ func (r *Directory) buildIndex() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *Directory) buildArchiveIndex(archiveTempDir string, archives []string, maxArchiveIndexDepth int) error {
|
||||||
|
archivesToIndex := make([]archiveAccessPath, len(archives))
|
||||||
|
for i, archive := range archives {
|
||||||
|
archivesToIndex[i] = archiveAccessPath{
|
||||||
|
realPath: r.Chroot.ToChrootPath(archive),
|
||||||
|
accessPath: r.Chroot.ToChrootPath(archive),
|
||||||
|
archiveRealPath: archive,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
loop:
|
||||||
|
for {
|
||||||
|
var currentArchivePath archiveAccessPath
|
||||||
|
switch len(archivesToIndex) {
|
||||||
|
case 0:
|
||||||
|
break loop
|
||||||
|
case 1:
|
||||||
|
currentArchivePath, archivesToIndex = archivesToIndex[0], nil
|
||||||
|
default:
|
||||||
|
currentArchivePath, archivesToIndex = archivesToIndex[0], archivesToIndex[1:]
|
||||||
|
}
|
||||||
|
|
||||||
|
if maxArchiveIndexDepth != -1 && currentArchivePath.archiveDepth >= maxArchiveIndexDepth {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
envelopedUnarchiver, err := archiver.ByExtension(currentArchivePath.archiveRealPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("could not determine archive envelope type: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver)
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("unarchiver %T is not an archiver", envelopedUnarchiver)
|
||||||
|
}
|
||||||
|
|
||||||
|
archiveDestinationTempDir, err := os.MkdirTemp(archiveTempDir, archiveContentTempPathPattern)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("unable to create tempdir for archive processing: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
archiveDestinationPath, err := filepath.EvalSymlinks(archiveDestinationTempDir)
|
||||||
|
if err != nil {
|
||||||
|
var pathErr *os.PathError
|
||||||
|
if errors.As(err, &pathErr) {
|
||||||
|
// we can't index the path, but we shouldn't consider this to be fatal
|
||||||
|
// TODO: known-unknowns
|
||||||
|
log.WithFields("archivePath", archiveDestinationTempDir, "error", err).Trace("unable to evaluate symlink while indexing branch")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = unarchiver.Unarchive(currentArchivePath.archiveRealPath, archiveDestinationPath); err != nil {
|
||||||
|
return fmt.Errorf("unable to unarchive nested file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := newFromDirectoryWithoutIndex(archiveDestinationPath, "")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err = d.buildIndex(); err != nil {
|
||||||
|
return fmt.Errorf("unable to build index of archive: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, archive := range d.indexer.archivePaths {
|
||||||
|
archivesToIndex = append(archivesToIndex, archiveAccessPath{
|
||||||
|
realPath: currentArchivePath.realPath,
|
||||||
|
accessPath: strings.Replace(archive, archiveDestinationPath, currentArchivePath.accessPath, 1),
|
||||||
|
archiveRealPath: archive,
|
||||||
|
archiveDepth: currentArchivePath.archiveDepth + 1,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
d.realPath = currentArchivePath.realPath
|
||||||
|
d.accessPath = currentArchivePath.accessPath
|
||||||
|
d.archiveRealPath = archiveDestinationPath
|
||||||
|
|
||||||
|
r.archives = append(r.archives, &d.FiletreeResolver)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Stringer to represent a directory path data source
|
// Stringer to represent a directory path data source
|
||||||
func (r *Directory) String() string {
|
func (r *Directory) String() string {
|
||||||
return fmt.Sprintf("dir:%s", r.path)
|
return fmt.Sprintf("dir:%s", r.path)
|
||||||
|
|||||||
@@ -9,13 +9,13 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/wagoodman/go-progress"
|
"github.com/anchore/archiver/v3"
|
||||||
|
|
||||||
"github.com/anchore/stereoscope/pkg/file"
|
"github.com/anchore/stereoscope/pkg/file"
|
||||||
"github.com/anchore/stereoscope/pkg/filetree"
|
"github.com/anchore/stereoscope/pkg/filetree"
|
||||||
"github.com/anchore/syft/internal/bus"
|
"github.com/anchore/syft/internal/bus"
|
||||||
"github.com/anchore/syft/internal/log"
|
"github.com/anchore/syft/internal/log"
|
||||||
"github.com/anchore/syft/syft/internal/windows"
|
"github.com/anchore/syft/syft/internal/windows"
|
||||||
|
"github.com/wagoodman/go-progress"
|
||||||
)
|
)
|
||||||
|
|
||||||
type PathIndexVisitor func(string, string, os.FileInfo, error) error
|
type PathIndexVisitor func(string, string, os.FileInfo, error) error
|
||||||
@@ -27,6 +27,7 @@ type directoryIndexer struct {
|
|||||||
errPaths map[string]error
|
errPaths map[string]error
|
||||||
tree filetree.ReadWriter
|
tree filetree.ReadWriter
|
||||||
index filetree.Index
|
index filetree.Index
|
||||||
|
archivePaths []string
|
||||||
}
|
}
|
||||||
|
|
||||||
func newDirectoryIndexer(path, base string, visitors ...PathIndexVisitor) *directoryIndexer {
|
func newDirectoryIndexer(path, base string, visitors ...PathIndexVisitor) *directoryIndexer {
|
||||||
@@ -144,6 +145,10 @@ func (r *directoryIndexer) indexTree(root string, stager *progress.AtomicStage)
|
|||||||
func(path string, info os.FileInfo, err error) error {
|
func(path string, info os.FileInfo, err error) error {
|
||||||
stager.Set(path)
|
stager.Set(path)
|
||||||
|
|
||||||
|
if isArchive(path, info) {
|
||||||
|
r.archivePaths = append(r.archivePaths, path)
|
||||||
|
}
|
||||||
|
|
||||||
newRoot, err := r.indexPath(path, info, err)
|
newRoot, err := r.indexPath(path, info, err)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -164,6 +169,17 @@ func (r *directoryIndexer) indexTree(root string, stager *progress.AtomicStage)
|
|||||||
return roots, nil
|
return roots, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isArchive returns true if the path appears to be an archive
|
||||||
|
func isArchive(path string, info os.FileInfo) bool {
|
||||||
|
if info == nil || !info.Mode().IsRegular() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
envelopedUnarchiver, err := archiver.ByExtension(path)
|
||||||
|
_, ok := envelopedUnarchiver.(archiver.Unarchiver)
|
||||||
|
return err == nil && ok
|
||||||
|
}
|
||||||
|
|
||||||
func isRealPath(root string) (bool, error) {
|
func isRealPath(root string) (bool, error) {
|
||||||
rootParent := filepath.Clean(filepath.Dir(root))
|
rootParent := filepath.Clean(filepath.Dir(root))
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||||
"github.com/anchore/stereoscope/pkg/filetree"
|
"github.com/anchore/stereoscope/pkg/filetree"
|
||||||
@@ -21,6 +22,11 @@ type FiletreeResolver struct {
|
|||||||
Index filetree.IndexReader
|
Index filetree.IndexReader
|
||||||
SearchContext filetree.Searcher
|
SearchContext filetree.Searcher
|
||||||
Opener func(stereoscopeFile.Reference) (io.ReadCloser, error)
|
Opener func(stereoscopeFile.Reference) (io.ReadCloser, error)
|
||||||
|
|
||||||
|
realPath string
|
||||||
|
accessPath string
|
||||||
|
archiveRealPath string
|
||||||
|
archives []*FiletreeResolver
|
||||||
}
|
}
|
||||||
|
|
||||||
func nativeOSFileOpener(ref stereoscopeFile.Reference) (io.ReadCloser, error) {
|
func nativeOSFileOpener(ref stereoscopeFile.Reference) (io.ReadCloser, error) {
|
||||||
@@ -35,7 +41,16 @@ func nativeOSFileOpener(ref stereoscopeFile.Reference) (io.ReadCloser, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *FiletreeResolver) requestPath(userPath string) (string, error) {
|
func (r *FiletreeResolver) requestPath(userPath string) (string, error) {
|
||||||
return r.Chroot.ToNativePath(userPath)
|
requestPath, err := r.Chroot.ToNativePath(userPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.accessPath != "" && r.archiveRealPath != "" {
|
||||||
|
return strings.Replace(requestPath, r.accessPath, r.archiveRealPath, 1), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return requestPath, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the file resolver.
|
// responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the file resolver.
|
||||||
@@ -43,13 +58,32 @@ func (r FiletreeResolver) responsePath(path string) string {
|
|||||||
return r.Chroot.ToChrootPath(path)
|
return r.Chroot.ToChrootPath(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r FiletreeResolver) responseAccessPath(path string) string {
|
||||||
|
if r.accessPath != "" && r.archiveRealPath != "" {
|
||||||
|
path = strings.Replace(path, r.archiveRealPath, r.accessPath, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r.Chroot.ToChrootPath(path)
|
||||||
|
}
|
||||||
|
|
||||||
// HasPath indicates if the given path exists in the underlying source.
|
// HasPath indicates if the given path exists in the underlying source.
|
||||||
func (r *FiletreeResolver) HasPath(userPath string) bool {
|
func (r *FiletreeResolver) HasPath(userPath string) bool {
|
||||||
requestPath, err := r.requestPath(userPath)
|
requestPath, err := r.requestPath(userPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return r.Tree.HasPath(stereoscopeFile.Path(requestPath))
|
|
||||||
|
if r.Tree.HasPath(stereoscopeFile.Path(requestPath)) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, archive := range r.archives {
|
||||||
|
if archive.HasPath(userPath) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// FilesByPath returns all file.References that match the given paths from the file index.
|
// FilesByPath returns all file.References that match the given paths from the file index.
|
||||||
@@ -93,18 +127,35 @@ func (r FiletreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err
|
|||||||
references = append(references,
|
references = append(references,
|
||||||
file.NewVirtualLocationFromDirectory(
|
file.NewVirtualLocationFromDirectory(
|
||||||
r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root
|
r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root
|
||||||
r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root
|
r.responseAccessPath(userStrPath), // the path used to access this file, relative to the resolver root
|
||||||
*ref.Reference,
|
*ref.Reference,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, archive := range r.archives {
|
||||||
|
locations, err := archive.FilesByPath(userPaths...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
references = append(references, locations...)
|
||||||
|
}
|
||||||
|
|
||||||
return references, nil
|
return references, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r FiletreeResolver) requestGlob(pattern string) (string, error) {
|
func (r FiletreeResolver) requestGlob(pattern string) (string, error) {
|
||||||
return r.Chroot.ToNativeGlob(pattern)
|
nativeGlob, err := r.Chroot.ToNativeGlob(pattern)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.accessPath != "" && r.archiveRealPath != "" {
|
||||||
|
return strings.Replace(nativeGlob, r.accessPath, r.archiveRealPath, 1), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nativeGlob, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
|
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
|
||||||
@@ -137,7 +188,7 @@ func (r FiletreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro
|
|||||||
|
|
||||||
loc := file.NewVirtualLocationFromDirectory(
|
loc := file.NewVirtualLocationFromDirectory(
|
||||||
r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root
|
r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root
|
||||||
r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
|
r.responseAccessPath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
|
||||||
*refVia.Reference,
|
*refVia.Reference,
|
||||||
)
|
)
|
||||||
uniqueFileIDs.Add(*refVia.Reference)
|
uniqueFileIDs.Add(*refVia.Reference)
|
||||||
@@ -145,6 +196,20 @@ func (r FiletreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, archive := range r.archives {
|
||||||
|
archiveUniqueLocations, err := archive.FilesByGlob(patterns...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, archiveUniqueLocation := range archiveUniqueLocations {
|
||||||
|
if uniqueFileIDs.Contains(archiveUniqueLocation.Reference()) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
uniqueLocations = append(uniqueLocations, archiveUniqueLocation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return uniqueLocations, nil
|
return uniqueLocations, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -170,6 +235,14 @@ func (r FiletreeResolver) FileContentsByLocation(location file.Location) (io.Rea
|
|||||||
}
|
}
|
||||||
|
|
||||||
entry, err := r.Index.Get(location.Reference())
|
entry, err := r.Index.Get(location.Reference())
|
||||||
|
if err != nil {
|
||||||
|
for _, archive := range r.archives {
|
||||||
|
entry, err = archive.Index.Get(location.Reference())
|
||||||
|
if err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -190,21 +263,39 @@ func (r *FiletreeResolver) AllLocations(ctx context.Context) <-chan file.Locatio
|
|||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
case results <- file.NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), "", ref):
|
case results <- file.NewVirtualLocationFromDirectory(r.responsePath(string(ref.RealPath)), r.responseAccessPath(string(ref.RealPath)), ref):
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, archive := range r.archives {
|
||||||
|
for location := range archive.AllLocations(ctx) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case results <- location:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}()
|
}()
|
||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *FiletreeResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
|
func (r *FiletreeResolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
|
||||||
entry, err := r.Index.Get(location.Reference())
|
entry, err := r.Index.Get(location.Reference())
|
||||||
if err != nil {
|
if err == nil {
|
||||||
return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
|
return entry.Metadata, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, archive := range r.archives {
|
||||||
|
entry, err = archive.Index.Get(location.Reference())
|
||||||
|
if err == nil {
|
||||||
return entry.Metadata, nil
|
return entry.Metadata, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *FiletreeResolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
|
func (r *FiletreeResolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
|
||||||
@@ -224,12 +315,26 @@ func (r *FiletreeResolver) FilesByMIMEType(types ...string) ([]file.Location, er
|
|||||||
}
|
}
|
||||||
location := file.NewVirtualLocationFromDirectory(
|
location := file.NewVirtualLocationFromDirectory(
|
||||||
r.responsePath(string(refVia.RealPath)),
|
r.responsePath(string(refVia.RealPath)),
|
||||||
r.responsePath(string(refVia.RequestPath)),
|
r.responseAccessPath(string(refVia.RequestPath)),
|
||||||
*refVia.Reference,
|
*refVia.Reference,
|
||||||
)
|
)
|
||||||
uniqueFileIDs.Add(*refVia.Reference)
|
uniqueFileIDs.Add(*refVia.Reference)
|
||||||
uniqueLocations = append(uniqueLocations, location)
|
uniqueLocations = append(uniqueLocations, location)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, archive := range r.archives {
|
||||||
|
archiveUniqueLocations, err := archive.FilesByMIMEType(types...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, archiveUniqueLocation := range archiveUniqueLocations {
|
||||||
|
if uniqueFileIDs.Contains(archiveUniqueLocation.Reference()) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
uniqueLocations = append(uniqueLocations, archiveUniqueLocation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return uniqueLocations, nil
|
return uniqueLocations, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -505,9 +505,12 @@ func TestDirectoryResolver_FilesByPath_request_response(t *testing.T) {
|
|||||||
require.NoError(t, os.Chdir(testDir))
|
require.NoError(t, os.Chdir(testDir))
|
||||||
})
|
})
|
||||||
|
|
||||||
resolver, err := NewFromDirectory(c.root, c.base)
|
resolver, cleanupFn, err := NewFromDirectory(c.root, c.base, 0)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NotNil(t, resolver)
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
refs, err := resolver.FilesByPath(c.input)
|
refs, err := resolver.FilesByPath(c.input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -557,8 +560,12 @@ func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
t.Run(c.name, func(t *testing.T) {
|
t.Run(c.name, func(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory(c.relativeRoot, "")
|
resolver, cleanupFn, err := NewFromDirectory(c.relativeRoot, "", 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
refs, err := resolver.FilesByPath(c.input)
|
refs, err := resolver.FilesByPath(c.input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -612,8 +619,12 @@ func TestDirectoryResolver_FilesByPath_absoluteRoot(t *testing.T) {
|
|||||||
absRoot, err := filepath.Abs(c.relativeRoot)
|
absRoot, err := filepath.Abs(c.relativeRoot)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
resolver, err := NewFromDirectory(absRoot, "")
|
resolver, cleanupFn, err := NewFromDirectory(absRoot, "", 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
refs, err := resolver.FilesByPath(c.input)
|
refs, err := resolver.FilesByPath(c.input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -673,8 +684,12 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
t.Run(c.name, func(t *testing.T) {
|
t.Run(c.name, func(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory(c.root, "")
|
resolver, cleanupFn, err := NewFromDirectory(c.root, "", 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
hasPath := resolver.HasPath(c.input)
|
hasPath := resolver.HasPath(c.input)
|
||||||
if !c.forcePositiveHasPath {
|
if !c.forcePositiveHasPath {
|
||||||
@@ -721,8 +736,13 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
t.Run(c.name, func(t *testing.T) {
|
t.Run(c.name, func(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory("./testdata", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata", "", 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
refs, err := resolver.FilesByPath(c.input...)
|
refs, err := resolver.FilesByPath(c.input...)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
@@ -734,8 +754,13 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
|
func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory("./testdata", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata", "", 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
refs, err := resolver.FilesByGlob("**/image-symlinks/file*")
|
refs, err := resolver.FilesByGlob("**/image-symlinks/file*")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
@@ -743,16 +768,26 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
|
func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory("./testdata/image-symlinks", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata/image-symlinks", "", 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
refs, err := resolver.FilesByGlob("**/*.txt")
|
refs, err := resolver.FilesByGlob("**/*.txt")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.Len(t, refs, 6)
|
assert.Len(t, refs, 6)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
|
func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory("./testdata", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata", "", 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
refs, err := resolver.FilesByGlob("**/image-symlinks/*1.txt")
|
refs, err := resolver.FilesByGlob("**/image-symlinks/*1.txt")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
|
|
||||||
@@ -778,8 +813,12 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {
|
|||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory("./testdata/symlinks-simple", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-simple", "", 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
refs, err := resolver.FilesByPath(test.fixture)
|
refs, err := resolver.FilesByPath(test.fixture)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -801,8 +840,12 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {
|
|||||||
|
|
||||||
func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) {
|
func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) {
|
||||||
// let's make certain that "dev/place" is not ignored, since it is not "/dev/place"
|
// let's make certain that "dev/place" is not ignored, since it is not "/dev/place"
|
||||||
resolver, err := NewFromDirectory("testdata/system_paths/target", "")
|
resolver, cleanupFn, err := NewFromDirectory("testdata/system_paths/target", "", 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
// all paths should be found (non filtering matches a path)
|
// all paths should be found (non filtering matches a path)
|
||||||
locations, err := resolver.FilesByGlob("**/place")
|
locations, err := resolver.FilesByGlob("**/place")
|
||||||
@@ -840,8 +883,13 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.fixturePath, func(t *testing.T) {
|
t.Run(test.fixturePath, func(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory(test.fixturePath, "")
|
resolver, cleanupFn, err := NewFromDirectory(test.fixturePath, "", 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
locations, err := resolver.FilesByMIMEType(test.mimeType)
|
locations, err := resolver.FilesByMIMEType(test.mimeType)
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.Equal(t, test.expectedPaths.Size(), len(locations))
|
assert.Equal(t, test.expectedPaths.Size(), len(locations))
|
||||||
@@ -853,8 +901,12 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func Test_IndexingNestedSymLinks(t *testing.T) {
|
func Test_IndexingNestedSymLinks(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory("./testdata/symlinks-simple", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-simple", "", 0)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
// check that we can get the real path
|
// check that we can get the real path
|
||||||
locations, err := resolver.FilesByPath("./readme")
|
locations, err := resolver.FilesByPath("./readme")
|
||||||
@@ -909,8 +961,12 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
resolver, err := NewFromDirectory("./testdata/symlinks-simple", "", filterFn)
|
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-simple", "", 0, filterFn)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
// the path to the real file is PRUNED from the index, so we should NOT expect a location returned
|
// the path to the real file is PRUNED from the index, so we should NOT expect a location returned
|
||||||
locations, err := resolver.FilesByPath("./readme")
|
locations, err := resolver.FilesByPath("./readme")
|
||||||
@@ -929,8 +985,12 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
|
func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory("./testdata/symlinks-multiple-roots/root", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-multiple-roots/root", "", 0)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
// check that we can get the real path
|
// check that we can get the real path
|
||||||
locations, err := resolver.FilesByPath("./readme")
|
locations, err := resolver.FilesByPath("./readme")
|
||||||
@@ -947,8 +1007,12 @@ func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func Test_RootViaSymlink(t *testing.T) {
|
func Test_RootViaSymlink(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory("./testdata/symlinked-root/nested/link-root", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinked-root/nested/link-root", "", 0)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
locations, err := resolver.FilesByPath("./file1.txt")
|
locations, err := resolver.FilesByPath("./file1.txt")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -967,10 +1031,14 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
|
|||||||
cwd, err := os.Getwd()
|
cwd, err := os.Getwd()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
r, err := NewFromDirectory(".", "")
|
resolver, cleanupFn, err := NewFromDirectory(".", "", 0)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
exists, existingPath, err := r.Tree.File(stereoscopeFile.Path(filepath.Join(cwd, "testdata/image-simple/file-1.txt")))
|
exists, existingPath, err := resolver.Tree.File(stereoscopeFile.Path(filepath.Join(cwd, "testdata/image-simple/file-1.txt")))
|
||||||
require.True(t, exists)
|
require.True(t, exists)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.True(t, existingPath.HasReference())
|
require.True(t, existingPath.HasReference())
|
||||||
@@ -994,8 +1062,7 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
actual, err := resolver.FileContentsByLocation(test.location)
|
||||||
actual, err := r.FileContentsByLocation(test.location)
|
|
||||||
if test.err {
|
if test.err {
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
return
|
return
|
||||||
@@ -1013,8 +1080,12 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
|
|||||||
|
|
||||||
func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) {
|
func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) {
|
||||||
test := func(t *testing.T) {
|
test := func(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory("./testdata/symlinks-loop", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-loop", "", 0)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
locations, err := resolver.FilesByGlob("**/file.target")
|
locations, err := resolver.FilesByGlob("**/file.target")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -1084,8 +1155,12 @@ func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
t.Run(c.name, func(t *testing.T) {
|
t.Run(c.name, func(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory(c.root, c.root)
|
resolver, cleanupFn, err := NewFromDirectory(c.root, c.root, 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
refs, err := resolver.FilesByPath(c.input)
|
refs, err := resolver.FilesByPath(c.input)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -1228,9 +1303,12 @@ func Test_directoryResolver_resolvesLinks(t *testing.T) {
|
|||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "", 0)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.NoError(t, err)
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
actual := test.runner(resolver)
|
actual := test.runner(resolver)
|
||||||
|
|
||||||
@@ -1240,8 +1318,12 @@ func Test_directoryResolver_resolvesLinks(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
|
func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory("./testdata/symlinks-prune-indexing", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-prune-indexing", "", 0)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
var allRealPaths []stereoscopeFile.Path
|
var allRealPaths []stereoscopeFile.Path
|
||||||
for l := range resolver.AllLocations(context.Background()) {
|
for l := range resolver.AllLocations(context.Background()) {
|
||||||
@@ -1263,8 +1345,12 @@ func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
|
|||||||
|
|
||||||
func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
|
func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
|
||||||
defer goleak.VerifyNone(t)
|
defer goleak.VerifyNone(t)
|
||||||
resolver, err := NewFromDirectory("./testdata/system_paths", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata/system_paths", "", 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
var dirLoc *file.Location
|
var dirLoc *file.Location
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
@@ -1286,8 +1372,12 @@ func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestDirectoryResolver_AllLocations(t *testing.T) {
|
func TestDirectoryResolver_AllLocations(t *testing.T) {
|
||||||
resolver, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "", 0)
|
||||||
assert.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
paths := strset.New()
|
paths := strset.New()
|
||||||
for loc := range resolver.AllLocations(context.Background()) {
|
for loc := range resolver.AllLocations(context.Background()) {
|
||||||
@@ -1319,8 +1409,13 @@ func TestDirectoryResolver_AllLocations(t *testing.T) {
|
|||||||
|
|
||||||
func TestAllLocationsDoesNotLeakGoRoutine(t *testing.T) {
|
func TestAllLocationsDoesNotLeakGoRoutine(t *testing.T) {
|
||||||
defer goleak.VerifyNone(t)
|
defer goleak.VerifyNone(t)
|
||||||
resolver, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "")
|
resolver, cleanupFn, err := NewFromDirectory("./testdata/symlinks-from-image-symlinks-fixture", "", 0)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
for range resolver.AllLocations(ctx) {
|
for range resolver.AllLocations(ctx) {
|
||||||
break
|
break
|
||||||
@@ -1553,6 +1648,7 @@ func Test_fileResolver_FileContentsByLocation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestFileResolver_AllLocations_errorOnDirRequest(t *testing.T) {
|
func TestFileResolver_AllLocations_errorOnDirRequest(t *testing.T) {
|
||||||
|
defer goleak.VerifyNone(t)
|
||||||
filePath := "./testdata/system_paths/target/home/place"
|
filePath := "./testdata/system_paths/target/home/place"
|
||||||
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
|
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -1579,11 +1675,10 @@ func TestFileResolver_AllLocations_errorOnDirRequest(t *testing.T) {
|
|||||||
reader, err := resolver.FileContentsByLocation(*dirLoc)
|
reader, err := resolver.FileContentsByLocation(*dirLoc)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
require.Nil(t, reader)
|
require.Nil(t, reader)
|
||||||
|
|
||||||
goleak.VerifyNone(t)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFileResolver_AllLocations(t *testing.T) {
|
func TestFileResolver_AllLocations(t *testing.T) {
|
||||||
|
defer goleak.VerifyNone(t)
|
||||||
// Verify both the parent and the file itself are indexed
|
// Verify both the parent and the file itself are indexed
|
||||||
filePath := "./testdata/system_paths/target/home/place"
|
filePath := "./testdata/system_paths/target/home/place"
|
||||||
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
|
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
|
||||||
@@ -1608,11 +1703,10 @@ func TestFileResolver_AllLocations(t *testing.T) {
|
|||||||
sort.Strings(pathsList)
|
sort.Strings(pathsList)
|
||||||
|
|
||||||
assert.ElementsMatchf(t, expected, pathsList, "expected all paths to be indexed, but found different paths: \n%s", cmp.Diff(expected, paths.List()))
|
assert.ElementsMatchf(t, expected, pathsList, "expected all paths to be indexed, but found different paths: \n%s", cmp.Diff(expected, paths.List()))
|
||||||
|
|
||||||
goleak.VerifyNone(t)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func Test_FileResolver_AllLocationsDoesNotLeakGoRoutine(t *testing.T) {
|
func Test_FileResolver_AllLocationsDoesNotLeakGoRoutine(t *testing.T) {
|
||||||
|
defer goleak.VerifyNone(t)
|
||||||
filePath := "./testdata/system_paths/target/home/place"
|
filePath := "./testdata/system_paths/target/home/place"
|
||||||
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
|
parentPath, err := absoluteSymlinkFreePathToParent(filePath)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -1629,6 +1723,4 @@ func Test_FileResolver_AllLocationsDoesNotLeakGoRoutine(t *testing.T) {
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
goleak.VerifyNone(t)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -273,8 +273,12 @@ func Test_mavenResolverRemote(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func Test_relativePathParent(t *testing.T) {
|
func Test_relativePathParent(t *testing.T) {
|
||||||
resolver, err := fileresolver.NewFromDirectory("testdata/local", "")
|
resolver, cleanupFn, err := fileresolver.NewFromDirectory("testdata/local", "", 0)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, resolver)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
require.NoError(t, cleanupFn())
|
||||||
|
})
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ type Config struct {
|
|||||||
Base string
|
Base string
|
||||||
Exclude source.ExcludeConfig
|
Exclude source.ExcludeConfig
|
||||||
Alias source.Alias
|
Alias source.Alias
|
||||||
|
MaxArchiveRecursiveIndexDepth int
|
||||||
}
|
}
|
||||||
|
|
||||||
type directorySource struct {
|
type directorySource struct {
|
||||||
@@ -32,6 +33,7 @@ type directorySource struct {
|
|||||||
config Config
|
config Config
|
||||||
resolver file.Resolver
|
resolver file.Resolver
|
||||||
mutex *sync.Mutex
|
mutex *sync.Mutex
|
||||||
|
closer func() error
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewFromPath(path string) (source.Source, error) {
|
func NewFromPath(path string) (source.Source, error) {
|
||||||
@@ -105,11 +107,12 @@ func (s *directorySource) FileResolver(_ source.Scope) (file.Resolver, error) {
|
|||||||
// this should be the only file resolver that might have overlap with where files are cached
|
// this should be the only file resolver that might have overlap with where files are cached
|
||||||
exclusionFunctions = append(exclusionFunctions, excludeCachePathVisitors()...)
|
exclusionFunctions = append(exclusionFunctions, excludeCachePathVisitors()...)
|
||||||
|
|
||||||
res, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, exclusionFunctions...)
|
res, cleanupFn, err := fileresolver.NewFromDirectory(s.config.Path, s.config.Base, s.config.MaxArchiveRecursiveIndexDepth, exclusionFunctions...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
|
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
s.closer = cleanupFn
|
||||||
s.resolver = res
|
s.resolver = res
|
||||||
return s.resolver, nil
|
return s.resolver, nil
|
||||||
}
|
}
|
||||||
@@ -119,7 +122,10 @@ func (s *directorySource) Close() error {
|
|||||||
defer s.mutex.Unlock()
|
defer s.mutex.Unlock()
|
||||||
|
|
||||||
s.resolver = nil
|
s.resolver = nil
|
||||||
|
if s.closer == nil {
|
||||||
return nil
|
return nil
|
||||||
|
}
|
||||||
|
return s.closer()
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileresolver.PathIndexVisitor, error) {
|
func GetDirectoryExclusionFunctions(root string, exclusions []string) ([]fileresolver.PathIndexVisitor, error) {
|
||||||
|
|||||||
@@ -10,12 +10,13 @@ import (
|
|||||||
"github.com/anchore/syft/syft/source"
|
"github.com/anchore/syft/syft/source"
|
||||||
)
|
)
|
||||||
|
|
||||||
func NewSourceProvider(path string, exclude source.ExcludeConfig, alias source.Alias, basePath string) source.Provider {
|
func NewSourceProvider(path string, exclude source.ExcludeConfig, alias source.Alias, basePath string, maxArchiveRecursiveIndexDepth int) source.Provider {
|
||||||
return &directorySourceProvider{
|
return &directorySourceProvider{
|
||||||
path: path,
|
path: path,
|
||||||
basePath: basePath,
|
basePath: basePath,
|
||||||
exclude: exclude,
|
exclude: exclude,
|
||||||
alias: alias,
|
alias: alias,
|
||||||
|
maxArchiveRecursiveIndexDepth: maxArchiveRecursiveIndexDepth,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -24,6 +25,7 @@ type directorySourceProvider struct {
|
|||||||
basePath string
|
basePath string
|
||||||
exclude source.ExcludeConfig
|
exclude source.ExcludeConfig
|
||||||
alias source.Alias
|
alias source.Alias
|
||||||
|
maxArchiveRecursiveIndexDepth int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l directorySourceProvider) Name() string {
|
func (l directorySourceProvider) Name() string {
|
||||||
@@ -52,6 +54,7 @@ func (l directorySourceProvider) Provide(_ context.Context) (source.Source, erro
|
|||||||
Base: basePath(l.basePath, location),
|
Base: basePath(l.basePath, location),
|
||||||
Exclude: l.exclude,
|
Exclude: l.exclude,
|
||||||
Alias: l.alias,
|
Alias: l.alias,
|
||||||
|
MaxArchiveRecursiveIndexDepth: l.maxArchiveRecursiveIndexDepth,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ type Config struct {
|
|||||||
DigestAlgorithms []crypto.Hash
|
DigestAlgorithms []crypto.Hash
|
||||||
Alias source.Alias
|
Alias source.Alias
|
||||||
SkipExtractArchive bool
|
SkipExtractArchive bool
|
||||||
|
MaxArchiveRecursiveIndexDepth int
|
||||||
}
|
}
|
||||||
|
|
||||||
type fileSource struct {
|
type fileSource struct {
|
||||||
@@ -153,12 +154,19 @@ func (s fileSource) FileResolver(_ source.Scope) (file.Resolver, error) {
|
|||||||
|
|
||||||
if isArchiveAnalysis := fi.IsDir(); isArchiveAnalysis {
|
if isArchiveAnalysis := fi.IsDir(); isArchiveAnalysis {
|
||||||
// this is an analysis of an archive file... we should scan the directory where the archive contents
|
// this is an analysis of an archive file... we should scan the directory where the archive contents
|
||||||
res, err := fileresolver.NewFromDirectory(s.analysisPath, "", exclusionFunctions...)
|
res, cleanupFn, err := fileresolver.NewFromDirectory(s.analysisPath, "", s.config.MaxArchiveRecursiveIndexDepth, exclusionFunctions...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
|
return nil, fmt.Errorf("unable to create directory resolver: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.resolver = res
|
s.resolver = res
|
||||||
|
s.closer = func() error {
|
||||||
|
if err := cleanupFn(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return s.closer()
|
||||||
|
}
|
||||||
return s.resolver, nil
|
return s.resolver, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,12 +11,13 @@ import (
|
|||||||
"github.com/anchore/syft/syft/source"
|
"github.com/anchore/syft/syft/source"
|
||||||
)
|
)
|
||||||
|
|
||||||
func NewSourceProvider(path string, exclude source.ExcludeConfig, digestAlgorithms []crypto.Hash, alias source.Alias) source.Provider {
|
func NewSourceProvider(path string, exclude source.ExcludeConfig, digestAlgorithms []crypto.Hash, alias source.Alias, maxArchiveRecursiveIndexDepth int) source.Provider {
|
||||||
return &fileSourceProvider{
|
return &fileSourceProvider{
|
||||||
path: path,
|
path: path,
|
||||||
exclude: exclude,
|
exclude: exclude,
|
||||||
digestAlgorithms: digestAlgorithms,
|
digestAlgorithms: digestAlgorithms,
|
||||||
alias: alias,
|
alias: alias,
|
||||||
|
maxArchiveRecursiveIndexDepth: maxArchiveRecursiveIndexDepth,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -25,6 +26,7 @@ type fileSourceProvider struct {
|
|||||||
exclude source.ExcludeConfig
|
exclude source.ExcludeConfig
|
||||||
digestAlgorithms []crypto.Hash
|
digestAlgorithms []crypto.Hash
|
||||||
alias source.Alias
|
alias source.Alias
|
||||||
|
maxArchiveRecursiveIndexDepth int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p fileSourceProvider) Name() string {
|
func (p fileSourceProvider) Name() string {
|
||||||
@@ -53,6 +55,7 @@ func (p fileSourceProvider) Provide(_ context.Context) (source.Source, error) {
|
|||||||
Exclude: p.exclude,
|
Exclude: p.exclude,
|
||||||
DigestAlgorithms: p.digestAlgorithms,
|
DigestAlgorithms: p.digestAlgorithms,
|
||||||
Alias: p.alias,
|
Alias: p.alias,
|
||||||
|
MaxArchiveRecursiveIndexDepth: p.maxArchiveRecursiveIndexDepth,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ type Config struct {
|
|||||||
Exclude source.ExcludeConfig
|
Exclude source.ExcludeConfig
|
||||||
DigestAlgorithms []crypto.Hash
|
DigestAlgorithms []crypto.Hash
|
||||||
BasePath string
|
BasePath string
|
||||||
|
MaxArchiveRecursiveIndexDepth int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Config) WithAlias(alias source.Alias) *Config {
|
func (c *Config) WithAlias(alias source.Alias) *Config {
|
||||||
@@ -47,6 +48,11 @@ func (c *Config) WithBasePath(basePath string) *Config {
|
|||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Config) WithMaxArchiveRecursiveIndexDepth(maxArchiveRecursiveIndexDepth int) *Config {
|
||||||
|
c.MaxArchiveRecursiveIndexDepth = maxArchiveRecursiveIndexDepth
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
func DefaultConfig() *Config {
|
func DefaultConfig() *Config {
|
||||||
return &Config{
|
return &Config{
|
||||||
DigestAlgorithms: []crypto.Hash{
|
DigestAlgorithms: []crypto.Hash{
|
||||||
|
|||||||
@@ -36,8 +36,8 @@ func All(userInput string, cfg *Config) []collections.TaggedValue[source.Provide
|
|||||||
Join(tagProvider(snapsource.NewLocalSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias), SnapTag)).
|
Join(tagProvider(snapsource.NewLocalSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias), SnapTag)).
|
||||||
|
|
||||||
// 2. try unspecific, local sources after other local sources last...
|
// 2. try unspecific, local sources after other local sources last...
|
||||||
Join(tagProvider(filesource.NewSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias), FileTag)).
|
Join(tagProvider(filesource.NewSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias, cfg.MaxArchiveRecursiveIndexDepth), FileTag)).
|
||||||
Join(tagProvider(directorysource.NewSourceProvider(userInput, cfg.Exclude, cfg.Alias, cfg.BasePath), DirTag)).
|
Join(tagProvider(directorysource.NewSourceProvider(userInput, cfg.Exclude, cfg.Alias, cfg.BasePath, cfg.MaxArchiveRecursiveIndexDepth), DirTag)).
|
||||||
|
|
||||||
// 3. try remote sources after everything else...
|
// 3. try remote sources after everything else...
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestArchiveScan(t *testing.T) {
|
func TestArchiveScan(t *testing.T) {
|
||||||
|
archive := createArchive(t, "testdata/archive", t.TempDir())
|
||||||
|
nestedArchive := createArchive(t, filepath.Dir(archive), t.TempDir())
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
args []string
|
args []string
|
||||||
@@ -23,7 +25,21 @@ func TestArchiveScan(t *testing.T) {
|
|||||||
"scan",
|
"scan",
|
||||||
"-o",
|
"-o",
|
||||||
"json",
|
"json",
|
||||||
"file:" + createArchive(t, "testdata/archive", t.TempDir()),
|
"file:" + archive,
|
||||||
|
},
|
||||||
|
assertions: []traitAssertion{
|
||||||
|
assertSuccessfulReturnCode,
|
||||||
|
assertJsonReport,
|
||||||
|
assertPackageCount(1),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "scan an nested archive within the temp dir",
|
||||||
|
args: []string{
|
||||||
|
"scan",
|
||||||
|
"-o",
|
||||||
|
"json",
|
||||||
|
"file:" + nestedArchive,
|
||||||
},
|
},
|
||||||
assertions: []traitAssertion{
|
assertions: []traitAssertion{
|
||||||
assertSuccessfulReturnCode,
|
assertSuccessfulReturnCode,
|
||||||
|
|||||||
Reference in New Issue
Block a user