Migrate location-related structs to the file package (#1751)

* migrate location structs to file package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* replace source.Location refs with file package call

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove hardlink test for file based catalogers

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove hardlink test for all-regular-files testing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* migrate file resolver implementations to separate package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* [wip] migrate resolvers to internal

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* migrate resolvers to syft/internal

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

---------

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
Signed-off-by: <>
This commit is contained in:
Alex Goodman 2023-05-24 17:06:38 -04:00 committed by GitHub
parent 4bf17a94b9
commit 07e76907f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
313 changed files with 2316 additions and 2191 deletions

View File

@ -167,12 +167,12 @@ always feel free to file an issue or reach out to us [on slack](https://anchore.
#### Searching for files #### Searching for files
All catalogers are provided an instance of the [`source.FileResolver`](https://github.com/anchore/syft/blob/v0.70.0/syft/source/file_resolver.go#L8) to interface with the image and search for files. The implementations for these All catalogers are provided an instance of the [`file.Resolver`](https://github.com/anchore/syft/blob/v0.70.0/syft/source/file_resolver.go#L8) to interface with the image and search for files. The implementations for these
abstractions leverage [`stereoscope`](https://github.com/anchore/stereoscope) in order to perform searching. Here is a abstractions leverage [`stereoscope`](https://github.com/anchore/stereoscope) in order to perform searching. Here is a
rough outline how that works: rough outline how that works:
1. a stereoscope `file.Index` is searched based on the input given (a path, glob, or MIME type). The index is relatively fast to search, but requires results to be filtered down to the files that exist in the specific layer(s) of interest. This is done automatically by the `filetree.Searcher` abstraction. This abstraction will fallback to searching directly against the raw `filetree.FileTree` if the index does not contain the file(s) of interest. Note: the `filetree.Searcher` is used by the `source.FileResolver` abstraction. 1. a stereoscope `file.Index` is searched based on the input given (a path, glob, or MIME type). The index is relatively fast to search, but requires results to be filtered down to the files that exist in the specific layer(s) of interest. This is done automatically by the `filetree.Searcher` abstraction. This abstraction will fallback to searching directly against the raw `filetree.FileTree` if the index does not contain the file(s) of interest. Note: the `filetree.Searcher` is used by the `file.Resolver` abstraction.
2. Once the set of files are returned from the `filetree.Searcher` the results are filtered down further to return the most unique file results. For example, you may have requested for files by a glob that returns multiple results. These results are filtered down to deduplicate by real files, so if a result contains two references to the same file, say one accessed via symlink and one accessed via the real path, then the real path reference is returned and the symlink reference is filtered out. If both were accessed by symlink then the first (by lexical order) is returned. This is done automatically by the `source.FileResolver` abstraction. 2. Once the set of files are returned from the `filetree.Searcher` the results are filtered down further to return the most unique file results. For example, you may have requested for files by a glob that returns multiple results. These results are filtered down to deduplicate by real files, so if a result contains two references to the same file, say one accessed via symlink and one accessed via the real path, then the real path reference is returned and the symlink reference is filtered out. If both were accessed by symlink then the first (by lexical order) is returned. This is done automatically by the `file.Resolver` abstraction.
3. By the time results reach the `pkg.Cataloger` you are guaranteed to have a set of unique files that exist in the layer(s) of interest (relative to what the resolver supports). 3. By the time results reach the `pkg.Cataloger` you are guaranteed to have a set of unique files that exist in the layer(s) of interest (relative to what the resolver supports).
## Testing ## Testing

View File

@ -8,6 +8,10 @@ import (
"github.com/anchore/syft/syft" "github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/file/cataloger/filecontent"
"github.com/anchore/syft/syft/file/cataloger/filedigest"
"github.com/anchore/syft/syft/file/cataloger/filemetadata"
"github.com/anchore/syft/syft/file/cataloger/secrets"
"github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
) )
@ -61,7 +65,7 @@ func generateCatalogFileMetadataTask(app *config.Application) (Task, error) {
return nil, nil return nil, nil
} }
metadataCataloger := file.NewMetadataCataloger() metadataCataloger := filemetadata.NewCataloger()
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt) resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt)
@ -104,10 +108,7 @@ func generateCatalogFileDigestsTask(app *config.Application) (Task, error) {
hashes = append(hashes, hashObj) hashes = append(hashes, hashObj)
} }
digestsCataloger, err := file.NewDigestsCataloger(hashes) digestsCataloger := filedigest.NewCataloger(hashes)
if err != nil {
return nil, err
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) { task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt) resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt)
@ -131,12 +132,12 @@ func generateCatalogSecretsTask(app *config.Application) (Task, error) {
return nil, nil return nil, nil
} }
patterns, err := file.GenerateSearchPatterns(file.DefaultSecretsPatterns, app.Secrets.AdditionalPatterns, app.Secrets.ExcludePatternNames) patterns, err := secrets.GenerateSearchPatterns(secrets.DefaultSecretsPatterns, app.Secrets.AdditionalPatterns, app.Secrets.ExcludePatternNames)
if err != nil { if err != nil {
return nil, err return nil, err
} }
secretsCataloger, err := file.NewSecretsCataloger(patterns, app.Secrets.RevealValues, app.Secrets.SkipFilesAboveSize) secretsCataloger, err := secrets.NewCataloger(patterns, app.Secrets.RevealValues, app.Secrets.SkipFilesAboveSize) //nolint:staticcheck
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -163,7 +164,7 @@ func generateCatalogContentsTask(app *config.Application) (Task, error) {
return nil, nil return nil, nil
} }
contentsCataloger, err := file.NewContentsCataloger(app.FileContents.Globs, app.FileContents.SkipFilesAboveSize) contentsCataloger, err := filecontent.NewCataloger(app.FileContents.Globs, app.FileContents.SkipFilesAboveSize) //nolint:staticcheck
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -5,9 +5,9 @@ import (
"github.com/google/licensecheck" "github.com/google/licensecheck"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/license" "github.com/anchore/syft/syft/license"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
) )
const ( const (
@ -16,7 +16,7 @@ const (
) )
// Parse scans the contents of a license file to attempt to determine the type of license it is // Parse scans the contents of a license file to attempt to determine the type of license it is
func Parse(reader io.Reader, l source.Location) (licenses []pkg.License, err error) { func Parse(reader io.Reader, l file.Location) (licenses []pkg.License, err error) {
licenses = make([]pkg.License, 0) licenses = make([]pkg.License, 0)
contents, err := io.ReadAll(reader) contents, err := io.ReadAll(reader)
if err != nil { if err != nil {

View File

@ -12,7 +12,7 @@ import (
"github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file/cataloger/secrets"
"github.com/anchore/syft/syft/pkg/cataloger" "github.com/anchore/syft/syft/pkg/cataloger"
) )
@ -54,12 +54,12 @@ func ParsePackageCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error)
return &monitor, nil return &monitor, nil
} }
func ParseSecretsCatalogingStarted(e partybus.Event) (*file.SecretsMonitor, error) { func ParseSecretsCatalogingStarted(e partybus.Event) (*secrets.Monitor, error) {
if err := checkEventType(e.Type, event.SecretsCatalogerStarted); err != nil { if err := checkEventType(e.Type, event.SecretsCatalogerStarted); err != nil {
return nil, err return nil, err
} }
monitor, ok := e.Value.(file.SecretsMonitor) monitor, ok := e.Value.(secrets.Monitor)
if !ok { if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value) return nil, newPayloadErr(e.Type, "Value", e.Value)
} }

View File

@ -1,4 +1,4 @@
package file package filecontent
import ( import (
"bytes" "bytes"
@ -8,24 +8,26 @@ import (
"github.com/anchore/syft/internal" "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/file"
) )
type ContentsCataloger struct { // Deprecated: will be removed in syft v1.0.0
type Cataloger struct {
globs []string globs []string
skipFilesAboveSizeInBytes int64 skipFilesAboveSizeInBytes int64
} }
func NewContentsCataloger(globs []string, skipFilesAboveSize int64) (*ContentsCataloger, error) { // Deprecated: will be removed in syft v1.0.0
return &ContentsCataloger{ func NewCataloger(globs []string, skipFilesAboveSize int64) (*Cataloger, error) {
return &Cataloger{
globs: globs, globs: globs,
skipFilesAboveSizeInBytes: skipFilesAboveSize, skipFilesAboveSizeInBytes: skipFilesAboveSize,
}, nil }, nil
} }
func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates]string, error) { func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]string, error) {
results := make(map[source.Coordinates]string) results := make(map[file.Coordinates]string)
var locations []source.Location var locations []file.Location
locations, err := resolver.FilesByGlob(i.globs...) locations, err := resolver.FilesByGlob(i.globs...)
if err != nil { if err != nil {
@ -56,7 +58,7 @@ func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Co
return results, nil return results, nil
} }
func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) (string, error) { func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Location) (string, error) {
contentReader, err := resolver.FileContentsByLocation(location) contentReader, err := resolver.FileContentsByLocation(location)
if err != nil { if err != nil {
return "", err return "", err

View File

@ -0,0 +1,80 @@
package filecontent
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/file"
)
func TestContentsCataloger(t *testing.T) {
allFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
tests := []struct {
name string
globs []string
maxSize int64
files []string
expected map[file.Coordinates]string
}{
{
name: "multi-pattern",
globs: []string{"test-fixtures/last/*.txt", "test-fixtures/*.txt"},
files: allFiles,
expected: map[file.Coordinates]string{
file.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
file.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
file.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "no-patterns",
globs: []string{},
files: []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"},
expected: map[file.Coordinates]string{},
},
{
name: "all-txt",
globs: []string{"**/*.txt"},
files: allFiles,
expected: map[file.Coordinates]string{
file.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
file.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
file.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "subpath",
globs: []string{"test-fixtures/*.txt"},
files: allFiles,
expected: map[file.Coordinates]string{
file.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
file.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "size-filter",
maxSize: 42,
globs: []string{"**/*.txt"},
files: allFiles,
expected: map[file.Coordinates]string{
file.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
file.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c, err := NewCataloger(test.globs, test.maxSize)
assert.NoError(t, err)
resolver := file.NewMockResolverForPaths(test.files...)
actual, err := c.Catalog(resolver)
assert.NoError(t, err)
assert.Equal(t, test.expected, actual, "mismatched contents")
})
}
}

View File

@ -0,0 +1,109 @@
package filedigest
import (
"crypto"
"errors"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/file"
internal2 "github.com/anchore/syft/syft/file/cataloger/internal"
)
var ErrUndigestableFile = errors.New("undigestable file")
type Cataloger struct {
hashes []crypto.Hash
}
func NewCataloger(hashes []crypto.Hash) *Cataloger {
return &Cataloger{
hashes: hashes,
}
}
func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates][]file.Digest, error) {
results := make(map[file.Coordinates][]file.Digest)
var locations []file.Location
if len(coordinates) == 0 {
locations = internal2.AllRegularFiles(resolver)
} else {
for _, c := range coordinates {
locations = append(locations, file.NewLocationFromCoordinates(c))
}
}
stage, prog := digestsCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
result, err := i.catalogLocation(resolver, location)
if errors.Is(err, ErrUndigestableFile) {
continue
}
if internal.IsErrPathPermission(err) {
log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
continue
}
if err != nil {
return nil, err
}
prog.Increment()
results[location.Coordinates] = result
}
log.Debugf("file digests cataloger processed %d files", prog.Current())
prog.SetCompleted()
return results, nil
}
func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Location) ([]file.Digest, error) {
meta, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
}
// we should only attempt to report digests for files that are regular files (don't attempt to resolve links)
if meta.Type != stereoscopeFile.TypeRegular {
return nil, ErrUndigestableFile
}
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, err
}
defer internal.CloseAndLogError(contentReader, location.VirtualPath)
digests, err := file.NewDigestsFromFile(contentReader, i.hashes)
if err != nil {
return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err}
}
return digests, nil
}
func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
prog := progress.NewManual(locations)
bus.Publish(partybus.Event{
Type: event.FileDigestsCatalogerStarted,
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
},
})
return stage, prog
}

View File

@ -1,9 +1,9 @@
package file package filedigest
import ( import (
"crypto" "crypto"
"fmt" "fmt"
"io/ioutil" "io"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@ -11,29 +11,36 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file" stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
) )
func testDigests(t testing.TB, root string, files []string, hashes ...crypto.Hash) map[source.Coordinates][]Digest { func testDigests(t testing.TB, root string, files []string, hashes ...crypto.Hash) map[file.Coordinates][]file.Digest {
digests := make(map[source.Coordinates][]Digest) digests := make(map[file.Coordinates][]file.Digest)
for _, f := range files { for _, f := range files {
fh, err := os.Open(filepath.Join(root, f)) fh, err := os.Open(filepath.Join(root, f))
if err != nil { if err != nil {
t.Fatalf("could not open %q : %+v", f, err) t.Fatalf("could not open %q : %+v", f, err)
} }
b, err := ioutil.ReadAll(fh) b, err := io.ReadAll(fh)
if err != nil { if err != nil {
t.Fatalf("could not read %q : %+v", f, err) t.Fatalf("could not read %q : %+v", f, err)
} }
if len(b) == 0 {
// we don't keep digests for empty files
digests[file.NewLocation(f).Coordinates] = []file.Digest{}
continue
}
for _, hash := range hashes { for _, hash := range hashes {
h := hash.New() h := hash.New()
h.Write(b) h.Write(b)
digests[source.NewLocation(f).Coordinates] = append(digests[source.NewLocation(f).Coordinates], Digest{ digests[file.NewLocation(f).Coordinates] = append(digests[file.NewLocation(f).Coordinates], file.Digest{
Algorithm: CleanDigestAlgorithmName(hash.String()), Algorithm: file.CleanDigestAlgorithmName(hash.String()),
Value: fmt.Sprintf("%x", h.Sum(nil)), Value: fmt.Sprintf("%x", h.Sum(nil)),
}) })
} }
@ -48,7 +55,7 @@ func TestDigestsCataloger(t *testing.T) {
name string name string
digests []crypto.Hash digests []crypto.Hash
files []string files []string
expected map[source.Coordinates][]Digest expected map[file.Coordinates][]file.Digest
}{ }{
{ {
name: "md5", name: "md5",
@ -66,8 +73,7 @@ func TestDigestsCataloger(t *testing.T) {
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
c, err := NewDigestsCataloger(test.digests) c := NewCataloger(test.digests)
require.NoError(t, err)
src, err := source.NewFromDirectory("test-fixtures/last/") src, err := source.NewFromDirectory("test-fixtures/last/")
require.NoError(t, err) require.NoError(t, err)
@ -86,11 +92,7 @@ func TestDigestsCataloger(t *testing.T) {
func TestDigestsCataloger_MixFileTypes(t *testing.T) { func TestDigestsCataloger_MixFileTypes(t *testing.T) {
testImage := "image-file-type-mix" testImage := "image-file-type-mix"
if *updateImageGoldenFiles { img := imagetest.GetFixtureImage(t, "docker-archive", testImage)
imagetest.UpdateGoldenFixtureImage(t, testImage)
}
img := imagetest.GetGoldenFixtureImage(t, testImage)
src, err := source.NewFromImage(img, "---") src, err := source.NewFromImage(img, "---")
if err != nil { if err != nil {
@ -110,9 +112,10 @@ func TestDigestsCataloger_MixFileTypes(t *testing.T) {
path: "/file-1.txt", path: "/file-1.txt",
expected: "888c139e550867814eb7c33b84d76e4d", expected: "888c139e550867814eb7c33b84d76e4d",
}, },
{ // this is difficult to reproduce in a cross-platform way
path: "/hardlink-1", //{
}, // path: "/hardlink-1",
//},
{ {
path: "/symlink-1", path: "/symlink-1",
}, },
@ -132,21 +135,18 @@ func TestDigestsCataloger_MixFileTypes(t *testing.T) {
for _, test := range tests { for _, test := range tests {
t.Run(test.path, func(t *testing.T) { t.Run(test.path, func(t *testing.T) {
c, err := NewDigestsCataloger([]crypto.Hash{crypto.MD5}) c := NewCataloger([]crypto.Hash{crypto.MD5})
if err != nil {
t.Fatalf("unable to get cataloger: %+v", err)
}
actual, err := c.Catalog(resolver) actual, err := c.Catalog(resolver)
if err != nil { if err != nil {
t.Fatalf("could not catalog: %+v", err) t.Fatalf("could not catalog: %+v", err)
} }
_, ref, err := img.SquashedTree().File(file.Path(test.path)) _, ref, err := img.SquashedTree().File(stereoscopeFile.Path(test.path))
if err != nil { if err != nil {
t.Fatalf("unable to get file=%q : %+v", test.path, err) t.Fatalf("unable to get file=%q : %+v", test.path, err)
} }
l := source.NewLocationFromImage(test.path, *ref.Reference, img) l := file.NewLocationFromImage(test.path, *ref.Reference, img)
if len(actual[l.Coordinates]) == 0 { if len(actual[l.Coordinates]) == 0 {
if test.expected != "" { if test.expected != "" {

View File

@ -1,4 +1,4 @@
FROM busybox:latest FROM busybox:1.28.1@sha256:c7b0a24019b0e6eda714ec0fa137ad42bc44a754d9cea17d14fba3a80ccc1ee4
ADD file-1.txt . ADD file-1.txt .
RUN chmod 644 file-1.txt RUN chmod 644 file-1.txt

View File

@ -0,0 +1 @@
test-fixtures/last/path.txt file contents!

View File

@ -1,4 +1,4 @@
package file package filemetadata
import ( import (
"github.com/wagoodman/go-partybus" "github.com/wagoodman/go-partybus"
@ -7,24 +7,37 @@ import (
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/file"
) )
type MetadataCataloger struct { type Cataloger struct {
} }
func NewMetadataCataloger() *MetadataCataloger { func NewCataloger() *Cataloger {
return &MetadataCataloger{} return &Cataloger{}
} }
func (i *MetadataCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates]source.FileMetadata, error) { func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates]file.Metadata, error) {
results := make(map[source.Coordinates]source.FileMetadata) results := make(map[file.Coordinates]file.Metadata)
var locations []source.Location var locations <-chan file.Location
for location := range resolver.AllLocations() {
locations = append(locations, location) if len(coordinates) == 0 {
locations = resolver.AllLocations()
} else {
locations = func() <-chan file.Location {
ch := make(chan file.Location)
go func() {
close(ch)
for _, c := range coordinates {
ch <- file.NewLocationFromCoordinates(c)
} }
}()
return ch
}()
}
stage, prog := metadataCatalogingProgress(int64(len(locations))) stage, prog := metadataCatalogingProgress(int64(len(locations)))
for _, location := range locations { for location := range locations {
stage.Current = location.RealPath stage.Current = location.RealPath
metadata, err := resolver.FileMetadataByLocation(location) metadata, err := resolver.FileMetadataByLocation(location)
if err != nil { if err != nil {

View File

@ -1,30 +1,24 @@
package file package filemetadata
import ( import (
"flag"
"os" "os"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file" stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
) )
var updateImageGoldenFiles = flag.Bool("update-image", false, "update the golden fixture images used for testing")
func TestFileMetadataCataloger(t *testing.T) { func TestFileMetadataCataloger(t *testing.T) {
testImage := "image-file-type-mix" testImage := "image-file-type-mix"
if *updateImageGoldenFiles { img := imagetest.GetFixtureImage(t, "docker-archive", testImage)
imagetest.UpdateGoldenFixtureImage(t, testImage)
}
img := imagetest.GetGoldenFixtureImage(t, testImage) c := NewCataloger()
c := NewMetadataCataloger()
src, err := source.NewFromImage(img, "---") src, err := source.NewFromImage(img, "---")
if err != nil { if err != nil {
@ -44,51 +38,36 @@ func TestFileMetadataCataloger(t *testing.T) {
tests := []struct { tests := []struct {
path string path string
exists bool exists bool
expected source.FileMetadata expected file.Metadata
err bool err bool
}{ }{
// note: it is difficult to add a hardlink-based test in a cross-platform way and is already covered well in stereoscope
{ {
path: "/file-1.txt", path: "/file-1.txt",
exists: true, exists: true,
expected: source.FileMetadata{ expected: file.Metadata{
FileInfo: file.ManualInfo{ FileInfo: stereoscopeFile.ManualInfo{
NameValue: "file-1.txt", NameValue: "file-1.txt",
ModeValue: 0644, ModeValue: 0644,
SizeValue: 7, SizeValue: 7,
}, },
Path: "/file-1.txt", Path: "/file-1.txt",
Type: file.TypeRegular, Type: stereoscopeFile.TypeRegular,
UserID: 1, UserID: 1,
GroupID: 2, GroupID: 2,
MIMEType: "text/plain", MIMEType: "text/plain",
}, },
}, },
{
path: "/hardlink-1",
exists: true,
expected: source.FileMetadata{
FileInfo: file.ManualInfo{
NameValue: "hardlink-1",
ModeValue: 0644,
},
Path: "/hardlink-1",
Type: file.TypeHardLink,
LinkDestination: "file-1.txt",
UserID: 1,
GroupID: 2,
MIMEType: "",
},
},
{ {
path: "/symlink-1", path: "/symlink-1",
exists: true, exists: true,
expected: source.FileMetadata{ expected: file.Metadata{
Path: "/symlink-1", Path: "/symlink-1",
FileInfo: file.ManualInfo{ FileInfo: stereoscopeFile.ManualInfo{
NameValue: "symlink-1", NameValue: "symlink-1",
ModeValue: 0777 | os.ModeSymlink, ModeValue: 0777 | os.ModeSymlink,
}, },
Type: file.TypeSymLink, Type: stereoscopeFile.TypeSymLink,
LinkDestination: "file-1.txt", LinkDestination: "file-1.txt",
UserID: 0, UserID: 0,
GroupID: 0, GroupID: 0,
@ -98,13 +77,13 @@ func TestFileMetadataCataloger(t *testing.T) {
{ {
path: "/char-device-1", path: "/char-device-1",
exists: true, exists: true,
expected: source.FileMetadata{ expected: file.Metadata{
Path: "/char-device-1", Path: "/char-device-1",
FileInfo: file.ManualInfo{ FileInfo: stereoscopeFile.ManualInfo{
NameValue: "char-device-1", NameValue: "char-device-1",
ModeValue: 0644 | os.ModeDevice | os.ModeCharDevice, ModeValue: 0644 | os.ModeDevice | os.ModeCharDevice,
}, },
Type: file.TypeCharacterDevice, Type: stereoscopeFile.TypeCharacterDevice,
UserID: 0, UserID: 0,
GroupID: 0, GroupID: 0,
MIMEType: "", MIMEType: "",
@ -113,13 +92,13 @@ func TestFileMetadataCataloger(t *testing.T) {
{ {
path: "/block-device-1", path: "/block-device-1",
exists: true, exists: true,
expected: source.FileMetadata{ expected: file.Metadata{
Path: "/block-device-1", Path: "/block-device-1",
FileInfo: file.ManualInfo{ FileInfo: stereoscopeFile.ManualInfo{
NameValue: "block-device-1", NameValue: "block-device-1",
ModeValue: 0644 | os.ModeDevice, ModeValue: 0644 | os.ModeDevice,
}, },
Type: file.TypeBlockDevice, Type: stereoscopeFile.TypeBlockDevice,
UserID: 0, UserID: 0,
GroupID: 0, GroupID: 0,
MIMEType: "", MIMEType: "",
@ -128,13 +107,13 @@ func TestFileMetadataCataloger(t *testing.T) {
{ {
path: "/fifo-1", path: "/fifo-1",
exists: true, exists: true,
expected: source.FileMetadata{ expected: file.Metadata{
Path: "/fifo-1", Path: "/fifo-1",
FileInfo: file.ManualInfo{ FileInfo: stereoscopeFile.ManualInfo{
NameValue: "fifo-1", NameValue: "fifo-1",
ModeValue: 0644 | os.ModeNamedPipe, ModeValue: 0644 | os.ModeNamedPipe,
}, },
Type: file.TypeFIFO, Type: stereoscopeFile.TypeFIFO,
UserID: 0, UserID: 0,
GroupID: 0, GroupID: 0,
MIMEType: "", MIMEType: "",
@ -143,13 +122,13 @@ func TestFileMetadataCataloger(t *testing.T) {
{ {
path: "/bin", path: "/bin",
exists: true, exists: true,
expected: source.FileMetadata{ expected: file.Metadata{
Path: "/bin", Path: "/bin",
FileInfo: file.ManualInfo{ FileInfo: stereoscopeFile.ManualInfo{
NameValue: "bin", NameValue: "bin",
ModeValue: 0755 | os.ModeDir, ModeValue: 0755 | os.ModeDir,
}, },
Type: file.TypeDirectory, Type: stereoscopeFile.TypeDirectory,
UserID: 0, UserID: 0,
GroupID: 0, GroupID: 0,
MIMEType: "", MIMEType: "",
@ -159,15 +138,15 @@ func TestFileMetadataCataloger(t *testing.T) {
for _, test := range tests { for _, test := range tests {
t.Run(test.path, func(t *testing.T) { t.Run(test.path, func(t *testing.T) {
_, ref, err := img.SquashedTree().File(file.Path(test.path)) _, ref, err := img.SquashedTree().File(stereoscopeFile.Path(test.path))
require.NoError(t, err) require.NoError(t, err)
l := source.NewLocationFromImage(test.path, *ref.Reference, img) l := file.NewLocationFromImage(test.path, *ref.Reference, img)
if _, ok := actual[l.Coordinates]; ok { if _, ok := actual[l.Coordinates]; ok {
// we're not interested in keeping the test fixtures up to date with the latest file modification times // we're not interested in keeping the test fixtures up to date with the latest file modification times
// thus ModTime is not under test // thus ModTime is not under test
fi := test.expected.FileInfo.(file.ManualInfo) fi := test.expected.FileInfo.(stereoscopeFile.ManualInfo)
fi.ModTimeValue = actual[l.Coordinates].ModTime() fi.ModTimeValue = actual[l.Coordinates].ModTime()
test.expected.FileInfo = fi test.expected.FileInfo = fi
} }

View File

@ -0,0 +1,13 @@
FROM busybox:1.28.1@sha256:c7b0a24019b0e6eda714ec0fa137ad42bc44a754d9cea17d14fba3a80ccc1ee4
ADD file-1.txt .
RUN chmod 644 file-1.txt
RUN chown 1:2 file-1.txt
RUN ln -s file-1.txt symlink-1
# note: hard links may behave inconsistently, this should be a golden image
RUN ln file-1.txt hardlink-1
RUN mknod char-device-1 c 89 1
RUN mknod block-device-1 b 0 1
RUN mknod fifo-1 p
RUN mkdir /dir
RUN rm -rf home etc/group etc/localtime etc/mtab etc/network etc/passwd etc/shadow var usr bin/*

View File

@ -1,12 +1,12 @@
package file package internal
import ( import (
"github.com/anchore/stereoscope/pkg/file" stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/file"
) )
func allRegularFiles(resolver source.FileResolver) (locations []source.Location) { func AllRegularFiles(resolver file.Resolver) (locations []file.Location) {
for location := range resolver.AllLocations() { for location := range resolver.AllLocations() {
resolvedLocations, err := resolver.FilesByPath(location.RealPath) resolvedLocations, err := resolver.FilesByPath(location.RealPath)
if err != nil { if err != nil {
@ -21,7 +21,7 @@ func allRegularFiles(resolver source.FileResolver) (locations []source.Location)
continue continue
} }
if metadata.Type != file.TypeRegular { if metadata.Type != stereoscopeFile.TypeRegular {
continue continue
} }
locations = append(locations, resolvedLocation) locations = append(locations, resolvedLocation)

View File

@ -1,4 +1,4 @@
package file package internal
import ( import (
"testing" "testing"
@ -9,30 +9,23 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
) )
func Test_allRegularFiles(t *testing.T) { func Test_allRegularFiles(t *testing.T) {
type access struct {
realPath string
virtualPath string
}
tests := []struct { tests := []struct {
name string name string
setup func() source.FileResolver setup func() file.Resolver
wantRealPaths *strset.Set wantRealPaths *strset.Set
wantVirtualPaths *strset.Set wantVirtualPaths *strset.Set
}{ }{
{ {
name: "image", name: "image",
setup: func() source.FileResolver { setup: func() file.Resolver {
testImage := "image-file-type-mix" testImage := "image-file-type-mix"
if *updateImageGoldenFiles { img := imagetest.GetFixtureImage(t, "docker-archive", testImage)
imagetest.UpdateGoldenFixtureImage(t, testImage)
}
img := imagetest.GetGoldenFixtureImage(t, testImage)
s, err := source.NewFromImage(img, "---") s, err := source.NewFromImage(img, "---")
require.NoError(t, err) require.NoError(t, err)
@ -47,7 +40,7 @@ func Test_allRegularFiles(t *testing.T) {
}, },
{ {
name: "directory", name: "directory",
setup: func() source.FileResolver { setup: func() file.Resolver {
s, err := source.NewFromDirectory("test-fixtures/symlinked-root/nested/link-root") s, err := source.NewFromDirectory("test-fixtures/symlinked-root/nested/link-root")
require.NoError(t, err) require.NoError(t, err)
r, err := s.FileResolver(source.SquashedScope) r, err := s.FileResolver(source.SquashedScope)
@ -61,7 +54,7 @@ func Test_allRegularFiles(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
resolver := tt.setup() resolver := tt.setup()
locations := allRegularFiles(resolver) locations := AllRegularFiles(resolver)
realLocations := strset.New() realLocations := strset.New()
virtualLocations := strset.New() virtualLocations := strset.New()
for _, l := range locations { for _, l := range locations {
@ -70,6 +63,13 @@ func Test_allRegularFiles(t *testing.T) {
virtualLocations.Add(l.VirtualPath) virtualLocations.Add(l.VirtualPath)
} }
} }
// this is difficult to reproduce in a cross-platform way
realLocations.Remove("/hardlink-1")
virtualLocations.Remove("/hardlink-1")
tt.wantRealPaths.Remove("/hardlink-1")
tt.wantVirtualPaths.Remove("/hardlink-1")
assert.ElementsMatch(t, tt.wantRealPaths.List(), realLocations.List(), "real paths differ: "+cmp.Diff(tt.wantRealPaths.List(), realLocations.List())) assert.ElementsMatch(t, tt.wantRealPaths.List(), realLocations.List(), "real paths differ: "+cmp.Diff(tt.wantRealPaths.List(), realLocations.List()))
assert.ElementsMatch(t, tt.wantVirtualPaths.List(), virtualLocations.List(), "virtual paths differ: "+cmp.Diff(tt.wantVirtualPaths.List(), virtualLocations.List())) assert.ElementsMatch(t, tt.wantVirtualPaths.List(), virtualLocations.List(), "virtual paths differ: "+cmp.Diff(tt.wantVirtualPaths.List(), virtualLocations.List()))
}) })

View File

@ -0,0 +1,13 @@
FROM busybox:1.28.1@sha256:c7b0a24019b0e6eda714ec0fa137ad42bc44a754d9cea17d14fba3a80ccc1ee4
ADD file-1.txt .
RUN chmod 644 file-1.txt
RUN chown 1:2 file-1.txt
RUN ln -s file-1.txt symlink-1
# note: hard links may behave inconsistently, this should be a golden image
RUN ln file-1.txt hardlink-1
RUN mknod char-device-1 c 89 1
RUN mknod block-device-1 b 0 1
RUN mknod fifo-1 p
RUN mkdir /dir
RUN rm -rf home etc/group etc/localtime etc/mtab etc/network etc/passwd etc/shadow var usr bin/*

View File

@ -1,4 +1,4 @@
package file package secrets
import ( import (
"bytes" "bytes"
@ -14,7 +14,8 @@ import (
"github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/file"
internal2 "github.com/anchore/syft/syft/file/cataloger/internal"
) )
var DefaultSecretsPatterns = map[string]string{ var DefaultSecretsPatterns = map[string]string{
@ -25,23 +26,25 @@ var DefaultSecretsPatterns = map[string]string{
"generic-api-key": `(?i)api(-|_)?key["'=:\s]*?(?P<value>[A-Z0-9]{20,60})["']?(\s|$)`, "generic-api-key": `(?i)api(-|_)?key["'=:\s]*?(?P<value>[A-Z0-9]{20,60})["']?(\s|$)`,
} }
type SecretsCataloger struct { // Deprecated: will be removed in syft v1.0.0
type Cataloger struct {
patterns map[string]*regexp.Regexp patterns map[string]*regexp.Regexp
revealValues bool revealValues bool
skipFilesAboveSize int64 skipFilesAboveSize int64
} }
func NewSecretsCataloger(patterns map[string]*regexp.Regexp, revealValues bool, maxFileSize int64) (*SecretsCataloger, error) { // Deprecated: will be removed in syft v1.0.0
return &SecretsCataloger{ func NewCataloger(patterns map[string]*regexp.Regexp, revealValues bool, maxFileSize int64) (*Cataloger, error) {
return &Cataloger{
patterns: patterns, patterns: patterns,
revealValues: revealValues, revealValues: revealValues,
skipFilesAboveSize: maxFileSize, skipFilesAboveSize: maxFileSize,
}, nil }, nil
} }
func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]SearchResult, error) { func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates][]file.SearchResult, error) {
results := make(map[source.Coordinates][]SearchResult) results := make(map[file.Coordinates][]file.SearchResult)
locations := allRegularFiles(resolver) locations := internal2.AllRegularFiles(resolver)
stage, prog, secretsDiscovered := secretsCatalogingProgress(int64(len(locations))) stage, prog, secretsDiscovered := secretsCatalogingProgress(int64(len(locations)))
for _, location := range locations { for _, location := range locations {
stage.Current = location.RealPath stage.Current = location.RealPath
@ -65,7 +68,7 @@ func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Coo
return results, nil return results, nil
} }
func (i *SecretsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]SearchResult, error) { func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Location) ([]file.SearchResult, error) {
metadata, err := resolver.FileMetadataByLocation(location) metadata, err := resolver.FileMetadataByLocation(location)
if err != nil { if err != nil {
return nil, err return nil, err
@ -103,7 +106,7 @@ func (i *SecretsCataloger) catalogLocation(resolver source.FileResolver, locatio
return secrets, nil return secrets, nil
} }
func extractValue(resolver source.FileResolver, location source.Location, start, length int64) (string, error) { func extractValue(resolver file.Resolver, location file.Location, start, length int64) (string, error) {
readCloser, err := resolver.FileContentsByLocation(location) readCloser, err := resolver.FileContentsByLocation(location)
if err != nil { if err != nil {
return "", fmt.Errorf("unable to fetch reader for location=%q : %w", location, err) return "", fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
@ -130,7 +133,7 @@ func extractValue(resolver source.FileResolver, location source.Location, start,
return buf.String(), nil return buf.String(), nil
} }
type SecretsMonitor struct { type Monitor struct {
progress.Stager progress.Stager
SecretsDiscovered progress.Monitorable SecretsDiscovered progress.Monitorable
progress.Progressable progress.Progressable
@ -144,7 +147,7 @@ func secretsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manu
bus.Publish(partybus.Event{ bus.Publish(partybus.Event{
Type: event.SecretsCatalogerStarted, Type: event.SecretsCatalogerStarted,
Source: secretsDiscovered, Source: secretsDiscovered,
Value: SecretsMonitor{ Value: Monitor{
Stager: progress.Stager(stage), Stager: progress.Stager(stage),
SecretsDiscovered: secretsDiscovered, SecretsDiscovered: secretsDiscovered,
Progressable: prog, Progressable: prog,

View File

@ -1,4 +1,4 @@
package file package secrets
import ( import (
"regexp" "regexp"
@ -6,8 +6,8 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/anchore/syft/internal/file" intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/file"
) )
func TestSecretsCataloger(t *testing.T) { func TestSecretsCataloger(t *testing.T) {
@ -17,7 +17,7 @@ func TestSecretsCataloger(t *testing.T) {
reveal bool reveal bool
maxSize int64 maxSize int64
patterns map[string]string patterns map[string]string
expected []SearchResult expected []file.SearchResult
constructorErr bool constructorErr bool
catalogErr bool catalogErr bool
}{ }{
@ -28,7 +28,7 @@ func TestSecretsCataloger(t *testing.T) {
patterns: map[string]string{ patterns: map[string]string{
"simple-secret-key": `^secret_key=.*`, "simple-secret-key": `^secret_key=.*`,
}, },
expected: []SearchResult{ expected: []file.SearchResult{
{ {
Classification: "simple-secret-key", Classification: "simple-secret-key",
LineNumber: 2, LineNumber: 2,
@ -46,7 +46,7 @@ func TestSecretsCataloger(t *testing.T) {
patterns: map[string]string{ patterns: map[string]string{
"simple-secret-key": `^secret_key=.*`, "simple-secret-key": `^secret_key=.*`,
}, },
expected: []SearchResult{ expected: []file.SearchResult{
{ {
Classification: "simple-secret-key", Classification: "simple-secret-key",
LineNumber: 2, LineNumber: 2,
@ -64,7 +64,7 @@ func TestSecretsCataloger(t *testing.T) {
patterns: map[string]string{ patterns: map[string]string{
"simple-secret-key": `^secret_key=(?P<value>.*)`, "simple-secret-key": `^secret_key=(?P<value>.*)`,
}, },
expected: []SearchResult{ expected: []file.SearchResult{
{ {
Classification: "simple-secret-key", Classification: "simple-secret-key",
LineNumber: 2, LineNumber: 2,
@ -82,7 +82,7 @@ func TestSecretsCataloger(t *testing.T) {
patterns: map[string]string{ patterns: map[string]string{
"simple-secret-key": `secret_key=.*`, "simple-secret-key": `secret_key=.*`,
}, },
expected: []SearchResult{ expected: []file.SearchResult{
{ {
Classification: "simple-secret-key", Classification: "simple-secret-key",
LineNumber: 1, LineNumber: 1,
@ -125,7 +125,7 @@ func TestSecretsCataloger(t *testing.T) {
patterns: map[string]string{ patterns: map[string]string{
"simple-secret-key": `secret_key=(?P<value>.*)`, "simple-secret-key": `secret_key=(?P<value>.*)`,
}, },
expected: []SearchResult{ expected: []file.SearchResult{
{ {
Classification: "simple-secret-key", Classification: "simple-secret-key",
LineNumber: 1, LineNumber: 1,
@ -176,7 +176,7 @@ func TestSecretsCataloger(t *testing.T) {
regexObjs[name] = obj regexObjs[name] = obj
} }
c, err := NewSecretsCataloger(regexObjs, test.reveal, test.maxSize) c, err := NewCataloger(regexObjs, test.reveal, test.maxSize)
if err != nil && !test.constructorErr { if err != nil && !test.constructorErr {
t.Fatalf("could not create cataloger (but should have been able to): %+v", err) t.Fatalf("could not create cataloger (but should have been able to): %+v", err)
} else if err == nil && test.constructorErr { } else if err == nil && test.constructorErr {
@ -185,7 +185,7 @@ func TestSecretsCataloger(t *testing.T) {
return return
} }
resolver := source.NewMockResolverForPaths(test.fixture) resolver := file.NewMockResolverForPaths(test.fixture)
actualResults, err := c.Catalog(resolver) actualResults, err := c.Catalog(resolver)
if err != nil && !test.catalogErr { if err != nil && !test.catalogErr {
@ -196,7 +196,7 @@ func TestSecretsCataloger(t *testing.T) {
return return
} }
loc := source.NewLocation(test.fixture) loc := file.NewLocation(test.fixture)
if _, exists := actualResults[loc.Coordinates]; !exists { if _, exists := actualResults[loc.Coordinates]; !exists {
t.Fatalf("could not find location=%q in results", loc) t.Fatalf("could not find location=%q in results", loc)
} }
@ -214,11 +214,11 @@ func TestSecretsCataloger_DefaultSecrets(t *testing.T) {
tests := []struct { tests := []struct {
fixture string fixture string
expected []SearchResult expected []file.SearchResult
}{ }{
{ {
fixture: "test-fixtures/secrets/default/aws.env", fixture: "test-fixtures/secrets/default/aws.env",
expected: []SearchResult{ expected: []file.SearchResult{
{ {
Classification: "aws-access-key", Classification: "aws-access-key",
LineNumber: 2, LineNumber: 2,
@ -239,7 +239,7 @@ func TestSecretsCataloger_DefaultSecrets(t *testing.T) {
}, },
{ {
fixture: "test-fixtures/secrets/default/aws.ini", fixture: "test-fixtures/secrets/default/aws.ini",
expected: []SearchResult{ expected: []file.SearchResult{
{ {
Classification: "aws-access-key", Classification: "aws-access-key",
LineNumber: 3, LineNumber: 3,
@ -260,7 +260,7 @@ func TestSecretsCataloger_DefaultSecrets(t *testing.T) {
}, },
{ {
fixture: "test-fixtures/secrets/default/private-key.pem", fixture: "test-fixtures/secrets/default/private-key.pem",
expected: []SearchResult{ expected: []file.SearchResult{
{ {
Classification: "pem-private-key", Classification: "pem-private-key",
LineNumber: 2, LineNumber: 2,
@ -280,7 +280,7 @@ z3P668YfhUbKdRF6S42Cg6zn
}, },
{ {
fixture: "test-fixtures/secrets/default/private-key-openssl.pem", fixture: "test-fixtures/secrets/default/private-key-openssl.pem",
expected: []SearchResult{ expected: []file.SearchResult{
{ {
Classification: "pem-private-key", Classification: "pem-private-key",
LineNumber: 2, LineNumber: 2,
@ -302,7 +302,7 @@ z3P668YfhUbKdRF6S42Cg6zn
// note: this test proves that the PEM regex matches the smallest possible match // note: this test proves that the PEM regex matches the smallest possible match
// since the test catches two adjacent secrets // since the test catches two adjacent secrets
fixture: "test-fixtures/secrets/default/private-keys.pem", fixture: "test-fixtures/secrets/default/private-keys.pem",
expected: []SearchResult{ expected: []file.SearchResult{
{ {
Classification: "pem-private-key", Classification: "pem-private-key",
LineNumber: 1, LineNumber: 1,
@ -345,7 +345,7 @@ j4f668YfhUbKdRF6S6734856
// 2. a named capture group with the correct line number and line offset case // 2. a named capture group with the correct line number and line offset case
// 3. the named capture group is in a different line than the match start, and both the match start and the capture group have different line offsets // 3. the named capture group is in a different line than the match start, and both the match start and the capture group have different line offsets
fixture: "test-fixtures/secrets/default/docker-config.json", fixture: "test-fixtures/secrets/default/docker-config.json",
expected: []SearchResult{ expected: []file.SearchResult{
{ {
Classification: "docker-config-auth", Classification: "docker-config-auth",
LineNumber: 5, LineNumber: 5,
@ -362,7 +362,7 @@ j4f668YfhUbKdRF6S6734856
}, },
{ {
fixture: "test-fixtures/secrets/default/api-key.txt", fixture: "test-fixtures/secrets/default/api-key.txt",
expected: []SearchResult{ expected: []file.SearchResult{
{ {
Classification: "generic-api-key", Classification: "generic-api-key",
LineNumber: 2, LineNumber: 2,
@ -418,19 +418,19 @@ j4f668YfhUbKdRF6S6734856
for _, test := range tests { for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) { t.Run(test.fixture, func(t *testing.T) {
c, err := NewSecretsCataloger(regexObjs, true, 10*file.MB) c, err := NewCataloger(regexObjs, true, 10*intFile.MB)
if err != nil { if err != nil {
t.Fatalf("could not create cataloger: %+v", err) t.Fatalf("could not create cataloger: %+v", err)
} }
resolver := source.NewMockResolverForPaths(test.fixture) resolver := file.NewMockResolverForPaths(test.fixture)
actualResults, err := c.Catalog(resolver) actualResults, err := c.Catalog(resolver)
if err != nil { if err != nil {
t.Fatalf("could not catalog: %+v", err) t.Fatalf("could not catalog: %+v", err)
} }
loc := source.NewLocation(test.fixture) loc := file.NewLocation(test.fixture)
if _, exists := actualResults[loc.Coordinates]; !exists && test.expected != nil { if _, exists := actualResults[loc.Coordinates]; !exists && test.expected != nil {
t.Fatalf("could not find location=%q in results", loc) t.Fatalf("could not find location=%q in results", loc)
} else if !exists && test.expected == nil { } else if !exists && test.expected == nil {

View File

@ -1,4 +1,4 @@
package file package secrets
import ( import (
"fmt" "fmt"

View File

@ -1,4 +1,4 @@
package file package secrets
import ( import (
"testing" "testing"

View File

@ -1,4 +1,4 @@
package file package secrets
import "io" import "io"

View File

@ -1,4 +1,4 @@
package file package secrets
import ( import (
"bufio" "bufio"

View File

@ -1,4 +1,4 @@
package file package secrets
import ( import (
"bufio" "bufio"
@ -8,10 +8,10 @@ import (
"regexp" "regexp"
"github.com/anchore/syft/internal" "github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/file"
) )
func catalogLocationByLine(resolver source.FileResolver, location source.Location, patterns map[string]*regexp.Regexp) ([]SearchResult, error) { func catalogLocationByLine(resolver file.Resolver, location file.Location, patterns map[string]*regexp.Regexp) ([]file.SearchResult, error) {
readCloser, err := resolver.FileContentsByLocation(location) readCloser, err := resolver.FileContentsByLocation(location)
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err) return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
@ -20,7 +20,7 @@ func catalogLocationByLine(resolver source.FileResolver, location source.Locatio
var scanner = bufio.NewReader(readCloser) var scanner = bufio.NewReader(readCloser)
var position int64 var position int64
var allSecrets []SearchResult var allSecrets []file.SearchResult
var lineNo int64 var lineNo int64
var readErr error var readErr error
for !errors.Is(readErr, io.EOF) { for !errors.Is(readErr, io.EOF) {
@ -43,8 +43,8 @@ func catalogLocationByLine(resolver source.FileResolver, location source.Locatio
return allSecrets, nil return allSecrets, nil
} }
func searchForSecretsWithinLine(resolver source.FileResolver, location source.Location, patterns map[string]*regexp.Regexp, line []byte, lineNo int64, position int64) ([]SearchResult, error) { func searchForSecretsWithinLine(resolver file.Resolver, location file.Location, patterns map[string]*regexp.Regexp, line []byte, lineNo int64, position int64) ([]file.SearchResult, error) {
var secrets []SearchResult var secrets []file.SearchResult
for name, pattern := range patterns { for name, pattern := range patterns {
matches := pattern.FindAllIndex(line, -1) matches := pattern.FindAllIndex(line, -1)
for i, match := range matches { for i, match := range matches {
@ -72,7 +72,7 @@ func searchForSecretsWithinLine(resolver source.FileResolver, location source.Lo
return secrets, nil return secrets, nil
} }
func readerAtPosition(resolver source.FileResolver, location source.Location, seekPosition int64) (io.ReadCloser, error) { func readerAtPosition(resolver file.Resolver, location file.Location, seekPosition int64) (io.ReadCloser, error) {
readCloser, err := resolver.FileContentsByLocation(location) readCloser, err := resolver.FileContentsByLocation(location)
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err) return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
@ -89,7 +89,7 @@ func readerAtPosition(resolver source.FileResolver, location source.Location, se
return readCloser, nil return readCloser, nil
} }
func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *regexp.Regexp, lineNo, lineOffset, seekPosition int64) *SearchResult { func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *regexp.Regexp, lineNo, lineOffset, seekPosition int64) *file.SearchResult {
reader := &newlineCounter{RuneReader: bufio.NewReader(readCloser)} reader := &newlineCounter{RuneReader: bufio.NewReader(readCloser)}
positions := pattern.FindReaderSubmatchIndex(reader) positions := pattern.FindReaderSubmatchIndex(reader)
if len(positions) == 0 { if len(positions) == 0 {
@ -125,7 +125,7 @@ func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *r
lineOffsetOfSecret += lineOffset lineOffsetOfSecret += lineOffset
} }
return &SearchResult{ return &file.SearchResult{
Classification: name, Classification: name,
SeekPosition: start + seekPosition, SeekPosition: start + seekPosition,
Length: stop - start, Length: stop - start,

View File

@ -1,80 +0,0 @@
package file
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/source"
)
func TestContentsCataloger(t *testing.T) {
allFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
tests := []struct {
name string
globs []string
maxSize int64
files []string
expected map[source.Coordinates]string
}{
{
name: "multi-pattern",
globs: []string{"test-fixtures/last/*.txt", "test-fixtures/*.txt"},
files: allFiles,
expected: map[source.Coordinates]string{
source.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "no-patterns",
globs: []string{},
files: []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"},
expected: map[source.Coordinates]string{},
},
{
name: "all-txt",
globs: []string{"**/*.txt"},
files: allFiles,
expected: map[source.Coordinates]string{
source.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "subpath",
globs: []string{"test-fixtures/*.txt"},
files: allFiles,
expected: map[source.Coordinates]string{
source.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "size-filter",
maxSize: 42,
globs: []string{"**/*.txt"},
files: allFiles,
expected: map[source.Coordinates]string{
source.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c, err := NewContentsCataloger(test.globs, test.maxSize)
assert.NoError(t, err)
resolver := source.NewMockResolverForPaths(test.files...)
actual, err := c.Catalog(resolver)
assert.NoError(t, err)
assert.Equal(t, test.expected, actual, "mismatched contents")
})
}
}

View File

@ -1,4 +1,4 @@
package source package file
import ( import (
"sort" "sort"

View File

@ -1,4 +1,4 @@
package source package file
import ( import (
"testing" "testing"

View File

@ -1,4 +1,4 @@
package source package file
import ( import (
"fmt" "fmt"

View File

@ -1,6 +1,76 @@
package file package file
import (
"crypto"
"fmt"
"hash"
"io"
"strings"
)
type Digest struct { type Digest struct {
Algorithm string `json:"algorithm"` Algorithm string `json:"algorithm"`
Value string `json:"value"` Value string `json:"value"`
} }
func NewDigestsFromFile(closer io.ReadCloser, hashes []crypto.Hash) ([]Digest, error) {
// create a set of hasher objects tied together with a single writer to feed content into
hashers := make([]hash.Hash, len(hashes))
writers := make([]io.Writer, len(hashes))
for idx, hashObj := range hashes {
hashers[idx] = hashObj.New()
writers[idx] = hashers[idx]
}
size, err := io.Copy(io.MultiWriter(writers...), closer)
if err != nil {
return nil, err
}
if size == 0 {
return make([]Digest, 0), nil
}
result := make([]Digest, len(hashes))
// only capture digests when there is content. It is important to do this based on SIZE and not
// FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only
// file type but a body is still allowed.
for idx, hasher := range hashers {
result[idx] = Digest{
Algorithm: DigestAlgorithmName(hashes[idx]),
Value: fmt.Sprintf("%+x", hasher.Sum(nil)),
}
}
return result, nil
}
func Hashers(names ...string) ([]crypto.Hash, error) {
supportedHashAlgorithms := make(map[string]crypto.Hash)
for _, h := range []crypto.Hash{
crypto.MD5,
crypto.SHA1,
crypto.SHA256,
} {
supportedHashAlgorithms[DigestAlgorithmName(h)] = h
}
var hashers []crypto.Hash
for _, hashStr := range names {
hashObj, ok := supportedHashAlgorithms[CleanDigestAlgorithmName(hashStr)]
if !ok {
return nil, fmt.Errorf("unsupported hash algorithm: %s", hashStr)
}
hashers = append(hashers, hashObj)
}
return hashers, nil
}
func DigestAlgorithmName(hash crypto.Hash) string {
return CleanDigestAlgorithmName(hash.String())
}
func CleanDigestAlgorithmName(name string) string {
lower := strings.ToLower(name)
return strings.ReplaceAll(lower, "-", "")
}

View File

@ -1,140 +0,0 @@
package file
import (
"crypto"
"errors"
"fmt"
"hash"
"io"
"strings"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/source"
)
var errUndigestableFile = errors.New("undigestable file")
type DigestsCataloger struct {
hashes []crypto.Hash
}
func NewDigestsCataloger(hashes []crypto.Hash) (*DigestsCataloger, error) {
return &DigestsCataloger{
hashes: hashes,
}, nil
}
func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]Digest, error) {
results := make(map[source.Coordinates][]Digest)
locations := allRegularFiles(resolver)
stage, prog := digestsCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
result, err := i.catalogLocation(resolver, location)
if errors.Is(err, errUndigestableFile) {
continue
}
if internal.IsErrPathPermission(err) {
log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
continue
}
if err != nil {
return nil, err
}
prog.Increment()
results[location.Coordinates] = result
}
log.Debugf("file digests cataloger processed %d files", prog.Current())
prog.SetCompleted()
return results, nil
}
func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]Digest, error) {
meta, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
}
// we should only attempt to report digests for files that are regular files (don't attempt to resolve links)
if meta.Type != file.TypeRegular {
return nil, errUndigestableFile
}
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, err
}
defer internal.CloseAndLogError(contentReader, location.VirtualPath)
digests, err := DigestsFromFile(contentReader, i.hashes)
if err != nil {
return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err}
}
return digests, nil
}
func DigestsFromFile(closer io.ReadCloser, hashes []crypto.Hash) ([]Digest, error) {
// create a set of hasher objects tied together with a single writer to feed content into
hashers := make([]hash.Hash, len(hashes))
writers := make([]io.Writer, len(hashes))
for idx, hashObj := range hashes {
hashers[idx] = hashObj.New()
writers[idx] = hashers[idx]
}
_, err := io.Copy(io.MultiWriter(writers...), closer)
if err != nil {
return nil, err
}
result := make([]Digest, len(hashes))
// only capture digests when there is content. It is important to do this based on SIZE and not
// FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only
// file type but a body is still allowed.
for idx, hasher := range hashers {
result[idx] = Digest{
Algorithm: DigestAlgorithmName(hashes[idx]),
Value: fmt.Sprintf("%+x", hasher.Sum(nil)),
}
}
return result, nil
}
func DigestAlgorithmName(hash crypto.Hash) string {
return CleanDigestAlgorithmName(hash.String())
}
func CleanDigestAlgorithmName(name string) string {
lower := strings.ToLower(name)
return strings.ReplaceAll(lower, "-", "")
}
func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
prog := progress.NewManual(locations)
bus.Publish(partybus.Event{
Type: event.FileDigestsCatalogerStarted,
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
},
})
return stage, prog
}

View File

@ -1,4 +1,4 @@
package source package file
import ( import (
"fmt" "fmt"
@ -24,6 +24,10 @@ type LocationData struct {
ref file.Reference `hash:"ignore"` // The file reference relative to the stereoscope.FileCatalog that has more information about this location. ref file.Reference `hash:"ignore"` // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
} }
func (l LocationData) Reference() file.Reference {
return l.ref
}
type LocationMetadata struct { type LocationMetadata struct {
Annotations map[string]string `json:"annotations,omitempty"` // Arbitrary key-value pairs that can be used to annotate a location Annotations map[string]string `json:"annotations,omitempty"` // Arbitrary key-value pairs that can be used to annotate a location
} }
@ -108,7 +112,7 @@ func NewVirtualLocationFromCoordinates(coordinates Coordinates, virtualPath stri
}} }}
} }
// NewLocationFromImage creates a new Location representing the given path (extracted from the ref) relative to the given image. // NewLocationFromImage creates a new Location representing the given path (extracted from the Reference) relative to the given image.
func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Image) Location { func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Image) Location {
layer := img.FileCatalog.Layer(ref) layer := img.FileCatalog.Layer(ref)
return Location{ return Location{
@ -126,7 +130,7 @@ func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Ima
} }
} }
// NewLocationFromDirectory creates a new Location representing the given path (extracted from the ref) relative to the given directory. // NewLocationFromDirectory creates a new Location representing the given path (extracted from the Reference) relative to the given directory.
func NewLocationFromDirectory(responsePath string, ref file.Reference) Location { func NewLocationFromDirectory(responsePath string, ref file.Reference) Location {
return Location{ return Location{
LocationData: LocationData{ LocationData: LocationData{
@ -141,7 +145,7 @@ func NewLocationFromDirectory(responsePath string, ref file.Reference) Location
} }
} }
// NewVirtualLocationFromDirectory creates a new Location representing the given path (extracted from the ref) relative to the given directory with a separate virtual access path. // NewVirtualLocationFromDirectory creates a new Location representing the given path (extracted from the Reference) relative to the given directory with a separate virtual access path.
func NewVirtualLocationFromDirectory(responsePath, virtualResponsePath string, ref file.Reference) Location { func NewVirtualLocationFromDirectory(responsePath, virtualResponsePath string, ref file.Reference) Location {
if responsePath == virtualResponsePath { if responsePath == virtualResponsePath {
return NewLocationFromDirectory(responsePath, ref) return NewLocationFromDirectory(responsePath, ref)

View File

@ -1,4 +1,4 @@
package source package file
import "io" import "io"

View File

@ -1,4 +1,4 @@
package source package file
import ( import (
"sort" "sort"

View File

@ -1,4 +1,4 @@
package source package file
import ( import (
"testing" "testing"

View File

@ -1,4 +1,4 @@
package source package file
import ( import (
"testing" "testing"

View File

@ -1,4 +1,4 @@
package source package file
type Locations []Location type Locations []Location

5
syft/file/metadata.go Normal file
View File

@ -0,0 +1,5 @@
package file
import "github.com/anchore/stereoscope/pkg/file"
type Metadata = file.Metadata

View File

@ -1,4 +1,4 @@
package source package file
import ( import (
"fmt" "fmt"
@ -11,14 +11,14 @@ import (
"github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/file"
) )
var _ FileResolver = (*MockResolver)(nil) var _ Resolver = (*MockResolver)(nil)
// MockResolver implements the FileResolver interface and is intended for use *only in test code*. // MockResolver implements the FileResolver interface and is intended for use *only in test code*.
// It provides an implementation that can resolve local filesystem paths using only a provided discrete list of file // It provides an implementation that can resolve local filesystem paths using only a provided discrete list of file
// paths, which are typically paths to test fixtures. // paths, which are typically paths to test fixtures.
type MockResolver struct { type MockResolver struct {
locations []Location locations []Location
metadata map[Coordinates]FileMetadata metadata map[Coordinates]Metadata
mimeTypeIndex map[string][]Location mimeTypeIndex map[string][]Location
extension map[string][]Location extension map[string][]Location
basename map[string][]Location basename map[string][]Location
@ -41,13 +41,13 @@ func NewMockResolverForPaths(paths ...string) *MockResolver {
return &MockResolver{ return &MockResolver{
locations: locations, locations: locations,
metadata: make(map[Coordinates]FileMetadata), metadata: make(map[Coordinates]Metadata),
extension: extension, extension: extension,
basename: basename, basename: basename,
} }
} }
func NewMockResolverForPathsWithMetadata(metadata map[Coordinates]FileMetadata) *MockResolver { func NewMockResolverForPathsWithMetadata(metadata map[Coordinates]Metadata) *MockResolver {
var locations []Location var locations []Location
var mimeTypeIndex = make(map[string][]Location) var mimeTypeIndex = make(map[string][]Location)
extension := make(map[string][]Location) extension := make(map[string][]Location)
@ -155,10 +155,10 @@ func (r MockResolver) AllLocations() <-chan Location {
return results return results
} }
func (r MockResolver) FileMetadataByLocation(l Location) (FileMetadata, error) { func (r MockResolver) FileMetadataByLocation(l Location) (Metadata, error) {
info, err := os.Stat(l.RealPath) info, err := os.Stat(l.RealPath)
if err != nil { if err != nil {
return FileMetadata{}, err return Metadata{}, err
} }
// other types not supported // other types not supported
@ -167,7 +167,7 @@ func (r MockResolver) FileMetadataByLocation(l Location) (FileMetadata, error) {
ty = file.TypeDirectory ty = file.TypeDirectory
} }
return FileMetadata{ return Metadata{
FileInfo: info, FileInfo: info,
Type: ty, Type: ty,
UserID: 0, // not supported UserID: 0, // not supported

View File

@ -1,28 +1,26 @@
package source package file
import ( import "io"
"io"
)
// FileResolver is an interface that encompasses how to get specific file references and file contents for a generic data source. // Resolver is an interface that encompasses how to get specific file references and file contents for a generic data source.
type FileResolver interface { type Resolver interface {
FileContentResolver ContentResolver
FilePathResolver PathResolver
FileLocationResolver LocationResolver
FileMetadataResolver MetadataResolver
} }
// FileContentResolver knows how to get file content for a given Location // ContentResolver knows how to get file content for a given Location
type FileContentResolver interface { type ContentResolver interface {
FileContentsByLocation(Location) (io.ReadCloser, error) FileContentsByLocation(Location) (io.ReadCloser, error)
} }
type FileMetadataResolver interface { type MetadataResolver interface {
FileMetadataByLocation(Location) (FileMetadata, error) FileMetadataByLocation(Location) (Metadata, error)
} }
// FilePathResolver knows how to get a Location for given string paths and globs // PathResolver knows how to get a Location for given string paths and globs
type FilePathResolver interface { type PathResolver interface {
// HasPath indicates if the given path exists in the underlying source. // HasPath indicates if the given path exists in the underlying source.
// The implementation for this may vary, however, generally the following considerations should be made: // The implementation for this may vary, however, generally the following considerations should be made:
// - full symlink resolution should be performed on all requests // - full symlink resolution should be performed on all requests
@ -50,7 +48,7 @@ type FilePathResolver interface {
RelativeFileByPath(_ Location, path string) *Location RelativeFileByPath(_ Location, path string) *Location
} }
type FileLocationResolver interface { type LocationResolver interface {
// AllLocations returns a channel of all file references from the underlying source. // AllLocations returns a channel of all file references from the underlying source.
// The implementation for this may vary, however, generally the following considerations should be made: // The implementation for this may vary, however, generally the following considerations should be made:
// - NO symlink resolution should be performed on results // - NO symlink resolution should be performed on results
@ -58,8 +56,8 @@ type FileLocationResolver interface {
AllLocations() <-chan Location AllLocations() <-chan Location
} }
type WritableFileResolver interface { type WritableResolver interface {
FileResolver Resolver
Write(location Location, reader io.Reader) error Write(location Location, reader io.Reader) error
} }

View File

@ -6,9 +6,9 @@ import (
"github.com/CycloneDX/cyclonedx-go" "github.com/CycloneDX/cyclonedx-go"
"github.com/anchore/packageurl-go" "github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/formats/common" "github.com/anchore/syft/syft/formats/common"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
) )
func encodeComponent(p pkg.Package) cyclonedx.Component { func encodeComponent(p pkg.Package) cyclonedx.Component {
@ -100,13 +100,13 @@ func decodeComponent(c *cyclonedx.Component) *pkg.Package {
return p return p
} }
func decodeLocations(vals map[string]string) source.LocationSet { func decodeLocations(vals map[string]string) file.LocationSet {
v := common.Decode(reflect.TypeOf([]source.Location{}), vals, "syft:location", CycloneDXFields) v := common.Decode(reflect.TypeOf([]file.Location{}), vals, "syft:location", CycloneDXFields)
out, ok := v.([]source.Location) out, ok := v.([]file.Location)
if !ok { if !ok {
out = nil out = nil
} }
return source.NewLocationSet(out...) return file.NewLocationSet(out...)
} }
func decodePackageMetadata(vals map[string]string, c *cyclonedx.Component, typ pkg.MetadataType) interface{} { func decodePackageMetadata(vals map[string]string, c *cyclonedx.Component, typ pkg.MetadataType) interface{} {

View File

@ -8,8 +8,8 @@ import (
"github.com/CycloneDX/cyclonedx-go" "github.com/CycloneDX/cyclonedx-go"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
) )
func Test_encodeComponentProperties(t *testing.T) { func Test_encodeComponentProperties(t *testing.T) {
@ -28,8 +28,8 @@ func Test_encodeComponentProperties(t *testing.T) {
name: "from apk", name: "from apk",
input: pkg.Package{ input: pkg.Package{
FoundBy: "cataloger", FoundBy: "cataloger",
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewLocationFromCoordinates(source.Coordinates{RealPath: "test"}), file.NewLocationFromCoordinates(file.Coordinates{RealPath: "test"}),
), ),
Metadata: pkg.ApkMetadata{ Metadata: pkg.ApkMetadata{
Package: "libc-utils", Package: "libc-utils",

View File

@ -5,8 +5,8 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
) )
func Test_SourceInfo(t *testing.T) { func Test_SourceInfo(t *testing.T) {
@ -19,9 +19,9 @@ func Test_SourceInfo(t *testing.T) {
name: "locations are captured", name: "locations are captured",
input: pkg.Package{ input: pkg.Package{
// note: no type given // note: no type given
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewVirtualLocation("/a-place", "/b-place"), file.NewVirtualLocation("/a-place", "/b-place"),
source.NewVirtualLocation("/c-place", "/d-place"), file.NewVirtualLocation("/c-place", "/d-place"),
), ),
}, },
expected: []string{ expected: []string{

View File

@ -21,7 +21,6 @@ import (
"github.com/anchore/syft/syft/formats/common/util" "github.com/anchore/syft/syft/formats/common/util"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
) )
const ( const (
@ -137,7 +136,7 @@ func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID {
switch it := identifiable.(type) { switch it := identifiable.(type) {
case pkg.Package: case pkg.Package:
id = SanitizeElementID(fmt.Sprintf("Package-%s-%s-%s", it.Type, it.Name, it.ID())) id = SanitizeElementID(fmt.Sprintf("Package-%s-%s-%s", it.Type, it.Name, it.ID()))
case source.Coordinates: case file.Coordinates:
p := "" p := ""
parts := strings.Split(it.RealPath, "/") parts := strings.Split(it.RealPath, "/")
for i := len(parts); i > 0; i-- { for i := len(parts); i > 0; i-- {
@ -437,7 +436,7 @@ func toFiles(s sbom.SBOM) (results []*spdx.File) {
artifacts := s.Artifacts artifacts := s.Artifacts
for _, coordinates := range s.AllCoordinates() { for _, coordinates := range s.AllCoordinates() {
var metadata *source.FileMetadata var metadata *file.Metadata
if metadataForLocation, exists := artifacts.FileMetadata[coordinates]; exists { if metadataForLocation, exists := artifacts.FileMetadata[coordinates]; exists {
metadata = &metadataForLocation metadata = &metadataForLocation
} }
@ -500,7 +499,7 @@ func toChecksumAlgorithm(algorithm string) spdx.ChecksumAlgorithm {
return spdx.ChecksumAlgorithm(strings.ToUpper(algorithm)) return spdx.ChecksumAlgorithm(strings.ToUpper(algorithm))
} }
func toFileTypes(metadata *source.FileMetadata) (ty []string) { func toFileTypes(metadata *file.Metadata) (ty []string) {
if metadata == nil { if metadata == nil {
return nil return nil
} }

View File

@ -115,12 +115,12 @@ func Test_toFileTypes(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
metadata source.FileMetadata metadata file.Metadata
expected []string expected []string
}{ }{
{ {
name: "application", name: "application",
metadata: source.FileMetadata{ metadata: file.Metadata{
MIMEType: "application/vnd.unknown", MIMEType: "application/vnd.unknown",
}, },
expected: []string{ expected: []string{
@ -129,7 +129,7 @@ func Test_toFileTypes(t *testing.T) {
}, },
{ {
name: "archive", name: "archive",
metadata: source.FileMetadata{ metadata: file.Metadata{
MIMEType: "application/zip", MIMEType: "application/zip",
}, },
expected: []string{ expected: []string{
@ -139,7 +139,7 @@ func Test_toFileTypes(t *testing.T) {
}, },
{ {
name: "audio", name: "audio",
metadata: source.FileMetadata{ metadata: file.Metadata{
MIMEType: "audio/ogg", MIMEType: "audio/ogg",
}, },
expected: []string{ expected: []string{
@ -148,7 +148,7 @@ func Test_toFileTypes(t *testing.T) {
}, },
{ {
name: "video", name: "video",
metadata: source.FileMetadata{ metadata: file.Metadata{
MIMEType: "video/3gpp", MIMEType: "video/3gpp",
}, },
expected: []string{ expected: []string{
@ -157,7 +157,7 @@ func Test_toFileTypes(t *testing.T) {
}, },
{ {
name: "text", name: "text",
metadata: source.FileMetadata{ metadata: file.Metadata{
MIMEType: "text/html", MIMEType: "text/html",
}, },
expected: []string{ expected: []string{
@ -166,7 +166,7 @@ func Test_toFileTypes(t *testing.T) {
}, },
{ {
name: "image", name: "image",
metadata: source.FileMetadata{ metadata: file.Metadata{
MIMEType: "image/png", MIMEType: "image/png",
}, },
expected: []string{ expected: []string{
@ -175,7 +175,7 @@ func Test_toFileTypes(t *testing.T) {
}, },
{ {
name: "binary", name: "binary",
metadata: source.FileMetadata{ metadata: file.Metadata{
MIMEType: "application/x-sharedlib", MIMEType: "application/x-sharedlib",
}, },
expected: []string{ expected: []string{
@ -276,7 +276,7 @@ func Test_fileIDsForPackage(t *testing.T) {
Name: "bogus", Name: "bogus",
} }
c := source.Coordinates{ c := file.Coordinates{
RealPath: "/path", RealPath: "/path",
FileSystemID: "nowhere", FileSystemID: "nowhere",
} }

View File

@ -35,8 +35,8 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) {
Source: src, Source: src,
Artifacts: sbom.Artifacts{ Artifacts: sbom.Artifacts{
Packages: pkg.NewCollection(), Packages: pkg.NewCollection(),
FileMetadata: map[source.Coordinates]source.FileMetadata{}, FileMetadata: map[file.Coordinates]file.Metadata{},
FileDigests: map[source.Coordinates][]file.Digest{}, FileDigests: map[file.Coordinates][]file.Digest{},
LinuxDistribution: findLinuxReleaseByPURL(doc), LinuxDistribution: findLinuxReleaseByPURL(doc),
}, },
} }
@ -135,7 +135,7 @@ func toFileDigests(f *spdx.File) (digests []file.Digest) {
return digests return digests
} }
func toFileMetadata(f *spdx.File) (meta source.FileMetadata) { func toFileMetadata(f *spdx.File) (meta file.Metadata) {
// FIXME Syft is currently lossy due to the SPDX 2.2.1 spec not supporting arbitrary mimetypes // FIXME Syft is currently lossy due to the SPDX 2.2.1 spec not supporting arbitrary mimetypes
for _, typ := range f.FileTypes { for _, typ := range f.FileTypes {
switch FileType(typ) { switch FileType(typ) {
@ -169,7 +169,7 @@ func toSyftRelationships(spdxIDMap map[string]interface{}, doc *spdx.Document) [
b := spdxIDMap[string(r.RefB.ElementRefID)] b := spdxIDMap[string(r.RefB.ElementRefID)]
from, fromOk := a.(*pkg.Package) from, fromOk := a.(*pkg.Package)
toPackage, toPackageOk := b.(*pkg.Package) toPackage, toPackageOk := b.(*pkg.Package)
toLocation, toLocationOk := b.(*source.Location) toLocation, toLocationOk := b.(*file.Location)
if !fromOk || !(toPackageOk || toLocationOk) { if !fromOk || !(toPackageOk || toLocationOk) {
log.Debugf("unable to find valid relationship mapping from SPDX 2.2 JSON, ignoring: (from: %+v) (to: %+v)", a, b) log.Debugf("unable to find valid relationship mapping from SPDX 2.2 JSON, ignoring: (from: %+v) (to: %+v)", a, b)
continue continue
@ -212,7 +212,7 @@ func toSyftRelationships(spdxIDMap map[string]interface{}, doc *spdx.Document) [
return out return out
} }
func toSyftCoordinates(f *spdx.File) source.Coordinates { func toSyftCoordinates(f *spdx.File) file.Coordinates {
const layerIDPrefix = "layerID: " const layerIDPrefix = "layerID: "
var fileSystemID string var fileSystemID string
if strings.Index(f.FileComment, layerIDPrefix) == 0 { if strings.Index(f.FileComment, layerIDPrefix) == 0 {
@ -221,14 +221,14 @@ func toSyftCoordinates(f *spdx.File) source.Coordinates {
if strings.Index(string(f.FileSPDXIdentifier), layerIDPrefix) == 0 { if strings.Index(string(f.FileSPDXIdentifier), layerIDPrefix) == 0 {
fileSystemID = strings.TrimPrefix(string(f.FileSPDXIdentifier), layerIDPrefix) fileSystemID = strings.TrimPrefix(string(f.FileSPDXIdentifier), layerIDPrefix)
} }
return source.Coordinates{ return file.Coordinates{
RealPath: f.FileName, RealPath: f.FileName,
FileSystemID: fileSystemID, FileSystemID: fileSystemID,
} }
} }
func toSyftLocation(f *spdx.File) *source.Location { func toSyftLocation(f *spdx.File) *file.Location {
l := source.NewVirtualLocationFromCoordinates(toSyftCoordinates(f), f.FileName) l := file.NewVirtualLocationFromCoordinates(toSyftCoordinates(f), f.FileName)
return &l return &l
} }

View File

@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
) )
@ -336,7 +337,7 @@ func Test_toSyftRelationships(t *testing.T) {
} }
pkg3.SetID() pkg3.SetID()
loc1 := source.NewLocationFromCoordinates(source.Coordinates{ loc1 := file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/somewhere/real", RealPath: "/somewhere/real",
FileSystemID: "abc", FileSystemID: "abc",
}) })

View File

@ -7,6 +7,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/anchore/packageurl-go" "github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/sbom"
@ -35,8 +36,8 @@ func Test_toGithubModel(t *testing.T) {
{ {
Name: "pkg-1", Name: "pkg-1",
Version: "1.0.1", Version: "1.0.1",
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewLocationFromCoordinates(source.Coordinates{ file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/usr/lib", RealPath: "/usr/lib",
FileSystemID: "fsid-1", FileSystemID: "fsid-1",
}), }),
@ -45,8 +46,8 @@ func Test_toGithubModel(t *testing.T) {
{ {
Name: "pkg-2", Name: "pkg-2",
Version: "2.0.2", Version: "2.0.2",
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewLocationFromCoordinates(source.Coordinates{ file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/usr/lib", RealPath: "/usr/lib",
FileSystemID: "fsid-1", FileSystemID: "fsid-1",
}), }),
@ -55,8 +56,8 @@ func Test_toGithubModel(t *testing.T) {
{ {
Name: "pkg-3", Name: "pkg-3",
Version: "3.0.3", Version: "3.0.3",
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewLocationFromCoordinates(source.Coordinates{ file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/etc", RealPath: "/etc",
FileSystemID: "fsid-1", FileSystemID: "fsid-1",
}), }),

View File

@ -17,6 +17,7 @@ import (
"github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/sbom"
@ -155,8 +156,8 @@ func populateImageCatalog(catalog *pkg.Collection, img *image.Image) {
catalog.Add(pkg.Package{ catalog.Add(pkg.Package{
Name: "package-1", Name: "package-1",
Version: "1.0.1", Version: "1.0.1",
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewLocationFromImage(string(ref1.RealPath), *ref1.Reference, img), file.NewLocationFromImage(string(ref1.RealPath), *ref1.Reference, img),
), ),
Type: pkg.PythonPkg, Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1", FoundBy: "the-cataloger-1",
@ -177,8 +178,8 @@ func populateImageCatalog(catalog *pkg.Collection, img *image.Image) {
catalog.Add(pkg.Package{ catalog.Add(pkg.Package{
Name: "package-2", Name: "package-2",
Version: "2.0.1", Version: "2.0.1",
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewLocationFromImage(string(ref2.RealPath), *ref2.Reference, img), file.NewLocationFromImage(string(ref2.RealPath), *ref2.Reference, img),
), ),
Type: pkg.DebPkg, Type: pkg.DebPkg,
FoundBy: "the-cataloger-2", FoundBy: "the-cataloger-2",
@ -265,8 +266,8 @@ func newDirectoryCatalog() *pkg.Collection {
Version: "1.0.1", Version: "1.0.1",
Type: pkg.PythonPkg, Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1", FoundBy: "the-cataloger-1",
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewLocation("/some/path/pkg1"), file.NewLocation("/some/path/pkg1"),
), ),
Language: pkg.Python, Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType, MetadataType: pkg.PythonPackageMetadataType,
@ -292,8 +293,8 @@ func newDirectoryCatalog() *pkg.Collection {
Version: "2.0.1", Version: "2.0.1",
Type: pkg.DebPkg, Type: pkg.DebPkg,
FoundBy: "the-cataloger-2", FoundBy: "the-cataloger-2",
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewLocation("/some/path/pkg1"), file.NewLocation("/some/path/pkg1"),
), ),
MetadataType: pkg.DpkgMetadataType, MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{ Metadata: pkg.DpkgMetadata{
@ -318,8 +319,8 @@ func newDirectoryCatalogWithAuthorField() *pkg.Collection {
Version: "1.0.1", Version: "1.0.1",
Type: pkg.PythonPkg, Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1", FoundBy: "the-cataloger-1",
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewLocation("/some/path/pkg1"), file.NewLocation("/some/path/pkg1"),
), ),
Language: pkg.Python, Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType, MetadataType: pkg.PythonPackageMetadataType,
@ -346,8 +347,8 @@ func newDirectoryCatalogWithAuthorField() *pkg.Collection {
Version: "2.0.1", Version: "2.0.1",
Type: pkg.DebPkg, Type: pkg.DebPkg,
FoundBy: "the-cataloger-2", FoundBy: "the-cataloger-2",
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewLocation("/some/path/pkg1"), file.NewLocation("/some/path/pkg1"),
), ),
MetadataType: pkg.DpkgMetadataType, MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{ Metadata: pkg.DpkgMetadata{
@ -366,15 +367,15 @@ func newDirectoryCatalogWithAuthorField() *pkg.Collection {
//nolint:gosec //nolint:gosec
func AddSampleFileRelationships(s *sbom.SBOM) { func AddSampleFileRelationships(s *sbom.SBOM) {
catalog := s.Artifacts.Packages.Sorted() catalog := s.Artifacts.Packages.Sorted()
s.Artifacts.FileMetadata = map[source.Coordinates]source.FileMetadata{} s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
files := []string{"/f1", "/f2", "/d1/f3", "/d2/f4", "/z1/f5", "/a1/f6"} files := []string{"/f1", "/f2", "/d1/f3", "/d2/f4", "/z1/f5", "/a1/f6"}
rnd := rand.New(rand.NewSource(time.Now().UnixNano())) rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
rnd.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] }) rnd.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] })
for _, f := range files { for _, f := range files {
meta := source.FileMetadata{} meta := file.Metadata{}
coords := source.Coordinates{RealPath: f} coords := file.Coordinates{RealPath: f}
s.Artifacts.FileMetadata[coords] = meta s.Artifacts.FileMetadata[coords] = meta
s.Relationships = append(s.Relationships, artifact.Relationship{ s.Relationships = append(s.Relationships, artifact.Relationship{

View File

@ -52,8 +52,8 @@ func TestEncodeFullJSONDocument(t *testing.T) {
p1 := pkg.Package{ p1 := pkg.Package{
Name: "package-1", Name: "package-1",
Version: "1.0.1", Version: "1.0.1",
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewLocationFromCoordinates(source.Coordinates{ file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a/place/a", RealPath: "/a/place/a",
}), }),
), ),
@ -76,8 +76,8 @@ func TestEncodeFullJSONDocument(t *testing.T) {
p2 := pkg.Package{ p2 := pkg.Package{
Name: "package-2", Name: "package-2",
Version: "2.0.1", Version: "2.0.1",
Locations: source.NewLocationSet( Locations: file.NewLocationSet(
source.NewLocationFromCoordinates(source.Coordinates{ file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b/place/b", RealPath: "/b/place/b",
}), }),
), ),
@ -101,8 +101,8 @@ func TestEncodeFullJSONDocument(t *testing.T) {
s := sbom.SBOM{ s := sbom.SBOM{
Artifacts: sbom.Artifacts{ Artifacts: sbom.Artifacts{
Packages: catalog, Packages: catalog,
FileMetadata: map[source.Coordinates]source.FileMetadata{ FileMetadata: map[file.Coordinates]file.Metadata{
source.NewLocation("/a/place").Coordinates: { file.NewLocation("/a/place").Coordinates: {
FileInfo: stereoFile.ManualInfo{ FileInfo: stereoFile.ManualInfo{
NameValue: "/a/place", NameValue: "/a/place",
ModeValue: 0775, ModeValue: 0775,
@ -111,7 +111,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
UserID: 0, UserID: 0,
GroupID: 0, GroupID: 0,
}, },
source.NewLocation("/a/place/a").Coordinates: { file.NewLocation("/a/place/a").Coordinates: {
FileInfo: stereoFile.ManualInfo{ FileInfo: stereoFile.ManualInfo{
NameValue: "/a/place/a", NameValue: "/a/place/a",
ModeValue: 0775, ModeValue: 0775,
@ -120,7 +120,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
UserID: 0, UserID: 0,
GroupID: 0, GroupID: 0,
}, },
source.NewLocation("/b").Coordinates: { file.NewLocation("/b").Coordinates: {
FileInfo: stereoFile.ManualInfo{ FileInfo: stereoFile.ManualInfo{
NameValue: "/b", NameValue: "/b",
ModeValue: 0775, ModeValue: 0775,
@ -130,7 +130,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
UserID: 0, UserID: 0,
GroupID: 0, GroupID: 0,
}, },
source.NewLocation("/b/place/b").Coordinates: { file.NewLocation("/b/place/b").Coordinates: {
FileInfo: stereoFile.ManualInfo{ FileInfo: stereoFile.ManualInfo{
NameValue: "/b/place/b", NameValue: "/b/place/b",
ModeValue: 0644, ModeValue: 0644,
@ -140,22 +140,22 @@ func TestEncodeFullJSONDocument(t *testing.T) {
GroupID: 2, GroupID: 2,
}, },
}, },
FileDigests: map[source.Coordinates][]file.Digest{ FileDigests: map[file.Coordinates][]file.Digest{
source.NewLocation("/a/place/a").Coordinates: { file.NewLocation("/a/place/a").Coordinates: {
{ {
Algorithm: "sha256", Algorithm: "sha256",
Value: "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703", Value: "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
}, },
}, },
source.NewLocation("/b/place/b").Coordinates: { file.NewLocation("/b/place/b").Coordinates: {
{ {
Algorithm: "sha256", Algorithm: "sha256",
Value: "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c", Value: "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c",
}, },
}, },
}, },
FileContents: map[source.Coordinates]string{ FileContents: map[file.Coordinates]string{
source.NewLocation("/a/place/a").Coordinates: "the-contents", file.NewLocation("/a/place/a").Coordinates: "the-contents",
}, },
LinuxDistribution: &linux.Release{ LinuxDistribution: &linux.Release{
ID: "redhat", ID: "redhat",

View File

@ -2,12 +2,11 @@ package model
import ( import (
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source"
) )
type File struct { type File struct {
ID string `json:"id"` ID string `json:"id"`
Location source.Coordinates `json:"location"` Location file.Coordinates `json:"location"`
Metadata *FileMetadataEntry `json:"metadata,omitempty"` Metadata *FileMetadataEntry `json:"metadata,omitempty"`
Contents string `json:"contents,omitempty"` Contents string `json:"contents,omitempty"`
Digests []file.Digest `json:"digests,omitempty"` Digests []file.Digest `json:"digests,omitempty"`

View File

@ -7,9 +7,9 @@ import (
"reflect" "reflect"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/license" "github.com/anchore/syft/syft/license"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
) )
var errUnknownMetadataType = errors.New("unknown metadata type") var errUnknownMetadataType = errors.New("unknown metadata type")
@ -27,7 +27,7 @@ type PackageBasicData struct {
Version string `json:"version"` Version string `json:"version"`
Type pkg.Type `json:"type"` Type pkg.Type `json:"type"`
FoundBy string `json:"foundBy"` FoundBy string `json:"foundBy"`
Locations []source.Location `json:"locations"` Locations []file.Location `json:"locations"`
Licenses licenses `json:"licenses"` Licenses licenses `json:"licenses"`
Language pkg.Language `json:"language"` Language pkg.Language `json:"language"`
CPEs []string `json:"cpes"` CPEs []string `json:"cpes"`
@ -41,7 +41,7 @@ type License struct {
SPDXExpression string `json:"spdxExpression"` SPDXExpression string `json:"spdxExpression"`
Type license.Type `json:"type"` Type license.Type `json:"type"`
URLs []string `json:"urls"` URLs []string `json:"urls"`
Locations []source.Location `json:"locations"` Locations []file.Location `json:"locations"`
} }
func newModelLicensesFromValues(licenses []string) (ml []License) { func newModelLicensesFromValues(licenses []string) (ml []License) {

View File

@ -2,10 +2,9 @@ package model
import ( import (
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source"
) )
type Secrets struct { type Secrets struct {
Location source.Coordinates `json:"location"` Location file.Coordinates `json:"location"`
Secrets []file.SearchResult `json:"secrets"` Secrets []file.SearchResult `json:"secrets"`
} }

View File

@ -74,7 +74,7 @@ func toDescriptor(d sbom.Descriptor) model.Descriptor {
} }
} }
func toSecrets(data map[source.Coordinates][]file.SearchResult) []model.Secrets { func toSecrets(data map[file.Coordinates][]file.SearchResult) []model.Secrets {
results := make([]model.Secrets, 0) results := make([]model.Secrets, 0)
for coordinates, secrets := range data { for coordinates, secrets := range data {
results = append(results, model.Secrets{ results = append(results, model.Secrets{
@ -95,7 +95,7 @@ func toFile(s sbom.SBOM) []model.File {
artifacts := s.Artifacts artifacts := s.Artifacts
for _, coordinates := range s.AllCoordinates() { for _, coordinates := range s.AllCoordinates() {
var metadata *source.FileMetadata var metadata *file.Metadata
if metadataForLocation, exists := artifacts.FileMetadata[coordinates]; exists { if metadataForLocation, exists := artifacts.FileMetadata[coordinates]; exists {
metadata = &metadataForLocation metadata = &metadataForLocation
} }
@ -126,7 +126,7 @@ func toFile(s sbom.SBOM) []model.File {
return results return results
} }
func toFileMetadataEntry(coordinates source.Coordinates, metadata *source.FileMetadata) *model.FileMetadataEntry { func toFileMetadataEntry(coordinates file.Coordinates, metadata *file.Metadata) *model.FileMetadataEntry {
if metadata == nil { if metadata == nil {
return nil return nil
} }
@ -195,7 +195,7 @@ func toPackageModels(catalog *pkg.Collection) []model.Package {
func toLicenseModel(pkgLicenses []pkg.License) (modelLicenses []model.License) { func toLicenseModel(pkgLicenses []pkg.License) (modelLicenses []model.License) {
for _, l := range pkgLicenses { for _, l := range pkgLicenses {
// guarantee collection // guarantee collection
locations := make([]source.Location, 0) locations := make([]file.Location, 0)
if v := l.Locations.ToSlice(); v != nil { if v := l.Locations.ToSlice(); v != nil {
locations = v locations = v
} }

View File

@ -7,7 +7,8 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file" stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/formats/syftjson/model" "github.com/anchore/syft/syft/formats/syftjson/model"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
) )
@ -94,46 +95,46 @@ func Test_toSourceModel(t *testing.T) {
func Test_toFileType(t *testing.T) { func Test_toFileType(t *testing.T) {
badType := file.Type(0x1337) badType := stereoscopeFile.Type(0x1337)
var allTypesTested []file.Type var allTypesTested []stereoscopeFile.Type
tests := []struct { tests := []struct {
ty file.Type ty stereoscopeFile.Type
name string name string
}{ }{
{ {
ty: file.TypeRegular, ty: stereoscopeFile.TypeRegular,
name: "RegularFile", name: "RegularFile",
}, },
{ {
ty: file.TypeDirectory, ty: stereoscopeFile.TypeDirectory,
name: "Directory", name: "Directory",
}, },
{ {
ty: file.TypeSymLink, ty: stereoscopeFile.TypeSymLink,
name: "SymbolicLink", name: "SymbolicLink",
}, },
{ {
ty: file.TypeHardLink, ty: stereoscopeFile.TypeHardLink,
name: "HardLink", name: "HardLink",
}, },
{ {
ty: file.TypeSocket, ty: stereoscopeFile.TypeSocket,
name: "Socket", name: "Socket",
}, },
{ {
ty: file.TypeCharacterDevice, ty: stereoscopeFile.TypeCharacterDevice,
name: "CharacterDevice", name: "CharacterDevice",
}, },
{ {
ty: file.TypeBlockDevice, ty: stereoscopeFile.TypeBlockDevice,
name: "BlockDevice", name: "BlockDevice",
}, },
{ {
ty: file.TypeFIFO, ty: stereoscopeFile.TypeFIFO,
name: "FIFONode", name: "FIFONode",
}, },
{ {
ty: file.TypeIrregular, ty: stereoscopeFile.TypeIrregular,
name: "IrregularFile", name: "IrregularFile",
}, },
{ {
@ -150,17 +151,17 @@ func Test_toFileType(t *testing.T) {
}) })
} }
assert.ElementsMatch(t, allTypesTested, file.AllTypes(), "not all file.Types are under test") assert.ElementsMatch(t, allTypesTested, stereoscopeFile.AllTypes(), "not all file.Types are under test")
} }
func Test_toFileMetadataEntry(t *testing.T) { func Test_toFileMetadataEntry(t *testing.T) {
coords := source.Coordinates{ coords := file.Coordinates{
RealPath: "/path", RealPath: "/path",
FileSystemID: "x", FileSystemID: "x",
} }
tests := []struct { tests := []struct {
name string name string
metadata *source.FileMetadata metadata *file.Metadata
want *model.FileMetadataEntry want *model.FileMetadataEntry
}{ }{
{ {
@ -168,23 +169,23 @@ func Test_toFileMetadataEntry(t *testing.T) {
}, },
{ {
name: "no file info", name: "no file info",
metadata: &source.FileMetadata{ metadata: &file.Metadata{
FileInfo: nil, FileInfo: nil,
}, },
want: &model.FileMetadataEntry{ want: &model.FileMetadataEntry{
Type: file.TypeRegular.String(), Type: stereoscopeFile.TypeRegular.String(),
}, },
}, },
{ {
name: "with file info", name: "with file info",
metadata: &source.FileMetadata{ metadata: &file.Metadata{
FileInfo: &file.ManualInfo{ FileInfo: &stereoscopeFile.ManualInfo{
ModeValue: 1, ModeValue: 1,
}, },
}, },
want: &model.FileMetadataEntry{ want: &model.FileMetadataEntry{
Mode: 1, Mode: 1,
Type: file.TypeRegular.String(), Type: stereoscopeFile.TypeRegular.String(),
}, },
}, },
} }

View File

@ -64,8 +64,8 @@ func deduplicateErrors(errors []error) []string {
func toSyftFiles(files []model.File) sbom.Artifacts { func toSyftFiles(files []model.File) sbom.Artifacts {
ret := sbom.Artifacts{ ret := sbom.Artifacts{
FileMetadata: make(map[source.Coordinates]source.FileMetadata), FileMetadata: make(map[file.Coordinates]file.Metadata),
FileDigests: make(map[source.Coordinates][]file.Digest), FileDigests: make(map[file.Coordinates][]file.Digest),
} }
for _, f := range files { for _, f := range files {
@ -79,7 +79,7 @@ func toSyftFiles(files []model.File) sbom.Artifacts {
fm := os.FileMode(mode) fm := os.FileMode(mode)
ret.FileMetadata[coord] = source.FileMetadata{ ret.FileMetadata[coord] = file.Metadata{
FileInfo: stereoscopeFile.ManualInfo{ FileInfo: stereoscopeFile.ManualInfo{
NameValue: path.Base(coord.RealPath), NameValue: path.Base(coord.RealPath),
SizeValue: f.Metadata.Size, SizeValue: f.Metadata.Size,
@ -112,7 +112,7 @@ func toSyftLicenses(m []model.License) (p []pkg.License) {
SPDXExpression: l.SPDXExpression, SPDXExpression: l.SPDXExpression,
Type: l.Type, Type: l.Type,
URLs: internal.NewStringSet(l.URLs...), URLs: internal.NewStringSet(l.URLs...),
Locations: source.NewLocationSet(l.Locations...), Locations: file.NewLocationSet(l.Locations...),
}) })
} }
return return
@ -320,7 +320,7 @@ func toSyftPackage(p model.Package, idAliases map[string]string) pkg.Package {
Name: p.Name, Name: p.Name,
Version: p.Version, Version: p.Version,
FoundBy: p.FoundBy, FoundBy: p.FoundBy,
Locations: source.NewLocationSet(p.Locations...), Locations: file.NewLocationSet(p.Locations...),
Licenses: pkg.NewLicenseSet(toSyftLicenses(p.Licenses)...), Licenses: pkg.NewLicenseSet(toSyftLicenses(p.Licenses)...),
Language: p.Language, Language: p.Language,
Type: p.Type, Type: p.Type,

View File

@ -131,7 +131,7 @@ func Test_idsHaveChanged(t *testing.T) {
} }
func Test_toSyftFiles(t *testing.T) { func Test_toSyftFiles(t *testing.T) {
coord := source.Coordinates{ coord := file.Coordinates{
RealPath: "/somerwhere/place", RealPath: "/somerwhere/place",
FileSystemID: "abc", FileSystemID: "abc",
} }
@ -145,8 +145,8 @@ func Test_toSyftFiles(t *testing.T) {
name: "empty", name: "empty",
files: []model.File{}, files: []model.File{},
want: sbom.Artifacts{ want: sbom.Artifacts{
FileMetadata: map[source.Coordinates]source.FileMetadata{}, FileMetadata: map[file.Coordinates]file.Metadata{},
FileDigests: map[source.Coordinates][]file.Digest{}, FileDigests: map[file.Coordinates][]file.Digest{},
}, },
}, },
{ {
@ -165,8 +165,8 @@ func Test_toSyftFiles(t *testing.T) {
}, },
}, },
want: sbom.Artifacts{ want: sbom.Artifacts{
FileMetadata: map[source.Coordinates]source.FileMetadata{}, FileMetadata: map[file.Coordinates]file.Metadata{},
FileDigests: map[source.Coordinates][]file.Digest{ FileDigests: map[file.Coordinates][]file.Digest{
coord: { coord: {
{ {
Algorithm: "sha256", Algorithm: "sha256",
@ -200,7 +200,7 @@ func Test_toSyftFiles(t *testing.T) {
}, },
}, },
want: sbom.Artifacts{ want: sbom.Artifacts{
FileMetadata: map[source.Coordinates]source.FileMetadata{ FileMetadata: map[file.Coordinates]file.Metadata{
coord: { coord: {
FileInfo: stereoFile.ManualInfo{ FileInfo: stereoFile.ManualInfo{
NameValue: "place", NameValue: "place",
@ -215,7 +215,7 @@ func Test_toSyftFiles(t *testing.T) {
MIMEType: "text/plain", MIMEType: "text/plain",
}, },
}, },
FileDigests: map[source.Coordinates][]file.Digest{ FileDigests: map[file.Coordinates][]file.Digest{
coord: { coord: {
{ {
Algorithm: "sha256", Algorithm: "sha256",

View File

@ -1,25 +1,26 @@
package source package fileresolver
import ( import (
"fmt" "fmt"
"io" "io"
"github.com/anchore/stereoscope/pkg/file" stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree" "github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/stereoscope/pkg/image" "github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
) )
var _ FileResolver = (*imageAllLayersResolver)(nil) var _ file.Resolver = (*ContainerImageAllLayers)(nil)
// imageAllLayersResolver implements path and content access for the AllLayers source option for container image data sources. // ContainerImageAllLayers implements path and content access for the AllLayers source option for container image data sources.
type imageAllLayersResolver struct { type ContainerImageAllLayers struct {
img *image.Image img *image.Image
layers []int layers []int
} }
// newAllLayersResolver returns a new resolver from the perspective of all image layers for the given image. // NewFromContainerImageAllLayers returns a new resolver from the perspective of all image layers for the given image.
func newAllLayersResolver(img *image.Image) (*imageAllLayersResolver, error) { func NewFromContainerImageAllLayers(img *image.Image) (*ContainerImageAllLayers, error) {
if len(img.Layers) == 0 { if len(img.Layers) == 0 {
return nil, fmt.Errorf("the image does not contain any layers") return nil, fmt.Errorf("the image does not contain any layers")
} }
@ -28,15 +29,15 @@ func newAllLayersResolver(img *image.Image) (*imageAllLayersResolver, error) {
for idx := range img.Layers { for idx := range img.Layers {
layers = append(layers, idx) layers = append(layers, idx)
} }
return &imageAllLayersResolver{ return &ContainerImageAllLayers{
img: img, img: img,
layers: layers, layers: layers,
}, nil }, nil
} }
// HasPath indicates if the given path exists in the underlying source. // HasPath indicates if the given path exists in the underlying source.
func (r *imageAllLayersResolver) HasPath(path string) bool { func (r *ContainerImageAllLayers) HasPath(path string) bool {
p := file.Path(path) p := stereoscopeFile.Path(path)
for _, layerIdx := range r.layers { for _, layerIdx := range r.layers {
tree := r.img.Layers[layerIdx].Tree tree := r.img.Layers[layerIdx].Tree
if tree.HasPath(p) { if tree.HasPath(p) {
@ -46,8 +47,8 @@ func (r *imageAllLayersResolver) HasPath(path string) bool {
return false return false
} }
func (r *imageAllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) { func (r *ContainerImageAllLayers) fileByRef(ref stereoscopeFile.Reference, uniqueFileIDs stereoscopeFile.ReferenceSet, layerIdx int) ([]stereoscopeFile.Reference, error) {
uniqueFiles := make([]file.Reference, 0) uniqueFiles := make([]stereoscopeFile.Reference, 0)
// since there is potentially considerable work for each symlink/hardlink that needs to be resolved, let's check to see if this is a symlink/hardlink first // since there is potentially considerable work for each symlink/hardlink that needs to be resolved, let's check to see if this is a symlink/hardlink first
entry, err := r.img.FileCatalog.Get(ref) entry, err := r.img.FileCatalog.Get(ref)
@ -55,7 +56,7 @@ func (r *imageAllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs fil
return nil, fmt.Errorf("unable to fetch metadata (ref=%+v): %w", ref, err) return nil, fmt.Errorf("unable to fetch metadata (ref=%+v): %w", ref, err)
} }
if entry.Metadata.Type == file.TypeHardLink || entry.Metadata.Type == file.TypeSymLink { if entry.Metadata.Type == stereoscopeFile.TypeHardLink || entry.Metadata.Type == stereoscopeFile.TypeSymLink {
// a link may resolve in this layer or higher, assuming a squashed tree is used to search // a link may resolve in this layer or higher, assuming a squashed tree is used to search
// we should search all possible resolutions within the valid source // we should search all possible resolutions within the valid source
for _, subLayerIdx := range r.layers[layerIdx:] { for _, subLayerIdx := range r.layers[layerIdx:] {
@ -77,9 +78,9 @@ func (r *imageAllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs fil
} }
// FilesByPath returns all file.References that match the given paths from any layer in the image. // FilesByPath returns all file.References that match the given paths from any layer in the image.
func (r *imageAllLayersResolver) FilesByPath(paths ...string) ([]Location, error) { func (r *ContainerImageAllLayers) FilesByPath(paths ...string) ([]file.Location, error) {
uniqueFileIDs := file.NewFileReferenceSet() uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]Location, 0) uniqueLocations := make([]file.Location, 0)
for _, path := range paths { for _, path := range paths {
for idx, layerIdx := range r.layers { for idx, layerIdx := range r.layers {
@ -110,7 +111,7 @@ func (r *imageAllLayersResolver) FilesByPath(paths ...string) ([]Location, error
return nil, err return nil, err
} }
for _, result := range results { for _, result := range results {
uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, result, r.img)) uniqueLocations = append(uniqueLocations, file.NewLocationFromImage(path, result, r.img))
} }
} }
} }
@ -119,9 +120,9 @@ func (r *imageAllLayersResolver) FilesByPath(paths ...string) ([]Location, error
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
// nolint:gocognit // nolint:gocognit
func (r *imageAllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) { func (r *ContainerImageAllLayers) FilesByGlob(patterns ...string) ([]file.Location, error) {
uniqueFileIDs := file.NewFileReferenceSet() uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]Location, 0) uniqueLocations := make([]file.Location, 0)
for _, pattern := range patterns { for _, pattern := range patterns {
for idx, layerIdx := range r.layers { for idx, layerIdx := range r.layers {
@ -153,7 +154,7 @@ func (r *imageAllLayersResolver) FilesByGlob(patterns ...string) ([]Location, er
return nil, err return nil, err
} }
for _, refResult := range refResults { for _, refResult := range refResults {
uniqueLocations = append(uniqueLocations, NewLocationFromImage(string(result.RequestPath), refResult, r.img)) uniqueLocations = append(uniqueLocations, file.NewLocationFromImage(string(result.RequestPath), refResult, r.img))
} }
} }
} }
@ -164,10 +165,10 @@ func (r *imageAllLayersResolver) FilesByGlob(patterns ...string) ([]Location, er
// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
// This is helpful when attempting to find a file that is in the same layer or lower as another file. // This is helpful when attempting to find a file that is in the same layer or lower as another file.
func (r *imageAllLayersResolver) RelativeFileByPath(location Location, path string) *Location { func (r *ContainerImageAllLayers) RelativeFileByPath(location file.Location, path string) *file.Location {
layer := r.img.FileCatalog.Layer(location.ref) layer := r.img.FileCatalog.Layer(location.Reference())
exists, relativeRef, err := layer.SquashedTree.File(file.Path(path), filetree.FollowBasenameLinks) exists, relativeRef, err := layer.SquashedTree.File(stereoscopeFile.Path(path), filetree.FollowBasenameLinks)
if err != nil { if err != nil {
log.Errorf("failed to find path=%q in squash: %+w", path, err) log.Errorf("failed to find path=%q in squash: %+w", path, err)
return nil return nil
@ -176,21 +177,21 @@ func (r *imageAllLayersResolver) RelativeFileByPath(location Location, path stri
return nil return nil
} }
relativeLocation := NewLocationFromImage(path, *relativeRef.Reference, r.img) relativeLocation := file.NewLocationFromImage(path, *relativeRef.Reference, r.img)
return &relativeLocation return &relativeLocation
} }
// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer. // FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
// If the path does not exist an error is returned. // If the path does not exist an error is returned.
func (r *imageAllLayersResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) { func (r *ContainerImageAllLayers) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
entry, err := r.img.FileCatalog.Get(location.ref) entry, err := r.img.FileCatalog.Get(location.Reference())
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err) return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err)
} }
switch entry.Metadata.Type { switch entry.Metadata.Type {
case file.TypeSymLink, file.TypeHardLink: case stereoscopeFile.TypeSymLink, stereoscopeFile.TypeHardLink:
// the location we are searching may be a symlink, we should always work with the resolved file // the location we are searching may be a symlink, we should always work with the resolved file
newLocation := r.RelativeFileByPath(location, location.VirtualPath) newLocation := r.RelativeFileByPath(location, location.VirtualPath)
if newLocation == nil { if newLocation == nil {
@ -198,16 +199,16 @@ func (r *imageAllLayersResolver) FileContentsByLocation(location Location) (io.R
return nil, fmt.Errorf("no contents for location=%q", location.VirtualPath) return nil, fmt.Errorf("no contents for location=%q", location.VirtualPath)
} }
location = *newLocation location = *newLocation
case file.TypeDirectory: case stereoscopeFile.TypeDirectory:
return nil, fmt.Errorf("cannot read contents of non-file %q", location.ref.RealPath) return nil, fmt.Errorf("cannot read contents of non-file %q", location.Reference().RealPath)
} }
return r.img.FileContentsByRef(location.ref) return r.img.OpenReference(location.Reference())
} }
func (r *imageAllLayersResolver) FilesByMIMEType(types ...string) ([]Location, error) { func (r *ContainerImageAllLayers) FilesByMIMEType(types ...string) ([]file.Location, error) {
uniqueFileIDs := file.NewFileReferenceSet() uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]Location, 0) uniqueLocations := make([]file.Location, 0)
for idx, layerIdx := range r.layers { for idx, layerIdx := range r.layers {
refs, err := r.img.Layers[layerIdx].SearchContext.SearchByMIMEType(types...) refs, err := r.img.Layers[layerIdx].SearchContext.SearchByMIMEType(types...)
@ -225,7 +226,7 @@ func (r *imageAllLayersResolver) FilesByMIMEType(types ...string) ([]Location, e
return nil, err return nil, err
} }
for _, refResult := range refResults { for _, refResult := range refResults {
uniqueLocations = append(uniqueLocations, NewLocationFromImage(string(ref.RequestPath), refResult, r.img)) uniqueLocations = append(uniqueLocations, file.NewLocationFromImage(string(ref.RequestPath), refResult, r.img))
} }
} }
} }
@ -233,20 +234,20 @@ func (r *imageAllLayersResolver) FilesByMIMEType(types ...string) ([]Location, e
return uniqueLocations, nil return uniqueLocations, nil
} }
func (r *imageAllLayersResolver) AllLocations() <-chan Location { func (r *ContainerImageAllLayers) AllLocations() <-chan file.Location {
results := make(chan Location) results := make(chan file.Location)
go func() { go func() {
defer close(results) defer close(results)
for _, layerIdx := range r.layers { for _, layerIdx := range r.layers {
tree := r.img.Layers[layerIdx].Tree tree := r.img.Layers[layerIdx].Tree
for _, ref := range tree.AllFiles(file.AllTypes()...) { for _, ref := range tree.AllFiles(stereoscopeFile.AllTypes()...) {
results <- NewLocationFromImage(string(ref.RealPath), ref, r.img) results <- file.NewLocationFromImage(string(ref.RealPath), ref, r.img)
} }
} }
}() }()
return results return results
} }
func (r *imageAllLayersResolver) FileMetadataByLocation(location Location) (FileMetadata, error) { func (r *ContainerImageAllLayers) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
return fileMetadataByLocation(r.img, location) return fileMetadataByLocation(r.img, location)
} }

View File

@ -1,4 +1,4 @@
package source package fileresolver
import ( import (
"fmt" "fmt"
@ -13,6 +13,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/file"
) )
type resolution struct { type resolution struct {
@ -93,7 +94,7 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newAllLayersResolver(img) resolver, err := NewFromContainerImageAllLayers(img)
if err != nil { if err != nil {
t.Fatalf("could not create resolver: %+v", err) t.Fatalf("could not create resolver: %+v", err)
} }
@ -121,15 +122,15 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
for idx, actual := range refs { for idx, actual := range refs {
expected := c.resolutions[idx] expected := c.resolutions[idx]
if string(actual.ref.RealPath) != expected.path { if string(actual.Reference().RealPath) != expected.path {
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path) t.Errorf("bad resolve path: '%s'!='%s'", string(actual.Reference().RealPath), expected.path)
} }
if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath { if expected.path != "" && string(actual.Reference().RealPath) != actual.RealPath {
t.Errorf("we should always prefer real paths over ones with links") t.Errorf("we should always prefer real paths over ones with links")
} }
layer := img.FileCatalog.Layer(actual.ref) layer := img.FileCatalog.Layer(actual.Reference())
if layer.Metadata.Index != expected.layer { if layer.Metadata.Index != expected.layer {
t.Errorf("bad resolve layer: '%d'!='%d'", layer.Metadata.Index, expected.layer) t.Errorf("bad resolve layer: '%d'!='%d'", layer.Metadata.Index, expected.layer)
} }
@ -207,7 +208,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newAllLayersResolver(img) resolver, err := NewFromContainerImageAllLayers(img)
if err != nil { if err != nil {
t.Fatalf("could not create resolver: %+v", err) t.Fatalf("could not create resolver: %+v", err)
} }
@ -224,15 +225,15 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
for idx, actual := range refs { for idx, actual := range refs {
expected := c.resolutions[idx] expected := c.resolutions[idx]
if string(actual.ref.RealPath) != expected.path { if string(actual.Reference().RealPath) != expected.path {
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path) t.Errorf("bad resolve path: '%s'!='%s'", string(actual.Reference().RealPath), expected.path)
} }
if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath { if expected.path != "" && string(actual.Reference().RealPath) != actual.RealPath {
t.Errorf("we should always prefer real paths over ones with links") t.Errorf("we should always prefer real paths over ones with links")
} }
layer := img.FileCatalog.Layer(actual.ref) layer := img.FileCatalog.Layer(actual.Reference())
if layer.Metadata.Index != expected.layer { if layer.Metadata.Index != expected.layer {
t.Errorf("bad resolve layer: '%d'!='%d'", layer.Metadata.Index, expected.layer) t.Errorf("bad resolve layer: '%d'!='%d'", layer.Metadata.Index, expected.layer)
@ -259,7 +260,7 @@ func Test_imageAllLayersResolver_FilesByMIMEType(t *testing.T) {
t.Run(test.fixtureName, func(t *testing.T) { t.Run(test.fixtureName, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureName) img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureName)
resolver, err := newAllLayersResolver(img) resolver, err := NewFromContainerImageAllLayers(img)
assert.NoError(t, err) assert.NoError(t, err)
locations, err := resolver.FilesByMIMEType(test.mimeType) locations, err := resolver.FilesByMIMEType(test.mimeType)
@ -276,7 +277,7 @@ func Test_imageAllLayersResolver_FilesByMIMEType(t *testing.T) {
func Test_imageAllLayersResolver_hasFilesystemIDInLocation(t *testing.T) { func Test_imageAllLayersResolver_hasFilesystemIDInLocation(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-duplicate-path") img := imagetest.GetFixtureImage(t, "docker-archive", "image-duplicate-path")
resolver, err := newAllLayersResolver(img) resolver, err := NewFromContainerImageAllLayers(img)
assert.NoError(t, err) assert.NoError(t, err)
locations, err := resolver.FilesByMIMEType("text/plain") locations, err := resolver.FilesByMIMEType("text/plain")
@ -336,7 +337,7 @@ func TestAllLayersImageResolver_FilesContents(t *testing.T) {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newAllLayersResolver(img) resolver, err := NewFromContainerImageAllLayers(img)
assert.NoError(t, err) assert.NoError(t, err)
refs, err := resolver.FilesByPath(test.fixture) refs, err := resolver.FilesByPath(test.fixture)
@ -363,12 +364,12 @@ func TestAllLayersImageResolver_FilesContents_errorOnDirRequest(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newAllLayersResolver(img) resolver, err := NewFromContainerImageAllLayers(img)
assert.NoError(t, err) assert.NoError(t, err)
var dirLoc *Location var dirLoc *file.Location
for loc := range resolver.AllLocations() { for loc := range resolver.AllLocations() {
entry, err := resolver.img.FileCatalog.Get(loc.ref) entry, err := resolver.img.FileCatalog.Get(loc.Reference())
require.NoError(t, err) require.NoError(t, err)
if entry.Metadata.IsDir() { if entry.Metadata.IsDir() {
dirLoc = &loc dirLoc = &loc
@ -386,119 +387,119 @@ func TestAllLayersImageResolver_FilesContents_errorOnDirRequest(t *testing.T) {
func Test_imageAllLayersResolver_resolvesLinks(t *testing.T) { func Test_imageAllLayersResolver_resolvesLinks(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
runner func(FileResolver) []Location runner func(file.Resolver) []file.Location
expected []Location expected []file.Location
}{ }{
{ {
name: "by mimetype", name: "by mimetype",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links should not show up when searching mimetype // links should not show up when searching mimetype
actualLocations, err := resolver.FilesByMIMEType("text/plain") actualLocations, err := resolver.FilesByMIMEType("text/plain")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewVirtualLocation("/etc/group", "/etc/group"), file.NewVirtualLocation("/etc/group", "/etc/group"),
NewVirtualLocation("/etc/passwd", "/etc/passwd"), file.NewVirtualLocation("/etc/passwd", "/etc/passwd"),
NewVirtualLocation("/etc/shadow", "/etc/shadow"), file.NewVirtualLocation("/etc/shadow", "/etc/shadow"),
NewVirtualLocation("/file-1.txt", "/file-1.txt"), file.NewVirtualLocation("/file-1.txt", "/file-1.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1 file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1
// note: we're de-duping the redundant access to file-3.txt // note: we're de-duping the redundant access to file-3.txt
// ... (there would usually be two copies) // ... (there would usually be two copies)
NewVirtualLocation("/file-3.txt", "/file-3.txt"), file.NewVirtualLocation("/file-3.txt", "/file-3.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2 file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // copy 1 file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // copy 1
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // copy 2 file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // copy 2
}, },
}, },
{ {
name: "by glob to links", name: "by glob to links",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files // links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("*ink-*") actualLocations, err := resolver.FilesByGlob("*ink-*")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewVirtualLocation("/file-1.txt", "/link-1"), file.NewVirtualLocation("/file-1.txt", "/link-1"),
NewVirtualLocation("/file-2.txt", "/link-2"), // copy 1 file.NewVirtualLocation("/file-2.txt", "/link-2"), // copy 1
NewVirtualLocation("/file-2.txt", "/link-2"), // copy 2 file.NewVirtualLocation("/file-2.txt", "/link-2"), // copy 2
NewVirtualLocation("/file-3.txt", "/link-within"), file.NewVirtualLocation("/file-3.txt", "/link-within"),
}, },
}, },
{ {
name: "by basename", name: "by basename",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files // links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/file-2.txt") actualLocations, err := resolver.FilesByGlob("**/file-2.txt")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1 file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2 file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2
}, },
}, },
{ {
name: "by basename glob", name: "by basename glob",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files // links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/file-?.txt") actualLocations, err := resolver.FilesByGlob("**/file-?.txt")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewVirtualLocation("/file-1.txt", "/file-1.txt"), file.NewVirtualLocation("/file-1.txt", "/file-1.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1 file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2 file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2
NewVirtualLocation("/file-3.txt", "/file-3.txt"), file.NewVirtualLocation("/file-3.txt", "/file-3.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // when we copy into the link path, the same file-4.txt is copied file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // when we copy into the link path, the same file-4.txt is copied
}, },
}, },
{ {
name: "by extension", name: "by extension",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files // links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/*.txt") actualLocations, err := resolver.FilesByGlob("**/*.txt")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewVirtualLocation("/file-1.txt", "/file-1.txt"), file.NewVirtualLocation("/file-1.txt", "/file-1.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1 file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2 file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2
NewVirtualLocation("/file-3.txt", "/file-3.txt"), file.NewVirtualLocation("/file-3.txt", "/file-3.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // when we copy into the link path, the same file-4.txt is copied file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // when we copy into the link path, the same file-4.txt is copied
}, },
}, },
{ {
name: "by path to degree 1 link", name: "by path to degree 1 link",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links resolve to the final file // links resolve to the final file
actualLocations, err := resolver.FilesByPath("/link-2") actualLocations, err := resolver.FilesByPath("/link-2")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
// we have multiple copies across layers // we have multiple copies across layers
NewVirtualLocation("/file-2.txt", "/link-2"), file.NewVirtualLocation("/file-2.txt", "/link-2"),
NewVirtualLocation("/file-2.txt", "/link-2"), file.NewVirtualLocation("/file-2.txt", "/link-2"),
}, },
}, },
{ {
name: "by path to degree 2 link", name: "by path to degree 2 link",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// multiple links resolves to the final file // multiple links resolves to the final file
actualLocations, err := resolver.FilesByPath("/link-indirect") actualLocations, err := resolver.FilesByPath("/link-indirect")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
// we have multiple copies across layers // we have multiple copies across layers
NewVirtualLocation("/file-2.txt", "/link-indirect"), file.NewVirtualLocation("/file-2.txt", "/link-indirect"),
NewVirtualLocation("/file-2.txt", "/link-indirect"), file.NewVirtualLocation("/file-2.txt", "/link-indirect"),
}, },
}, },
} }
@ -508,7 +509,7 @@ func Test_imageAllLayersResolver_resolvesLinks(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newAllLayersResolver(img) resolver, err := NewFromContainerImageAllLayers(img)
assert.NoError(t, err) assert.NoError(t, err)
actual := test.runner(resolver) actual := test.runner(resolver)
@ -527,7 +528,7 @@ func TestAllLayersResolver_AllLocations(t *testing.T) {
arch = "aarch64" arch = "aarch64"
} }
resolver, err := newAllLayersResolver(img) resolver, err := NewFromContainerImageAllLayers(img)
assert.NoError(t, err) assert.NoError(t, err)
paths := strset.New() paths := strset.New()

View File

@ -1,41 +1,42 @@
package source package fileresolver
import ( import (
"fmt" "fmt"
"io" "io"
"github.com/anchore/stereoscope/pkg/file" stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree" "github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/stereoscope/pkg/image" "github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/syft/file"
) )
var _ FileResolver = (*imageSquashResolver)(nil) var _ file.Resolver = (*ContainerImageSquash)(nil)
// imageSquashResolver implements path and content access for the Squashed source option for container image data sources. // ContainerImageSquash implements path and content access for the Squashed source option for container image data sources.
type imageSquashResolver struct { type ContainerImageSquash struct {
img *image.Image img *image.Image
} }
// newImageSquashResolver returns a new resolver from the perspective of the squashed representation for the given image. // NewFromContainerImageSquash returns a new resolver from the perspective of the squashed representation for the given image.
func newImageSquashResolver(img *image.Image) (*imageSquashResolver, error) { func NewFromContainerImageSquash(img *image.Image) (*ContainerImageSquash, error) {
if img.SquashedTree() == nil { if img.SquashedTree() == nil {
return nil, fmt.Errorf("the image does not have have a squashed tree") return nil, fmt.Errorf("the image does not have have a squashed tree")
} }
return &imageSquashResolver{ return &ContainerImageSquash{
img: img, img: img,
}, nil }, nil
} }
// HasPath indicates if the given path exists in the underlying source. // HasPath indicates if the given path exists in the underlying source.
func (r *imageSquashResolver) HasPath(path string) bool { func (r *ContainerImageSquash) HasPath(path string) bool {
return r.img.SquashedTree().HasPath(file.Path(path)) return r.img.SquashedTree().HasPath(stereoscopeFile.Path(path))
} }
// FilesByPath returns all file.References that match the given paths within the squashed representation of the image. // FilesByPath returns all file.References that match the given paths within the squashed representation of the image.
func (r *imageSquashResolver) FilesByPath(paths ...string) ([]Location, error) { func (r *ContainerImageSquash) FilesByPath(paths ...string) ([]file.Location, error) {
uniqueFileIDs := file.NewFileReferenceSet() uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]Location, 0) uniqueLocations := make([]file.Location, 0)
for _, path := range paths { for _, path := range paths {
ref, err := r.img.SquashedSearchContext.SearchByPath(path, filetree.FollowBasenameLinks) ref, err := r.img.SquashedSearchContext.SearchByPath(path, filetree.FollowBasenameLinks)
@ -69,7 +70,7 @@ func (r *imageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
if resolvedRef.HasReference() && !uniqueFileIDs.Contains(*resolvedRef.Reference) { if resolvedRef.HasReference() && !uniqueFileIDs.Contains(*resolvedRef.Reference) {
uniqueFileIDs.Add(*resolvedRef.Reference) uniqueFileIDs.Add(*resolvedRef.Reference)
uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, *resolvedRef.Reference, r.img)) uniqueLocations = append(uniqueLocations, file.NewLocationFromImage(path, *resolvedRef.Reference, r.img))
} }
} }
@ -78,9 +79,9 @@ func (r *imageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
// FilesByGlob returns all file.References that match the given path glob pattern within the squashed representation of the image. // FilesByGlob returns all file.References that match the given path glob pattern within the squashed representation of the image.
// nolint:gocognit // nolint:gocognit
func (r *imageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error) { func (r *ContainerImageSquash) FilesByGlob(patterns ...string) ([]file.Location, error) {
uniqueFileIDs := file.NewFileReferenceSet() uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]Location, 0) uniqueLocations := make([]file.Location, 0)
for _, pattern := range patterns { for _, pattern := range patterns {
results, err := r.img.SquashedSearchContext.SearchByGlob(pattern, filetree.FollowBasenameLinks) results, err := r.img.SquashedSearchContext.SearchByGlob(pattern, filetree.FollowBasenameLinks)
@ -113,10 +114,10 @@ func (r *imageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error
return nil, fmt.Errorf("failed to find files by path (result=%+v): %w", result, err) return nil, fmt.Errorf("failed to find files by path (result=%+v): %w", result, err)
} }
for _, resolvedLocation := range resolvedLocations { for _, resolvedLocation := range resolvedLocations {
if uniqueFileIDs.Contains(resolvedLocation.ref) { if uniqueFileIDs.Contains(resolvedLocation.Reference()) {
continue continue
} }
uniqueFileIDs.Add(resolvedLocation.ref) uniqueFileIDs.Add(resolvedLocation.Reference())
uniqueLocations = append(uniqueLocations, resolvedLocation) uniqueLocations = append(uniqueLocations, resolvedLocation)
} }
} }
@ -127,8 +128,8 @@ func (r *imageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error
// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
// This is helpful when attempting to find a file that is in the same layer or lower as another file. For the // This is helpful when attempting to find a file that is in the same layer or lower as another file. For the
// imageSquashResolver, this is a simple path lookup. // ContainerImageSquash, this is a simple path lookup.
func (r *imageSquashResolver) RelativeFileByPath(_ Location, path string) *Location { func (r *ContainerImageSquash) RelativeFileByPath(_ file.Location, path string) *file.Location {
paths, err := r.FilesByPath(path) paths, err := r.FilesByPath(path)
if err != nil { if err != nil {
return nil return nil
@ -142,14 +143,14 @@ func (r *imageSquashResolver) RelativeFileByPath(_ Location, path string) *Locat
// FileContentsByLocation fetches file contents for a single file reference, regardless of the source layer. // FileContentsByLocation fetches file contents for a single file reference, regardless of the source layer.
// If the path does not exist an error is returned. // If the path does not exist an error is returned.
func (r *imageSquashResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) { func (r *ContainerImageSquash) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
entry, err := r.img.FileCatalog.Get(location.ref) entry, err := r.img.FileCatalog.Get(location.Reference())
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err) return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err)
} }
switch entry.Metadata.Type { switch entry.Metadata.Type {
case file.TypeSymLink, file.TypeHardLink: case stereoscopeFile.TypeSymLink, stereoscopeFile.TypeHardLink:
// the location we are searching may be a symlink, we should always work with the resolved file // the location we are searching may be a symlink, we should always work with the resolved file
locations, err := r.FilesByPath(location.RealPath) locations, err := r.FilesByPath(location.RealPath)
if err != nil { if err != nil {
@ -164,39 +165,39 @@ func (r *imageSquashResolver) FileContentsByLocation(location Location) (io.Read
default: default:
return nil, fmt.Errorf("link resolution resulted in multiple results while resolving content location: %+v", location) return nil, fmt.Errorf("link resolution resulted in multiple results while resolving content location: %+v", location)
} }
case file.TypeDirectory: case stereoscopeFile.TypeDirectory:
return nil, fmt.Errorf("unable to get file contents for directory: %+v", location) return nil, fmt.Errorf("unable to get file contents for directory: %+v", location)
} }
return r.img.FileContentsByRef(location.ref) return r.img.OpenReference(location.Reference())
} }
func (r *imageSquashResolver) AllLocations() <-chan Location { func (r *ContainerImageSquash) AllLocations() <-chan file.Location {
results := make(chan Location) results := make(chan file.Location)
go func() { go func() {
defer close(results) defer close(results)
for _, ref := range r.img.SquashedTree().AllFiles(file.AllTypes()...) { for _, ref := range r.img.SquashedTree().AllFiles(stereoscopeFile.AllTypes()...) {
results <- NewLocationFromImage(string(ref.RealPath), ref, r.img) results <- file.NewLocationFromImage(string(ref.RealPath), ref, r.img)
} }
}() }()
return results return results
} }
func (r *imageSquashResolver) FilesByMIMEType(types ...string) ([]Location, error) { func (r *ContainerImageSquash) FilesByMIMEType(types ...string) ([]file.Location, error) {
refs, err := r.img.SquashedSearchContext.SearchByMIMEType(types...) refs, err := r.img.SquashedSearchContext.SearchByMIMEType(types...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
uniqueFileIDs := file.NewFileReferenceSet() uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]Location, 0) uniqueLocations := make([]file.Location, 0)
for _, ref := range refs { for _, ref := range refs {
if ref.HasReference() { if ref.HasReference() {
if uniqueFileIDs.Contains(*ref.Reference) { if uniqueFileIDs.Contains(*ref.Reference) {
continue continue
} }
location := NewLocationFromImage(string(ref.RequestPath), *ref.Reference, r.img) location := file.NewLocationFromImage(string(ref.RequestPath), *ref.Reference, r.img)
uniqueFileIDs.Add(*ref.Reference) uniqueFileIDs.Add(*ref.Reference)
uniqueLocations = append(uniqueLocations, location) uniqueLocations = append(uniqueLocations, location)
@ -206,6 +207,6 @@ func (r *imageSquashResolver) FilesByMIMEType(types ...string) ([]Location, erro
return uniqueLocations, nil return uniqueLocations, nil
} }
func (r *imageSquashResolver) FileMetadataByLocation(location Location) (FileMetadata, error) { func (r *ContainerImageSquash) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
return fileMetadataByLocation(r.img, location) return fileMetadataByLocation(r.img, location)
} }

View File

@ -1,4 +1,4 @@
package source package fileresolver
import ( import (
"io" "io"
@ -6,13 +6,12 @@ import (
"testing" "testing"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/scylladb/go-set/strset" "github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/file"
) )
func TestImageSquashResolver_FilesByPath(t *testing.T) { func TestImageSquashResolver_FilesByPath(t *testing.T) {
@ -73,7 +72,7 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newImageSquashResolver(img) resolver, err := NewFromContainerImageSquash(img)
if err != nil { if err != nil {
t.Fatalf("could not create resolver: %+v", err) t.Fatalf("could not create resolver: %+v", err)
} }
@ -110,15 +109,15 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
actual := refs[0] actual := refs[0]
if string(actual.ref.RealPath) != c.resolvePath { if string(actual.Reference().RealPath) != c.resolvePath {
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath) t.Errorf("bad resolve path: '%s'!='%s'", string(actual.Reference().RealPath), c.resolvePath)
} }
if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath { if c.resolvePath != "" && string(actual.Reference().RealPath) != actual.RealPath {
t.Errorf("we should always prefer real paths over ones with links") t.Errorf("we should always prefer real paths over ones with links")
} }
layer := img.FileCatalog.Layer(actual.ref) layer := img.FileCatalog.Layer(actual.Reference())
if layer.Metadata.Index != c.resolveLayer { if layer.Metadata.Index != c.resolveLayer {
t.Errorf("bad resolve layer: '%d'!='%d'", layer.Metadata.Index, c.resolveLayer) t.Errorf("bad resolve layer: '%d'!='%d'", layer.Metadata.Index, c.resolveLayer)
@ -186,7 +185,7 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newImageSquashResolver(img) resolver, err := NewFromContainerImageSquash(img)
if err != nil { if err != nil {
t.Fatalf("could not create resolver: %+v", err) t.Fatalf("could not create resolver: %+v", err)
} }
@ -212,15 +211,15 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
actual := refs[0] actual := refs[0]
if string(actual.ref.RealPath) != c.resolvePath { if string(actual.Reference().RealPath) != c.resolvePath {
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath) t.Errorf("bad resolve path: '%s'!='%s'", string(actual.Reference().RealPath), c.resolvePath)
} }
if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath { if c.resolvePath != "" && string(actual.Reference().RealPath) != actual.RealPath {
t.Errorf("we should always prefer real paths over ones with links") t.Errorf("we should always prefer real paths over ones with links")
} }
layer := img.FileCatalog.Layer(actual.ref) layer := img.FileCatalog.Layer(actual.Reference())
if layer.Metadata.Index != c.resolveLayer { if layer.Metadata.Index != c.resolveLayer {
t.Errorf("bad resolve layer: '%d'!='%d'", layer.Metadata.Index, c.resolveLayer) t.Errorf("bad resolve layer: '%d'!='%d'", layer.Metadata.Index, c.resolveLayer)
@ -247,7 +246,7 @@ func Test_imageSquashResolver_FilesByMIMEType(t *testing.T) {
t.Run(test.fixtureName, func(t *testing.T) { t.Run(test.fixtureName, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureName) img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureName)
resolver, err := newImageSquashResolver(img) resolver, err := NewFromContainerImageSquash(img)
assert.NoError(t, err) assert.NoError(t, err)
locations, err := resolver.FilesByMIMEType(test.mimeType) locations, err := resolver.FilesByMIMEType(test.mimeType)
@ -264,7 +263,7 @@ func Test_imageSquashResolver_FilesByMIMEType(t *testing.T) {
func Test_imageSquashResolver_hasFilesystemIDInLocation(t *testing.T) { func Test_imageSquashResolver_hasFilesystemIDInLocation(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-duplicate-path") img := imagetest.GetFixtureImage(t, "docker-archive", "image-duplicate-path")
resolver, err := newImageSquashResolver(img) resolver, err := NewFromContainerImageSquash(img)
assert.NoError(t, err) assert.NoError(t, err)
locations, err := resolver.FilesByMIMEType("text/plain") locations, err := resolver.FilesByMIMEType("text/plain")
@ -322,7 +321,7 @@ func TestSquashImageResolver_FilesContents(t *testing.T) {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newImageSquashResolver(img) resolver, err := NewFromContainerImageSquash(img)
assert.NoError(t, err) assert.NoError(t, err)
refs, err := resolver.FilesByPath(test.path) refs, err := resolver.FilesByPath(test.path)
@ -347,12 +346,12 @@ func TestSquashImageResolver_FilesContents_errorOnDirRequest(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newImageSquashResolver(img) resolver, err := NewFromContainerImageSquash(img)
assert.NoError(t, err) assert.NoError(t, err)
var dirLoc *Location var dirLoc *file.Location
for loc := range resolver.AllLocations() { for loc := range resolver.AllLocations() {
entry, err := resolver.img.FileCatalog.Get(loc.ref) entry, err := resolver.img.FileCatalog.Get(loc.Reference())
require.NoError(t, err) require.NoError(t, err)
if entry.Metadata.IsDir() { if entry.Metadata.IsDir() {
dirLoc = &loc dirLoc = &loc
@ -370,162 +369,130 @@ func TestSquashImageResolver_FilesContents_errorOnDirRequest(t *testing.T) {
func Test_imageSquashResolver_resolvesLinks(t *testing.T) { func Test_imageSquashResolver_resolvesLinks(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
runner func(FileResolver) []Location runner func(file.Resolver) []file.Location
expected []Location expected []file.Location
}{ }{
{ {
name: "by mimetype", name: "by mimetype",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links should not show up when searching mimetype // links should not show up when searching mimetype
actualLocations, err := resolver.FilesByMIMEType("text/plain") actualLocations, err := resolver.FilesByMIMEType("text/plain")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewVirtualLocation("/etc/group", "/etc/group"), file.NewVirtualLocation("/etc/group", "/etc/group"),
NewVirtualLocation("/etc/passwd", "/etc/passwd"), file.NewVirtualLocation("/etc/passwd", "/etc/passwd"),
NewVirtualLocation("/etc/shadow", "/etc/shadow"), file.NewVirtualLocation("/etc/shadow", "/etc/shadow"),
NewVirtualLocation("/file-1.txt", "/file-1.txt"), file.NewVirtualLocation("/file-1.txt", "/file-1.txt"),
NewVirtualLocation("/file-3.txt", "/file-3.txt"), file.NewVirtualLocation("/file-3.txt", "/file-3.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"), file.NewVirtualLocation("/file-2.txt", "/file-2.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
}, },
}, },
{ {
name: "by glob to links", name: "by glob to links",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files // links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("*ink-*") actualLocations, err := resolver.FilesByGlob("*ink-*")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewVirtualLocation("/file-1.txt", "/link-1"), file.NewVirtualLocation("/file-1.txt", "/link-1"),
NewVirtualLocation("/file-2.txt", "/link-2"), file.NewVirtualLocation("/file-2.txt", "/link-2"),
// though this is a link, and it matches to the file, the resolver de-duplicates files // though this is a link, and it matches to the file, the resolver de-duplicates files
// by the real path, so it is not included in the results // by the real path, so it is not included in the results
//NewVirtualLocation("/file-2.txt", "/link-indirect"), //file.NewVirtualLocation("/file-2.txt", "/link-indirect"),
NewVirtualLocation("/file-3.txt", "/link-within"), file.NewVirtualLocation("/file-3.txt", "/link-within"),
}, },
}, },
{ {
name: "by basename", name: "by basename",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files // links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/file-2.txt") actualLocations, err := resolver.FilesByGlob("**/file-2.txt")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
// this has two copies in the base image, which overwrites the same location // this has two copies in the base image, which overwrites the same location
NewVirtualLocation("/file-2.txt", "/file-2.txt"), file.NewVirtualLocation("/file-2.txt", "/file-2.txt"),
}, },
}, },
{ {
name: "by basename glob", name: "by basename glob",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files // links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/file-?.txt") actualLocations, err := resolver.FilesByGlob("**/file-?.txt")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewVirtualLocation("/file-1.txt", "/file-1.txt"), file.NewVirtualLocation("/file-1.txt", "/file-1.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"), file.NewVirtualLocation("/file-2.txt", "/file-2.txt"),
NewVirtualLocation("/file-3.txt", "/file-3.txt"), file.NewVirtualLocation("/file-3.txt", "/file-3.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
}, },
}, },
{ {
name: "by basename glob to links", name: "by basename glob to links",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
actualLocations, err := resolver.FilesByGlob("**/link-*") actualLocations, err := resolver.FilesByGlob("**/link-*")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
file.NewVirtualLocation("/file-1.txt", "/link-1"),
file.NewVirtualLocation("/file-2.txt", "/link-2"),
{
LocationData: LocationData{
Coordinates: Coordinates{
RealPath: "/file-1.txt",
},
VirtualPath: "/link-1",
ref: file.Reference{RealPath: "/file-1.txt"},
},
},
{
LocationData: LocationData{
Coordinates: Coordinates{
RealPath: "/file-2.txt",
},
VirtualPath: "/link-2",
ref: file.Reference{RealPath: "/file-2.txt"},
},
},
// we already have this real file path via another link, so only one is returned // we already have this real file path via another link, so only one is returned
//{ // file.NewVirtualLocation("/file-2.txt", "/link-indirect"),
// LocationData: LocationData{
// Coordinates: Coordinates{ file.NewVirtualLocation("/file-3.txt", "/link-within"),
// RealPath: "/file-2.txt",
// },
// VirtualPath: "/link-indirect",
// ref: file.Reference{RealPath: "/file-2.txt"},
// },
//},
{
LocationData: LocationData{
Coordinates: Coordinates{
RealPath: "/file-3.txt",
},
VirtualPath: "/link-within",
ref: file.Reference{RealPath: "/file-3.txt"},
},
},
}, },
}, },
{ {
name: "by extension", name: "by extension",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files // links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/*.txt") actualLocations, err := resolver.FilesByGlob("**/*.txt")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewVirtualLocation("/file-1.txt", "/file-1.txt"), file.NewVirtualLocation("/file-1.txt", "/file-1.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"), file.NewVirtualLocation("/file-2.txt", "/file-2.txt"),
NewVirtualLocation("/file-3.txt", "/file-3.txt"), file.NewVirtualLocation("/file-3.txt", "/file-3.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
}, },
}, },
{ {
name: "by path to degree 1 link", name: "by path to degree 1 link",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links resolve to the final file // links resolve to the final file
actualLocations, err := resolver.FilesByPath("/link-2") actualLocations, err := resolver.FilesByPath("/link-2")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
// we have multiple copies across layers // we have multiple copies across layers
NewVirtualLocation("/file-2.txt", "/link-2"), file.NewVirtualLocation("/file-2.txt", "/link-2"),
}, },
}, },
{ {
name: "by path to degree 2 link", name: "by path to degree 2 link",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// multiple links resolves to the final file // multiple links resolves to the final file
actualLocations, err := resolver.FilesByPath("/link-indirect") actualLocations, err := resolver.FilesByPath("/link-indirect")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
// we have multiple copies across layers // we have multiple copies across layers
NewVirtualLocation("/file-2.txt", "/link-indirect"), file.NewVirtualLocation("/file-2.txt", "/link-indirect"),
}, },
}, },
} }
@ -535,7 +502,7 @@ func Test_imageSquashResolver_resolvesLinks(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks") img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newImageSquashResolver(img) resolver, err := NewFromContainerImageSquash(img)
assert.NoError(t, err) assert.NoError(t, err)
actual := test.runner(resolver) actual := test.runner(resolver)
@ -546,30 +513,10 @@ func Test_imageSquashResolver_resolvesLinks(t *testing.T) {
} }
func compareLocations(t *testing.T, expected, actual []Location) {
t.Helper()
ignoreUnexported := cmpopts.IgnoreFields(LocationData{}, "ref")
ignoreMetadata := cmpopts.IgnoreFields(LocationMetadata{}, "Annotations")
ignoreFS := cmpopts.IgnoreFields(Coordinates{}, "FileSystemID")
sort.Sort(Locations(expected))
sort.Sort(Locations(actual))
if d := cmp.Diff(expected, actual,
ignoreUnexported,
ignoreFS,
ignoreMetadata,
); d != "" {
t.Errorf("unexpected locations (-want +got):\n%s", d)
}
}
func TestSquashResolver_AllLocations(t *testing.T) { func TestSquashResolver_AllLocations(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-files-deleted") img := imagetest.GetFixtureImage(t, "docker-archive", "image-files-deleted")
resolver, err := newImageSquashResolver(img) resolver, err := NewFromContainerImageSquash(img)
assert.NoError(t, err) assert.NoError(t, err)
paths := strset.New() paths := strset.New()

View File

@ -0,0 +1,98 @@
package fileresolver
import (
"io"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
)
var _ file.Resolver = (*Deferred)(nil)
func NewDeferred(creator func() (file.Resolver, error)) *Deferred {
return &Deferred{
creator: creator,
}
}
type Deferred struct {
creator func() (file.Resolver, error)
resolver file.Resolver
}
func (d *Deferred) getResolver() (file.Resolver, error) {
if d.resolver == nil {
resolver, err := d.creator()
if err != nil {
return nil, err
}
d.resolver = resolver
}
return d.resolver, nil
}
func (d *Deferred) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
r, err := d.getResolver()
if err != nil {
return nil, err
}
return r.FileContentsByLocation(location)
}
func (d *Deferred) HasPath(s string) bool {
r, err := d.getResolver()
if err != nil {
log.Debug("unable to get resolver: %v", err)
return false
}
return r.HasPath(s)
}
func (d *Deferred) FilesByPath(paths ...string) ([]file.Location, error) {
r, err := d.getResolver()
if err != nil {
return nil, err
}
return r.FilesByPath(paths...)
}
func (d *Deferred) FilesByGlob(patterns ...string) ([]file.Location, error) {
r, err := d.getResolver()
if err != nil {
return nil, err
}
return r.FilesByGlob(patterns...)
}
func (d *Deferred) FilesByMIMEType(types ...string) ([]file.Location, error) {
r, err := d.getResolver()
if err != nil {
return nil, err
}
return r.FilesByMIMEType(types...)
}
func (d *Deferred) RelativeFileByPath(location file.Location, path string) *file.Location {
r, err := d.getResolver()
if err != nil {
return nil
}
return r.RelativeFileByPath(location, path)
}
func (d *Deferred) AllLocations() <-chan file.Location {
r, err := d.getResolver()
if err != nil {
log.Debug("unable to get resolver: %v", err)
return nil
}
return r.AllLocations()
}
func (d *Deferred) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
r, err := d.getResolver()
if err != nil {
return file.Metadata{}, err
}
return r.FileMetadataByLocation(location)
}

View File

@ -1,17 +1,19 @@
package source package fileresolver
import ( import (
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/file"
) )
func Test_NewDeferredResolver(t *testing.T) { func Test_NewDeferredResolver(t *testing.T) {
creatorCalled := false creatorCalled := false
deferredResolver := NewDeferredResolver(func() (FileResolver, error) { deferredResolver := NewDeferred(func() (file.Resolver, error) {
creatorCalled = true creatorCalled = true
return NewMockResolverForPaths(), nil return file.NewMockResolverForPaths(), nil
}) })
require.False(t, creatorCalled) require.False(t, creatorCalled)

View File

@ -1,4 +1,4 @@
package source package fileresolver
import ( import (
"errors" "errors"
@ -10,9 +10,10 @@ import (
"runtime" "runtime"
"strings" "strings"
"github.com/anchore/stereoscope/pkg/file" stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree" "github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
) )
const WindowsOS = "windows" const WindowsOS = "windows"
@ -23,12 +24,12 @@ var unixSystemRuntimePrefixes = []string{
"/sys", "/sys",
} }
var errSkipPath = errors.New("skip path") var ErrSkipPath = errors.New("skip path")
var _ FileResolver = (*directoryResolver)(nil) var _ file.Resolver = (*Directory)(nil)
// directoryResolver implements path and content access for the directory data source. // Directory implements path and content access for the directory data source.
type directoryResolver struct { type Directory struct {
path string path string
base string base string
currentWdRelativeToRoot string currentWdRelativeToRoot string
@ -39,8 +40,8 @@ type directoryResolver struct {
indexer *directoryIndexer indexer *directoryIndexer
} }
func newDirectoryResolver(root string, base string, pathFilters ...pathIndexVisitor) (*directoryResolver, error) { func NewFromDirectory(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
r, err := newDirectoryResolverWithoutIndex(root, base, pathFilters...) r, err := newFromDirectoryWithoutIndex(root, base, pathFilters...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -48,7 +49,7 @@ func newDirectoryResolver(root string, base string, pathFilters ...pathIndexVisi
return r, r.buildIndex() return r, r.buildIndex()
} }
func newDirectoryResolverWithoutIndex(root string, base string, pathFilters ...pathIndexVisitor) (*directoryResolver, error) { func newFromDirectoryWithoutIndex(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
currentWD, err := os.Getwd() currentWD, err := os.Getwd()
if err != nil { if err != nil {
return nil, fmt.Errorf("could not get CWD: %w", err) return nil, fmt.Errorf("could not get CWD: %w", err)
@ -87,7 +88,7 @@ func newDirectoryResolverWithoutIndex(root string, base string, pathFilters ...p
currentWdRelRoot = filepath.Clean(cleanRoot) currentWdRelRoot = filepath.Clean(cleanRoot)
} }
return &directoryResolver{ return &Directory{
path: cleanRoot, path: cleanRoot,
base: cleanBase, base: cleanBase,
currentWd: cleanCWD, currentWd: cleanCWD,
@ -98,7 +99,7 @@ func newDirectoryResolverWithoutIndex(root string, base string, pathFilters ...p
}, nil }, nil
} }
func (r *directoryResolver) buildIndex() error { func (r *Directory) buildIndex() error {
if r.indexer == nil { if r.indexer == nil {
return fmt.Errorf("no directory indexer configured") return fmt.Errorf("no directory indexer configured")
} }
@ -114,7 +115,7 @@ func (r *directoryResolver) buildIndex() error {
return nil return nil
} }
func (r directoryResolver) requestPath(userPath string) (string, error) { func (r Directory) requestPath(userPath string) (string, error) {
if filepath.IsAbs(userPath) { if filepath.IsAbs(userPath) {
// don't allow input to potentially hop above root path // don't allow input to potentially hop above root path
userPath = path.Join(r.path, userPath) userPath = path.Join(r.path, userPath)
@ -131,7 +132,7 @@ func (r directoryResolver) requestPath(userPath string) (string, error) {
return userPath, nil return userPath, nil
} }
func (r directoryResolver) responsePath(path string) string { func (r Directory) responsePath(path string) string {
// check to see if we need to encode back to Windows from posix // check to see if we need to encode back to Windows from posix
if runtime.GOOS == WindowsOS { if runtime.GOOS == WindowsOS {
path = posixToWindows(path) path = posixToWindows(path)
@ -154,22 +155,22 @@ func (r directoryResolver) responsePath(path string) string {
} }
// HasPath indicates if the given path exists in the underlying source. // HasPath indicates if the given path exists in the underlying source.
func (r *directoryResolver) HasPath(userPath string) bool { func (r *Directory) HasPath(userPath string) bool {
requestPath, err := r.requestPath(userPath) requestPath, err := r.requestPath(userPath)
if err != nil { if err != nil {
return false return false
} }
return r.tree.HasPath(file.Path(requestPath)) return r.tree.HasPath(stereoscopeFile.Path(requestPath))
} }
// Stringer to represent a directory path data source // Stringer to represent a directory path data source
func (r directoryResolver) String() string { func (r Directory) String() string {
return fmt.Sprintf("dir:%s", r.path) return fmt.Sprintf("dir:%s", r.path)
} }
// FilesByPath returns all file.References that match the given paths from the directory. // FilesByPath returns all file.References that match the given paths from the directory.
func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error) { func (r Directory) FilesByPath(userPaths ...string) ([]file.Location, error) {
var references = make([]Location, 0) var references = make([]file.Location, 0)
for _, userPath := range userPaths { for _, userPath := range userPaths {
userStrPath, err := r.requestPath(userPath) userStrPath, err := r.requestPath(userPath)
@ -206,7 +207,7 @@ func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
if ref.HasReference() { if ref.HasReference() {
references = append(references, references = append(references,
NewVirtualLocationFromDirectory( file.NewVirtualLocationFromDirectory(
r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root
r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root
*ref.Reference, *ref.Reference,
@ -219,9 +220,9 @@ func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
} }
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
func (r directoryResolver) FilesByGlob(patterns ...string) ([]Location, error) { func (r Directory) FilesByGlob(patterns ...string) ([]file.Location, error) {
uniqueFileIDs := file.NewFileReferenceSet() uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]Location, 0) uniqueLocations := make([]file.Location, 0)
for _, pattern := range patterns { for _, pattern := range patterns {
refVias, err := r.searchContext.SearchByGlob(pattern, filetree.FollowBasenameLinks) refVias, err := r.searchContext.SearchByGlob(pattern, filetree.FollowBasenameLinks)
@ -242,7 +243,7 @@ func (r directoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
continue continue
} }
loc := NewVirtualLocationFromDirectory( loc := file.NewVirtualLocationFromDirectory(
r.responsePath(string(refVia.Reference.RealPath)), // the actual path relative to the resolver root r.responsePath(string(refVia.Reference.RealPath)), // the actual path relative to the resolver root
r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
*refVia.Reference, *refVia.Reference,
@ -257,8 +258,8 @@ func (r directoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
// This is helpful when attempting to find a file that is in the same layer or lower as another file. For the // This is helpful when attempting to find a file that is in the same layer or lower as another file. For the
// directoryResolver, this is a simple path lookup. // Directory, this is a simple path lookup.
func (r *directoryResolver) RelativeFileByPath(_ Location, path string) *Location { func (r *Directory) RelativeFileByPath(_ file.Location, path string) *file.Location {
paths, err := r.FilesByPath(path) paths, err := r.FilesByPath(path)
if err != nil { if err != nil {
return nil return nil
@ -272,54 +273,54 @@ func (r *directoryResolver) RelativeFileByPath(_ Location, path string) *Locatio
// FileContentsByLocation fetches file contents for a single file reference relative to a directory. // FileContentsByLocation fetches file contents for a single file reference relative to a directory.
// If the path does not exist an error is returned. // If the path does not exist an error is returned.
func (r directoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) { func (r Directory) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
if location.ref.RealPath == "" { if location.RealPath == "" {
return nil, errors.New("empty path given") return nil, errors.New("empty path given")
} }
entry, err := r.index.Get(location.ref) entry, err := r.index.Get(location.Reference())
if err != nil { if err != nil {
return nil, err return nil, err
} }
// don't consider directories // don't consider directories
if entry.Type == file.TypeDirectory { if entry.Type == stereoscopeFile.TypeDirectory {
return nil, fmt.Errorf("cannot read contents of non-file %q", location.ref.RealPath) return nil, fmt.Errorf("cannot read contents of non-file %q", location.Reference().RealPath)
} }
// RealPath is posix so for windows directory resolver we need to translate // RealPath is posix so for windows directory resolver we need to translate
// to its true on disk path. // to its true on disk path.
filePath := string(location.ref.RealPath) filePath := string(location.Reference().RealPath)
if runtime.GOOS == WindowsOS { if runtime.GOOS == WindowsOS {
filePath = posixToWindows(filePath) filePath = posixToWindows(filePath)
} }
return file.NewLazyReadCloser(filePath), nil return stereoscopeFile.NewLazyReadCloser(filePath), nil
} }
func (r *directoryResolver) AllLocations() <-chan Location { func (r *Directory) AllLocations() <-chan file.Location {
results := make(chan Location) results := make(chan file.Location)
go func() { go func() {
defer close(results) defer close(results)
for _, ref := range r.tree.AllFiles(file.AllTypes()...) { for _, ref := range r.tree.AllFiles(stereoscopeFile.AllTypes()...) {
results <- NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), ref) results <- file.NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), ref)
} }
}() }()
return results return results
} }
func (r *directoryResolver) FileMetadataByLocation(location Location) (FileMetadata, error) { func (r *Directory) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
entry, err := r.index.Get(location.ref) entry, err := r.index.Get(location.Reference())
if err != nil { if err != nil {
return FileMetadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist) return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
} }
return entry.Metadata, nil return entry.Metadata, nil
} }
func (r *directoryResolver) FilesByMIMEType(types ...string) ([]Location, error) { func (r *Directory) FilesByMIMEType(types ...string) ([]file.Location, error) {
uniqueFileIDs := file.NewFileReferenceSet() uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]Location, 0) uniqueLocations := make([]file.Location, 0)
refVias, err := r.searchContext.SearchByMIMEType(types...) refVias, err := r.searchContext.SearchByMIMEType(types...)
if err != nil { if err != nil {
@ -332,7 +333,7 @@ func (r *directoryResolver) FilesByMIMEType(types ...string) ([]Location, error)
if uniqueFileIDs.Contains(*refVia.Reference) { if uniqueFileIDs.Contains(*refVia.Reference) {
continue continue
} }
location := NewLocationFromDirectory( location := file.NewLocationFromDirectory(
r.responsePath(string(refVia.Reference.RealPath)), r.responsePath(string(refVia.Reference.RealPath)),
*refVia.Reference, *refVia.Reference,
) )

View File

@ -1,4 +1,4 @@
package source package fileresolver
import ( import (
"errors" "errors"
@ -20,30 +20,30 @@ import (
"github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event"
) )
type pathIndexVisitor func(string, os.FileInfo, error) error type PathIndexVisitor func(string, os.FileInfo, error) error
type directoryIndexer struct { type directoryIndexer struct {
path string path string
base string base string
pathIndexVisitors []pathIndexVisitor pathIndexVisitors []PathIndexVisitor
errPaths map[string]error errPaths map[string]error
tree filetree.ReadWriter tree filetree.ReadWriter
index filetree.Index index filetree.Index
} }
func newDirectoryIndexer(path, base string, visitors ...pathIndexVisitor) *directoryIndexer { func newDirectoryIndexer(path, base string, visitors ...PathIndexVisitor) *directoryIndexer {
i := &directoryIndexer{ i := &directoryIndexer{
path: path, path: path,
base: base, base: base,
tree: filetree.New(), tree: filetree.New(),
index: filetree.NewIndex(), index: filetree.NewIndex(),
pathIndexVisitors: append([]pathIndexVisitor{requireFileInfo, disallowByFileType, disallowUnixSystemRuntimePath}, visitors...), pathIndexVisitors: append([]PathIndexVisitor{requireFileInfo, disallowByFileType, disallowUnixSystemRuntimePath}, visitors...),
errPaths: make(map[string]error), errPaths: make(map[string]error),
} }
// these additional stateful visitors should be the first thing considered when walking / indexing // these additional stateful visitors should be the first thing considered when walking / indexing
i.pathIndexVisitors = append( i.pathIndexVisitors = append(
[]pathIndexVisitor{ []PathIndexVisitor{
i.disallowRevisitingVisitor, i.disallowRevisitingVisitor,
i.disallowFileAccessErr, i.disallowFileAccessErr,
}, },
@ -181,7 +181,7 @@ func (r *directoryIndexer) indexPath(path string, info os.FileInfo, err error) (
func (r *directoryIndexer) disallowFileAccessErr(path string, _ os.FileInfo, err error) error { func (r *directoryIndexer) disallowFileAccessErr(path string, _ os.FileInfo, err error) error {
if r.isFileAccessErr(path, err) { if r.isFileAccessErr(path, err) {
return errSkipPath return ErrSkipPath
} }
return nil return nil
} }
@ -311,7 +311,7 @@ func (r *directoryIndexer) disallowRevisitingVisitor(path string, _ os.FileInfo,
// signal to walk() that we should skip this directory entirely // signal to walk() that we should skip this directory entirely
return fs.SkipDir return fs.SkipDir
} }
return errSkipPath return ErrSkipPath
} }
return nil return nil
} }
@ -330,7 +330,7 @@ func disallowByFileType(_ string, info os.FileInfo, _ error) error {
} }
switch file.TypeFromMode(info.Mode()) { switch file.TypeFromMode(info.Mode()) {
case file.TypeCharacterDevice, file.TypeSocket, file.TypeBlockDevice, file.TypeFIFO, file.TypeIrregular: case file.TypeCharacterDevice, file.TypeSocket, file.TypeBlockDevice, file.TypeFIFO, file.TypeIrregular:
return errSkipPath return ErrSkipPath
// note: symlinks that point to these files may still get by. // note: symlinks that point to these files may still get by.
// We handle this later in processing to help prevent against infinite links traversal. // We handle this later in processing to help prevent against infinite links traversal.
} }
@ -340,7 +340,7 @@ func disallowByFileType(_ string, info os.FileInfo, _ error) error {
func requireFileInfo(_ string, info os.FileInfo, _ error) error { func requireFileInfo(_ string, info os.FileInfo, _ error) error {
if info == nil { if info == nil {
return errSkipPath return ErrSkipPath
} }
return nil return nil
} }

View File

@ -1,4 +1,4 @@
package source package fileresolver
import ( import (
"io/fs" "io/fs"
@ -172,7 +172,7 @@ func TestDirectoryIndexer_indexPath_skipsNilFileInfo(t *testing.T) {
} }
func TestDirectoryIndexer_index(t *testing.T) { func TestDirectoryIndexer_index(t *testing.T) {
// note: this test is testing the effects from newDirectoryResolver, indexTree, and addPathToIndex // note: this test is testing the effects from NewFromDirectory, indexTree, and addPathToIndex
indexer := newDirectoryIndexer("test-fixtures/system_paths/target", "") indexer := newDirectoryIndexer("test-fixtures/system_paths/target", "")
tree, index, err := indexer.build() tree, index, err := indexer.build()
require.NoError(t, err) require.NoError(t, err)
@ -237,7 +237,7 @@ func TestDirectoryIndexer_SkipsAlreadyVisitedLinkDestinations(t *testing.T) {
} }
resolver := newDirectoryIndexer("./test-fixtures/symlinks-prune-indexing", "") resolver := newDirectoryIndexer("./test-fixtures/symlinks-prune-indexing", "")
// we want to cut ahead of any possible filters to see what paths are considered for indexing (closest to walking) // we want to cut ahead of any possible filters to see what paths are considered for indexing (closest to walking)
resolver.pathIndexVisitors = append([]pathIndexVisitor{pathObserver}, resolver.pathIndexVisitors...) resolver.pathIndexVisitors = append([]PathIndexVisitor{pathObserver}, resolver.pathIndexVisitors...)
// note: this test is NOT about the effects left on the tree or the index, but rather the WHICH paths that are // note: this test is NOT about the effects left on the tree or the index, but rather the WHICH paths that are
// considered for indexing and HOW traversal prunes paths that have already been visited // considered for indexing and HOW traversal prunes paths that have already been visited

View File

@ -1,12 +1,11 @@
//go:build !windows //go:build !windows
// +build !windows // +build !windows
package source package fileresolver
import ( import (
"io" "io"
"io/fs" "io/fs"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
@ -19,7 +18,8 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file" stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/file"
) )
func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) { func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) {
@ -47,16 +47,17 @@ func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) {
}, },
{ {
name: "should find a file from a relative path (root above cwd)", name: "should find a file from a relative path (root above cwd)",
// TODO: refactor me! this test depends on the structure of the source dir not changing, which isn't great
relativeRoot: "../", relativeRoot: "../",
input: "sbom/sbom.go", input: "fileresolver/directory.go",
expected: []string{ expected: []string{
"sbom/sbom.go", "fileresolver/directory.go",
}, },
}, },
} }
for _, c := range cases { for _, c := range cases {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
resolver, err := newDirectoryResolver(c.relativeRoot, "") resolver, err := NewFromDirectory(c.relativeRoot, "")
assert.NoError(t, err) assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input) refs, err := resolver.FilesByPath(c.input)
@ -96,10 +97,11 @@ func TestDirectoryResolver_FilesByPath_absoluteRoot(t *testing.T) {
}, },
{ {
name: "should find a file from a relative path (root above cwd)", name: "should find a file from a relative path (root above cwd)",
// TODO: refactor me! this test depends on the structure of the source dir not changing, which isn't great
relativeRoot: "../", relativeRoot: "../",
input: "sbom/sbom.go", input: "fileresolver/directory.go",
expected: []string{ expected: []string{
"sbom/sbom.go", "fileresolver/directory.go",
}, },
}, },
} }
@ -110,7 +112,7 @@ func TestDirectoryResolver_FilesByPath_absoluteRoot(t *testing.T) {
absRoot, err := filepath.Abs(c.relativeRoot) absRoot, err := filepath.Abs(c.relativeRoot)
require.NoError(t, err) require.NoError(t, err)
resolver, err := newDirectoryResolver(absRoot, "") resolver, err := NewFromDirectory(absRoot, "")
assert.NoError(t, err) assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input) refs, err := resolver.FilesByPath(c.input)
@ -171,7 +173,7 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
} }
for _, c := range cases { for _, c := range cases {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
resolver, err := newDirectoryResolver(c.root, "") resolver, err := NewFromDirectory(c.root, "")
assert.NoError(t, err) assert.NoError(t, err)
hasPath := resolver.HasPath(c.input) hasPath := resolver.HasPath(c.input)
@ -219,7 +221,7 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
} }
for _, c := range cases { for _, c := range cases {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures", "") resolver, err := NewFromDirectory("./test-fixtures", "")
assert.NoError(t, err) assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input...) refs, err := resolver.FilesByPath(c.input...)
assert.NoError(t, err) assert.NoError(t, err)
@ -232,7 +234,7 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
} }
func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) { func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures", "") resolver, err := NewFromDirectory("./test-fixtures", "")
assert.NoError(t, err) assert.NoError(t, err)
refs, err := resolver.FilesByGlob("**/image-symlinks/file*") refs, err := resolver.FilesByGlob("**/image-symlinks/file*")
assert.NoError(t, err) assert.NoError(t, err)
@ -241,7 +243,7 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
} }
func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) { func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/image-symlinks", "") resolver, err := NewFromDirectory("./test-fixtures/image-symlinks", "")
assert.NoError(t, err) assert.NoError(t, err)
refs, err := resolver.FilesByGlob("**/*.txt") refs, err := resolver.FilesByGlob("**/*.txt")
assert.NoError(t, err) assert.NoError(t, err)
@ -249,7 +251,7 @@ func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
} }
func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) { func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures", "") resolver, err := NewFromDirectory("./test-fixtures", "")
assert.NoError(t, err) assert.NoError(t, err)
refs, err := resolver.FilesByGlob("**/image-symlinks/*1.txt") refs, err := resolver.FilesByGlob("**/image-symlinks/*1.txt")
assert.NoError(t, err) assert.NoError(t, err)
@ -276,7 +278,7 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple", "") resolver, err := NewFromDirectory("./test-fixtures/symlinks-simple", "")
assert.NoError(t, err) assert.NoError(t, err)
refs, err := resolver.FilesByPath(test.fixture) refs, err := resolver.FilesByPath(test.fixture)
@ -299,7 +301,7 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {
func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) { func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) {
// let's make certain that "dev/place" is not ignored, since it is not "/dev/place" // let's make certain that "dev/place" is not ignored, since it is not "/dev/place"
resolver, err := newDirectoryResolver("test-fixtures/system_paths/target", "") resolver, err := NewFromDirectory("test-fixtures/system_paths/target", "")
assert.NoError(t, err) assert.NoError(t, err)
// all paths should be found (non filtering matches a path) // all paths should be found (non filtering matches a path)
@ -383,35 +385,35 @@ func Test_isUnallowableFileType(t *testing.T) {
info: testFileInfo{ info: testFileInfo{
mode: os.ModeSocket, mode: os.ModeSocket,
}, },
expected: errSkipPath, expected: ErrSkipPath,
}, },
{ {
name: "named pipe", name: "named pipe",
info: testFileInfo{ info: testFileInfo{
mode: os.ModeNamedPipe, mode: os.ModeNamedPipe,
}, },
expected: errSkipPath, expected: ErrSkipPath,
}, },
{ {
name: "char device", name: "char device",
info: testFileInfo{ info: testFileInfo{
mode: os.ModeCharDevice, mode: os.ModeCharDevice,
}, },
expected: errSkipPath, expected: ErrSkipPath,
}, },
{ {
name: "block device", name: "block device",
info: testFileInfo{ info: testFileInfo{
mode: os.ModeDevice, mode: os.ModeDevice,
}, },
expected: errSkipPath, expected: ErrSkipPath,
}, },
{ {
name: "irregular", name: "irregular",
info: testFileInfo{ info: testFileInfo{
mode: os.ModeIrregular, mode: os.ModeIrregular,
}, },
expected: errSkipPath, expected: ErrSkipPath,
}, },
} }
for _, test := range tests { for _, test := range tests {
@ -435,7 +437,7 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
t.Run(test.fixturePath, func(t *testing.T) { t.Run(test.fixturePath, func(t *testing.T) {
resolver, err := newDirectoryResolver(test.fixturePath, "") resolver, err := NewFromDirectory(test.fixturePath, "")
assert.NoError(t, err) assert.NoError(t, err)
locations, err := resolver.FilesByMIMEType(test.mimeType) locations, err := resolver.FilesByMIMEType(test.mimeType)
assert.NoError(t, err) assert.NoError(t, err)
@ -448,7 +450,7 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
} }
func Test_IndexingNestedSymLinks(t *testing.T) { func Test_IndexingNestedSymLinks(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple", "") resolver, err := NewFromDirectory("./test-fixtures/symlinks-simple", "")
require.NoError(t, err) require.NoError(t, err)
// check that we can get the real path // check that we can get the real path
@ -499,12 +501,12 @@ func Test_IndexingNestedSymLinks(t *testing.T) {
func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) { func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
filterFn := func(path string, _ os.FileInfo, _ error) error { filterFn := func(path string, _ os.FileInfo, _ error) error {
if strings.HasSuffix(path, string(filepath.Separator)+"readme") { if strings.HasSuffix(path, string(filepath.Separator)+"readme") {
return errSkipPath return ErrSkipPath
} }
return nil return nil
} }
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple", "", filterFn) resolver, err := NewFromDirectory("./test-fixtures/symlinks-simple", "", filterFn)
require.NoError(t, err) require.NoError(t, err)
// the path to the real file is PRUNED from the index, so we should NOT expect a location returned // the path to the real file is PRUNED from the index, so we should NOT expect a location returned
@ -524,7 +526,7 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
} }
func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) { func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-multiple-roots/root", "") resolver, err := NewFromDirectory("./test-fixtures/symlinks-multiple-roots/root", "")
require.NoError(t, err) require.NoError(t, err)
// check that we can get the real path // check that we can get the real path
@ -542,7 +544,7 @@ func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
} }
func Test_RootViaSymlink(t *testing.T) { func Test_RootViaSymlink(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinked-root/nested/link-root", "") resolver, err := NewFromDirectory("./test-fixtures/symlinked-root/nested/link-root", "")
require.NoError(t, err) require.NoError(t, err)
locations, err := resolver.FilesByPath("./file1.txt") locations, err := resolver.FilesByPath("./file1.txt")
@ -562,28 +564,28 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
cwd, err := os.Getwd() cwd, err := os.Getwd()
require.NoError(t, err) require.NoError(t, err)
r, err := newDirectoryResolver(".", "") r, err := NewFromDirectory(".", "")
require.NoError(t, err) require.NoError(t, err)
exists, existingPath, err := r.tree.File(file.Path(filepath.Join(cwd, "test-fixtures/image-simple/file-1.txt"))) exists, existingPath, err := r.tree.File(stereoscopeFile.Path(filepath.Join(cwd, "test-fixtures/image-simple/file-1.txt")))
require.True(t, exists) require.True(t, exists)
require.NoError(t, err) require.NoError(t, err)
require.True(t, existingPath.HasReference()) require.True(t, existingPath.HasReference())
tests := []struct { tests := []struct {
name string name string
location Location location file.Location
expects string expects string
err bool err bool
}{ }{
{ {
name: "use file reference for content requests", name: "use file reference for content requests",
location: NewLocationFromDirectory("some/place", *existingPath.Reference), location: file.NewLocationFromDirectory("some/place", *existingPath.Reference),
expects: "this file has contents", expects: "this file has contents",
}, },
{ {
name: "error on empty file reference", name: "error on empty file reference",
location: NewLocationFromDirectory("doesn't matter", file.Reference{}), location: file.NewLocationFromDirectory("doesn't matter", stereoscopeFile.Reference{}),
err: true, err: true,
}, },
} }
@ -598,7 +600,7 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
if test.expects != "" { if test.expects != "" {
b, err := ioutil.ReadAll(actual) b, err := io.ReadAll(actual)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, test.expects, string(b)) assert.Equal(t, test.expects, string(b))
} }
@ -649,7 +651,7 @@ func Test_isUnixSystemRuntimePath(t *testing.T) {
func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) { func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) {
test := func(t *testing.T) { test := func(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-loop", "") resolver, err := NewFromDirectory("./test-fixtures/symlinks-loop", "")
require.NoError(t, err) require.NoError(t, err)
locations, err := resolver.FilesByGlob("**/file.target") locations, err := resolver.FilesByGlob("**/file.target")
@ -662,20 +664,6 @@ func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) {
testWithTimeout(t, 5*time.Second, test) testWithTimeout(t, 5*time.Second, test)
} }
func testWithTimeout(t *testing.T, timeout time.Duration, test func(*testing.T)) {
done := make(chan bool)
go func() {
test(t)
done <- true
}()
select {
case <-time.After(timeout):
t.Fatal("test timed out")
case <-done:
}
}
func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) { func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) {
cases := []struct { cases := []struct {
name string name string
@ -734,7 +722,7 @@ func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) {
} }
for _, c := range cases { for _, c := range cases {
t.Run(c.name, func(t *testing.T) { t.Run(c.name, func(t *testing.T) {
resolver, err := newDirectoryResolver(c.root, c.root) resolver, err := NewFromDirectory(c.root, c.root)
assert.NoError(t, err) assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input) refs, err := resolver.FilesByPath(c.input)
@ -753,162 +741,132 @@ func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) {
func Test_directoryResolver_resolvesLinks(t *testing.T) { func Test_directoryResolver_resolvesLinks(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
runner func(FileResolver) []Location runner func(file.Resolver) []file.Location
expected []Location expected []file.Location
}{ }{
{ {
name: "by mimetype", name: "by mimetype",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links should not show up when searching mimetype // links should not show up when searching mimetype
actualLocations, err := resolver.FilesByMIMEType("text/plain") actualLocations, err := resolver.FilesByMIMEType("text/plain")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewLocation("file-1.txt"), // note: missing virtual path "file-1.txt" file.NewLocation("file-1.txt"), // note: missing virtual path "file-1.txt"
NewLocation("file-3.txt"), // note: missing virtual path "file-3.txt" file.NewLocation("file-3.txt"), // note: missing virtual path "file-3.txt"
NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt" file.NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt"
NewLocation("parent/file-4.txt"), // note: missing virtual path "file-4.txt" file.NewLocation("parent/file-4.txt"), // note: missing virtual path "file-4.txt"
}, },
}, },
{ {
name: "by glob to links", name: "by glob to links",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files // links are searched, but resolve to the real files
// for that reason we need to place **/ in front (which is not the same for other resolvers) // for that reason we need to place **/ in front (which is not the same for other resolvers)
actualLocations, err := resolver.FilesByGlob("**/*ink-*") actualLocations, err := resolver.FilesByGlob("**/*ink-*")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewVirtualLocation("file-1.txt", "link-1"), file.NewVirtualLocation("file-1.txt", "link-1"),
NewVirtualLocation("file-2.txt", "link-2"), file.NewVirtualLocation("file-2.txt", "link-2"),
// we already have this real file path via another link, so only one is returned // we already have this real file path via another link, so only one is returned
//NewVirtualLocation("file-2.txt", "link-indirect"), //file.NewVirtualLocation("file-2.txt", "link-indirect"),
NewVirtualLocation("file-3.txt", "link-within"), file.NewVirtualLocation("file-3.txt", "link-within"),
}, },
}, },
{ {
name: "by basename", name: "by basename",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files // links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/file-2.txt") actualLocations, err := resolver.FilesByGlob("**/file-2.txt")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
// this has two copies in the base image, which overwrites the same location // this has two copies in the base image, which overwrites the same location
NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt", file.NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt",
}, },
}, },
{ {
name: "by basename glob", name: "by basename glob",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files // links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/file-?.txt") actualLocations, err := resolver.FilesByGlob("**/file-?.txt")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewLocation("file-1.txt"), // note: missing virtual path "file-1.txt" file.NewLocation("file-1.txt"), // note: missing virtual path "file-1.txt"
NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt" file.NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt"
NewLocation("file-3.txt"), // note: missing virtual path "file-3.txt" file.NewLocation("file-3.txt"), // note: missing virtual path "file-3.txt"
NewLocation("parent/file-4.txt"), // note: missing virtual path "parent/file-4.txt" file.NewLocation("parent/file-4.txt"), // note: missing virtual path "parent/file-4.txt"
}, },
}, },
{ {
name: "by basename glob to links", name: "by basename glob to links",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
actualLocations, err := resolver.FilesByGlob("**/link-*") actualLocations, err := resolver.FilesByGlob("**/link-*")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
{ file.NewVirtualLocation("file-1.txt", "link-1"),
LocationData: LocationData{ file.NewVirtualLocation("file-2.txt", "link-2"),
Coordinates: Coordinates{
RealPath: "file-1.txt",
},
VirtualPath: "link-1",
ref: file.Reference{RealPath: "file-1.txt"},
},
},
{
LocationData: LocationData{
Coordinates: Coordinates{
RealPath: "file-2.txt",
},
VirtualPath: "link-2",
ref: file.Reference{RealPath: "file-2.txt"},
},
},
// we already have this real file path via another link, so only one is returned // we already have this real file path via another link, so only one is returned
//{ //file.NewVirtualLocation("file-2.txt", "link-indirect"),
// LocationData: LocationData{
// Coordinates: Coordinates{ file.NewVirtualLocation("file-3.txt", "link-within"),
// RealPath: "file-2.txt",
// },
// VirtualPath: "link-indirect",
// ref: file.Reference{RealPath: "file-2.txt"},
// },
//},
{
LocationData: LocationData{
Coordinates: Coordinates{
RealPath: "file-3.txt",
},
VirtualPath: "link-within",
ref: file.Reference{RealPath: "file-3.txt"},
},
},
}, },
}, },
{ {
name: "by extension", name: "by extension",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files // links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/*.txt") actualLocations, err := resolver.FilesByGlob("**/*.txt")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
NewLocation("file-1.txt"), // note: missing virtual path "file-1.txt" file.NewLocation("file-1.txt"), // note: missing virtual path "file-1.txt"
NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt" file.NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt"
NewLocation("file-3.txt"), // note: missing virtual path "file-3.txt" file.NewLocation("file-3.txt"), // note: missing virtual path "file-3.txt"
NewLocation("parent/file-4.txt"), // note: missing virtual path "parent/file-4.txt" file.NewLocation("parent/file-4.txt"), // note: missing virtual path "parent/file-4.txt"
}, },
}, },
{ {
name: "by path to degree 1 link", name: "by path to degree 1 link",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// links resolve to the final file // links resolve to the final file
actualLocations, err := resolver.FilesByPath("/link-2") actualLocations, err := resolver.FilesByPath("/link-2")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
// we have multiple copies across layers // we have multiple copies across layers
NewVirtualLocation("file-2.txt", "link-2"), file.NewVirtualLocation("file-2.txt", "link-2"),
}, },
}, },
{ {
name: "by path to degree 2 link", name: "by path to degree 2 link",
runner: func(resolver FileResolver) []Location { runner: func(resolver file.Resolver) []file.Location {
// multiple links resolves to the final file // multiple links resolves to the final file
actualLocations, err := resolver.FilesByPath("/link-indirect") actualLocations, err := resolver.FilesByPath("/link-indirect")
assert.NoError(t, err) assert.NoError(t, err)
return actualLocations return actualLocations
}, },
expected: []Location{ expected: []file.Location{
// we have multiple copies across layers // we have multiple copies across layers
NewVirtualLocation("file-2.txt", "link-indirect"), file.NewVirtualLocation("file-2.txt", "link-indirect"),
}, },
}, },
} }
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-from-image-symlinks-fixture", "") resolver, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "")
require.NoError(t, err) require.NoError(t, err)
assert.NoError(t, err) assert.NoError(t, err)
@ -920,14 +878,14 @@ func Test_directoryResolver_resolvesLinks(t *testing.T) {
} }
func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) { func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-prune-indexing", "") resolver, err := NewFromDirectory("./test-fixtures/symlinks-prune-indexing", "")
require.NoError(t, err) require.NoError(t, err)
var allRealPaths []file.Path var allRealPaths []stereoscopeFile.Path
for l := range resolver.AllLocations() { for l := range resolver.AllLocations() {
allRealPaths = append(allRealPaths, file.Path(l.RealPath)) allRealPaths = append(allRealPaths, stereoscopeFile.Path(l.RealPath))
} }
pathSet := file.NewPathSet(allRealPaths...) pathSet := stereoscopeFile.NewPathSet(allRealPaths...)
assert.False(t, assert.False(t,
pathSet.Contains("before-path/file.txt"), pathSet.Contains("before-path/file.txt"),
@ -942,12 +900,12 @@ func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
} }
func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) { func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/system_paths", "") resolver, err := NewFromDirectory("./test-fixtures/system_paths", "")
assert.NoError(t, err) assert.NoError(t, err)
var dirLoc *Location var dirLoc *file.Location
for loc := range resolver.AllLocations() { for loc := range resolver.AllLocations() {
entry, err := resolver.index.Get(loc.ref) entry, err := resolver.index.Get(loc.Reference())
require.NoError(t, err) require.NoError(t, err)
if entry.Metadata.IsDir() { if entry.Metadata.IsDir() {
dirLoc = &loc dirLoc = &loc
@ -963,13 +921,13 @@ func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
} }
func TestDirectoryResolver_AllLocations(t *testing.T) { func TestDirectoryResolver_AllLocations(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-from-image-symlinks-fixture", "") resolver, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "")
assert.NoError(t, err) assert.NoError(t, err)
paths := strset.New() paths := strset.New()
for loc := range resolver.AllLocations() { for loc := range resolver.AllLocations() {
if strings.HasPrefix(loc.RealPath, "/") { if strings.HasPrefix(loc.RealPath, "/") {
// ignore outside of the fixture root for now // ignore outside the fixture root for now
continue continue
} }
paths.Add(loc.RealPath) paths.Add(loc.RealPath)

View File

@ -1,4 +1,4 @@
package source package fileresolver
import "testing" import "testing"

View File

@ -0,0 +1,47 @@
package fileresolver
import (
"io"
"github.com/anchore/syft/syft/file"
)
var _ file.WritableResolver = (*Empty)(nil)
type Empty struct{}
func (e Empty) FileContentsByLocation(_ file.Location) (io.ReadCloser, error) {
return nil, nil
}
func (e Empty) HasPath(_ string) bool {
return false
}
func (e Empty) FilesByPath(_ ...string) ([]file.Location, error) {
return nil, nil
}
func (e Empty) FilesByGlob(_ ...string) ([]file.Location, error) {
return nil, nil
}
func (e Empty) FilesByMIMEType(_ ...string) ([]file.Location, error) {
return nil, nil
}
func (e Empty) RelativeFileByPath(_ file.Location, _ string) *file.Location {
return nil
}
func (e Empty) AllLocations() <-chan file.Location {
return nil
}
func (e Empty) FileMetadataByLocation(_ file.Location) (file.Metadata, error) {
return file.Metadata{}, nil
}
func (e Empty) Write(_ file.Location, _ io.Reader) error {
return nil
}

View File

@ -1,65 +1,67 @@
package source package fileresolver
import ( import (
"fmt" "fmt"
"io" "io"
"github.com/anchore/syft/syft/file"
) )
type excludeFn func(string) bool type excludeFn func(string) bool
// excludingResolver decorates a resolver with an exclusion function that is used to // excluding decorates a resolver with an exclusion function that is used to
// filter out entries in the delegate resolver // filter out entries in the delegate resolver
type excludingResolver struct { type excluding struct {
delegate FileResolver delegate file.Resolver
excludeFn excludeFn excludeFn excludeFn
} }
// NewExcludingResolver create a new resolver which wraps the provided delegate and excludes // NewExcluding create a new resolver which wraps the provided delegate and excludes
// entries based on a provided path exclusion function // entries based on a provided path exclusion function
func NewExcludingResolver(delegate FileResolver, excludeFn excludeFn) FileResolver { func NewExcluding(delegate file.Resolver, excludeFn excludeFn) file.Resolver {
return &excludingResolver{ return &excluding{
delegate, delegate,
excludeFn, excludeFn,
} }
} }
func (r *excludingResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) { func (r *excluding) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
if locationMatches(&location, r.excludeFn) { if locationMatches(&location, r.excludeFn) {
return nil, fmt.Errorf("no such location: %+v", location.RealPath) return nil, fmt.Errorf("no such location: %+v", location.RealPath)
} }
return r.delegate.FileContentsByLocation(location) return r.delegate.FileContentsByLocation(location)
} }
func (r *excludingResolver) FileMetadataByLocation(location Location) (FileMetadata, error) { func (r *excluding) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
if locationMatches(&location, r.excludeFn) { if locationMatches(&location, r.excludeFn) {
return FileMetadata{}, fmt.Errorf("no such location: %+v", location.RealPath) return file.Metadata{}, fmt.Errorf("no such location: %+v", location.RealPath)
} }
return r.delegate.FileMetadataByLocation(location) return r.delegate.FileMetadataByLocation(location)
} }
func (r *excludingResolver) HasPath(path string) bool { func (r *excluding) HasPath(path string) bool {
if r.excludeFn(path) { if r.excludeFn(path) {
return false return false
} }
return r.delegate.HasPath(path) return r.delegate.HasPath(path)
} }
func (r *excludingResolver) FilesByPath(paths ...string) ([]Location, error) { func (r *excluding) FilesByPath(paths ...string) ([]file.Location, error) {
locations, err := r.delegate.FilesByPath(paths...) locations, err := r.delegate.FilesByPath(paths...)
return filterLocations(locations, err, r.excludeFn) return filterLocations(locations, err, r.excludeFn)
} }
func (r *excludingResolver) FilesByGlob(patterns ...string) ([]Location, error) { func (r *excluding) FilesByGlob(patterns ...string) ([]file.Location, error) {
locations, err := r.delegate.FilesByGlob(patterns...) locations, err := r.delegate.FilesByGlob(patterns...)
return filterLocations(locations, err, r.excludeFn) return filterLocations(locations, err, r.excludeFn)
} }
func (r *excludingResolver) FilesByMIMEType(types ...string) ([]Location, error) { func (r *excluding) FilesByMIMEType(types ...string) ([]file.Location, error) {
locations, err := r.delegate.FilesByMIMEType(types...) locations, err := r.delegate.FilesByMIMEType(types...)
return filterLocations(locations, err, r.excludeFn) return filterLocations(locations, err, r.excludeFn)
} }
func (r *excludingResolver) RelativeFileByPath(location Location, path string) *Location { func (r *excluding) RelativeFileByPath(location file.Location, path string) *file.Location {
l := r.delegate.RelativeFileByPath(location, path) l := r.delegate.RelativeFileByPath(location, path)
if l != nil && locationMatches(l, r.excludeFn) { if l != nil && locationMatches(l, r.excludeFn) {
return nil return nil
@ -67,8 +69,8 @@ func (r *excludingResolver) RelativeFileByPath(location Location, path string) *
return l return l
} }
func (r *excludingResolver) AllLocations() <-chan Location { func (r *excluding) AllLocations() <-chan file.Location {
c := make(chan Location) c := make(chan file.Location)
go func() { go func() {
defer close(c) defer close(c)
for location := range r.delegate.AllLocations() { for location := range r.delegate.AllLocations() {
@ -80,11 +82,11 @@ func (r *excludingResolver) AllLocations() <-chan Location {
return c return c
} }
func locationMatches(location *Location, exclusionFn excludeFn) bool { func locationMatches(location *file.Location, exclusionFn excludeFn) bool {
return exclusionFn(location.RealPath) || exclusionFn(location.VirtualPath) return exclusionFn(location.RealPath) || exclusionFn(location.VirtualPath)
} }
func filterLocations(locations []Location, err error, exclusionFn excludeFn) ([]Location, error) { func filterLocations(locations []file.Location, err error, exclusionFn excludeFn) ([]file.Location, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -1,4 +1,4 @@
package source package fileresolver
import ( import (
"io" "io"
@ -6,6 +6,8 @@ import (
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/file"
) )
func TestExcludingResolver(t *testing.T) { func TestExcludingResolver(t *testing.T) {
@ -54,7 +56,7 @@ func TestExcludingResolver(t *testing.T) {
resolver := &mockResolver{ resolver := &mockResolver{
locations: test.locations, locations: test.locations,
} }
er := NewExcludingResolver(resolver, test.excludeFn) er := NewExcluding(resolver, test.excludeFn)
locations, _ := er.FilesByPath() locations, _ := er.FilesByPath()
assert.ElementsMatch(t, locationPaths(locations), test.expected) assert.ElementsMatch(t, locationPaths(locations), test.expected)
@ -65,7 +67,7 @@ func TestExcludingResolver(t *testing.T) {
locations, _ = er.FilesByMIMEType() locations, _ = er.FilesByMIMEType()
assert.ElementsMatch(t, locationPaths(locations), test.expected) assert.ElementsMatch(t, locationPaths(locations), test.expected)
locations = []Location{} locations = []file.Location{}
channel := er.AllLocations() channel := er.AllLocations()
for location := range channel { for location := range channel {
@ -77,25 +79,25 @@ func TestExcludingResolver(t *testing.T) {
for _, path := range diff { for _, path := range diff {
assert.False(t, er.HasPath(path)) assert.False(t, er.HasPath(path))
c, err := er.FileContentsByLocation(NewLocation(path)) c, err := er.FileContentsByLocation(file.NewLocation(path))
assert.Nil(t, c) assert.Nil(t, c)
assert.Error(t, err) assert.Error(t, err)
m, err := er.FileMetadataByLocation(NewLocation(path)) m, err := er.FileMetadataByLocation(file.NewLocation(path))
assert.Empty(t, m.LinkDestination) assert.Empty(t, m.LinkDestination)
assert.Error(t, err) assert.Error(t, err)
l := er.RelativeFileByPath(NewLocation(""), path) l := er.RelativeFileByPath(file.NewLocation(""), path)
assert.Nil(t, l) assert.Nil(t, l)
} }
for _, path := range test.expected { for _, path := range test.expected {
assert.True(t, er.HasPath(path)) assert.True(t, er.HasPath(path))
c, err := er.FileContentsByLocation(NewLocation(path)) c, err := er.FileContentsByLocation(file.NewLocation(path))
assert.NotNil(t, c) assert.NotNil(t, c)
assert.Nil(t, err) assert.Nil(t, err)
m, err := er.FileMetadataByLocation(NewLocation(path)) m, err := er.FileMetadataByLocation(file.NewLocation(path))
assert.NotEmpty(t, m.LinkDestination) assert.NotEmpty(t, m.LinkDestination)
assert.Nil(t, err) assert.Nil(t, err)
l := er.RelativeFileByPath(NewLocation(""), path) l := er.RelativeFileByPath(file.NewLocation(""), path)
assert.NotNil(t, l) assert.NotNil(t, l)
} }
}) })
@ -117,7 +119,7 @@ func difference(a, b []string) []string {
return diff return diff
} }
func locationPaths(locations []Location) []string { func locationPaths(locations []file.Location) []string {
paths := []string{} paths := []string{}
for _, l := range locations { for _, l := range locations {
paths = append(paths, l.RealPath) paths = append(paths, l.RealPath)
@ -129,20 +131,20 @@ type mockResolver struct {
locations []string locations []string
} }
func (r *mockResolver) getLocations() ([]Location, error) { func (r *mockResolver) getLocations() ([]file.Location, error) {
out := []Location{} out := []file.Location{}
for _, path := range r.locations { for _, path := range r.locations {
out = append(out, NewLocation(path)) out = append(out, file.NewLocation(path))
} }
return out, nil return out, nil
} }
func (r *mockResolver) FileContentsByLocation(_ Location) (io.ReadCloser, error) { func (r *mockResolver) FileContentsByLocation(_ file.Location) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("Hello, world!")), nil return io.NopCloser(strings.NewReader("Hello, world!")), nil
} }
func (r *mockResolver) FileMetadataByLocation(_ Location) (FileMetadata, error) { func (r *mockResolver) FileMetadataByLocation(_ file.Location) (file.Metadata, error) {
return FileMetadata{ return file.Metadata{
LinkDestination: "MOCK", LinkDestination: "MOCK",
}, nil }, nil
} }
@ -151,37 +153,37 @@ func (r *mockResolver) HasPath(_ string) bool {
return true return true
} }
func (r *mockResolver) FilesByPath(_ ...string) ([]Location, error) { func (r *mockResolver) FilesByPath(_ ...string) ([]file.Location, error) {
return r.getLocations() return r.getLocations()
} }
func (r *mockResolver) FilesByGlob(_ ...string) ([]Location, error) { func (r *mockResolver) FilesByGlob(_ ...string) ([]file.Location, error) {
return r.getLocations() return r.getLocations()
} }
func (r *mockResolver) FilesByMIMEType(_ ...string) ([]Location, error) { func (r *mockResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) {
return r.getLocations() return r.getLocations()
} }
func (r *mockResolver) FilesByExtension(_ ...string) ([]Location, error) { func (r *mockResolver) FilesByExtension(_ ...string) ([]file.Location, error) {
return r.getLocations() return r.getLocations()
} }
func (r *mockResolver) FilesByBasename(_ ...string) ([]Location, error) { func (r *mockResolver) FilesByBasename(_ ...string) ([]file.Location, error) {
return r.getLocations() return r.getLocations()
} }
func (r *mockResolver) FilesByBasenameGlob(_ ...string) ([]Location, error) { func (r *mockResolver) FilesByBasenameGlob(_ ...string) ([]file.Location, error) {
return r.getLocations() return r.getLocations()
} }
func (r *mockResolver) RelativeFileByPath(_ Location, path string) *Location { func (r *mockResolver) RelativeFileByPath(_ file.Location, path string) *file.Location {
l := NewLocation(path) l := file.NewLocation(path)
return &l return &l
} }
func (r *mockResolver) AllLocations() <-chan Location { func (r *mockResolver) AllLocations() <-chan file.Location {
c := make(chan Location) c := make(chan file.Location)
go func() { go func() {
defer close(c) defer close(c)
locations, _ := r.getLocations() locations, _ := r.getLocations()

View File

@ -0,0 +1,15 @@
package fileresolver
import (
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/syft/file"
)
func fileMetadataByLocation(img *image.Image, location file.Location) (file.Metadata, error) {
entry, err := img.FileCatalog.Get(location.Reference())
if err != nil {
return file.Metadata{}, err
}
return entry.Metadata, nil
}

View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -eux
# $1 —— absolute path to destination file, should end with .tar
# $2 —— absolute path to directory from which to add entries to the archive
pushd "$2"
tar -cvf "$1" .
popd

Some files were not shown because too many files have changed in this diff Show More