Migrate location-related structs to the file package (#1751)

* migrate location structs to file package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* replace source.Location refs with file package call

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove hardlink test for file based catalogers

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove hardlink test for all-regular-files testing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* migrate file resolver implementations to separate package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* [wip] migrate resolvers to internal

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* migrate resolvers to syft/internal

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

---------

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
Signed-off-by: <>
This commit is contained in:
Alex Goodman 2023-05-24 17:06:38 -04:00 committed by GitHub
parent 4bf17a94b9
commit 07e76907f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
313 changed files with 2316 additions and 2191 deletions

View File

@ -167,12 +167,12 @@ always feel free to file an issue or reach out to us [on slack](https://anchore.
#### Searching for files
All catalogers are provided an instance of the [`source.FileResolver`](https://github.com/anchore/syft/blob/v0.70.0/syft/source/file_resolver.go#L8) to interface with the image and search for files. The implementations for these
All catalogers are provided an instance of the [`file.Resolver`](https://github.com/anchore/syft/blob/v0.70.0/syft/source/file_resolver.go#L8) to interface with the image and search for files. The implementations for these
abstractions leverage [`stereoscope`](https://github.com/anchore/stereoscope) in order to perform searching. Here is a
rough outline how that works:
1. a stereoscope `file.Index` is searched based on the input given (a path, glob, or MIME type). The index is relatively fast to search, but requires results to be filtered down to the files that exist in the specific layer(s) of interest. This is done automatically by the `filetree.Searcher` abstraction. This abstraction will fallback to searching directly against the raw `filetree.FileTree` if the index does not contain the file(s) of interest. Note: the `filetree.Searcher` is used by the `source.FileResolver` abstraction.
2. Once the set of files are returned from the `filetree.Searcher` the results are filtered down further to return the most unique file results. For example, you may have requested for files by a glob that returns multiple results. These results are filtered down to deduplicate by real files, so if a result contains two references to the same file, say one accessed via symlink and one accessed via the real path, then the real path reference is returned and the symlink reference is filtered out. If both were accessed by symlink then the first (by lexical order) is returned. This is done automatically by the `source.FileResolver` abstraction.
1. a stereoscope `file.Index` is searched based on the input given (a path, glob, or MIME type). The index is relatively fast to search, but requires results to be filtered down to the files that exist in the specific layer(s) of interest. This is done automatically by the `filetree.Searcher` abstraction. This abstraction will fallback to searching directly against the raw `filetree.FileTree` if the index does not contain the file(s) of interest. Note: the `filetree.Searcher` is used by the `file.Resolver` abstraction.
2. Once the set of files are returned from the `filetree.Searcher` the results are filtered down further to return the most unique file results. For example, you may have requested for files by a glob that returns multiple results. These results are filtered down to deduplicate by real files, so if a result contains two references to the same file, say one accessed via symlink and one accessed via the real path, then the real path reference is returned and the symlink reference is filtered out. If both were accessed by symlink then the first (by lexical order) is returned. This is done automatically by the `file.Resolver` abstraction.
3. By the time results reach the `pkg.Cataloger` you are guaranteed to have a set of unique files that exist in the layer(s) of interest (relative to what the resolver supports).
## Testing

View File

@ -8,6 +8,10 @@ import (
"github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/file/cataloger/filecontent"
"github.com/anchore/syft/syft/file/cataloger/filedigest"
"github.com/anchore/syft/syft/file/cataloger/filemetadata"
"github.com/anchore/syft/syft/file/cataloger/secrets"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
@ -61,7 +65,7 @@ func generateCatalogFileMetadataTask(app *config.Application) (Task, error) {
return nil, nil
}
metadataCataloger := file.NewMetadataCataloger()
metadataCataloger := filemetadata.NewCataloger()
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt)
@ -104,10 +108,7 @@ func generateCatalogFileDigestsTask(app *config.Application) (Task, error) {
hashes = append(hashes, hashObj)
}
digestsCataloger, err := file.NewDigestsCataloger(hashes)
if err != nil {
return nil, err
}
digestsCataloger := filedigest.NewCataloger(hashes)
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
resolver, err := src.FileResolver(app.FileMetadata.Cataloger.ScopeOpt)
@ -131,12 +132,12 @@ func generateCatalogSecretsTask(app *config.Application) (Task, error) {
return nil, nil
}
patterns, err := file.GenerateSearchPatterns(file.DefaultSecretsPatterns, app.Secrets.AdditionalPatterns, app.Secrets.ExcludePatternNames)
patterns, err := secrets.GenerateSearchPatterns(secrets.DefaultSecretsPatterns, app.Secrets.AdditionalPatterns, app.Secrets.ExcludePatternNames)
if err != nil {
return nil, err
}
secretsCataloger, err := file.NewSecretsCataloger(patterns, app.Secrets.RevealValues, app.Secrets.SkipFilesAboveSize)
secretsCataloger, err := secrets.NewCataloger(patterns, app.Secrets.RevealValues, app.Secrets.SkipFilesAboveSize) //nolint:staticcheck
if err != nil {
return nil, err
}
@ -163,7 +164,7 @@ func generateCatalogContentsTask(app *config.Application) (Task, error) {
return nil, nil
}
contentsCataloger, err := file.NewContentsCataloger(app.FileContents.Globs, app.FileContents.SkipFilesAboveSize)
contentsCataloger, err := filecontent.NewCataloger(app.FileContents.Globs, app.FileContents.SkipFilesAboveSize) //nolint:staticcheck
if err != nil {
return nil, err
}

View File

@ -5,9 +5,9 @@ import (
"github.com/google/licensecheck"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/license"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
const (
@ -16,7 +16,7 @@ const (
)
// Parse scans the contents of a license file to attempt to determine the type of license it is
func Parse(reader io.Reader, l source.Location) (licenses []pkg.License, err error) {
func Parse(reader io.Reader, l file.Location) (licenses []pkg.License, err error) {
licenses = make([]pkg.License, 0)
contents, err := io.ReadAll(reader)
if err != nil {

View File

@ -12,7 +12,7 @@ import (
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/file/cataloger/secrets"
"github.com/anchore/syft/syft/pkg/cataloger"
)
@ -54,12 +54,12 @@ func ParsePackageCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error)
return &monitor, nil
}
func ParseSecretsCatalogingStarted(e partybus.Event) (*file.SecretsMonitor, error) {
func ParseSecretsCatalogingStarted(e partybus.Event) (*secrets.Monitor, error) {
if err := checkEventType(e.Type, event.SecretsCatalogerStarted); err != nil {
return nil, err
}
monitor, ok := e.Value.(file.SecretsMonitor)
monitor, ok := e.Value.(secrets.Monitor)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}

View File

@ -1,4 +1,4 @@
package file
package filecontent
import (
"bytes"
@ -8,24 +8,26 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/file"
)
type ContentsCataloger struct {
// Deprecated: will be removed in syft v1.0.0
type Cataloger struct {
globs []string
skipFilesAboveSizeInBytes int64
}
func NewContentsCataloger(globs []string, skipFilesAboveSize int64) (*ContentsCataloger, error) {
return &ContentsCataloger{
// Deprecated: will be removed in syft v1.0.0
func NewCataloger(globs []string, skipFilesAboveSize int64) (*Cataloger, error) {
return &Cataloger{
globs: globs,
skipFilesAboveSizeInBytes: skipFilesAboveSize,
}, nil
}
func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates]string, error) {
results := make(map[source.Coordinates]string)
var locations []source.Location
func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates]string, error) {
results := make(map[file.Coordinates]string)
var locations []file.Location
locations, err := resolver.FilesByGlob(i.globs...)
if err != nil {
@ -56,7 +58,7 @@ func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Co
return results, nil
}
func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) (string, error) {
func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Location) (string, error) {
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
return "", err

View File

@ -0,0 +1,80 @@
package filecontent
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/file"
)
func TestContentsCataloger(t *testing.T) {
allFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
tests := []struct {
name string
globs []string
maxSize int64
files []string
expected map[file.Coordinates]string
}{
{
name: "multi-pattern",
globs: []string{"test-fixtures/last/*.txt", "test-fixtures/*.txt"},
files: allFiles,
expected: map[file.Coordinates]string{
file.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
file.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
file.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "no-patterns",
globs: []string{},
files: []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"},
expected: map[file.Coordinates]string{},
},
{
name: "all-txt",
globs: []string{"**/*.txt"},
files: allFiles,
expected: map[file.Coordinates]string{
file.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
file.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
file.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "subpath",
globs: []string{"test-fixtures/*.txt"},
files: allFiles,
expected: map[file.Coordinates]string{
file.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
file.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "size-filter",
maxSize: 42,
globs: []string{"**/*.txt"},
files: allFiles,
expected: map[file.Coordinates]string{
file.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
file.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c, err := NewCataloger(test.globs, test.maxSize)
assert.NoError(t, err)
resolver := file.NewMockResolverForPaths(test.files...)
actual, err := c.Catalog(resolver)
assert.NoError(t, err)
assert.Equal(t, test.expected, actual, "mismatched contents")
})
}
}

View File

@ -0,0 +1,109 @@
package filedigest
import (
"crypto"
"errors"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/file"
internal2 "github.com/anchore/syft/syft/file/cataloger/internal"
)
var ErrUndigestableFile = errors.New("undigestable file")
type Cataloger struct {
hashes []crypto.Hash
}
func NewCataloger(hashes []crypto.Hash) *Cataloger {
return &Cataloger{
hashes: hashes,
}
}
func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates][]file.Digest, error) {
results := make(map[file.Coordinates][]file.Digest)
var locations []file.Location
if len(coordinates) == 0 {
locations = internal2.AllRegularFiles(resolver)
} else {
for _, c := range coordinates {
locations = append(locations, file.NewLocationFromCoordinates(c))
}
}
stage, prog := digestsCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
result, err := i.catalogLocation(resolver, location)
if errors.Is(err, ErrUndigestableFile) {
continue
}
if internal.IsErrPathPermission(err) {
log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
continue
}
if err != nil {
return nil, err
}
prog.Increment()
results[location.Coordinates] = result
}
log.Debugf("file digests cataloger processed %d files", prog.Current())
prog.SetCompleted()
return results, nil
}
func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Location) ([]file.Digest, error) {
meta, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
}
// we should only attempt to report digests for files that are regular files (don't attempt to resolve links)
if meta.Type != stereoscopeFile.TypeRegular {
return nil, ErrUndigestableFile
}
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, err
}
defer internal.CloseAndLogError(contentReader, location.VirtualPath)
digests, err := file.NewDigestsFromFile(contentReader, i.hashes)
if err != nil {
return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err}
}
return digests, nil
}
func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
prog := progress.NewManual(locations)
bus.Publish(partybus.Event{
Type: event.FileDigestsCatalogerStarted,
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
},
})
return stage, prog
}

View File

@ -1,9 +1,9 @@
package file
package filedigest
import (
"crypto"
"fmt"
"io/ioutil"
"io"
"os"
"path/filepath"
"testing"
@ -11,29 +11,36 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source"
)
func testDigests(t testing.TB, root string, files []string, hashes ...crypto.Hash) map[source.Coordinates][]Digest {
digests := make(map[source.Coordinates][]Digest)
func testDigests(t testing.TB, root string, files []string, hashes ...crypto.Hash) map[file.Coordinates][]file.Digest {
digests := make(map[file.Coordinates][]file.Digest)
for _, f := range files {
fh, err := os.Open(filepath.Join(root, f))
if err != nil {
t.Fatalf("could not open %q : %+v", f, err)
}
b, err := ioutil.ReadAll(fh)
b, err := io.ReadAll(fh)
if err != nil {
t.Fatalf("could not read %q : %+v", f, err)
}
if len(b) == 0 {
// we don't keep digests for empty files
digests[file.NewLocation(f).Coordinates] = []file.Digest{}
continue
}
for _, hash := range hashes {
h := hash.New()
h.Write(b)
digests[source.NewLocation(f).Coordinates] = append(digests[source.NewLocation(f).Coordinates], Digest{
Algorithm: CleanDigestAlgorithmName(hash.String()),
digests[file.NewLocation(f).Coordinates] = append(digests[file.NewLocation(f).Coordinates], file.Digest{
Algorithm: file.CleanDigestAlgorithmName(hash.String()),
Value: fmt.Sprintf("%x", h.Sum(nil)),
})
}
@ -48,7 +55,7 @@ func TestDigestsCataloger(t *testing.T) {
name string
digests []crypto.Hash
files []string
expected map[source.Coordinates][]Digest
expected map[file.Coordinates][]file.Digest
}{
{
name: "md5",
@ -66,8 +73,7 @@ func TestDigestsCataloger(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c, err := NewDigestsCataloger(test.digests)
require.NoError(t, err)
c := NewCataloger(test.digests)
src, err := source.NewFromDirectory("test-fixtures/last/")
require.NoError(t, err)
@ -86,11 +92,7 @@ func TestDigestsCataloger(t *testing.T) {
func TestDigestsCataloger_MixFileTypes(t *testing.T) {
testImage := "image-file-type-mix"
if *updateImageGoldenFiles {
imagetest.UpdateGoldenFixtureImage(t, testImage)
}
img := imagetest.GetGoldenFixtureImage(t, testImage)
img := imagetest.GetFixtureImage(t, "docker-archive", testImage)
src, err := source.NewFromImage(img, "---")
if err != nil {
@ -110,9 +112,10 @@ func TestDigestsCataloger_MixFileTypes(t *testing.T) {
path: "/file-1.txt",
expected: "888c139e550867814eb7c33b84d76e4d",
},
{
path: "/hardlink-1",
},
// this is difficult to reproduce in a cross-platform way
//{
// path: "/hardlink-1",
//},
{
path: "/symlink-1",
},
@ -132,21 +135,18 @@ func TestDigestsCataloger_MixFileTypes(t *testing.T) {
for _, test := range tests {
t.Run(test.path, func(t *testing.T) {
c, err := NewDigestsCataloger([]crypto.Hash{crypto.MD5})
if err != nil {
t.Fatalf("unable to get cataloger: %+v", err)
}
c := NewCataloger([]crypto.Hash{crypto.MD5})
actual, err := c.Catalog(resolver)
if err != nil {
t.Fatalf("could not catalog: %+v", err)
}
_, ref, err := img.SquashedTree().File(file.Path(test.path))
_, ref, err := img.SquashedTree().File(stereoscopeFile.Path(test.path))
if err != nil {
t.Fatalf("unable to get file=%q : %+v", test.path, err)
}
l := source.NewLocationFromImage(test.path, *ref.Reference, img)
l := file.NewLocationFromImage(test.path, *ref.Reference, img)
if len(actual[l.Coordinates]) == 0 {
if test.expected != "" {

View File

@ -1,4 +1,4 @@
FROM busybox:latest
FROM busybox:1.28.1@sha256:c7b0a24019b0e6eda714ec0fa137ad42bc44a754d9cea17d14fba3a80ccc1ee4
ADD file-1.txt .
RUN chmod 644 file-1.txt

View File

@ -0,0 +1 @@
test-fixtures/last/path.txt file contents!

View File

@ -1,4 +1,4 @@
package file
package filemetadata
import (
"github.com/wagoodman/go-partybus"
@ -7,24 +7,37 @@ import (
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/file"
)
type MetadataCataloger struct {
type Cataloger struct {
}
func NewMetadataCataloger() *MetadataCataloger {
return &MetadataCataloger{}
func NewCataloger() *Cataloger {
return &Cataloger{}
}
func (i *MetadataCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates]source.FileMetadata, error) {
results := make(map[source.Coordinates]source.FileMetadata)
var locations []source.Location
for location := range resolver.AllLocations() {
locations = append(locations, location)
func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates]file.Metadata, error) {
results := make(map[file.Coordinates]file.Metadata)
var locations <-chan file.Location
if len(coordinates) == 0 {
locations = resolver.AllLocations()
} else {
locations = func() <-chan file.Location {
ch := make(chan file.Location)
go func() {
close(ch)
for _, c := range coordinates {
ch <- file.NewLocationFromCoordinates(c)
}
}()
return ch
}()
}
stage, prog := metadataCatalogingProgress(int64(len(locations)))
for _, location := range locations {
for location := range locations {
stage.Current = location.RealPath
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {

View File

@ -1,30 +1,24 @@
package file
package filemetadata
import (
"flag"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source"
)
var updateImageGoldenFiles = flag.Bool("update-image", false, "update the golden fixture images used for testing")
func TestFileMetadataCataloger(t *testing.T) {
testImage := "image-file-type-mix"
if *updateImageGoldenFiles {
imagetest.UpdateGoldenFixtureImage(t, testImage)
}
img := imagetest.GetFixtureImage(t, "docker-archive", testImage)
img := imagetest.GetGoldenFixtureImage(t, testImage)
c := NewMetadataCataloger()
c := NewCataloger()
src, err := source.NewFromImage(img, "---")
if err != nil {
@ -44,51 +38,36 @@ func TestFileMetadataCataloger(t *testing.T) {
tests := []struct {
path string
exists bool
expected source.FileMetadata
expected file.Metadata
err bool
}{
// note: it is difficult to add a hardlink-based test in a cross-platform way and is already covered well in stereoscope
{
path: "/file-1.txt",
exists: true,
expected: source.FileMetadata{
FileInfo: file.ManualInfo{
expected: file.Metadata{
FileInfo: stereoscopeFile.ManualInfo{
NameValue: "file-1.txt",
ModeValue: 0644,
SizeValue: 7,
},
Path: "/file-1.txt",
Type: file.TypeRegular,
Type: stereoscopeFile.TypeRegular,
UserID: 1,
GroupID: 2,
MIMEType: "text/plain",
},
},
{
path: "/hardlink-1",
exists: true,
expected: source.FileMetadata{
FileInfo: file.ManualInfo{
NameValue: "hardlink-1",
ModeValue: 0644,
},
Path: "/hardlink-1",
Type: file.TypeHardLink,
LinkDestination: "file-1.txt",
UserID: 1,
GroupID: 2,
MIMEType: "",
},
},
{
path: "/symlink-1",
exists: true,
expected: source.FileMetadata{
expected: file.Metadata{
Path: "/symlink-1",
FileInfo: file.ManualInfo{
FileInfo: stereoscopeFile.ManualInfo{
NameValue: "symlink-1",
ModeValue: 0777 | os.ModeSymlink,
},
Type: file.TypeSymLink,
Type: stereoscopeFile.TypeSymLink,
LinkDestination: "file-1.txt",
UserID: 0,
GroupID: 0,
@ -98,13 +77,13 @@ func TestFileMetadataCataloger(t *testing.T) {
{
path: "/char-device-1",
exists: true,
expected: source.FileMetadata{
expected: file.Metadata{
Path: "/char-device-1",
FileInfo: file.ManualInfo{
FileInfo: stereoscopeFile.ManualInfo{
NameValue: "char-device-1",
ModeValue: 0644 | os.ModeDevice | os.ModeCharDevice,
},
Type: file.TypeCharacterDevice,
Type: stereoscopeFile.TypeCharacterDevice,
UserID: 0,
GroupID: 0,
MIMEType: "",
@ -113,13 +92,13 @@ func TestFileMetadataCataloger(t *testing.T) {
{
path: "/block-device-1",
exists: true,
expected: source.FileMetadata{
expected: file.Metadata{
Path: "/block-device-1",
FileInfo: file.ManualInfo{
FileInfo: stereoscopeFile.ManualInfo{
NameValue: "block-device-1",
ModeValue: 0644 | os.ModeDevice,
},
Type: file.TypeBlockDevice,
Type: stereoscopeFile.TypeBlockDevice,
UserID: 0,
GroupID: 0,
MIMEType: "",
@ -128,13 +107,13 @@ func TestFileMetadataCataloger(t *testing.T) {
{
path: "/fifo-1",
exists: true,
expected: source.FileMetadata{
expected: file.Metadata{
Path: "/fifo-1",
FileInfo: file.ManualInfo{
FileInfo: stereoscopeFile.ManualInfo{
NameValue: "fifo-1",
ModeValue: 0644 | os.ModeNamedPipe,
},
Type: file.TypeFIFO,
Type: stereoscopeFile.TypeFIFO,
UserID: 0,
GroupID: 0,
MIMEType: "",
@ -143,13 +122,13 @@ func TestFileMetadataCataloger(t *testing.T) {
{
path: "/bin",
exists: true,
expected: source.FileMetadata{
expected: file.Metadata{
Path: "/bin",
FileInfo: file.ManualInfo{
FileInfo: stereoscopeFile.ManualInfo{
NameValue: "bin",
ModeValue: 0755 | os.ModeDir,
},
Type: file.TypeDirectory,
Type: stereoscopeFile.TypeDirectory,
UserID: 0,
GroupID: 0,
MIMEType: "",
@ -159,15 +138,15 @@ func TestFileMetadataCataloger(t *testing.T) {
for _, test := range tests {
t.Run(test.path, func(t *testing.T) {
_, ref, err := img.SquashedTree().File(file.Path(test.path))
_, ref, err := img.SquashedTree().File(stereoscopeFile.Path(test.path))
require.NoError(t, err)
l := source.NewLocationFromImage(test.path, *ref.Reference, img)
l := file.NewLocationFromImage(test.path, *ref.Reference, img)
if _, ok := actual[l.Coordinates]; ok {
// we're not interested in keeping the test fixtures up to date with the latest file modification times
// thus ModTime is not under test
fi := test.expected.FileInfo.(file.ManualInfo)
fi := test.expected.FileInfo.(stereoscopeFile.ManualInfo)
fi.ModTimeValue = actual[l.Coordinates].ModTime()
test.expected.FileInfo = fi
}

View File

@ -0,0 +1,13 @@
FROM busybox:1.28.1@sha256:c7b0a24019b0e6eda714ec0fa137ad42bc44a754d9cea17d14fba3a80ccc1ee4
ADD file-1.txt .
RUN chmod 644 file-1.txt
RUN chown 1:2 file-1.txt
RUN ln -s file-1.txt symlink-1
# note: hard links may behave inconsistently, this should be a golden image
RUN ln file-1.txt hardlink-1
RUN mknod char-device-1 c 89 1
RUN mknod block-device-1 b 0 1
RUN mknod fifo-1 p
RUN mkdir /dir
RUN rm -rf home etc/group etc/localtime etc/mtab etc/network etc/passwd etc/shadow var usr bin/*

View File

@ -1,12 +1,12 @@
package file
package internal
import (
"github.com/anchore/stereoscope/pkg/file"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/file"
)
func allRegularFiles(resolver source.FileResolver) (locations []source.Location) {
func AllRegularFiles(resolver file.Resolver) (locations []file.Location) {
for location := range resolver.AllLocations() {
resolvedLocations, err := resolver.FilesByPath(location.RealPath)
if err != nil {
@ -21,7 +21,7 @@ func allRegularFiles(resolver source.FileResolver) (locations []source.Location)
continue
}
if metadata.Type != file.TypeRegular {
if metadata.Type != stereoscopeFile.TypeRegular {
continue
}
locations = append(locations, resolvedLocation)

View File

@ -1,4 +1,4 @@
package file
package internal
import (
"testing"
@ -9,30 +9,23 @@ import (
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source"
)
func Test_allRegularFiles(t *testing.T) {
type access struct {
realPath string
virtualPath string
}
tests := []struct {
name string
setup func() source.FileResolver
setup func() file.Resolver
wantRealPaths *strset.Set
wantVirtualPaths *strset.Set
}{
{
name: "image",
setup: func() source.FileResolver {
setup: func() file.Resolver {
testImage := "image-file-type-mix"
if *updateImageGoldenFiles {
imagetest.UpdateGoldenFixtureImage(t, testImage)
}
img := imagetest.GetGoldenFixtureImage(t, testImage)
img := imagetest.GetFixtureImage(t, "docker-archive", testImage)
s, err := source.NewFromImage(img, "---")
require.NoError(t, err)
@ -47,7 +40,7 @@ func Test_allRegularFiles(t *testing.T) {
},
{
name: "directory",
setup: func() source.FileResolver {
setup: func() file.Resolver {
s, err := source.NewFromDirectory("test-fixtures/symlinked-root/nested/link-root")
require.NoError(t, err)
r, err := s.FileResolver(source.SquashedScope)
@ -61,7 +54,7 @@ func Test_allRegularFiles(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resolver := tt.setup()
locations := allRegularFiles(resolver)
locations := AllRegularFiles(resolver)
realLocations := strset.New()
virtualLocations := strset.New()
for _, l := range locations {
@ -70,6 +63,13 @@ func Test_allRegularFiles(t *testing.T) {
virtualLocations.Add(l.VirtualPath)
}
}
// this is difficult to reproduce in a cross-platform way
realLocations.Remove("/hardlink-1")
virtualLocations.Remove("/hardlink-1")
tt.wantRealPaths.Remove("/hardlink-1")
tt.wantVirtualPaths.Remove("/hardlink-1")
assert.ElementsMatch(t, tt.wantRealPaths.List(), realLocations.List(), "real paths differ: "+cmp.Diff(tt.wantRealPaths.List(), realLocations.List()))
assert.ElementsMatch(t, tt.wantVirtualPaths.List(), virtualLocations.List(), "virtual paths differ: "+cmp.Diff(tt.wantVirtualPaths.List(), virtualLocations.List()))
})

View File

@ -0,0 +1,13 @@
FROM busybox:1.28.1@sha256:c7b0a24019b0e6eda714ec0fa137ad42bc44a754d9cea17d14fba3a80ccc1ee4
ADD file-1.txt .
RUN chmod 644 file-1.txt
RUN chown 1:2 file-1.txt
RUN ln -s file-1.txt symlink-1
# note: hard links may behave inconsistently, this should be a golden image
RUN ln file-1.txt hardlink-1
RUN mknod char-device-1 c 89 1
RUN mknod block-device-1 b 0 1
RUN mknod fifo-1 p
RUN mkdir /dir
RUN rm -rf home etc/group etc/localtime etc/mtab etc/network etc/passwd etc/shadow var usr bin/*

View File

@ -1,4 +1,4 @@
package file
package secrets
import (
"bytes"
@ -14,7 +14,8 @@ import (
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/file"
internal2 "github.com/anchore/syft/syft/file/cataloger/internal"
)
var DefaultSecretsPatterns = map[string]string{
@ -25,23 +26,25 @@ var DefaultSecretsPatterns = map[string]string{
"generic-api-key": `(?i)api(-|_)?key["'=:\s]*?(?P<value>[A-Z0-9]{20,60})["']?(\s|$)`,
}
type SecretsCataloger struct {
// Deprecated: will be removed in syft v1.0.0
type Cataloger struct {
patterns map[string]*regexp.Regexp
revealValues bool
skipFilesAboveSize int64
}
func NewSecretsCataloger(patterns map[string]*regexp.Regexp, revealValues bool, maxFileSize int64) (*SecretsCataloger, error) {
return &SecretsCataloger{
// Deprecated: will be removed in syft v1.0.0
func NewCataloger(patterns map[string]*regexp.Regexp, revealValues bool, maxFileSize int64) (*Cataloger, error) {
return &Cataloger{
patterns: patterns,
revealValues: revealValues,
skipFilesAboveSize: maxFileSize,
}, nil
}
func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]SearchResult, error) {
results := make(map[source.Coordinates][]SearchResult)
locations := allRegularFiles(resolver)
func (i *Cataloger) Catalog(resolver file.Resolver) (map[file.Coordinates][]file.SearchResult, error) {
results := make(map[file.Coordinates][]file.SearchResult)
locations := internal2.AllRegularFiles(resolver)
stage, prog, secretsDiscovered := secretsCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
@ -65,7 +68,7 @@ func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Coo
return results, nil
}
func (i *SecretsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]SearchResult, error) {
func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Location) ([]file.SearchResult, error) {
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
@ -103,7 +106,7 @@ func (i *SecretsCataloger) catalogLocation(resolver source.FileResolver, locatio
return secrets, nil
}
func extractValue(resolver source.FileResolver, location source.Location, start, length int64) (string, error) {
func extractValue(resolver file.Resolver, location file.Location, start, length int64) (string, error) {
readCloser, err := resolver.FileContentsByLocation(location)
if err != nil {
return "", fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
@ -130,7 +133,7 @@ func extractValue(resolver source.FileResolver, location source.Location, start,
return buf.String(), nil
}
type SecretsMonitor struct {
type Monitor struct {
progress.Stager
SecretsDiscovered progress.Monitorable
progress.Progressable
@ -144,7 +147,7 @@ func secretsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manu
bus.Publish(partybus.Event{
Type: event.SecretsCatalogerStarted,
Source: secretsDiscovered,
Value: SecretsMonitor{
Value: Monitor{
Stager: progress.Stager(stage),
SecretsDiscovered: secretsDiscovered,
Progressable: prog,

View File

@ -1,4 +1,4 @@
package file
package secrets
import (
"regexp"
@ -6,8 +6,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/internal/file"
"github.com/anchore/syft/syft/source"
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/syft/file"
)
func TestSecretsCataloger(t *testing.T) {
@ -17,7 +17,7 @@ func TestSecretsCataloger(t *testing.T) {
reveal bool
maxSize int64
patterns map[string]string
expected []SearchResult
expected []file.SearchResult
constructorErr bool
catalogErr bool
}{
@ -28,7 +28,7 @@ func TestSecretsCataloger(t *testing.T) {
patterns: map[string]string{
"simple-secret-key": `^secret_key=.*`,
},
expected: []SearchResult{
expected: []file.SearchResult{
{
Classification: "simple-secret-key",
LineNumber: 2,
@ -46,7 +46,7 @@ func TestSecretsCataloger(t *testing.T) {
patterns: map[string]string{
"simple-secret-key": `^secret_key=.*`,
},
expected: []SearchResult{
expected: []file.SearchResult{
{
Classification: "simple-secret-key",
LineNumber: 2,
@ -64,7 +64,7 @@ func TestSecretsCataloger(t *testing.T) {
patterns: map[string]string{
"simple-secret-key": `^secret_key=(?P<value>.*)`,
},
expected: []SearchResult{
expected: []file.SearchResult{
{
Classification: "simple-secret-key",
LineNumber: 2,
@ -82,7 +82,7 @@ func TestSecretsCataloger(t *testing.T) {
patterns: map[string]string{
"simple-secret-key": `secret_key=.*`,
},
expected: []SearchResult{
expected: []file.SearchResult{
{
Classification: "simple-secret-key",
LineNumber: 1,
@ -125,7 +125,7 @@ func TestSecretsCataloger(t *testing.T) {
patterns: map[string]string{
"simple-secret-key": `secret_key=(?P<value>.*)`,
},
expected: []SearchResult{
expected: []file.SearchResult{
{
Classification: "simple-secret-key",
LineNumber: 1,
@ -176,7 +176,7 @@ func TestSecretsCataloger(t *testing.T) {
regexObjs[name] = obj
}
c, err := NewSecretsCataloger(regexObjs, test.reveal, test.maxSize)
c, err := NewCataloger(regexObjs, test.reveal, test.maxSize)
if err != nil && !test.constructorErr {
t.Fatalf("could not create cataloger (but should have been able to): %+v", err)
} else if err == nil && test.constructorErr {
@ -185,7 +185,7 @@ func TestSecretsCataloger(t *testing.T) {
return
}
resolver := source.NewMockResolverForPaths(test.fixture)
resolver := file.NewMockResolverForPaths(test.fixture)
actualResults, err := c.Catalog(resolver)
if err != nil && !test.catalogErr {
@ -196,7 +196,7 @@ func TestSecretsCataloger(t *testing.T) {
return
}
loc := source.NewLocation(test.fixture)
loc := file.NewLocation(test.fixture)
if _, exists := actualResults[loc.Coordinates]; !exists {
t.Fatalf("could not find location=%q in results", loc)
}
@ -214,11 +214,11 @@ func TestSecretsCataloger_DefaultSecrets(t *testing.T) {
tests := []struct {
fixture string
expected []SearchResult
expected []file.SearchResult
}{
{
fixture: "test-fixtures/secrets/default/aws.env",
expected: []SearchResult{
expected: []file.SearchResult{
{
Classification: "aws-access-key",
LineNumber: 2,
@ -239,7 +239,7 @@ func TestSecretsCataloger_DefaultSecrets(t *testing.T) {
},
{
fixture: "test-fixtures/secrets/default/aws.ini",
expected: []SearchResult{
expected: []file.SearchResult{
{
Classification: "aws-access-key",
LineNumber: 3,
@ -260,7 +260,7 @@ func TestSecretsCataloger_DefaultSecrets(t *testing.T) {
},
{
fixture: "test-fixtures/secrets/default/private-key.pem",
expected: []SearchResult{
expected: []file.SearchResult{
{
Classification: "pem-private-key",
LineNumber: 2,
@ -280,7 +280,7 @@ z3P668YfhUbKdRF6S42Cg6zn
},
{
fixture: "test-fixtures/secrets/default/private-key-openssl.pem",
expected: []SearchResult{
expected: []file.SearchResult{
{
Classification: "pem-private-key",
LineNumber: 2,
@ -302,7 +302,7 @@ z3P668YfhUbKdRF6S42Cg6zn
// note: this test proves that the PEM regex matches the smallest possible match
// since the test catches two adjacent secrets
fixture: "test-fixtures/secrets/default/private-keys.pem",
expected: []SearchResult{
expected: []file.SearchResult{
{
Classification: "pem-private-key",
LineNumber: 1,
@ -345,7 +345,7 @@ j4f668YfhUbKdRF6S6734856
// 2. a named capture group with the correct line number and line offset case
// 3. the named capture group is in a different line than the match start, and both the match start and the capture group have different line offsets
fixture: "test-fixtures/secrets/default/docker-config.json",
expected: []SearchResult{
expected: []file.SearchResult{
{
Classification: "docker-config-auth",
LineNumber: 5,
@ -362,7 +362,7 @@ j4f668YfhUbKdRF6S6734856
},
{
fixture: "test-fixtures/secrets/default/api-key.txt",
expected: []SearchResult{
expected: []file.SearchResult{
{
Classification: "generic-api-key",
LineNumber: 2,
@ -418,19 +418,19 @@ j4f668YfhUbKdRF6S6734856
for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) {
c, err := NewSecretsCataloger(regexObjs, true, 10*file.MB)
c, err := NewCataloger(regexObjs, true, 10*intFile.MB)
if err != nil {
t.Fatalf("could not create cataloger: %+v", err)
}
resolver := source.NewMockResolverForPaths(test.fixture)
resolver := file.NewMockResolverForPaths(test.fixture)
actualResults, err := c.Catalog(resolver)
if err != nil {
t.Fatalf("could not catalog: %+v", err)
}
loc := source.NewLocation(test.fixture)
loc := file.NewLocation(test.fixture)
if _, exists := actualResults[loc.Coordinates]; !exists && test.expected != nil {
t.Fatalf("could not find location=%q in results", loc)
} else if !exists && test.expected == nil {

View File

@ -1,4 +1,4 @@
package file
package secrets
import (
"fmt"

View File

@ -1,4 +1,4 @@
package file
package secrets
import (
"testing"

View File

@ -1,4 +1,4 @@
package file
package secrets
import "io"

View File

@ -1,4 +1,4 @@
package file
package secrets
import (
"bufio"

View File

@ -1,4 +1,4 @@
package file
package secrets
import (
"bufio"
@ -8,10 +8,10 @@ import (
"regexp"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/file"
)
func catalogLocationByLine(resolver source.FileResolver, location source.Location, patterns map[string]*regexp.Regexp) ([]SearchResult, error) {
func catalogLocationByLine(resolver file.Resolver, location file.Location, patterns map[string]*regexp.Regexp) ([]file.SearchResult, error) {
readCloser, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
@ -20,7 +20,7 @@ func catalogLocationByLine(resolver source.FileResolver, location source.Locatio
var scanner = bufio.NewReader(readCloser)
var position int64
var allSecrets []SearchResult
var allSecrets []file.SearchResult
var lineNo int64
var readErr error
for !errors.Is(readErr, io.EOF) {
@ -43,8 +43,8 @@ func catalogLocationByLine(resolver source.FileResolver, location source.Locatio
return allSecrets, nil
}
func searchForSecretsWithinLine(resolver source.FileResolver, location source.Location, patterns map[string]*regexp.Regexp, line []byte, lineNo int64, position int64) ([]SearchResult, error) {
var secrets []SearchResult
func searchForSecretsWithinLine(resolver file.Resolver, location file.Location, patterns map[string]*regexp.Regexp, line []byte, lineNo int64, position int64) ([]file.SearchResult, error) {
var secrets []file.SearchResult
for name, pattern := range patterns {
matches := pattern.FindAllIndex(line, -1)
for i, match := range matches {
@ -72,7 +72,7 @@ func searchForSecretsWithinLine(resolver source.FileResolver, location source.Lo
return secrets, nil
}
func readerAtPosition(resolver source.FileResolver, location source.Location, seekPosition int64) (io.ReadCloser, error) {
func readerAtPosition(resolver file.Resolver, location file.Location, seekPosition int64) (io.ReadCloser, error) {
readCloser, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, fmt.Errorf("unable to fetch reader for location=%q : %w", location, err)
@ -89,7 +89,7 @@ func readerAtPosition(resolver source.FileResolver, location source.Location, se
return readCloser, nil
}
func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *regexp.Regexp, lineNo, lineOffset, seekPosition int64) *SearchResult {
func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *regexp.Regexp, lineNo, lineOffset, seekPosition int64) *file.SearchResult {
reader := &newlineCounter{RuneReader: bufio.NewReader(readCloser)}
positions := pattern.FindReaderSubmatchIndex(reader)
if len(positions) == 0 {
@ -125,7 +125,7 @@ func extractSecretFromPosition(readCloser io.ReadCloser, name string, pattern *r
lineOffsetOfSecret += lineOffset
}
return &SearchResult{
return &file.SearchResult{
Classification: name,
SeekPosition: start + seekPosition,
Length: stop - start,

View File

@ -1,80 +0,0 @@
package file
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/source"
)
func TestContentsCataloger(t *testing.T) {
allFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
tests := []struct {
name string
globs []string
maxSize int64
files []string
expected map[source.Coordinates]string
}{
{
name: "multi-pattern",
globs: []string{"test-fixtures/last/*.txt", "test-fixtures/*.txt"},
files: allFiles,
expected: map[source.Coordinates]string{
source.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "no-patterns",
globs: []string{},
files: []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"},
expected: map[source.Coordinates]string{},
},
{
name: "all-txt",
globs: []string{"**/*.txt"},
files: allFiles,
expected: map[source.Coordinates]string{
source.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "subpath",
globs: []string{"test-fixtures/*.txt"},
files: allFiles,
expected: map[source.Coordinates]string{
source.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "size-filter",
maxSize: 42,
globs: []string{"**/*.txt"},
files: allFiles,
expected: map[source.Coordinates]string{
source.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c, err := NewContentsCataloger(test.globs, test.maxSize)
assert.NoError(t, err)
resolver := source.NewMockResolverForPaths(test.files...)
actual, err := c.Catalog(resolver)
assert.NoError(t, err)
assert.Equal(t, test.expected, actual, "mismatched contents")
})
}
}

View File

@ -1,4 +1,4 @@
package source
package file
import (
"sort"

View File

@ -1,4 +1,4 @@
package source
package file
import (
"testing"

View File

@ -1,4 +1,4 @@
package source
package file
import (
"fmt"

View File

@ -1,6 +1,76 @@
package file
import (
"crypto"
"fmt"
"hash"
"io"
"strings"
)
type Digest struct {
Algorithm string `json:"algorithm"`
Value string `json:"value"`
}
func NewDigestsFromFile(closer io.ReadCloser, hashes []crypto.Hash) ([]Digest, error) {
// create a set of hasher objects tied together with a single writer to feed content into
hashers := make([]hash.Hash, len(hashes))
writers := make([]io.Writer, len(hashes))
for idx, hashObj := range hashes {
hashers[idx] = hashObj.New()
writers[idx] = hashers[idx]
}
size, err := io.Copy(io.MultiWriter(writers...), closer)
if err != nil {
return nil, err
}
if size == 0 {
return make([]Digest, 0), nil
}
result := make([]Digest, len(hashes))
// only capture digests when there is content. It is important to do this based on SIZE and not
// FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only
// file type but a body is still allowed.
for idx, hasher := range hashers {
result[idx] = Digest{
Algorithm: DigestAlgorithmName(hashes[idx]),
Value: fmt.Sprintf("%+x", hasher.Sum(nil)),
}
}
return result, nil
}
func Hashers(names ...string) ([]crypto.Hash, error) {
supportedHashAlgorithms := make(map[string]crypto.Hash)
for _, h := range []crypto.Hash{
crypto.MD5,
crypto.SHA1,
crypto.SHA256,
} {
supportedHashAlgorithms[DigestAlgorithmName(h)] = h
}
var hashers []crypto.Hash
for _, hashStr := range names {
hashObj, ok := supportedHashAlgorithms[CleanDigestAlgorithmName(hashStr)]
if !ok {
return nil, fmt.Errorf("unsupported hash algorithm: %s", hashStr)
}
hashers = append(hashers, hashObj)
}
return hashers, nil
}
func DigestAlgorithmName(hash crypto.Hash) string {
return CleanDigestAlgorithmName(hash.String())
}
func CleanDigestAlgorithmName(name string) string {
lower := strings.ToLower(name)
return strings.ReplaceAll(lower, "-", "")
}

View File

@ -1,140 +0,0 @@
package file
import (
"crypto"
"errors"
"fmt"
"hash"
"io"
"strings"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/source"
)
var errUndigestableFile = errors.New("undigestable file")
type DigestsCataloger struct {
hashes []crypto.Hash
}
func NewDigestsCataloger(hashes []crypto.Hash) (*DigestsCataloger, error) {
return &DigestsCataloger{
hashes: hashes,
}, nil
}
func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]Digest, error) {
results := make(map[source.Coordinates][]Digest)
locations := allRegularFiles(resolver)
stage, prog := digestsCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
result, err := i.catalogLocation(resolver, location)
if errors.Is(err, errUndigestableFile) {
continue
}
if internal.IsErrPathPermission(err) {
log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
continue
}
if err != nil {
return nil, err
}
prog.Increment()
results[location.Coordinates] = result
}
log.Debugf("file digests cataloger processed %d files", prog.Current())
prog.SetCompleted()
return results, nil
}
func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]Digest, error) {
meta, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
}
// we should only attempt to report digests for files that are regular files (don't attempt to resolve links)
if meta.Type != file.TypeRegular {
return nil, errUndigestableFile
}
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, err
}
defer internal.CloseAndLogError(contentReader, location.VirtualPath)
digests, err := DigestsFromFile(contentReader, i.hashes)
if err != nil {
return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err}
}
return digests, nil
}
func DigestsFromFile(closer io.ReadCloser, hashes []crypto.Hash) ([]Digest, error) {
// create a set of hasher objects tied together with a single writer to feed content into
hashers := make([]hash.Hash, len(hashes))
writers := make([]io.Writer, len(hashes))
for idx, hashObj := range hashes {
hashers[idx] = hashObj.New()
writers[idx] = hashers[idx]
}
_, err := io.Copy(io.MultiWriter(writers...), closer)
if err != nil {
return nil, err
}
result := make([]Digest, len(hashes))
// only capture digests when there is content. It is important to do this based on SIZE and not
// FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only
// file type but a body is still allowed.
for idx, hasher := range hashers {
result[idx] = Digest{
Algorithm: DigestAlgorithmName(hashes[idx]),
Value: fmt.Sprintf("%+x", hasher.Sum(nil)),
}
}
return result, nil
}
func DigestAlgorithmName(hash crypto.Hash) string {
return CleanDigestAlgorithmName(hash.String())
}
func CleanDigestAlgorithmName(name string) string {
lower := strings.ToLower(name)
return strings.ReplaceAll(lower, "-", "")
}
func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
prog := progress.NewManual(locations)
bus.Publish(partybus.Event{
Type: event.FileDigestsCatalogerStarted,
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
},
})
return stage, prog
}

View File

@ -1,4 +1,4 @@
package source
package file
import (
"fmt"
@ -24,6 +24,10 @@ type LocationData struct {
ref file.Reference `hash:"ignore"` // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
}
func (l LocationData) Reference() file.Reference {
return l.ref
}
type LocationMetadata struct {
Annotations map[string]string `json:"annotations,omitempty"` // Arbitrary key-value pairs that can be used to annotate a location
}
@ -108,7 +112,7 @@ func NewVirtualLocationFromCoordinates(coordinates Coordinates, virtualPath stri
}}
}
// NewLocationFromImage creates a new Location representing the given path (extracted from the ref) relative to the given image.
// NewLocationFromImage creates a new Location representing the given path (extracted from the Reference) relative to the given image.
func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Image) Location {
layer := img.FileCatalog.Layer(ref)
return Location{
@ -126,7 +130,7 @@ func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Ima
}
}
// NewLocationFromDirectory creates a new Location representing the given path (extracted from the ref) relative to the given directory.
// NewLocationFromDirectory creates a new Location representing the given path (extracted from the Reference) relative to the given directory.
func NewLocationFromDirectory(responsePath string, ref file.Reference) Location {
return Location{
LocationData: LocationData{
@ -141,7 +145,7 @@ func NewLocationFromDirectory(responsePath string, ref file.Reference) Location
}
}
// NewVirtualLocationFromDirectory creates a new Location representing the given path (extracted from the ref) relative to the given directory with a separate virtual access path.
// NewVirtualLocationFromDirectory creates a new Location representing the given path (extracted from the Reference) relative to the given directory with a separate virtual access path.
func NewVirtualLocationFromDirectory(responsePath, virtualResponsePath string, ref file.Reference) Location {
if responsePath == virtualResponsePath {
return NewLocationFromDirectory(responsePath, ref)

View File

@ -1,4 +1,4 @@
package source
package file
import "io"

View File

@ -1,4 +1,4 @@
package source
package file
import (
"sort"

View File

@ -1,4 +1,4 @@
package source
package file
import (
"testing"

View File

@ -1,4 +1,4 @@
package source
package file
import (
"testing"

View File

@ -1,4 +1,4 @@
package source
package file
type Locations []Location

5
syft/file/metadata.go Normal file
View File

@ -0,0 +1,5 @@
package file
import "github.com/anchore/stereoscope/pkg/file"
type Metadata = file.Metadata

View File

@ -1,4 +1,4 @@
package source
package file
import (
"fmt"
@ -11,14 +11,14 @@ import (
"github.com/anchore/stereoscope/pkg/file"
)
var _ FileResolver = (*MockResolver)(nil)
var _ Resolver = (*MockResolver)(nil)
// MockResolver implements the FileResolver interface and is intended for use *only in test code*.
// It provides an implementation that can resolve local filesystem paths using only a provided discrete list of file
// paths, which are typically paths to test fixtures.
type MockResolver struct {
locations []Location
metadata map[Coordinates]FileMetadata
metadata map[Coordinates]Metadata
mimeTypeIndex map[string][]Location
extension map[string][]Location
basename map[string][]Location
@ -41,13 +41,13 @@ func NewMockResolverForPaths(paths ...string) *MockResolver {
return &MockResolver{
locations: locations,
metadata: make(map[Coordinates]FileMetadata),
metadata: make(map[Coordinates]Metadata),
extension: extension,
basename: basename,
}
}
func NewMockResolverForPathsWithMetadata(metadata map[Coordinates]FileMetadata) *MockResolver {
func NewMockResolverForPathsWithMetadata(metadata map[Coordinates]Metadata) *MockResolver {
var locations []Location
var mimeTypeIndex = make(map[string][]Location)
extension := make(map[string][]Location)
@ -155,10 +155,10 @@ func (r MockResolver) AllLocations() <-chan Location {
return results
}
func (r MockResolver) FileMetadataByLocation(l Location) (FileMetadata, error) {
func (r MockResolver) FileMetadataByLocation(l Location) (Metadata, error) {
info, err := os.Stat(l.RealPath)
if err != nil {
return FileMetadata{}, err
return Metadata{}, err
}
// other types not supported
@ -167,7 +167,7 @@ func (r MockResolver) FileMetadataByLocation(l Location) (FileMetadata, error) {
ty = file.TypeDirectory
}
return FileMetadata{
return Metadata{
FileInfo: info,
Type: ty,
UserID: 0, // not supported

View File

@ -1,28 +1,26 @@
package source
package file
import (
"io"
)
import "io"
// FileResolver is an interface that encompasses how to get specific file references and file contents for a generic data source.
type FileResolver interface {
FileContentResolver
FilePathResolver
FileLocationResolver
FileMetadataResolver
// Resolver is an interface that encompasses how to get specific file references and file contents for a generic data source.
type Resolver interface {
ContentResolver
PathResolver
LocationResolver
MetadataResolver
}
// FileContentResolver knows how to get file content for a given Location
type FileContentResolver interface {
// ContentResolver knows how to get file content for a given Location
type ContentResolver interface {
FileContentsByLocation(Location) (io.ReadCloser, error)
}
type FileMetadataResolver interface {
FileMetadataByLocation(Location) (FileMetadata, error)
type MetadataResolver interface {
FileMetadataByLocation(Location) (Metadata, error)
}
// FilePathResolver knows how to get a Location for given string paths and globs
type FilePathResolver interface {
// PathResolver knows how to get a Location for given string paths and globs
type PathResolver interface {
// HasPath indicates if the given path exists in the underlying source.
// The implementation for this may vary, however, generally the following considerations should be made:
// - full symlink resolution should be performed on all requests
@ -50,7 +48,7 @@ type FilePathResolver interface {
RelativeFileByPath(_ Location, path string) *Location
}
type FileLocationResolver interface {
type LocationResolver interface {
// AllLocations returns a channel of all file references from the underlying source.
// The implementation for this may vary, however, generally the following considerations should be made:
// - NO symlink resolution should be performed on results
@ -58,8 +56,8 @@ type FileLocationResolver interface {
AllLocations() <-chan Location
}
type WritableFileResolver interface {
FileResolver
type WritableResolver interface {
Resolver
Write(location Location, reader io.Reader) error
}

View File

@ -6,9 +6,9 @@ import (
"github.com/CycloneDX/cyclonedx-go"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/formats/common"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func encodeComponent(p pkg.Package) cyclonedx.Component {
@ -100,13 +100,13 @@ func decodeComponent(c *cyclonedx.Component) *pkg.Package {
return p
}
func decodeLocations(vals map[string]string) source.LocationSet {
v := common.Decode(reflect.TypeOf([]source.Location{}), vals, "syft:location", CycloneDXFields)
out, ok := v.([]source.Location)
func decodeLocations(vals map[string]string) file.LocationSet {
v := common.Decode(reflect.TypeOf([]file.Location{}), vals, "syft:location", CycloneDXFields)
out, ok := v.([]file.Location)
if !ok {
out = nil
}
return source.NewLocationSet(out...)
return file.NewLocationSet(out...)
}
func decodePackageMetadata(vals map[string]string, c *cyclonedx.Component, typ pkg.MetadataType) interface{} {

View File

@ -8,8 +8,8 @@ import (
"github.com/CycloneDX/cyclonedx-go"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func Test_encodeComponentProperties(t *testing.T) {
@ -28,8 +28,8 @@ func Test_encodeComponentProperties(t *testing.T) {
name: "from apk",
input: pkg.Package{
FoundBy: "cataloger",
Locations: source.NewLocationSet(
source.NewLocationFromCoordinates(source.Coordinates{RealPath: "test"}),
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{RealPath: "test"}),
),
Metadata: pkg.ApkMetadata{
Package: "libc-utils",

View File

@ -5,8 +5,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func Test_SourceInfo(t *testing.T) {
@ -19,9 +19,9 @@ func Test_SourceInfo(t *testing.T) {
name: "locations are captured",
input: pkg.Package{
// note: no type given
Locations: source.NewLocationSet(
source.NewVirtualLocation("/a-place", "/b-place"),
source.NewVirtualLocation("/c-place", "/d-place"),
Locations: file.NewLocationSet(
file.NewVirtualLocation("/a-place", "/b-place"),
file.NewVirtualLocation("/c-place", "/d-place"),
),
},
expected: []string{

View File

@ -21,7 +21,6 @@ import (
"github.com/anchore/syft/syft/formats/common/util"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
const (
@ -137,7 +136,7 @@ func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID {
switch it := identifiable.(type) {
case pkg.Package:
id = SanitizeElementID(fmt.Sprintf("Package-%s-%s-%s", it.Type, it.Name, it.ID()))
case source.Coordinates:
case file.Coordinates:
p := ""
parts := strings.Split(it.RealPath, "/")
for i := len(parts); i > 0; i-- {
@ -437,7 +436,7 @@ func toFiles(s sbom.SBOM) (results []*spdx.File) {
artifacts := s.Artifacts
for _, coordinates := range s.AllCoordinates() {
var metadata *source.FileMetadata
var metadata *file.Metadata
if metadataForLocation, exists := artifacts.FileMetadata[coordinates]; exists {
metadata = &metadataForLocation
}
@ -500,7 +499,7 @@ func toChecksumAlgorithm(algorithm string) spdx.ChecksumAlgorithm {
return spdx.ChecksumAlgorithm(strings.ToUpper(algorithm))
}
func toFileTypes(metadata *source.FileMetadata) (ty []string) {
func toFileTypes(metadata *file.Metadata) (ty []string) {
if metadata == nil {
return nil
}

View File

@ -115,12 +115,12 @@ func Test_toFileTypes(t *testing.T) {
tests := []struct {
name string
metadata source.FileMetadata
metadata file.Metadata
expected []string
}{
{
name: "application",
metadata: source.FileMetadata{
metadata: file.Metadata{
MIMEType: "application/vnd.unknown",
},
expected: []string{
@ -129,7 +129,7 @@ func Test_toFileTypes(t *testing.T) {
},
{
name: "archive",
metadata: source.FileMetadata{
metadata: file.Metadata{
MIMEType: "application/zip",
},
expected: []string{
@ -139,7 +139,7 @@ func Test_toFileTypes(t *testing.T) {
},
{
name: "audio",
metadata: source.FileMetadata{
metadata: file.Metadata{
MIMEType: "audio/ogg",
},
expected: []string{
@ -148,7 +148,7 @@ func Test_toFileTypes(t *testing.T) {
},
{
name: "video",
metadata: source.FileMetadata{
metadata: file.Metadata{
MIMEType: "video/3gpp",
},
expected: []string{
@ -157,7 +157,7 @@ func Test_toFileTypes(t *testing.T) {
},
{
name: "text",
metadata: source.FileMetadata{
metadata: file.Metadata{
MIMEType: "text/html",
},
expected: []string{
@ -166,7 +166,7 @@ func Test_toFileTypes(t *testing.T) {
},
{
name: "image",
metadata: source.FileMetadata{
metadata: file.Metadata{
MIMEType: "image/png",
},
expected: []string{
@ -175,7 +175,7 @@ func Test_toFileTypes(t *testing.T) {
},
{
name: "binary",
metadata: source.FileMetadata{
metadata: file.Metadata{
MIMEType: "application/x-sharedlib",
},
expected: []string{
@ -276,7 +276,7 @@ func Test_fileIDsForPackage(t *testing.T) {
Name: "bogus",
}
c := source.Coordinates{
c := file.Coordinates{
RealPath: "/path",
FileSystemID: "nowhere",
}

View File

@ -35,8 +35,8 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) {
Source: src,
Artifacts: sbom.Artifacts{
Packages: pkg.NewCollection(),
FileMetadata: map[source.Coordinates]source.FileMetadata{},
FileDigests: map[source.Coordinates][]file.Digest{},
FileMetadata: map[file.Coordinates]file.Metadata{},
FileDigests: map[file.Coordinates][]file.Digest{},
LinuxDistribution: findLinuxReleaseByPURL(doc),
},
}
@ -135,7 +135,7 @@ func toFileDigests(f *spdx.File) (digests []file.Digest) {
return digests
}
func toFileMetadata(f *spdx.File) (meta source.FileMetadata) {
func toFileMetadata(f *spdx.File) (meta file.Metadata) {
// FIXME Syft is currently lossy due to the SPDX 2.2.1 spec not supporting arbitrary mimetypes
for _, typ := range f.FileTypes {
switch FileType(typ) {
@ -169,7 +169,7 @@ func toSyftRelationships(spdxIDMap map[string]interface{}, doc *spdx.Document) [
b := spdxIDMap[string(r.RefB.ElementRefID)]
from, fromOk := a.(*pkg.Package)
toPackage, toPackageOk := b.(*pkg.Package)
toLocation, toLocationOk := b.(*source.Location)
toLocation, toLocationOk := b.(*file.Location)
if !fromOk || !(toPackageOk || toLocationOk) {
log.Debugf("unable to find valid relationship mapping from SPDX 2.2 JSON, ignoring: (from: %+v) (to: %+v)", a, b)
continue
@ -212,7 +212,7 @@ func toSyftRelationships(spdxIDMap map[string]interface{}, doc *spdx.Document) [
return out
}
func toSyftCoordinates(f *spdx.File) source.Coordinates {
func toSyftCoordinates(f *spdx.File) file.Coordinates {
const layerIDPrefix = "layerID: "
var fileSystemID string
if strings.Index(f.FileComment, layerIDPrefix) == 0 {
@ -221,14 +221,14 @@ func toSyftCoordinates(f *spdx.File) source.Coordinates {
if strings.Index(string(f.FileSPDXIdentifier), layerIDPrefix) == 0 {
fileSystemID = strings.TrimPrefix(string(f.FileSPDXIdentifier), layerIDPrefix)
}
return source.Coordinates{
return file.Coordinates{
RealPath: f.FileName,
FileSystemID: fileSystemID,
}
}
func toSyftLocation(f *spdx.File) *source.Location {
l := source.NewVirtualLocationFromCoordinates(toSyftCoordinates(f), f.FileName)
func toSyftLocation(f *spdx.File) *file.Location {
l := file.NewVirtualLocationFromCoordinates(toSyftCoordinates(f), f.FileName)
return &l
}

View File

@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
@ -336,7 +337,7 @@ func Test_toSyftRelationships(t *testing.T) {
}
pkg3.SetID()
loc1 := source.NewLocationFromCoordinates(source.Coordinates{
loc1 := file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/somewhere/real",
FileSystemID: "abc",
})

View File

@ -7,6 +7,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
@ -35,8 +36,8 @@ func Test_toGithubModel(t *testing.T) {
{
Name: "pkg-1",
Version: "1.0.1",
Locations: source.NewLocationSet(
source.NewLocationFromCoordinates(source.Coordinates{
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/usr/lib",
FileSystemID: "fsid-1",
}),
@ -45,8 +46,8 @@ func Test_toGithubModel(t *testing.T) {
{
Name: "pkg-2",
Version: "2.0.2",
Locations: source.NewLocationSet(
source.NewLocationFromCoordinates(source.Coordinates{
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/usr/lib",
FileSystemID: "fsid-1",
}),
@ -55,8 +56,8 @@ func Test_toGithubModel(t *testing.T) {
{
Name: "pkg-3",
Version: "3.0.3",
Locations: source.NewLocationSet(
source.NewLocationFromCoordinates(source.Coordinates{
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/etc",
FileSystemID: "fsid-1",
}),

View File

@ -17,6 +17,7 @@ import (
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
@ -155,8 +156,8 @@ func populateImageCatalog(catalog *pkg.Collection, img *image.Image) {
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Locations: source.NewLocationSet(
source.NewLocationFromImage(string(ref1.RealPath), *ref1.Reference, img),
Locations: file.NewLocationSet(
file.NewLocationFromImage(string(ref1.RealPath), *ref1.Reference, img),
),
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
@ -177,8 +178,8 @@ func populateImageCatalog(catalog *pkg.Collection, img *image.Image) {
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Locations: source.NewLocationSet(
source.NewLocationFromImage(string(ref2.RealPath), *ref2.Reference, img),
Locations: file.NewLocationSet(
file.NewLocationFromImage(string(ref2.RealPath), *ref2.Reference, img),
),
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
@ -265,8 +266,8 @@ func newDirectoryCatalog() *pkg.Collection {
Version: "1.0.1",
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Locations: source.NewLocationSet(
source.NewLocation("/some/path/pkg1"),
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
@ -292,8 +293,8 @@ func newDirectoryCatalog() *pkg.Collection {
Version: "2.0.1",
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Locations: source.NewLocationSet(
source.NewLocation("/some/path/pkg1"),
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
@ -318,8 +319,8 @@ func newDirectoryCatalogWithAuthorField() *pkg.Collection {
Version: "1.0.1",
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Locations: source.NewLocationSet(
source.NewLocation("/some/path/pkg1"),
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
@ -346,8 +347,8 @@ func newDirectoryCatalogWithAuthorField() *pkg.Collection {
Version: "2.0.1",
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Locations: source.NewLocationSet(
source.NewLocation("/some/path/pkg1"),
Locations: file.NewLocationSet(
file.NewLocation("/some/path/pkg1"),
),
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
@ -366,15 +367,15 @@ func newDirectoryCatalogWithAuthorField() *pkg.Collection {
//nolint:gosec
func AddSampleFileRelationships(s *sbom.SBOM) {
catalog := s.Artifacts.Packages.Sorted()
s.Artifacts.FileMetadata = map[source.Coordinates]source.FileMetadata{}
s.Artifacts.FileMetadata = map[file.Coordinates]file.Metadata{}
files := []string{"/f1", "/f2", "/d1/f3", "/d2/f4", "/z1/f5", "/a1/f6"}
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
rnd.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] })
for _, f := range files {
meta := source.FileMetadata{}
coords := source.Coordinates{RealPath: f}
meta := file.Metadata{}
coords := file.Coordinates{RealPath: f}
s.Artifacts.FileMetadata[coords] = meta
s.Relationships = append(s.Relationships, artifact.Relationship{

View File

@ -52,8 +52,8 @@ func TestEncodeFullJSONDocument(t *testing.T) {
p1 := pkg.Package{
Name: "package-1",
Version: "1.0.1",
Locations: source.NewLocationSet(
source.NewLocationFromCoordinates(source.Coordinates{
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/a/place/a",
}),
),
@ -76,8 +76,8 @@ func TestEncodeFullJSONDocument(t *testing.T) {
p2 := pkg.Package{
Name: "package-2",
Version: "2.0.1",
Locations: source.NewLocationSet(
source.NewLocationFromCoordinates(source.Coordinates{
Locations: file.NewLocationSet(
file.NewLocationFromCoordinates(file.Coordinates{
RealPath: "/b/place/b",
}),
),
@ -101,8 +101,8 @@ func TestEncodeFullJSONDocument(t *testing.T) {
s := sbom.SBOM{
Artifacts: sbom.Artifacts{
Packages: catalog,
FileMetadata: map[source.Coordinates]source.FileMetadata{
source.NewLocation("/a/place").Coordinates: {
FileMetadata: map[file.Coordinates]file.Metadata{
file.NewLocation("/a/place").Coordinates: {
FileInfo: stereoFile.ManualInfo{
NameValue: "/a/place",
ModeValue: 0775,
@ -111,7 +111,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
UserID: 0,
GroupID: 0,
},
source.NewLocation("/a/place/a").Coordinates: {
file.NewLocation("/a/place/a").Coordinates: {
FileInfo: stereoFile.ManualInfo{
NameValue: "/a/place/a",
ModeValue: 0775,
@ -120,7 +120,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
UserID: 0,
GroupID: 0,
},
source.NewLocation("/b").Coordinates: {
file.NewLocation("/b").Coordinates: {
FileInfo: stereoFile.ManualInfo{
NameValue: "/b",
ModeValue: 0775,
@ -130,7 +130,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
UserID: 0,
GroupID: 0,
},
source.NewLocation("/b/place/b").Coordinates: {
file.NewLocation("/b/place/b").Coordinates: {
FileInfo: stereoFile.ManualInfo{
NameValue: "/b/place/b",
ModeValue: 0644,
@ -140,22 +140,22 @@ func TestEncodeFullJSONDocument(t *testing.T) {
GroupID: 2,
},
},
FileDigests: map[source.Coordinates][]file.Digest{
source.NewLocation("/a/place/a").Coordinates: {
FileDigests: map[file.Coordinates][]file.Digest{
file.NewLocation("/a/place/a").Coordinates: {
{
Algorithm: "sha256",
Value: "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
},
},
source.NewLocation("/b/place/b").Coordinates: {
file.NewLocation("/b/place/b").Coordinates: {
{
Algorithm: "sha256",
Value: "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c",
},
},
},
FileContents: map[source.Coordinates]string{
source.NewLocation("/a/place/a").Coordinates: "the-contents",
FileContents: map[file.Coordinates]string{
file.NewLocation("/a/place/a").Coordinates: "the-contents",
},
LinuxDistribution: &linux.Release{
ID: "redhat",

View File

@ -2,12 +2,11 @@ package model
import (
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source"
)
type File struct {
ID string `json:"id"`
Location source.Coordinates `json:"location"`
Location file.Coordinates `json:"location"`
Metadata *FileMetadataEntry `json:"metadata,omitempty"`
Contents string `json:"contents,omitempty"`
Digests []file.Digest `json:"digests,omitempty"`

View File

@ -7,9 +7,9 @@ import (
"reflect"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/license"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
var errUnknownMetadataType = errors.New("unknown metadata type")
@ -22,26 +22,26 @@ type Package struct {
// PackageBasicData contains non-ambiguous values (type-wise) from pkg.Package.
type PackageBasicData struct {
ID string `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
Type pkg.Type `json:"type"`
FoundBy string `json:"foundBy"`
Locations []source.Location `json:"locations"`
Licenses licenses `json:"licenses"`
Language pkg.Language `json:"language"`
CPEs []string `json:"cpes"`
PURL string `json:"purl"`
ID string `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
Type pkg.Type `json:"type"`
FoundBy string `json:"foundBy"`
Locations []file.Location `json:"locations"`
Licenses licenses `json:"licenses"`
Language pkg.Language `json:"language"`
CPEs []string `json:"cpes"`
PURL string `json:"purl"`
}
type licenses []License
type License struct {
Value string `json:"value"`
SPDXExpression string `json:"spdxExpression"`
Type license.Type `json:"type"`
URLs []string `json:"urls"`
Locations []source.Location `json:"locations"`
Value string `json:"value"`
SPDXExpression string `json:"spdxExpression"`
Type license.Type `json:"type"`
URLs []string `json:"urls"`
Locations []file.Location `json:"locations"`
}
func newModelLicensesFromValues(licenses []string) (ml []License) {

View File

@ -2,10 +2,9 @@ package model
import (
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source"
)
type Secrets struct {
Location source.Coordinates `json:"location"`
Location file.Coordinates `json:"location"`
Secrets []file.SearchResult `json:"secrets"`
}

View File

@ -74,7 +74,7 @@ func toDescriptor(d sbom.Descriptor) model.Descriptor {
}
}
func toSecrets(data map[source.Coordinates][]file.SearchResult) []model.Secrets {
func toSecrets(data map[file.Coordinates][]file.SearchResult) []model.Secrets {
results := make([]model.Secrets, 0)
for coordinates, secrets := range data {
results = append(results, model.Secrets{
@ -95,7 +95,7 @@ func toFile(s sbom.SBOM) []model.File {
artifacts := s.Artifacts
for _, coordinates := range s.AllCoordinates() {
var metadata *source.FileMetadata
var metadata *file.Metadata
if metadataForLocation, exists := artifacts.FileMetadata[coordinates]; exists {
metadata = &metadataForLocation
}
@ -126,7 +126,7 @@ func toFile(s sbom.SBOM) []model.File {
return results
}
func toFileMetadataEntry(coordinates source.Coordinates, metadata *source.FileMetadata) *model.FileMetadataEntry {
func toFileMetadataEntry(coordinates file.Coordinates, metadata *file.Metadata) *model.FileMetadataEntry {
if metadata == nil {
return nil
}
@ -195,7 +195,7 @@ func toPackageModels(catalog *pkg.Collection) []model.Package {
func toLicenseModel(pkgLicenses []pkg.License) (modelLicenses []model.License) {
for _, l := range pkgLicenses {
// guarantee collection
locations := make([]source.Location, 0)
locations := make([]file.Location, 0)
if v := l.Locations.ToSlice(); v != nil {
locations = v
}

View File

@ -7,7 +7,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/formats/syftjson/model"
"github.com/anchore/syft/syft/source"
)
@ -94,46 +95,46 @@ func Test_toSourceModel(t *testing.T) {
func Test_toFileType(t *testing.T) {
badType := file.Type(0x1337)
var allTypesTested []file.Type
badType := stereoscopeFile.Type(0x1337)
var allTypesTested []stereoscopeFile.Type
tests := []struct {
ty file.Type
ty stereoscopeFile.Type
name string
}{
{
ty: file.TypeRegular,
ty: stereoscopeFile.TypeRegular,
name: "RegularFile",
},
{
ty: file.TypeDirectory,
ty: stereoscopeFile.TypeDirectory,
name: "Directory",
},
{
ty: file.TypeSymLink,
ty: stereoscopeFile.TypeSymLink,
name: "SymbolicLink",
},
{
ty: file.TypeHardLink,
ty: stereoscopeFile.TypeHardLink,
name: "HardLink",
},
{
ty: file.TypeSocket,
ty: stereoscopeFile.TypeSocket,
name: "Socket",
},
{
ty: file.TypeCharacterDevice,
ty: stereoscopeFile.TypeCharacterDevice,
name: "CharacterDevice",
},
{
ty: file.TypeBlockDevice,
ty: stereoscopeFile.TypeBlockDevice,
name: "BlockDevice",
},
{
ty: file.TypeFIFO,
ty: stereoscopeFile.TypeFIFO,
name: "FIFONode",
},
{
ty: file.TypeIrregular,
ty: stereoscopeFile.TypeIrregular,
name: "IrregularFile",
},
{
@ -150,17 +151,17 @@ func Test_toFileType(t *testing.T) {
})
}
assert.ElementsMatch(t, allTypesTested, file.AllTypes(), "not all file.Types are under test")
assert.ElementsMatch(t, allTypesTested, stereoscopeFile.AllTypes(), "not all file.Types are under test")
}
func Test_toFileMetadataEntry(t *testing.T) {
coords := source.Coordinates{
coords := file.Coordinates{
RealPath: "/path",
FileSystemID: "x",
}
tests := []struct {
name string
metadata *source.FileMetadata
metadata *file.Metadata
want *model.FileMetadataEntry
}{
{
@ -168,23 +169,23 @@ func Test_toFileMetadataEntry(t *testing.T) {
},
{
name: "no file info",
metadata: &source.FileMetadata{
metadata: &file.Metadata{
FileInfo: nil,
},
want: &model.FileMetadataEntry{
Type: file.TypeRegular.String(),
Type: stereoscopeFile.TypeRegular.String(),
},
},
{
name: "with file info",
metadata: &source.FileMetadata{
FileInfo: &file.ManualInfo{
metadata: &file.Metadata{
FileInfo: &stereoscopeFile.ManualInfo{
ModeValue: 1,
},
},
want: &model.FileMetadataEntry{
Mode: 1,
Type: file.TypeRegular.String(),
Type: stereoscopeFile.TypeRegular.String(),
},
},
}

View File

@ -64,8 +64,8 @@ func deduplicateErrors(errors []error) []string {
func toSyftFiles(files []model.File) sbom.Artifacts {
ret := sbom.Artifacts{
FileMetadata: make(map[source.Coordinates]source.FileMetadata),
FileDigests: make(map[source.Coordinates][]file.Digest),
FileMetadata: make(map[file.Coordinates]file.Metadata),
FileDigests: make(map[file.Coordinates][]file.Digest),
}
for _, f := range files {
@ -79,7 +79,7 @@ func toSyftFiles(files []model.File) sbom.Artifacts {
fm := os.FileMode(mode)
ret.FileMetadata[coord] = source.FileMetadata{
ret.FileMetadata[coord] = file.Metadata{
FileInfo: stereoscopeFile.ManualInfo{
NameValue: path.Base(coord.RealPath),
SizeValue: f.Metadata.Size,
@ -112,7 +112,7 @@ func toSyftLicenses(m []model.License) (p []pkg.License) {
SPDXExpression: l.SPDXExpression,
Type: l.Type,
URLs: internal.NewStringSet(l.URLs...),
Locations: source.NewLocationSet(l.Locations...),
Locations: file.NewLocationSet(l.Locations...),
})
}
return
@ -320,7 +320,7 @@ func toSyftPackage(p model.Package, idAliases map[string]string) pkg.Package {
Name: p.Name,
Version: p.Version,
FoundBy: p.FoundBy,
Locations: source.NewLocationSet(p.Locations...),
Locations: file.NewLocationSet(p.Locations...),
Licenses: pkg.NewLicenseSet(toSyftLicenses(p.Licenses)...),
Language: p.Language,
Type: p.Type,

View File

@ -131,7 +131,7 @@ func Test_idsHaveChanged(t *testing.T) {
}
func Test_toSyftFiles(t *testing.T) {
coord := source.Coordinates{
coord := file.Coordinates{
RealPath: "/somerwhere/place",
FileSystemID: "abc",
}
@ -145,8 +145,8 @@ func Test_toSyftFiles(t *testing.T) {
name: "empty",
files: []model.File{},
want: sbom.Artifacts{
FileMetadata: map[source.Coordinates]source.FileMetadata{},
FileDigests: map[source.Coordinates][]file.Digest{},
FileMetadata: map[file.Coordinates]file.Metadata{},
FileDigests: map[file.Coordinates][]file.Digest{},
},
},
{
@ -165,8 +165,8 @@ func Test_toSyftFiles(t *testing.T) {
},
},
want: sbom.Artifacts{
FileMetadata: map[source.Coordinates]source.FileMetadata{},
FileDigests: map[source.Coordinates][]file.Digest{
FileMetadata: map[file.Coordinates]file.Metadata{},
FileDigests: map[file.Coordinates][]file.Digest{
coord: {
{
Algorithm: "sha256",
@ -200,7 +200,7 @@ func Test_toSyftFiles(t *testing.T) {
},
},
want: sbom.Artifacts{
FileMetadata: map[source.Coordinates]source.FileMetadata{
FileMetadata: map[file.Coordinates]file.Metadata{
coord: {
FileInfo: stereoFile.ManualInfo{
NameValue: "place",
@ -215,7 +215,7 @@ func Test_toSyftFiles(t *testing.T) {
MIMEType: "text/plain",
},
},
FileDigests: map[source.Coordinates][]file.Digest{
FileDigests: map[file.Coordinates][]file.Digest{
coord: {
{
Algorithm: "sha256",

View File

@ -1,25 +1,26 @@
package source
package fileresolver
import (
"fmt"
"io"
"github.com/anchore/stereoscope/pkg/file"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
)
var _ FileResolver = (*imageAllLayersResolver)(nil)
var _ file.Resolver = (*ContainerImageAllLayers)(nil)
// imageAllLayersResolver implements path and content access for the AllLayers source option for container image data sources.
type imageAllLayersResolver struct {
// ContainerImageAllLayers implements path and content access for the AllLayers source option for container image data sources.
type ContainerImageAllLayers struct {
img *image.Image
layers []int
}
// newAllLayersResolver returns a new resolver from the perspective of all image layers for the given image.
func newAllLayersResolver(img *image.Image) (*imageAllLayersResolver, error) {
// NewFromContainerImageAllLayers returns a new resolver from the perspective of all image layers for the given image.
func NewFromContainerImageAllLayers(img *image.Image) (*ContainerImageAllLayers, error) {
if len(img.Layers) == 0 {
return nil, fmt.Errorf("the image does not contain any layers")
}
@ -28,15 +29,15 @@ func newAllLayersResolver(img *image.Image) (*imageAllLayersResolver, error) {
for idx := range img.Layers {
layers = append(layers, idx)
}
return &imageAllLayersResolver{
return &ContainerImageAllLayers{
img: img,
layers: layers,
}, nil
}
// HasPath indicates if the given path exists in the underlying source.
func (r *imageAllLayersResolver) HasPath(path string) bool {
p := file.Path(path)
func (r *ContainerImageAllLayers) HasPath(path string) bool {
p := stereoscopeFile.Path(path)
for _, layerIdx := range r.layers {
tree := r.img.Layers[layerIdx].Tree
if tree.HasPath(p) {
@ -46,8 +47,8 @@ func (r *imageAllLayersResolver) HasPath(path string) bool {
return false
}
func (r *imageAllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) {
uniqueFiles := make([]file.Reference, 0)
func (r *ContainerImageAllLayers) fileByRef(ref stereoscopeFile.Reference, uniqueFileIDs stereoscopeFile.ReferenceSet, layerIdx int) ([]stereoscopeFile.Reference, error) {
uniqueFiles := make([]stereoscopeFile.Reference, 0)
// since there is potentially considerable work for each symlink/hardlink that needs to be resolved, let's check to see if this is a symlink/hardlink first
entry, err := r.img.FileCatalog.Get(ref)
@ -55,7 +56,7 @@ func (r *imageAllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs fil
return nil, fmt.Errorf("unable to fetch metadata (ref=%+v): %w", ref, err)
}
if entry.Metadata.Type == file.TypeHardLink || entry.Metadata.Type == file.TypeSymLink {
if entry.Metadata.Type == stereoscopeFile.TypeHardLink || entry.Metadata.Type == stereoscopeFile.TypeSymLink {
// a link may resolve in this layer or higher, assuming a squashed tree is used to search
// we should search all possible resolutions within the valid source
for _, subLayerIdx := range r.layers[layerIdx:] {
@ -77,9 +78,9 @@ func (r *imageAllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs fil
}
// FilesByPath returns all file.References that match the given paths from any layer in the image.
func (r *imageAllLayersResolver) FilesByPath(paths ...string) ([]Location, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueLocations := make([]Location, 0)
func (r *ContainerImageAllLayers) FilesByPath(paths ...string) ([]file.Location, error) {
uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]file.Location, 0)
for _, path := range paths {
for idx, layerIdx := range r.layers {
@ -110,7 +111,7 @@ func (r *imageAllLayersResolver) FilesByPath(paths ...string) ([]Location, error
return nil, err
}
for _, result := range results {
uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, result, r.img))
uniqueLocations = append(uniqueLocations, file.NewLocationFromImage(path, result, r.img))
}
}
}
@ -119,9 +120,9 @@ func (r *imageAllLayersResolver) FilesByPath(paths ...string) ([]Location, error
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
// nolint:gocognit
func (r *imageAllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueLocations := make([]Location, 0)
func (r *ContainerImageAllLayers) FilesByGlob(patterns ...string) ([]file.Location, error) {
uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]file.Location, 0)
for _, pattern := range patterns {
for idx, layerIdx := range r.layers {
@ -153,7 +154,7 @@ func (r *imageAllLayersResolver) FilesByGlob(patterns ...string) ([]Location, er
return nil, err
}
for _, refResult := range refResults {
uniqueLocations = append(uniqueLocations, NewLocationFromImage(string(result.RequestPath), refResult, r.img))
uniqueLocations = append(uniqueLocations, file.NewLocationFromImage(string(result.RequestPath), refResult, r.img))
}
}
}
@ -164,10 +165,10 @@ func (r *imageAllLayersResolver) FilesByGlob(patterns ...string) ([]Location, er
// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
// This is helpful when attempting to find a file that is in the same layer or lower as another file.
func (r *imageAllLayersResolver) RelativeFileByPath(location Location, path string) *Location {
layer := r.img.FileCatalog.Layer(location.ref)
func (r *ContainerImageAllLayers) RelativeFileByPath(location file.Location, path string) *file.Location {
layer := r.img.FileCatalog.Layer(location.Reference())
exists, relativeRef, err := layer.SquashedTree.File(file.Path(path), filetree.FollowBasenameLinks)
exists, relativeRef, err := layer.SquashedTree.File(stereoscopeFile.Path(path), filetree.FollowBasenameLinks)
if err != nil {
log.Errorf("failed to find path=%q in squash: %+w", path, err)
return nil
@ -176,21 +177,21 @@ func (r *imageAllLayersResolver) RelativeFileByPath(location Location, path stri
return nil
}
relativeLocation := NewLocationFromImage(path, *relativeRef.Reference, r.img)
relativeLocation := file.NewLocationFromImage(path, *relativeRef.Reference, r.img)
return &relativeLocation
}
// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
// If the path does not exist an error is returned.
func (r *imageAllLayersResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
entry, err := r.img.FileCatalog.Get(location.ref)
func (r *ContainerImageAllLayers) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
entry, err := r.img.FileCatalog.Get(location.Reference())
if err != nil {
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err)
}
switch entry.Metadata.Type {
case file.TypeSymLink, file.TypeHardLink:
case stereoscopeFile.TypeSymLink, stereoscopeFile.TypeHardLink:
// the location we are searching may be a symlink, we should always work with the resolved file
newLocation := r.RelativeFileByPath(location, location.VirtualPath)
if newLocation == nil {
@ -198,16 +199,16 @@ func (r *imageAllLayersResolver) FileContentsByLocation(location Location) (io.R
return nil, fmt.Errorf("no contents for location=%q", location.VirtualPath)
}
location = *newLocation
case file.TypeDirectory:
return nil, fmt.Errorf("cannot read contents of non-file %q", location.ref.RealPath)
case stereoscopeFile.TypeDirectory:
return nil, fmt.Errorf("cannot read contents of non-file %q", location.Reference().RealPath)
}
return r.img.FileContentsByRef(location.ref)
return r.img.OpenReference(location.Reference())
}
func (r *imageAllLayersResolver) FilesByMIMEType(types ...string) ([]Location, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueLocations := make([]Location, 0)
func (r *ContainerImageAllLayers) FilesByMIMEType(types ...string) ([]file.Location, error) {
uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]file.Location, 0)
for idx, layerIdx := range r.layers {
refs, err := r.img.Layers[layerIdx].SearchContext.SearchByMIMEType(types...)
@ -225,7 +226,7 @@ func (r *imageAllLayersResolver) FilesByMIMEType(types ...string) ([]Location, e
return nil, err
}
for _, refResult := range refResults {
uniqueLocations = append(uniqueLocations, NewLocationFromImage(string(ref.RequestPath), refResult, r.img))
uniqueLocations = append(uniqueLocations, file.NewLocationFromImage(string(ref.RequestPath), refResult, r.img))
}
}
}
@ -233,20 +234,20 @@ func (r *imageAllLayersResolver) FilesByMIMEType(types ...string) ([]Location, e
return uniqueLocations, nil
}
func (r *imageAllLayersResolver) AllLocations() <-chan Location {
results := make(chan Location)
func (r *ContainerImageAllLayers) AllLocations() <-chan file.Location {
results := make(chan file.Location)
go func() {
defer close(results)
for _, layerIdx := range r.layers {
tree := r.img.Layers[layerIdx].Tree
for _, ref := range tree.AllFiles(file.AllTypes()...) {
results <- NewLocationFromImage(string(ref.RealPath), ref, r.img)
for _, ref := range tree.AllFiles(stereoscopeFile.AllTypes()...) {
results <- file.NewLocationFromImage(string(ref.RealPath), ref, r.img)
}
}
}()
return results
}
func (r *imageAllLayersResolver) FileMetadataByLocation(location Location) (FileMetadata, error) {
func (r *ContainerImageAllLayers) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
return fileMetadataByLocation(r.img, location)
}

View File

@ -1,4 +1,4 @@
package source
package fileresolver
import (
"fmt"
@ -13,6 +13,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/file"
)
type resolution struct {
@ -93,7 +94,7 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
t.Run(c.name, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newAllLayersResolver(img)
resolver, err := NewFromContainerImageAllLayers(img)
if err != nil {
t.Fatalf("could not create resolver: %+v", err)
}
@ -121,15 +122,15 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
for idx, actual := range refs {
expected := c.resolutions[idx]
if string(actual.ref.RealPath) != expected.path {
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path)
if string(actual.Reference().RealPath) != expected.path {
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.Reference().RealPath), expected.path)
}
if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath {
if expected.path != "" && string(actual.Reference().RealPath) != actual.RealPath {
t.Errorf("we should always prefer real paths over ones with links")
}
layer := img.FileCatalog.Layer(actual.ref)
layer := img.FileCatalog.Layer(actual.Reference())
if layer.Metadata.Index != expected.layer {
t.Errorf("bad resolve layer: '%d'!='%d'", layer.Metadata.Index, expected.layer)
}
@ -207,7 +208,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
t.Run(c.name, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newAllLayersResolver(img)
resolver, err := NewFromContainerImageAllLayers(img)
if err != nil {
t.Fatalf("could not create resolver: %+v", err)
}
@ -224,15 +225,15 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
for idx, actual := range refs {
expected := c.resolutions[idx]
if string(actual.ref.RealPath) != expected.path {
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path)
if string(actual.Reference().RealPath) != expected.path {
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.Reference().RealPath), expected.path)
}
if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath {
if expected.path != "" && string(actual.Reference().RealPath) != actual.RealPath {
t.Errorf("we should always prefer real paths over ones with links")
}
layer := img.FileCatalog.Layer(actual.ref)
layer := img.FileCatalog.Layer(actual.Reference())
if layer.Metadata.Index != expected.layer {
t.Errorf("bad resolve layer: '%d'!='%d'", layer.Metadata.Index, expected.layer)
@ -259,7 +260,7 @@ func Test_imageAllLayersResolver_FilesByMIMEType(t *testing.T) {
t.Run(test.fixtureName, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureName)
resolver, err := newAllLayersResolver(img)
resolver, err := NewFromContainerImageAllLayers(img)
assert.NoError(t, err)
locations, err := resolver.FilesByMIMEType(test.mimeType)
@ -276,7 +277,7 @@ func Test_imageAllLayersResolver_FilesByMIMEType(t *testing.T) {
func Test_imageAllLayersResolver_hasFilesystemIDInLocation(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-duplicate-path")
resolver, err := newAllLayersResolver(img)
resolver, err := NewFromContainerImageAllLayers(img)
assert.NoError(t, err)
locations, err := resolver.FilesByMIMEType("text/plain")
@ -336,7 +337,7 @@ func TestAllLayersImageResolver_FilesContents(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newAllLayersResolver(img)
resolver, err := NewFromContainerImageAllLayers(img)
assert.NoError(t, err)
refs, err := resolver.FilesByPath(test.fixture)
@ -363,12 +364,12 @@ func TestAllLayersImageResolver_FilesContents_errorOnDirRequest(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newAllLayersResolver(img)
resolver, err := NewFromContainerImageAllLayers(img)
assert.NoError(t, err)
var dirLoc *Location
var dirLoc *file.Location
for loc := range resolver.AllLocations() {
entry, err := resolver.img.FileCatalog.Get(loc.ref)
entry, err := resolver.img.FileCatalog.Get(loc.Reference())
require.NoError(t, err)
if entry.Metadata.IsDir() {
dirLoc = &loc
@ -386,119 +387,119 @@ func TestAllLayersImageResolver_FilesContents_errorOnDirRequest(t *testing.T) {
func Test_imageAllLayersResolver_resolvesLinks(t *testing.T) {
tests := []struct {
name string
runner func(FileResolver) []Location
expected []Location
runner func(file.Resolver) []file.Location
expected []file.Location
}{
{
name: "by mimetype",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links should not show up when searching mimetype
actualLocations, err := resolver.FilesByMIMEType("text/plain")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewVirtualLocation("/etc/group", "/etc/group"),
NewVirtualLocation("/etc/passwd", "/etc/passwd"),
NewVirtualLocation("/etc/shadow", "/etc/shadow"),
NewVirtualLocation("/file-1.txt", "/file-1.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1
expected: []file.Location{
file.NewVirtualLocation("/etc/group", "/etc/group"),
file.NewVirtualLocation("/etc/passwd", "/etc/passwd"),
file.NewVirtualLocation("/etc/shadow", "/etc/shadow"),
file.NewVirtualLocation("/file-1.txt", "/file-1.txt"),
file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1
// note: we're de-duping the redundant access to file-3.txt
// ... (there would usually be two copies)
NewVirtualLocation("/file-3.txt", "/file-3.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // copy 1
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // copy 2
file.NewVirtualLocation("/file-3.txt", "/file-3.txt"),
file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2
file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // copy 1
file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // copy 2
},
},
{
name: "by glob to links",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("*ink-*")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewVirtualLocation("/file-1.txt", "/link-1"),
NewVirtualLocation("/file-2.txt", "/link-2"), // copy 1
NewVirtualLocation("/file-2.txt", "/link-2"), // copy 2
NewVirtualLocation("/file-3.txt", "/link-within"),
expected: []file.Location{
file.NewVirtualLocation("/file-1.txt", "/link-1"),
file.NewVirtualLocation("/file-2.txt", "/link-2"), // copy 1
file.NewVirtualLocation("/file-2.txt", "/link-2"), // copy 2
file.NewVirtualLocation("/file-3.txt", "/link-within"),
},
},
{
name: "by basename",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/file-2.txt")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2
expected: []file.Location{
file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1
file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2
},
},
{
name: "by basename glob",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/file-?.txt")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewVirtualLocation("/file-1.txt", "/file-1.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2
NewVirtualLocation("/file-3.txt", "/file-3.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // when we copy into the link path, the same file-4.txt is copied
expected: []file.Location{
file.NewVirtualLocation("/file-1.txt", "/file-1.txt"),
file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1
file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2
file.NewVirtualLocation("/file-3.txt", "/file-3.txt"),
file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // when we copy into the link path, the same file-4.txt is copied
},
},
{
name: "by extension",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/*.txt")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewVirtualLocation("/file-1.txt", "/file-1.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1
NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2
NewVirtualLocation("/file-3.txt", "/file-3.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // when we copy into the link path, the same file-4.txt is copied
expected: []file.Location{
file.NewVirtualLocation("/file-1.txt", "/file-1.txt"),
file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 1
file.NewVirtualLocation("/file-2.txt", "/file-2.txt"), // copy 2
file.NewVirtualLocation("/file-3.txt", "/file-3.txt"),
file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"), // when we copy into the link path, the same file-4.txt is copied
},
},
{
name: "by path to degree 1 link",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links resolve to the final file
actualLocations, err := resolver.FilesByPath("/link-2")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
expected: []file.Location{
// we have multiple copies across layers
NewVirtualLocation("/file-2.txt", "/link-2"),
NewVirtualLocation("/file-2.txt", "/link-2"),
file.NewVirtualLocation("/file-2.txt", "/link-2"),
file.NewVirtualLocation("/file-2.txt", "/link-2"),
},
},
{
name: "by path to degree 2 link",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// multiple links resolves to the final file
actualLocations, err := resolver.FilesByPath("/link-indirect")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
expected: []file.Location{
// we have multiple copies across layers
NewVirtualLocation("/file-2.txt", "/link-indirect"),
NewVirtualLocation("/file-2.txt", "/link-indirect"),
file.NewVirtualLocation("/file-2.txt", "/link-indirect"),
file.NewVirtualLocation("/file-2.txt", "/link-indirect"),
},
},
}
@ -508,7 +509,7 @@ func Test_imageAllLayersResolver_resolvesLinks(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newAllLayersResolver(img)
resolver, err := NewFromContainerImageAllLayers(img)
assert.NoError(t, err)
actual := test.runner(resolver)
@ -527,7 +528,7 @@ func TestAllLayersResolver_AllLocations(t *testing.T) {
arch = "aarch64"
}
resolver, err := newAllLayersResolver(img)
resolver, err := NewFromContainerImageAllLayers(img)
assert.NoError(t, err)
paths := strset.New()

View File

@ -1,41 +1,42 @@
package source
package fileresolver
import (
"fmt"
"io"
"github.com/anchore/stereoscope/pkg/file"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/syft/file"
)
var _ FileResolver = (*imageSquashResolver)(nil)
var _ file.Resolver = (*ContainerImageSquash)(nil)
// imageSquashResolver implements path and content access for the Squashed source option for container image data sources.
type imageSquashResolver struct {
// ContainerImageSquash implements path and content access for the Squashed source option for container image data sources.
type ContainerImageSquash struct {
img *image.Image
}
// newImageSquashResolver returns a new resolver from the perspective of the squashed representation for the given image.
func newImageSquashResolver(img *image.Image) (*imageSquashResolver, error) {
// NewFromContainerImageSquash returns a new resolver from the perspective of the squashed representation for the given image.
func NewFromContainerImageSquash(img *image.Image) (*ContainerImageSquash, error) {
if img.SquashedTree() == nil {
return nil, fmt.Errorf("the image does not have have a squashed tree")
}
return &imageSquashResolver{
return &ContainerImageSquash{
img: img,
}, nil
}
// HasPath indicates if the given path exists in the underlying source.
func (r *imageSquashResolver) HasPath(path string) bool {
return r.img.SquashedTree().HasPath(file.Path(path))
func (r *ContainerImageSquash) HasPath(path string) bool {
return r.img.SquashedTree().HasPath(stereoscopeFile.Path(path))
}
// FilesByPath returns all file.References that match the given paths within the squashed representation of the image.
func (r *imageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueLocations := make([]Location, 0)
func (r *ContainerImageSquash) FilesByPath(paths ...string) ([]file.Location, error) {
uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]file.Location, 0)
for _, path := range paths {
ref, err := r.img.SquashedSearchContext.SearchByPath(path, filetree.FollowBasenameLinks)
@ -69,7 +70,7 @@ func (r *imageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
if resolvedRef.HasReference() && !uniqueFileIDs.Contains(*resolvedRef.Reference) {
uniqueFileIDs.Add(*resolvedRef.Reference)
uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, *resolvedRef.Reference, r.img))
uniqueLocations = append(uniqueLocations, file.NewLocationFromImage(path, *resolvedRef.Reference, r.img))
}
}
@ -78,9 +79,9 @@ func (r *imageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
// FilesByGlob returns all file.References that match the given path glob pattern within the squashed representation of the image.
// nolint:gocognit
func (r *imageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueLocations := make([]Location, 0)
func (r *ContainerImageSquash) FilesByGlob(patterns ...string) ([]file.Location, error) {
uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]file.Location, 0)
for _, pattern := range patterns {
results, err := r.img.SquashedSearchContext.SearchByGlob(pattern, filetree.FollowBasenameLinks)
@ -113,10 +114,10 @@ func (r *imageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error
return nil, fmt.Errorf("failed to find files by path (result=%+v): %w", result, err)
}
for _, resolvedLocation := range resolvedLocations {
if uniqueFileIDs.Contains(resolvedLocation.ref) {
if uniqueFileIDs.Contains(resolvedLocation.Reference()) {
continue
}
uniqueFileIDs.Add(resolvedLocation.ref)
uniqueFileIDs.Add(resolvedLocation.Reference())
uniqueLocations = append(uniqueLocations, resolvedLocation)
}
}
@ -127,8 +128,8 @@ func (r *imageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error
// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
// This is helpful when attempting to find a file that is in the same layer or lower as another file. For the
// imageSquashResolver, this is a simple path lookup.
func (r *imageSquashResolver) RelativeFileByPath(_ Location, path string) *Location {
// ContainerImageSquash, this is a simple path lookup.
func (r *ContainerImageSquash) RelativeFileByPath(_ file.Location, path string) *file.Location {
paths, err := r.FilesByPath(path)
if err != nil {
return nil
@ -142,14 +143,14 @@ func (r *imageSquashResolver) RelativeFileByPath(_ Location, path string) *Locat
// FileContentsByLocation fetches file contents for a single file reference, regardless of the source layer.
// If the path does not exist an error is returned.
func (r *imageSquashResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
entry, err := r.img.FileCatalog.Get(location.ref)
func (r *ContainerImageSquash) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
entry, err := r.img.FileCatalog.Get(location.Reference())
if err != nil {
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err)
}
switch entry.Metadata.Type {
case file.TypeSymLink, file.TypeHardLink:
case stereoscopeFile.TypeSymLink, stereoscopeFile.TypeHardLink:
// the location we are searching may be a symlink, we should always work with the resolved file
locations, err := r.FilesByPath(location.RealPath)
if err != nil {
@ -164,39 +165,39 @@ func (r *imageSquashResolver) FileContentsByLocation(location Location) (io.Read
default:
return nil, fmt.Errorf("link resolution resulted in multiple results while resolving content location: %+v", location)
}
case file.TypeDirectory:
case stereoscopeFile.TypeDirectory:
return nil, fmt.Errorf("unable to get file contents for directory: %+v", location)
}
return r.img.FileContentsByRef(location.ref)
return r.img.OpenReference(location.Reference())
}
func (r *imageSquashResolver) AllLocations() <-chan Location {
results := make(chan Location)
func (r *ContainerImageSquash) AllLocations() <-chan file.Location {
results := make(chan file.Location)
go func() {
defer close(results)
for _, ref := range r.img.SquashedTree().AllFiles(file.AllTypes()...) {
results <- NewLocationFromImage(string(ref.RealPath), ref, r.img)
for _, ref := range r.img.SquashedTree().AllFiles(stereoscopeFile.AllTypes()...) {
results <- file.NewLocationFromImage(string(ref.RealPath), ref, r.img)
}
}()
return results
}
func (r *imageSquashResolver) FilesByMIMEType(types ...string) ([]Location, error) {
func (r *ContainerImageSquash) FilesByMIMEType(types ...string) ([]file.Location, error) {
refs, err := r.img.SquashedSearchContext.SearchByMIMEType(types...)
if err != nil {
return nil, err
}
uniqueFileIDs := file.NewFileReferenceSet()
uniqueLocations := make([]Location, 0)
uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]file.Location, 0)
for _, ref := range refs {
if ref.HasReference() {
if uniqueFileIDs.Contains(*ref.Reference) {
continue
}
location := NewLocationFromImage(string(ref.RequestPath), *ref.Reference, r.img)
location := file.NewLocationFromImage(string(ref.RequestPath), *ref.Reference, r.img)
uniqueFileIDs.Add(*ref.Reference)
uniqueLocations = append(uniqueLocations, location)
@ -206,6 +207,6 @@ func (r *imageSquashResolver) FilesByMIMEType(types ...string) ([]Location, erro
return uniqueLocations, nil
}
func (r *imageSquashResolver) FileMetadataByLocation(location Location) (FileMetadata, error) {
func (r *ContainerImageSquash) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
return fileMetadataByLocation(r.img, location)
}

View File

@ -1,4 +1,4 @@
package source
package fileresolver
import (
"io"
@ -6,13 +6,12 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/file"
)
func TestImageSquashResolver_FilesByPath(t *testing.T) {
@ -73,7 +72,7 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
t.Run(c.name, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newImageSquashResolver(img)
resolver, err := NewFromContainerImageSquash(img)
if err != nil {
t.Fatalf("could not create resolver: %+v", err)
}
@ -110,15 +109,15 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
actual := refs[0]
if string(actual.ref.RealPath) != c.resolvePath {
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath)
if string(actual.Reference().RealPath) != c.resolvePath {
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.Reference().RealPath), c.resolvePath)
}
if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath {
if c.resolvePath != "" && string(actual.Reference().RealPath) != actual.RealPath {
t.Errorf("we should always prefer real paths over ones with links")
}
layer := img.FileCatalog.Layer(actual.ref)
layer := img.FileCatalog.Layer(actual.Reference())
if layer.Metadata.Index != c.resolveLayer {
t.Errorf("bad resolve layer: '%d'!='%d'", layer.Metadata.Index, c.resolveLayer)
@ -186,7 +185,7 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
t.Run(c.name, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newImageSquashResolver(img)
resolver, err := NewFromContainerImageSquash(img)
if err != nil {
t.Fatalf("could not create resolver: %+v", err)
}
@ -212,15 +211,15 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
actual := refs[0]
if string(actual.ref.RealPath) != c.resolvePath {
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath)
if string(actual.Reference().RealPath) != c.resolvePath {
t.Errorf("bad resolve path: '%s'!='%s'", string(actual.Reference().RealPath), c.resolvePath)
}
if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath {
if c.resolvePath != "" && string(actual.Reference().RealPath) != actual.RealPath {
t.Errorf("we should always prefer real paths over ones with links")
}
layer := img.FileCatalog.Layer(actual.ref)
layer := img.FileCatalog.Layer(actual.Reference())
if layer.Metadata.Index != c.resolveLayer {
t.Errorf("bad resolve layer: '%d'!='%d'", layer.Metadata.Index, c.resolveLayer)
@ -247,7 +246,7 @@ func Test_imageSquashResolver_FilesByMIMEType(t *testing.T) {
t.Run(test.fixtureName, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureName)
resolver, err := newImageSquashResolver(img)
resolver, err := NewFromContainerImageSquash(img)
assert.NoError(t, err)
locations, err := resolver.FilesByMIMEType(test.mimeType)
@ -264,7 +263,7 @@ func Test_imageSquashResolver_FilesByMIMEType(t *testing.T) {
func Test_imageSquashResolver_hasFilesystemIDInLocation(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-duplicate-path")
resolver, err := newImageSquashResolver(img)
resolver, err := NewFromContainerImageSquash(img)
assert.NoError(t, err)
locations, err := resolver.FilesByMIMEType("text/plain")
@ -322,7 +321,7 @@ func TestSquashImageResolver_FilesContents(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newImageSquashResolver(img)
resolver, err := NewFromContainerImageSquash(img)
assert.NoError(t, err)
refs, err := resolver.FilesByPath(test.path)
@ -347,12 +346,12 @@ func TestSquashImageResolver_FilesContents_errorOnDirRequest(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newImageSquashResolver(img)
resolver, err := NewFromContainerImageSquash(img)
assert.NoError(t, err)
var dirLoc *Location
var dirLoc *file.Location
for loc := range resolver.AllLocations() {
entry, err := resolver.img.FileCatalog.Get(loc.ref)
entry, err := resolver.img.FileCatalog.Get(loc.Reference())
require.NoError(t, err)
if entry.Metadata.IsDir() {
dirLoc = &loc
@ -370,162 +369,130 @@ func TestSquashImageResolver_FilesContents_errorOnDirRequest(t *testing.T) {
func Test_imageSquashResolver_resolvesLinks(t *testing.T) {
tests := []struct {
name string
runner func(FileResolver) []Location
expected []Location
runner func(file.Resolver) []file.Location
expected []file.Location
}{
{
name: "by mimetype",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links should not show up when searching mimetype
actualLocations, err := resolver.FilesByMIMEType("text/plain")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewVirtualLocation("/etc/group", "/etc/group"),
NewVirtualLocation("/etc/passwd", "/etc/passwd"),
NewVirtualLocation("/etc/shadow", "/etc/shadow"),
NewVirtualLocation("/file-1.txt", "/file-1.txt"),
NewVirtualLocation("/file-3.txt", "/file-3.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
expected: []file.Location{
file.NewVirtualLocation("/etc/group", "/etc/group"),
file.NewVirtualLocation("/etc/passwd", "/etc/passwd"),
file.NewVirtualLocation("/etc/shadow", "/etc/shadow"),
file.NewVirtualLocation("/file-1.txt", "/file-1.txt"),
file.NewVirtualLocation("/file-3.txt", "/file-3.txt"),
file.NewVirtualLocation("/file-2.txt", "/file-2.txt"),
file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
},
},
{
name: "by glob to links",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("*ink-*")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewVirtualLocation("/file-1.txt", "/link-1"),
NewVirtualLocation("/file-2.txt", "/link-2"),
expected: []file.Location{
file.NewVirtualLocation("/file-1.txt", "/link-1"),
file.NewVirtualLocation("/file-2.txt", "/link-2"),
// though this is a link, and it matches to the file, the resolver de-duplicates files
// by the real path, so it is not included in the results
//NewVirtualLocation("/file-2.txt", "/link-indirect"),
//file.NewVirtualLocation("/file-2.txt", "/link-indirect"),
NewVirtualLocation("/file-3.txt", "/link-within"),
file.NewVirtualLocation("/file-3.txt", "/link-within"),
},
},
{
name: "by basename",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/file-2.txt")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
expected: []file.Location{
// this has two copies in the base image, which overwrites the same location
NewVirtualLocation("/file-2.txt", "/file-2.txt"),
file.NewVirtualLocation("/file-2.txt", "/file-2.txt"),
},
},
{
name: "by basename glob",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/file-?.txt")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewVirtualLocation("/file-1.txt", "/file-1.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"),
NewVirtualLocation("/file-3.txt", "/file-3.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
expected: []file.Location{
file.NewVirtualLocation("/file-1.txt", "/file-1.txt"),
file.NewVirtualLocation("/file-2.txt", "/file-2.txt"),
file.NewVirtualLocation("/file-3.txt", "/file-3.txt"),
file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
},
},
{
name: "by basename glob to links",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
actualLocations, err := resolver.FilesByGlob("**/link-*")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
expected: []file.Location{
file.NewVirtualLocation("/file-1.txt", "/link-1"),
file.NewVirtualLocation("/file-2.txt", "/link-2"),
{
LocationData: LocationData{
Coordinates: Coordinates{
RealPath: "/file-1.txt",
},
VirtualPath: "/link-1",
ref: file.Reference{RealPath: "/file-1.txt"},
},
},
{
LocationData: LocationData{
Coordinates: Coordinates{
RealPath: "/file-2.txt",
},
VirtualPath: "/link-2",
ref: file.Reference{RealPath: "/file-2.txt"},
},
},
// we already have this real file path via another link, so only one is returned
//{
// LocationData: LocationData{
// Coordinates: Coordinates{
// RealPath: "/file-2.txt",
// },
// VirtualPath: "/link-indirect",
// ref: file.Reference{RealPath: "/file-2.txt"},
// },
//},
{
LocationData: LocationData{
Coordinates: Coordinates{
RealPath: "/file-3.txt",
},
VirtualPath: "/link-within",
ref: file.Reference{RealPath: "/file-3.txt"},
},
},
// file.NewVirtualLocation("/file-2.txt", "/link-indirect"),
file.NewVirtualLocation("/file-3.txt", "/link-within"),
},
},
{
name: "by extension",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/*.txt")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewVirtualLocation("/file-1.txt", "/file-1.txt"),
NewVirtualLocation("/file-2.txt", "/file-2.txt"),
NewVirtualLocation("/file-3.txt", "/file-3.txt"),
NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
expected: []file.Location{
file.NewVirtualLocation("/file-1.txt", "/file-1.txt"),
file.NewVirtualLocation("/file-2.txt", "/file-2.txt"),
file.NewVirtualLocation("/file-3.txt", "/file-3.txt"),
file.NewVirtualLocation("/parent/file-4.txt", "/parent/file-4.txt"),
},
},
{
name: "by path to degree 1 link",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links resolve to the final file
actualLocations, err := resolver.FilesByPath("/link-2")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
expected: []file.Location{
// we have multiple copies across layers
NewVirtualLocation("/file-2.txt", "/link-2"),
file.NewVirtualLocation("/file-2.txt", "/link-2"),
},
},
{
name: "by path to degree 2 link",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// multiple links resolves to the final file
actualLocations, err := resolver.FilesByPath("/link-indirect")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
expected: []file.Location{
// we have multiple copies across layers
NewVirtualLocation("/file-2.txt", "/link-indirect"),
file.NewVirtualLocation("/file-2.txt", "/link-indirect"),
},
},
}
@ -535,7 +502,7 @@ func Test_imageSquashResolver_resolvesLinks(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-symlinks")
resolver, err := newImageSquashResolver(img)
resolver, err := NewFromContainerImageSquash(img)
assert.NoError(t, err)
actual := test.runner(resolver)
@ -546,30 +513,10 @@ func Test_imageSquashResolver_resolvesLinks(t *testing.T) {
}
func compareLocations(t *testing.T, expected, actual []Location) {
t.Helper()
ignoreUnexported := cmpopts.IgnoreFields(LocationData{}, "ref")
ignoreMetadata := cmpopts.IgnoreFields(LocationMetadata{}, "Annotations")
ignoreFS := cmpopts.IgnoreFields(Coordinates{}, "FileSystemID")
sort.Sort(Locations(expected))
sort.Sort(Locations(actual))
if d := cmp.Diff(expected, actual,
ignoreUnexported,
ignoreFS,
ignoreMetadata,
); d != "" {
t.Errorf("unexpected locations (-want +got):\n%s", d)
}
}
func TestSquashResolver_AllLocations(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-files-deleted")
resolver, err := newImageSquashResolver(img)
resolver, err := NewFromContainerImageSquash(img)
assert.NoError(t, err)
paths := strset.New()

View File

@ -0,0 +1,98 @@
package fileresolver
import (
"io"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
)
var _ file.Resolver = (*Deferred)(nil)
func NewDeferred(creator func() (file.Resolver, error)) *Deferred {
return &Deferred{
creator: creator,
}
}
type Deferred struct {
creator func() (file.Resolver, error)
resolver file.Resolver
}
func (d *Deferred) getResolver() (file.Resolver, error) {
if d.resolver == nil {
resolver, err := d.creator()
if err != nil {
return nil, err
}
d.resolver = resolver
}
return d.resolver, nil
}
func (d *Deferred) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
r, err := d.getResolver()
if err != nil {
return nil, err
}
return r.FileContentsByLocation(location)
}
func (d *Deferred) HasPath(s string) bool {
r, err := d.getResolver()
if err != nil {
log.Debug("unable to get resolver: %v", err)
return false
}
return r.HasPath(s)
}
func (d *Deferred) FilesByPath(paths ...string) ([]file.Location, error) {
r, err := d.getResolver()
if err != nil {
return nil, err
}
return r.FilesByPath(paths...)
}
func (d *Deferred) FilesByGlob(patterns ...string) ([]file.Location, error) {
r, err := d.getResolver()
if err != nil {
return nil, err
}
return r.FilesByGlob(patterns...)
}
func (d *Deferred) FilesByMIMEType(types ...string) ([]file.Location, error) {
r, err := d.getResolver()
if err != nil {
return nil, err
}
return r.FilesByMIMEType(types...)
}
func (d *Deferred) RelativeFileByPath(location file.Location, path string) *file.Location {
r, err := d.getResolver()
if err != nil {
return nil
}
return r.RelativeFileByPath(location, path)
}
func (d *Deferred) AllLocations() <-chan file.Location {
r, err := d.getResolver()
if err != nil {
log.Debug("unable to get resolver: %v", err)
return nil
}
return r.AllLocations()
}
func (d *Deferred) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
r, err := d.getResolver()
if err != nil {
return file.Metadata{}, err
}
return r.FileMetadataByLocation(location)
}

View File

@ -1,17 +1,19 @@
package source
package fileresolver
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/file"
)
func Test_NewDeferredResolver(t *testing.T) {
creatorCalled := false
deferredResolver := NewDeferredResolver(func() (FileResolver, error) {
deferredResolver := NewDeferred(func() (file.Resolver, error) {
creatorCalled = true
return NewMockResolverForPaths(), nil
return file.NewMockResolverForPaths(), nil
})
require.False(t, creatorCalled)

View File

@ -1,4 +1,4 @@
package source
package fileresolver
import (
"errors"
@ -10,9 +10,10 @@ import (
"runtime"
"strings"
"github.com/anchore/stereoscope/pkg/file"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
)
const WindowsOS = "windows"
@ -23,12 +24,12 @@ var unixSystemRuntimePrefixes = []string{
"/sys",
}
var errSkipPath = errors.New("skip path")
var ErrSkipPath = errors.New("skip path")
var _ FileResolver = (*directoryResolver)(nil)
var _ file.Resolver = (*Directory)(nil)
// directoryResolver implements path and content access for the directory data source.
type directoryResolver struct {
// Directory implements path and content access for the directory data source.
type Directory struct {
path string
base string
currentWdRelativeToRoot string
@ -39,8 +40,8 @@ type directoryResolver struct {
indexer *directoryIndexer
}
func newDirectoryResolver(root string, base string, pathFilters ...pathIndexVisitor) (*directoryResolver, error) {
r, err := newDirectoryResolverWithoutIndex(root, base, pathFilters...)
func NewFromDirectory(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
r, err := newFromDirectoryWithoutIndex(root, base, pathFilters...)
if err != nil {
return nil, err
}
@ -48,7 +49,7 @@ func newDirectoryResolver(root string, base string, pathFilters ...pathIndexVisi
return r, r.buildIndex()
}
func newDirectoryResolverWithoutIndex(root string, base string, pathFilters ...pathIndexVisitor) (*directoryResolver, error) {
func newFromDirectoryWithoutIndex(root string, base string, pathFilters ...PathIndexVisitor) (*Directory, error) {
currentWD, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("could not get CWD: %w", err)
@ -87,7 +88,7 @@ func newDirectoryResolverWithoutIndex(root string, base string, pathFilters ...p
currentWdRelRoot = filepath.Clean(cleanRoot)
}
return &directoryResolver{
return &Directory{
path: cleanRoot,
base: cleanBase,
currentWd: cleanCWD,
@ -98,7 +99,7 @@ func newDirectoryResolverWithoutIndex(root string, base string, pathFilters ...p
}, nil
}
func (r *directoryResolver) buildIndex() error {
func (r *Directory) buildIndex() error {
if r.indexer == nil {
return fmt.Errorf("no directory indexer configured")
}
@ -114,7 +115,7 @@ func (r *directoryResolver) buildIndex() error {
return nil
}
func (r directoryResolver) requestPath(userPath string) (string, error) {
func (r Directory) requestPath(userPath string) (string, error) {
if filepath.IsAbs(userPath) {
// don't allow input to potentially hop above root path
userPath = path.Join(r.path, userPath)
@ -131,7 +132,7 @@ func (r directoryResolver) requestPath(userPath string) (string, error) {
return userPath, nil
}
func (r directoryResolver) responsePath(path string) string {
func (r Directory) responsePath(path string) string {
// check to see if we need to encode back to Windows from posix
if runtime.GOOS == WindowsOS {
path = posixToWindows(path)
@ -154,22 +155,22 @@ func (r directoryResolver) responsePath(path string) string {
}
// HasPath indicates if the given path exists in the underlying source.
func (r *directoryResolver) HasPath(userPath string) bool {
func (r *Directory) HasPath(userPath string) bool {
requestPath, err := r.requestPath(userPath)
if err != nil {
return false
}
return r.tree.HasPath(file.Path(requestPath))
return r.tree.HasPath(stereoscopeFile.Path(requestPath))
}
// Stringer to represent a directory path data source
func (r directoryResolver) String() string {
func (r Directory) String() string {
return fmt.Sprintf("dir:%s", r.path)
}
// FilesByPath returns all file.References that match the given paths from the directory.
func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error) {
var references = make([]Location, 0)
func (r Directory) FilesByPath(userPaths ...string) ([]file.Location, error) {
var references = make([]file.Location, 0)
for _, userPath := range userPaths {
userStrPath, err := r.requestPath(userPath)
@ -206,7 +207,7 @@ func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
if ref.HasReference() {
references = append(references,
NewVirtualLocationFromDirectory(
file.NewVirtualLocationFromDirectory(
r.responsePath(string(ref.RealPath)), // the actual path relative to the resolver root
r.responsePath(userStrPath), // the path used to access this file, relative to the resolver root
*ref.Reference,
@ -219,9 +220,9 @@ func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
}
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
func (r directoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueLocations := make([]Location, 0)
func (r Directory) FilesByGlob(patterns ...string) ([]file.Location, error) {
uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]file.Location, 0)
for _, pattern := range patterns {
refVias, err := r.searchContext.SearchByGlob(pattern, filetree.FollowBasenameLinks)
@ -242,7 +243,7 @@ func (r directoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
continue
}
loc := NewVirtualLocationFromDirectory(
loc := file.NewVirtualLocationFromDirectory(
r.responsePath(string(refVia.Reference.RealPath)), // the actual path relative to the resolver root
r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
*refVia.Reference,
@ -257,8 +258,8 @@ func (r directoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
// This is helpful when attempting to find a file that is in the same layer or lower as another file. For the
// directoryResolver, this is a simple path lookup.
func (r *directoryResolver) RelativeFileByPath(_ Location, path string) *Location {
// Directory, this is a simple path lookup.
func (r *Directory) RelativeFileByPath(_ file.Location, path string) *file.Location {
paths, err := r.FilesByPath(path)
if err != nil {
return nil
@ -272,54 +273,54 @@ func (r *directoryResolver) RelativeFileByPath(_ Location, path string) *Locatio
// FileContentsByLocation fetches file contents for a single file reference relative to a directory.
// If the path does not exist an error is returned.
func (r directoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
if location.ref.RealPath == "" {
func (r Directory) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
if location.RealPath == "" {
return nil, errors.New("empty path given")
}
entry, err := r.index.Get(location.ref)
entry, err := r.index.Get(location.Reference())
if err != nil {
return nil, err
}
// don't consider directories
if entry.Type == file.TypeDirectory {
return nil, fmt.Errorf("cannot read contents of non-file %q", location.ref.RealPath)
if entry.Type == stereoscopeFile.TypeDirectory {
return nil, fmt.Errorf("cannot read contents of non-file %q", location.Reference().RealPath)
}
// RealPath is posix so for windows directory resolver we need to translate
// to its true on disk path.
filePath := string(location.ref.RealPath)
filePath := string(location.Reference().RealPath)
if runtime.GOOS == WindowsOS {
filePath = posixToWindows(filePath)
}
return file.NewLazyReadCloser(filePath), nil
return stereoscopeFile.NewLazyReadCloser(filePath), nil
}
func (r *directoryResolver) AllLocations() <-chan Location {
results := make(chan Location)
func (r *Directory) AllLocations() <-chan file.Location {
results := make(chan file.Location)
go func() {
defer close(results)
for _, ref := range r.tree.AllFiles(file.AllTypes()...) {
results <- NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), ref)
for _, ref := range r.tree.AllFiles(stereoscopeFile.AllTypes()...) {
results <- file.NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), ref)
}
}()
return results
}
func (r *directoryResolver) FileMetadataByLocation(location Location) (FileMetadata, error) {
entry, err := r.index.Get(location.ref)
func (r *Directory) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
entry, err := r.index.Get(location.Reference())
if err != nil {
return FileMetadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
return file.Metadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
}
return entry.Metadata, nil
}
func (r *directoryResolver) FilesByMIMEType(types ...string) ([]Location, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueLocations := make([]Location, 0)
func (r *Directory) FilesByMIMEType(types ...string) ([]file.Location, error) {
uniqueFileIDs := stereoscopeFile.NewFileReferenceSet()
uniqueLocations := make([]file.Location, 0)
refVias, err := r.searchContext.SearchByMIMEType(types...)
if err != nil {
@ -332,7 +333,7 @@ func (r *directoryResolver) FilesByMIMEType(types ...string) ([]Location, error)
if uniqueFileIDs.Contains(*refVia.Reference) {
continue
}
location := NewLocationFromDirectory(
location := file.NewLocationFromDirectory(
r.responsePath(string(refVia.Reference.RealPath)),
*refVia.Reference,
)

View File

@ -1,4 +1,4 @@
package source
package fileresolver
import (
"errors"
@ -20,30 +20,30 @@ import (
"github.com/anchore/syft/syft/event"
)
type pathIndexVisitor func(string, os.FileInfo, error) error
type PathIndexVisitor func(string, os.FileInfo, error) error
type directoryIndexer struct {
path string
base string
pathIndexVisitors []pathIndexVisitor
pathIndexVisitors []PathIndexVisitor
errPaths map[string]error
tree filetree.ReadWriter
index filetree.Index
}
func newDirectoryIndexer(path, base string, visitors ...pathIndexVisitor) *directoryIndexer {
func newDirectoryIndexer(path, base string, visitors ...PathIndexVisitor) *directoryIndexer {
i := &directoryIndexer{
path: path,
base: base,
tree: filetree.New(),
index: filetree.NewIndex(),
pathIndexVisitors: append([]pathIndexVisitor{requireFileInfo, disallowByFileType, disallowUnixSystemRuntimePath}, visitors...),
pathIndexVisitors: append([]PathIndexVisitor{requireFileInfo, disallowByFileType, disallowUnixSystemRuntimePath}, visitors...),
errPaths: make(map[string]error),
}
// these additional stateful visitors should be the first thing considered when walking / indexing
i.pathIndexVisitors = append(
[]pathIndexVisitor{
[]PathIndexVisitor{
i.disallowRevisitingVisitor,
i.disallowFileAccessErr,
},
@ -181,7 +181,7 @@ func (r *directoryIndexer) indexPath(path string, info os.FileInfo, err error) (
func (r *directoryIndexer) disallowFileAccessErr(path string, _ os.FileInfo, err error) error {
if r.isFileAccessErr(path, err) {
return errSkipPath
return ErrSkipPath
}
return nil
}
@ -311,7 +311,7 @@ func (r *directoryIndexer) disallowRevisitingVisitor(path string, _ os.FileInfo,
// signal to walk() that we should skip this directory entirely
return fs.SkipDir
}
return errSkipPath
return ErrSkipPath
}
return nil
}
@ -330,7 +330,7 @@ func disallowByFileType(_ string, info os.FileInfo, _ error) error {
}
switch file.TypeFromMode(info.Mode()) {
case file.TypeCharacterDevice, file.TypeSocket, file.TypeBlockDevice, file.TypeFIFO, file.TypeIrregular:
return errSkipPath
return ErrSkipPath
// note: symlinks that point to these files may still get by.
// We handle this later in processing to help prevent against infinite links traversal.
}
@ -340,7 +340,7 @@ func disallowByFileType(_ string, info os.FileInfo, _ error) error {
func requireFileInfo(_ string, info os.FileInfo, _ error) error {
if info == nil {
return errSkipPath
return ErrSkipPath
}
return nil
}

View File

@ -1,4 +1,4 @@
package source
package fileresolver
import (
"io/fs"
@ -172,7 +172,7 @@ func TestDirectoryIndexer_indexPath_skipsNilFileInfo(t *testing.T) {
}
func TestDirectoryIndexer_index(t *testing.T) {
// note: this test is testing the effects from newDirectoryResolver, indexTree, and addPathToIndex
// note: this test is testing the effects from NewFromDirectory, indexTree, and addPathToIndex
indexer := newDirectoryIndexer("test-fixtures/system_paths/target", "")
tree, index, err := indexer.build()
require.NoError(t, err)
@ -237,7 +237,7 @@ func TestDirectoryIndexer_SkipsAlreadyVisitedLinkDestinations(t *testing.T) {
}
resolver := newDirectoryIndexer("./test-fixtures/symlinks-prune-indexing", "")
// we want to cut ahead of any possible filters to see what paths are considered for indexing (closest to walking)
resolver.pathIndexVisitors = append([]pathIndexVisitor{pathObserver}, resolver.pathIndexVisitors...)
resolver.pathIndexVisitors = append([]PathIndexVisitor{pathObserver}, resolver.pathIndexVisitors...)
// note: this test is NOT about the effects left on the tree or the index, but rather the WHICH paths that are
// considered for indexing and HOW traversal prunes paths that have already been visited

View File

@ -1,12 +1,11 @@
//go:build !windows
// +build !windows
package source
package fileresolver
import (
"io"
"io/fs"
"io/ioutil"
"os"
"path/filepath"
"sort"
@ -19,7 +18,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/file"
)
func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) {
@ -46,17 +46,18 @@ func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) {
},
},
{
name: "should find a file from a relative path (root above cwd)",
name: "should find a file from a relative path (root above cwd)",
// TODO: refactor me! this test depends on the structure of the source dir not changing, which isn't great
relativeRoot: "../",
input: "sbom/sbom.go",
input: "fileresolver/directory.go",
expected: []string{
"sbom/sbom.go",
"fileresolver/directory.go",
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, err := newDirectoryResolver(c.relativeRoot, "")
resolver, err := NewFromDirectory(c.relativeRoot, "")
assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input)
@ -95,11 +96,12 @@ func TestDirectoryResolver_FilesByPath_absoluteRoot(t *testing.T) {
},
},
{
name: "should find a file from a relative path (root above cwd)",
name: "should find a file from a relative path (root above cwd)",
// TODO: refactor me! this test depends on the structure of the source dir not changing, which isn't great
relativeRoot: "../",
input: "sbom/sbom.go",
input: "fileresolver/directory.go",
expected: []string{
"sbom/sbom.go",
"fileresolver/directory.go",
},
},
}
@ -110,7 +112,7 @@ func TestDirectoryResolver_FilesByPath_absoluteRoot(t *testing.T) {
absRoot, err := filepath.Abs(c.relativeRoot)
require.NoError(t, err)
resolver, err := newDirectoryResolver(absRoot, "")
resolver, err := NewFromDirectory(absRoot, "")
assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input)
@ -171,7 +173,7 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, err := newDirectoryResolver(c.root, "")
resolver, err := NewFromDirectory(c.root, "")
assert.NoError(t, err)
hasPath := resolver.HasPath(c.input)
@ -219,7 +221,7 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures", "")
resolver, err := NewFromDirectory("./test-fixtures", "")
assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input...)
assert.NoError(t, err)
@ -232,7 +234,7 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
}
func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures", "")
resolver, err := NewFromDirectory("./test-fixtures", "")
assert.NoError(t, err)
refs, err := resolver.FilesByGlob("**/image-symlinks/file*")
assert.NoError(t, err)
@ -241,7 +243,7 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
}
func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/image-symlinks", "")
resolver, err := NewFromDirectory("./test-fixtures/image-symlinks", "")
assert.NoError(t, err)
refs, err := resolver.FilesByGlob("**/*.txt")
assert.NoError(t, err)
@ -249,7 +251,7 @@ func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
}
func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures", "")
resolver, err := NewFromDirectory("./test-fixtures", "")
assert.NoError(t, err)
refs, err := resolver.FilesByGlob("**/image-symlinks/*1.txt")
assert.NoError(t, err)
@ -276,7 +278,7 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple", "")
resolver, err := NewFromDirectory("./test-fixtures/symlinks-simple", "")
assert.NoError(t, err)
refs, err := resolver.FilesByPath(test.fixture)
@ -299,7 +301,7 @@ func TestDirectoryResolver_FilesByPath_ResolvesSymlinks(t *testing.T) {
func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) {
// let's make certain that "dev/place" is not ignored, since it is not "/dev/place"
resolver, err := newDirectoryResolver("test-fixtures/system_paths/target", "")
resolver, err := NewFromDirectory("test-fixtures/system_paths/target", "")
assert.NoError(t, err)
// all paths should be found (non filtering matches a path)
@ -383,35 +385,35 @@ func Test_isUnallowableFileType(t *testing.T) {
info: testFileInfo{
mode: os.ModeSocket,
},
expected: errSkipPath,
expected: ErrSkipPath,
},
{
name: "named pipe",
info: testFileInfo{
mode: os.ModeNamedPipe,
},
expected: errSkipPath,
expected: ErrSkipPath,
},
{
name: "char device",
info: testFileInfo{
mode: os.ModeCharDevice,
},
expected: errSkipPath,
expected: ErrSkipPath,
},
{
name: "block device",
info: testFileInfo{
mode: os.ModeDevice,
},
expected: errSkipPath,
expected: ErrSkipPath,
},
{
name: "irregular",
info: testFileInfo{
mode: os.ModeIrregular,
},
expected: errSkipPath,
expected: ErrSkipPath,
},
}
for _, test := range tests {
@ -435,7 +437,7 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
}
for _, test := range tests {
t.Run(test.fixturePath, func(t *testing.T) {
resolver, err := newDirectoryResolver(test.fixturePath, "")
resolver, err := NewFromDirectory(test.fixturePath, "")
assert.NoError(t, err)
locations, err := resolver.FilesByMIMEType(test.mimeType)
assert.NoError(t, err)
@ -448,7 +450,7 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
}
func Test_IndexingNestedSymLinks(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple", "")
resolver, err := NewFromDirectory("./test-fixtures/symlinks-simple", "")
require.NoError(t, err)
// check that we can get the real path
@ -499,12 +501,12 @@ func Test_IndexingNestedSymLinks(t *testing.T) {
func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
filterFn := func(path string, _ os.FileInfo, _ error) error {
if strings.HasSuffix(path, string(filepath.Separator)+"readme") {
return errSkipPath
return ErrSkipPath
}
return nil
}
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple", "", filterFn)
resolver, err := NewFromDirectory("./test-fixtures/symlinks-simple", "", filterFn)
require.NoError(t, err)
// the path to the real file is PRUNED from the index, so we should NOT expect a location returned
@ -524,7 +526,7 @@ func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
}
func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-multiple-roots/root", "")
resolver, err := NewFromDirectory("./test-fixtures/symlinks-multiple-roots/root", "")
require.NoError(t, err)
// check that we can get the real path
@ -542,7 +544,7 @@ func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
}
func Test_RootViaSymlink(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinked-root/nested/link-root", "")
resolver, err := NewFromDirectory("./test-fixtures/symlinked-root/nested/link-root", "")
require.NoError(t, err)
locations, err := resolver.FilesByPath("./file1.txt")
@ -562,28 +564,28 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
cwd, err := os.Getwd()
require.NoError(t, err)
r, err := newDirectoryResolver(".", "")
r, err := NewFromDirectory(".", "")
require.NoError(t, err)
exists, existingPath, err := r.tree.File(file.Path(filepath.Join(cwd, "test-fixtures/image-simple/file-1.txt")))
exists, existingPath, err := r.tree.File(stereoscopeFile.Path(filepath.Join(cwd, "test-fixtures/image-simple/file-1.txt")))
require.True(t, exists)
require.NoError(t, err)
require.True(t, existingPath.HasReference())
tests := []struct {
name string
location Location
location file.Location
expects string
err bool
}{
{
name: "use file reference for content requests",
location: NewLocationFromDirectory("some/place", *existingPath.Reference),
location: file.NewLocationFromDirectory("some/place", *existingPath.Reference),
expects: "this file has contents",
},
{
name: "error on empty file reference",
location: NewLocationFromDirectory("doesn't matter", file.Reference{}),
location: file.NewLocationFromDirectory("doesn't matter", stereoscopeFile.Reference{}),
err: true,
},
}
@ -598,7 +600,7 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
require.NoError(t, err)
if test.expects != "" {
b, err := ioutil.ReadAll(actual)
b, err := io.ReadAll(actual)
require.NoError(t, err)
assert.Equal(t, test.expects, string(b))
}
@ -649,7 +651,7 @@ func Test_isUnixSystemRuntimePath(t *testing.T) {
func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) {
test := func(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-loop", "")
resolver, err := NewFromDirectory("./test-fixtures/symlinks-loop", "")
require.NoError(t, err)
locations, err := resolver.FilesByGlob("**/file.target")
@ -662,20 +664,6 @@ func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) {
testWithTimeout(t, 5*time.Second, test)
}
func testWithTimeout(t *testing.T, timeout time.Duration, test func(*testing.T)) {
done := make(chan bool)
go func() {
test(t)
done <- true
}()
select {
case <-time.After(timeout):
t.Fatal("test timed out")
case <-done:
}
}
func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) {
cases := []struct {
name string
@ -734,7 +722,7 @@ func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver, err := newDirectoryResolver(c.root, c.root)
resolver, err := NewFromDirectory(c.root, c.root)
assert.NoError(t, err)
refs, err := resolver.FilesByPath(c.input)
@ -753,162 +741,132 @@ func TestDirectoryResolver_FilesByPath_baseRoot(t *testing.T) {
func Test_directoryResolver_resolvesLinks(t *testing.T) {
tests := []struct {
name string
runner func(FileResolver) []Location
expected []Location
runner func(file.Resolver) []file.Location
expected []file.Location
}{
{
name: "by mimetype",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links should not show up when searching mimetype
actualLocations, err := resolver.FilesByMIMEType("text/plain")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewLocation("file-1.txt"), // note: missing virtual path "file-1.txt"
NewLocation("file-3.txt"), // note: missing virtual path "file-3.txt"
NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt"
NewLocation("parent/file-4.txt"), // note: missing virtual path "file-4.txt"
expected: []file.Location{
file.NewLocation("file-1.txt"), // note: missing virtual path "file-1.txt"
file.NewLocation("file-3.txt"), // note: missing virtual path "file-3.txt"
file.NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt"
file.NewLocation("parent/file-4.txt"), // note: missing virtual path "file-4.txt"
},
},
{
name: "by glob to links",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files
// for that reason we need to place **/ in front (which is not the same for other resolvers)
actualLocations, err := resolver.FilesByGlob("**/*ink-*")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewVirtualLocation("file-1.txt", "link-1"),
NewVirtualLocation("file-2.txt", "link-2"),
expected: []file.Location{
file.NewVirtualLocation("file-1.txt", "link-1"),
file.NewVirtualLocation("file-2.txt", "link-2"),
// we already have this real file path via another link, so only one is returned
//NewVirtualLocation("file-2.txt", "link-indirect"),
NewVirtualLocation("file-3.txt", "link-within"),
//file.NewVirtualLocation("file-2.txt", "link-indirect"),
file.NewVirtualLocation("file-3.txt", "link-within"),
},
},
{
name: "by basename",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/file-2.txt")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
expected: []file.Location{
// this has two copies in the base image, which overwrites the same location
NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt",
file.NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt",
},
},
{
name: "by basename glob",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/file-?.txt")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewLocation("file-1.txt"), // note: missing virtual path "file-1.txt"
NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt"
NewLocation("file-3.txt"), // note: missing virtual path "file-3.txt"
NewLocation("parent/file-4.txt"), // note: missing virtual path "parent/file-4.txt"
expected: []file.Location{
file.NewLocation("file-1.txt"), // note: missing virtual path "file-1.txt"
file.NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt"
file.NewLocation("file-3.txt"), // note: missing virtual path "file-3.txt"
file.NewLocation("parent/file-4.txt"), // note: missing virtual path "parent/file-4.txt"
},
},
{
name: "by basename glob to links",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
actualLocations, err := resolver.FilesByGlob("**/link-*")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
{
LocationData: LocationData{
Coordinates: Coordinates{
RealPath: "file-1.txt",
},
VirtualPath: "link-1",
ref: file.Reference{RealPath: "file-1.txt"},
},
},
{
LocationData: LocationData{
Coordinates: Coordinates{
RealPath: "file-2.txt",
},
VirtualPath: "link-2",
ref: file.Reference{RealPath: "file-2.txt"},
},
},
expected: []file.Location{
file.NewVirtualLocation("file-1.txt", "link-1"),
file.NewVirtualLocation("file-2.txt", "link-2"),
// we already have this real file path via another link, so only one is returned
//{
// LocationData: LocationData{
// Coordinates: Coordinates{
// RealPath: "file-2.txt",
// },
// VirtualPath: "link-indirect",
// ref: file.Reference{RealPath: "file-2.txt"},
// },
//},
{
LocationData: LocationData{
Coordinates: Coordinates{
RealPath: "file-3.txt",
},
VirtualPath: "link-within",
ref: file.Reference{RealPath: "file-3.txt"},
},
},
//file.NewVirtualLocation("file-2.txt", "link-indirect"),
file.NewVirtualLocation("file-3.txt", "link-within"),
},
},
{
name: "by extension",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links are searched, but resolve to the real files
actualLocations, err := resolver.FilesByGlob("**/*.txt")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
NewLocation("file-1.txt"), // note: missing virtual path "file-1.txt"
NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt"
NewLocation("file-3.txt"), // note: missing virtual path "file-3.txt"
NewLocation("parent/file-4.txt"), // note: missing virtual path "parent/file-4.txt"
expected: []file.Location{
file.NewLocation("file-1.txt"), // note: missing virtual path "file-1.txt"
file.NewLocation("file-2.txt"), // note: missing virtual path "file-2.txt"
file.NewLocation("file-3.txt"), // note: missing virtual path "file-3.txt"
file.NewLocation("parent/file-4.txt"), // note: missing virtual path "parent/file-4.txt"
},
},
{
name: "by path to degree 1 link",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// links resolve to the final file
actualLocations, err := resolver.FilesByPath("/link-2")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
expected: []file.Location{
// we have multiple copies across layers
NewVirtualLocation("file-2.txt", "link-2"),
file.NewVirtualLocation("file-2.txt", "link-2"),
},
},
{
name: "by path to degree 2 link",
runner: func(resolver FileResolver) []Location {
runner: func(resolver file.Resolver) []file.Location {
// multiple links resolves to the final file
actualLocations, err := resolver.FilesByPath("/link-indirect")
assert.NoError(t, err)
return actualLocations
},
expected: []Location{
expected: []file.Location{
// we have multiple copies across layers
NewVirtualLocation("file-2.txt", "link-indirect"),
file.NewVirtualLocation("file-2.txt", "link-indirect"),
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-from-image-symlinks-fixture", "")
resolver, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "")
require.NoError(t, err)
assert.NoError(t, err)
@ -920,14 +878,14 @@ func Test_directoryResolver_resolvesLinks(t *testing.T) {
}
func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-prune-indexing", "")
resolver, err := NewFromDirectory("./test-fixtures/symlinks-prune-indexing", "")
require.NoError(t, err)
var allRealPaths []file.Path
var allRealPaths []stereoscopeFile.Path
for l := range resolver.AllLocations() {
allRealPaths = append(allRealPaths, file.Path(l.RealPath))
allRealPaths = append(allRealPaths, stereoscopeFile.Path(l.RealPath))
}
pathSet := file.NewPathSet(allRealPaths...)
pathSet := stereoscopeFile.NewPathSet(allRealPaths...)
assert.False(t,
pathSet.Contains("before-path/file.txt"),
@ -942,12 +900,12 @@ func TestDirectoryResolver_DoNotAddVirtualPathsToTree(t *testing.T) {
}
func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/system_paths", "")
resolver, err := NewFromDirectory("./test-fixtures/system_paths", "")
assert.NoError(t, err)
var dirLoc *Location
var dirLoc *file.Location
for loc := range resolver.AllLocations() {
entry, err := resolver.index.Get(loc.ref)
entry, err := resolver.index.Get(loc.Reference())
require.NoError(t, err)
if entry.Metadata.IsDir() {
dirLoc = &loc
@ -963,13 +921,13 @@ func TestDirectoryResolver_FilesContents_errorOnDirRequest(t *testing.T) {
}
func TestDirectoryResolver_AllLocations(t *testing.T) {
resolver, err := newDirectoryResolver("./test-fixtures/symlinks-from-image-symlinks-fixture", "")
resolver, err := NewFromDirectory("./test-fixtures/symlinks-from-image-symlinks-fixture", "")
assert.NoError(t, err)
paths := strset.New()
for loc := range resolver.AllLocations() {
if strings.HasPrefix(loc.RealPath, "/") {
// ignore outside of the fixture root for now
// ignore outside the fixture root for now
continue
}
paths.Add(loc.RealPath)

View File

@ -1,4 +1,4 @@
package source
package fileresolver
import "testing"

View File

@ -0,0 +1,47 @@
package fileresolver
import (
"io"
"github.com/anchore/syft/syft/file"
)
var _ file.WritableResolver = (*Empty)(nil)
type Empty struct{}
func (e Empty) FileContentsByLocation(_ file.Location) (io.ReadCloser, error) {
return nil, nil
}
func (e Empty) HasPath(_ string) bool {
return false
}
func (e Empty) FilesByPath(_ ...string) ([]file.Location, error) {
return nil, nil
}
func (e Empty) FilesByGlob(_ ...string) ([]file.Location, error) {
return nil, nil
}
func (e Empty) FilesByMIMEType(_ ...string) ([]file.Location, error) {
return nil, nil
}
func (e Empty) RelativeFileByPath(_ file.Location, _ string) *file.Location {
return nil
}
func (e Empty) AllLocations() <-chan file.Location {
return nil
}
func (e Empty) FileMetadataByLocation(_ file.Location) (file.Metadata, error) {
return file.Metadata{}, nil
}
func (e Empty) Write(_ file.Location, _ io.Reader) error {
return nil
}

View File

@ -1,65 +1,67 @@
package source
package fileresolver
import (
"fmt"
"io"
"github.com/anchore/syft/syft/file"
)
type excludeFn func(string) bool
// excludingResolver decorates a resolver with an exclusion function that is used to
// excluding decorates a resolver with an exclusion function that is used to
// filter out entries in the delegate resolver
type excludingResolver struct {
delegate FileResolver
type excluding struct {
delegate file.Resolver
excludeFn excludeFn
}
// NewExcludingResolver create a new resolver which wraps the provided delegate and excludes
// NewExcluding create a new resolver which wraps the provided delegate and excludes
// entries based on a provided path exclusion function
func NewExcludingResolver(delegate FileResolver, excludeFn excludeFn) FileResolver {
return &excludingResolver{
func NewExcluding(delegate file.Resolver, excludeFn excludeFn) file.Resolver {
return &excluding{
delegate,
excludeFn,
}
}
func (r *excludingResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
func (r *excluding) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
if locationMatches(&location, r.excludeFn) {
return nil, fmt.Errorf("no such location: %+v", location.RealPath)
}
return r.delegate.FileContentsByLocation(location)
}
func (r *excludingResolver) FileMetadataByLocation(location Location) (FileMetadata, error) {
func (r *excluding) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
if locationMatches(&location, r.excludeFn) {
return FileMetadata{}, fmt.Errorf("no such location: %+v", location.RealPath)
return file.Metadata{}, fmt.Errorf("no such location: %+v", location.RealPath)
}
return r.delegate.FileMetadataByLocation(location)
}
func (r *excludingResolver) HasPath(path string) bool {
func (r *excluding) HasPath(path string) bool {
if r.excludeFn(path) {
return false
}
return r.delegate.HasPath(path)
}
func (r *excludingResolver) FilesByPath(paths ...string) ([]Location, error) {
func (r *excluding) FilesByPath(paths ...string) ([]file.Location, error) {
locations, err := r.delegate.FilesByPath(paths...)
return filterLocations(locations, err, r.excludeFn)
}
func (r *excludingResolver) FilesByGlob(patterns ...string) ([]Location, error) {
func (r *excluding) FilesByGlob(patterns ...string) ([]file.Location, error) {
locations, err := r.delegate.FilesByGlob(patterns...)
return filterLocations(locations, err, r.excludeFn)
}
func (r *excludingResolver) FilesByMIMEType(types ...string) ([]Location, error) {
func (r *excluding) FilesByMIMEType(types ...string) ([]file.Location, error) {
locations, err := r.delegate.FilesByMIMEType(types...)
return filterLocations(locations, err, r.excludeFn)
}
func (r *excludingResolver) RelativeFileByPath(location Location, path string) *Location {
func (r *excluding) RelativeFileByPath(location file.Location, path string) *file.Location {
l := r.delegate.RelativeFileByPath(location, path)
if l != nil && locationMatches(l, r.excludeFn) {
return nil
@ -67,8 +69,8 @@ func (r *excludingResolver) RelativeFileByPath(location Location, path string) *
return l
}
func (r *excludingResolver) AllLocations() <-chan Location {
c := make(chan Location)
func (r *excluding) AllLocations() <-chan file.Location {
c := make(chan file.Location)
go func() {
defer close(c)
for location := range r.delegate.AllLocations() {
@ -80,11 +82,11 @@ func (r *excludingResolver) AllLocations() <-chan Location {
return c
}
func locationMatches(location *Location, exclusionFn excludeFn) bool {
func locationMatches(location *file.Location, exclusionFn excludeFn) bool {
return exclusionFn(location.RealPath) || exclusionFn(location.VirtualPath)
}
func filterLocations(locations []Location, err error, exclusionFn excludeFn) ([]Location, error) {
func filterLocations(locations []file.Location, err error, exclusionFn excludeFn) ([]file.Location, error) {
if err != nil {
return nil, err
}

View File

@ -1,4 +1,4 @@
package source
package fileresolver
import (
"io"
@ -6,6 +6,8 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/file"
)
func TestExcludingResolver(t *testing.T) {
@ -54,7 +56,7 @@ func TestExcludingResolver(t *testing.T) {
resolver := &mockResolver{
locations: test.locations,
}
er := NewExcludingResolver(resolver, test.excludeFn)
er := NewExcluding(resolver, test.excludeFn)
locations, _ := er.FilesByPath()
assert.ElementsMatch(t, locationPaths(locations), test.expected)
@ -65,7 +67,7 @@ func TestExcludingResolver(t *testing.T) {
locations, _ = er.FilesByMIMEType()
assert.ElementsMatch(t, locationPaths(locations), test.expected)
locations = []Location{}
locations = []file.Location{}
channel := er.AllLocations()
for location := range channel {
@ -77,25 +79,25 @@ func TestExcludingResolver(t *testing.T) {
for _, path := range diff {
assert.False(t, er.HasPath(path))
c, err := er.FileContentsByLocation(NewLocation(path))
c, err := er.FileContentsByLocation(file.NewLocation(path))
assert.Nil(t, c)
assert.Error(t, err)
m, err := er.FileMetadataByLocation(NewLocation(path))
m, err := er.FileMetadataByLocation(file.NewLocation(path))
assert.Empty(t, m.LinkDestination)
assert.Error(t, err)
l := er.RelativeFileByPath(NewLocation(""), path)
l := er.RelativeFileByPath(file.NewLocation(""), path)
assert.Nil(t, l)
}
for _, path := range test.expected {
assert.True(t, er.HasPath(path))
c, err := er.FileContentsByLocation(NewLocation(path))
c, err := er.FileContentsByLocation(file.NewLocation(path))
assert.NotNil(t, c)
assert.Nil(t, err)
m, err := er.FileMetadataByLocation(NewLocation(path))
m, err := er.FileMetadataByLocation(file.NewLocation(path))
assert.NotEmpty(t, m.LinkDestination)
assert.Nil(t, err)
l := er.RelativeFileByPath(NewLocation(""), path)
l := er.RelativeFileByPath(file.NewLocation(""), path)
assert.NotNil(t, l)
}
})
@ -117,7 +119,7 @@ func difference(a, b []string) []string {
return diff
}
func locationPaths(locations []Location) []string {
func locationPaths(locations []file.Location) []string {
paths := []string{}
for _, l := range locations {
paths = append(paths, l.RealPath)
@ -129,20 +131,20 @@ type mockResolver struct {
locations []string
}
func (r *mockResolver) getLocations() ([]Location, error) {
out := []Location{}
func (r *mockResolver) getLocations() ([]file.Location, error) {
out := []file.Location{}
for _, path := range r.locations {
out = append(out, NewLocation(path))
out = append(out, file.NewLocation(path))
}
return out, nil
}
func (r *mockResolver) FileContentsByLocation(_ Location) (io.ReadCloser, error) {
func (r *mockResolver) FileContentsByLocation(_ file.Location) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("Hello, world!")), nil
}
func (r *mockResolver) FileMetadataByLocation(_ Location) (FileMetadata, error) {
return FileMetadata{
func (r *mockResolver) FileMetadataByLocation(_ file.Location) (file.Metadata, error) {
return file.Metadata{
LinkDestination: "MOCK",
}, nil
}
@ -151,37 +153,37 @@ func (r *mockResolver) HasPath(_ string) bool {
return true
}
func (r *mockResolver) FilesByPath(_ ...string) ([]Location, error) {
func (r *mockResolver) FilesByPath(_ ...string) ([]file.Location, error) {
return r.getLocations()
}
func (r *mockResolver) FilesByGlob(_ ...string) ([]Location, error) {
func (r *mockResolver) FilesByGlob(_ ...string) ([]file.Location, error) {
return r.getLocations()
}
func (r *mockResolver) FilesByMIMEType(_ ...string) ([]Location, error) {
func (r *mockResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) {
return r.getLocations()
}
func (r *mockResolver) FilesByExtension(_ ...string) ([]Location, error) {
func (r *mockResolver) FilesByExtension(_ ...string) ([]file.Location, error) {
return r.getLocations()
}
func (r *mockResolver) FilesByBasename(_ ...string) ([]Location, error) {
func (r *mockResolver) FilesByBasename(_ ...string) ([]file.Location, error) {
return r.getLocations()
}
func (r *mockResolver) FilesByBasenameGlob(_ ...string) ([]Location, error) {
func (r *mockResolver) FilesByBasenameGlob(_ ...string) ([]file.Location, error) {
return r.getLocations()
}
func (r *mockResolver) RelativeFileByPath(_ Location, path string) *Location {
l := NewLocation(path)
func (r *mockResolver) RelativeFileByPath(_ file.Location, path string) *file.Location {
l := file.NewLocation(path)
return &l
}
func (r *mockResolver) AllLocations() <-chan Location {
c := make(chan Location)
func (r *mockResolver) AllLocations() <-chan file.Location {
c := make(chan file.Location)
go func() {
defer close(c)
locations, _ := r.getLocations()

View File

@ -0,0 +1,15 @@
package fileresolver
import (
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/syft/file"
)
func fileMetadataByLocation(img *image.Image, location file.Location) (file.Metadata, error) {
entry, err := img.FileCatalog.Get(location.Reference())
if err != nil {
return file.Metadata{}, err
}
return entry.Metadata, nil
}

View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -eux
# $1 —— absolute path to destination file, should end with .tar
# $2 —— absolute path to directory from which to add entries to the archive
pushd "$2"
tar -cvf "$1" .
popd

Some files were not shown because too many files have changed in this diff Show More