From b694dacb21299fbd9f78b75a5eefe78b6681efdc Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 13 Nov 2020 12:34:22 -0500 Subject: [PATCH] add source.Location + reorient Resolvers to use it Signed-off-by: Alex Goodman --- syft/source/all_layers_resolver.go | 71 ++++++++++++++++++++------- syft/source/directory_resolver.go | 72 ++++++++++++++-------------- syft/source/image_squash_resolver.go | 46 +++++++++--------- syft/source/location.go | 46 ++++++++++++++++++ syft/source/resolver.go | 17 ++++--- 5 files changed, 168 insertions(+), 84 deletions(-) create mode 100644 syft/source/location.go diff --git a/syft/source/all_layers_resolver.go b/syft/source/all_layers_resolver.go index a050ad616..1c2be3749 100644 --- a/syft/source/all_layers_resolver.go +++ b/syft/source/all_layers_resolver.go @@ -8,6 +8,8 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) +var _ Resolver = &AllLayersResolver{} + // AllLayersResolver implements path and content access for the AllLayers source option for container image data sources. type AllLayersResolver struct { img *image.Image @@ -61,14 +63,14 @@ func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.Ref } // FilesByPath returns all file.References that match the given paths from any layer in the image. -func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) { +func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() - uniqueFiles := make([]file.Reference, 0) + uniqueLocations := make([]Location, 0) for _, path := range paths { for idx, layerIdx := range r.layers { tree := r.img.Layers[layerIdx].Tree - ref := tree.File(path) + ref := tree.File(file.Path(path)) if ref == nil { // no file found, keep looking through layers continue @@ -91,17 +93,18 @@ func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, e if err != nil { return nil, err } - uniqueFiles = append(uniqueFiles, results...) + for _, result := range results { + uniqueLocations = append(uniqueLocations, newLocationFromImage(result, r.img)) + } } } - - return uniqueFiles, nil + return uniqueLocations, nil } // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. -func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) { +func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() - uniqueFiles := make([]file.Reference, 0) + uniqueLocations := make([]Location, 0) for _, pattern := range patterns { for idx, layerIdx := range r.layers { @@ -128,31 +131,63 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, e if err != nil { return nil, err } - uniqueFiles = append(uniqueFiles, results...) + for _, result := range results { + uniqueLocations = append(uniqueLocations, newLocationFromImage(result, r.img)) + } } } } - return uniqueFiles, nil + return uniqueLocations, nil } -func (r *AllLayersResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) { - entry, err := r.img.FileCatalog.Get(reference) +func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *Location { + entry, err := r.img.FileCatalog.Get(location.ref) if err != nil { - return nil, err + return nil } - return entry.Source.SquashedTree.File(file.Path(path)), nil + relativeRef := entry.Source.SquashedTree.File(file.Path(path)) + if relativeRef == nil { + return nil + } + + relativeLocation := newLocationFromImage(*relativeRef, r.img) + + return &relativeLocation } // MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a // file.Reference is a path relative to a particular layer. -func (r *AllLayersResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { - return r.img.MultipleFileContentsByRef(f...) +func (r *AllLayersResolver) MultipleFileContentsByRef(locations []Location) (map[Location]string, error) { + return mapLocationRefs(r.img.MultipleFileContentsByRef, locations) } // FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer. // If the path does not exist an error is returned. -func (r *AllLayersResolver) FileContentsByRef(ref file.Reference) (string, error) { - return r.img.FileContentsByRef(ref) +func (r *AllLayersResolver) FileContentsByRef(location Location) (string, error) { + return r.img.FileContentsByRef(location.ref) +} + +type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]string, error) + +func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]string, error) { + var fileRefs = make([]file.Reference, len(locations)) + var locationByRefs = make(map[file.Reference]Location) + var results = make(map[Location]string) + + for i, location := range locations { + locationByRefs[location.ref] = location + fileRefs[i] = location.ref + } + + contentsByRef, err := callback(fileRefs...) + if err != nil { + return nil, err + } + + for ref, content := range contentsByRef { + results[locationByRefs[ref]] = content + } + return results, nil } diff --git a/syft/source/directory_resolver.go b/syft/source/directory_resolver.go index 89b717db0..9cbacf6bf 100644 --- a/syft/source/directory_resolver.go +++ b/syft/source/directory_resolver.go @@ -7,11 +7,14 @@ import ( "path" "path/filepath" - "github.com/anchore/stereoscope/pkg/file" + "github.com/docker/distribution/reference" + "github.com/anchore/syft/internal/log" "github.com/bmatcuk/doublestar" ) +var _ Resolver = &DirectoryResolver{} + // DirectoryResolver implements path and content access for the directory data source. type DirectoryResolver struct { Path string @@ -23,11 +26,11 @@ func (s DirectoryResolver) String() string { } // FilesByPath returns all file.References that match the given paths from the directory. -func (s DirectoryResolver) FilesByPath(userPaths ...file.Path) ([]file.Reference, error) { - var references = make([]file.Reference, 0) +func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) { + var references = make([]Location, 0) for _, userPath := range userPaths { - userStrPath := string(userPath) + userStrPath := userPath if filepath.IsAbs(userStrPath) { // a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is @@ -45,33 +48,24 @@ func (s DirectoryResolver) FilesByPath(userPaths ...file.Path) ([]file.Reference continue } - references = append(references, file.NewFileReference(file.Path(userStrPath))) + references = append(references, newLocation(userStrPath)) } return references, nil } -func fileContents(path file.Path) ([]byte, error) { - contents, err := ioutil.ReadFile(string(path)) - - if err != nil { - return nil, err - } - return contents, nil -} - // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. -func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) { - result := make([]file.Reference, 0) +func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) { + result := make([]Location, 0) for _, pattern := range patterns { pathPattern := path.Join(s.Path, pattern) - matches, err := doublestar.Glob(pathPattern) + pathMatches, err := doublestar.Glob(pathPattern) if err != nil { - return result, err + return nil, err } - for _, match := range matches { - fileMeta, err := os.Stat(match) + for _, matchedPath := range pathMatches { + fileMeta, err := os.Stat(matchedPath) if err != nil { continue } @@ -81,47 +75,55 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, er continue } - matchedPath := file.Path(match) - result = append(result, file.NewFileReference(matchedPath)) + result = append(result, newLocation(matchedPath)) } } return result, nil } -func (s *DirectoryResolver) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) { - paths, err := s.FilesByPath(file.Path(path)) +func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location { + paths, err := s.FilesByPath(path) if err != nil { - return nil, err + return nil } if len(paths) == 0 { - return nil, nil + return nil } - return &paths[0], nil + return &paths[0] } // MultipleFileContentsByRef returns the file contents for all file.References relative a directory. -func (s DirectoryResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { - refContents := make(map[file.Reference]string) - for _, fileRef := range f { - contents, err := fileContents(fileRef.Path) +func (s DirectoryResolver) MultipleFileContentsByRef(locations []Location) (map[Location]string, error) { + refContents := make(map[Location]string) + for _, location := range locations { + contents, err := fileContents(location.Path) if err != nil { - return nil, fmt.Errorf("could not read contents of file: %s", fileRef.Path) + return nil, fmt.Errorf("could not read contents of file: %s", location.Path) } - refContents[fileRef] = string(contents) + refContents[location] = string(contents) } return refContents, nil } // FileContentsByRef fetches file contents for a single file reference relative to a directory. // If the path does not exist an error is returned. -func (s DirectoryResolver) FileContentsByRef(reference file.Reference) (string, error) { - contents, err := fileContents(reference.Path) +func (s DirectoryResolver) FileContentsByRef(location Location) (string, error) { + contents, err := fileContents(location.Path) if err != nil { return "", fmt.Errorf("could not read contents of file: %s", reference.Path) } return string(contents), nil } + +func fileContents(path string) ([]byte, error) { + contents, err := ioutil.ReadFile(path) + + if err != nil { + return nil, err + } + return contents, nil +} diff --git a/syft/source/image_squash_resolver.go b/syft/source/image_squash_resolver.go index 07a9a54d9..ea4f3ffd6 100644 --- a/syft/source/image_squash_resolver.go +++ b/syft/source/image_squash_resolver.go @@ -7,6 +7,8 @@ import ( "github.com/anchore/stereoscope/pkg/image" ) +var _ Resolver = &ImageSquashResolver{} + // ImageSquashResolver implements path and content access for the Squashed source option for container image data sources. type ImageSquashResolver struct { img *image.Image @@ -21,13 +23,13 @@ func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) { } // FilesByPath returns all file.References that match the given paths within the squashed representation of the image. -func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) { +func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() - uniqueFiles := make([]file.Reference, 0) + uniqueLocations := make([]Location, 0) for _, path := range paths { tree := r.img.SquashedTree() - ref := tree.File(path) + ref := tree.File(file.Path(path)) if ref == nil { // no file found, keep looking through layers continue @@ -54,17 +56,17 @@ func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) { uniqueFileIDs.Add(*resolvedRef) - uniqueFiles = append(uniqueFiles, *resolvedRef) + uniqueLocations = append(uniqueLocations, newLocationFromImage(*resolvedRef, r.img)) } } - return uniqueFiles, nil + return uniqueLocations, nil } // FilesByGlob returns all file.References that match the given path glob pattern within the squashed representation of the image. -func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) { +func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error) { uniqueFileIDs := file.NewFileReferenceSet() - uniqueFiles := make([]file.Reference, 0) + uniqueLocations := make([]Location, 0) for _, pattern := range patterns { refs, err := r.img.SquashedTree().FilesByGlob(pattern) @@ -86,42 +88,42 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, } } - resolvedRefs, err := r.FilesByPath(ref.Path) + resolvedLocations, err := r.FilesByPath(string(ref.Path)) if err != nil { return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err) } - for _, resolvedRef := range resolvedRefs { - if !uniqueFileIDs.Contains(resolvedRef) { - uniqueFileIDs.Add(resolvedRef) - uniqueFiles = append(uniqueFiles, resolvedRef) + for _, resolvedLocation := range resolvedLocations { + if !uniqueFileIDs.Contains(resolvedLocation.ref) { + uniqueFileIDs.Add(resolvedLocation.ref) + uniqueLocations = append(uniqueLocations, resolvedLocation) } } } } - return uniqueFiles, nil + return uniqueLocations, nil } -func (r *ImageSquashResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) { - paths, err := r.FilesByPath(file.Path(path)) +func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Location { + paths, err := r.FilesByPath(path) if err != nil { - return nil, err + return nil } if len(paths) == 0 { - return nil, nil + return nil } - return &paths[0], nil + return &paths[0] } // MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a // file.Reference is a path relative to a particular layer, in this case only from the squashed representation. -func (r *ImageSquashResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) { - return r.img.MultipleFileContentsByRef(f...) +func (r *ImageSquashResolver) MultipleFileContentsByRef(locations []Location) (map[Location]string, error) { + return mapLocationRefs(r.img.MultipleFileContentsByRef, locations) } // FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer. // If the path does not exist an error is returned. -func (r *ImageSquashResolver) FileContentsByRef(ref file.Reference) (string, error) { - return r.img.FileContentsByRef(ref) +func (r *ImageSquashResolver) FileContentsByRef(location Location) (string, error) { + return r.img.FileContentsByRef(location.ref) } diff --git a/syft/source/location.go b/syft/source/location.go new file mode 100644 index 000000000..1bc5e5ce0 --- /dev/null +++ b/syft/source/location.go @@ -0,0 +1,46 @@ +package source + +import ( + "github.com/anchore/syft/internal/log" + + "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/stereoscope/pkg/image" +) + +type Location struct { + Path string `json:"path"` + LayerIndex uint `json:"layerIndex"` + LayerID string `json:"layerID"` + ref file.Reference +} + +func newLocation(path string) Location { + return Location{ + Path: path, + } +} + +func newLocationFromRef(ref file.Reference) Location { + return Location{ + Path: string(ref.Path), + ref: ref, + } +} + +func newLocationFromImage(ref file.Reference, img *image.Image) Location { + entry, err := img.FileCatalog.Get(ref) + if err != nil { + log.Warnf("unable to find file catalog entry for ref=%+v", ref) + return Location{ + Path: string(ref.Path), + ref: ref, + } + } + + return Location{ + Path: string(ref.Path), + LayerIndex: entry.Source.Metadata.Index, + LayerID: entry.Source.Metadata.Digest, + ref: ref, + } +} diff --git a/syft/source/resolver.go b/syft/source/resolver.go index 6af9feaa2..2bab3bc11 100644 --- a/syft/source/resolver.go +++ b/syft/source/resolver.go @@ -3,7 +3,6 @@ package source import ( "fmt" - "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/image" ) @@ -15,30 +14,30 @@ type Resolver interface { // ContentResolver knows how to get file content for given file.References type ContentResolver interface { - FileContentsByRef(ref file.Reference) (string, error) - MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) + FileContentsByRef(Location) (string, error) + MultipleFileContentsByRef([]Location) (map[Location]string, error) // TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering). } // FileResolver knows how to get file.References for given string paths and globs type FileResolver interface { // FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches) - FilesByPath(paths ...file.Path) ([]file.Reference, error) + FilesByPath(paths ...string) ([]Location, error) // FilesByGlob fetches a set of file references which the given glob matches - FilesByGlob(patterns ...string) ([]file.Reference, error) + FilesByGlob(patterns ...string) ([]Location, error) // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference. // This is helpful when attempting to find a file that is in the same layer or lower as another file. - RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) + RelativeFileByPath(_ Location, path string) *Location } // getImageResolver returns the appropriate resolve for a container image given the source option -func getImageResolver(img *image.Image, option Scope) (Resolver, error) { - switch option { +func getImageResolver(img *image.Image, scope Scope) (Resolver, error) { + switch scope { case SquashedScope: return NewImageSquashResolver(img) case AllLayersScope: return NewAllLayersResolver(img) default: - return nil, fmt.Errorf("bad option provided: %+v", option) + return nil, fmt.Errorf("bad scope provided: %+v", scope) } }