bump stereoscope to pull in content API refactors

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
-	github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e
+	github.com/anchore/stereoscope v0.0.0-20201219143203-af397ece87bf
 	github.com/antihax/optional v1.0.0
 	github.com/bmatcuk/doublestar v1.3.3
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible

go.sum

@@ -138,6 +138,8 @@ github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZV
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
 github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e h1:vHUqHTvH9/oxdDDh1fxS9Ls9gWGytKO7XbbzcQ9MBwI=
 github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e/go.mod h1:/dHAFjYflH/1tzhdHAcnMCjprMch+YzHJKi59m/1KCM=
+github.com/anchore/stereoscope v0.0.0-20201219143203-af397ece87bf h1:4sN/HJ6whcrK/HxORFGAQUWM58Q7EFiPmoxRKcEs76A=
+github.com/anchore/stereoscope v0.0.0-20201219143203-af397ece87bf/go.mod h1:/dHAFjYflH/1tzhdHAcnMCjprMch+YzHJKi59m/1KCM=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=

syft/cataloger/common/generic_cataloger.go

@@ -4,7 +4,7 @@ Package common provides generic utilities used by multiple catalogers.
 package common
 
 import (
-	"strings"
+	"io"
 
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/pkg"
@@ -89,7 +89,7 @@ func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []source.Lo
 }
 
 // catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
-func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.Package, error) {
+func (c *GenericCataloger) catalog(contents map[source.Location]io.ReadCloser) ([]pkg.Package, error) {
 	defer c.clear()
 
 	packages := make([]pkg.Package, 0)
@@ -101,7 +101,7 @@ func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.P
 			continue
 		}
 
-		entries, err := parser(location.Path, strings.NewReader(content))
+		entries, err := parser(location.Path, content)
 		if err != nil {
 			// TODO: should we fail? or only log?
 			log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)

syft/cataloger/common/generic_cataloger_test.go

@@ -4,28 +4,28 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"strings"
 	"testing"
 
-	"github.com/anchore/syft/syft/source"
-
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
 )
 
 type testResolverMock struct {
-	contents map[source.Location]string
+	contents map[source.Location]io.ReadCloser
 }
 
 func newTestResolver() *testResolverMock {
 	return &testResolverMock{
-		contents: make(map[source.Location]string),
+		contents: make(map[source.Location]io.ReadCloser),
 	}
 }
 
-func (r *testResolverMock) FileContentsByLocation(_ source.Location) (string, error) {
+func (r *testResolverMock) FileContentsByLocation(_ source.Location) (io.ReadCloser, error) {
-	return "", fmt.Errorf("not implemented")
+	return nil, fmt.Errorf("not implemented")
 }
 
-func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]string, error) {
+func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]io.ReadCloser, error) {
 	return r.contents, nil
 }
 
@@ -34,7 +34,7 @@ func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, erro
 
 	for idx, p := range paths {
 		results[idx] = source.NewLocation(p)
-		r.contents[results[idx]] = fmt.Sprintf("%s file contents!", p)
+		r.contents[results[idx]] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", p)))
 	}
 
 	return results, nil
@@ -43,7 +43,7 @@ func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, erro
 func (r *testResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) {
 	path := "/a-path.txt"
 	location := source.NewLocation(path)
-	r.contents[location] = fmt.Sprintf("%s file contents!", path)
+	r.contents[location] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", path)))
 	return []source.Location{location}, nil
 }
 

syft/cataloger/deb/cataloger.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"path"
 	"path/filepath"
-	"strings"
 
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
@@ -47,7 +46,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 			return nil, err
 		}
 
-		pkgs, err = parseDpkgStatus(strings.NewReader(dbContents))
+		pkgs, err = parseDpkgStatus(dbContents)
 		if err != nil {
 			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err)
 		}
@@ -138,7 +137,7 @@ func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs
 	var refsByName = make(map[string]source.Location)
 	for location, contents := range md5ContentsByLocation {
 		name := nameByRef[location]
-		contentsByName[name] = strings.NewReader(contents)
+		contentsByName[name] = contents
 		refsByName[name] = location
 	}
 
@@ -174,7 +173,7 @@ func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location
 	var refsByName = make(map[string]source.Location)
 	for location, contents := range copyrightContentsByLocation {
 		name := nameByLocation[location]
-		contentsByName[name] = strings.NewReader(contents)
+		contentsByName[name] = contents
 		refsByName[name] = location
 	}
 

syft/cataloger/python/package_cataloger.go

@@ -3,7 +3,6 @@ package python
 import (
 	"bufio"
 	"fmt"
-	"strings"
 
 	"github.com/anchore/syft/syft/pkg"
 
@@ -119,7 +118,7 @@ func (c *PackageCataloger) catalogEggOrWheel(entry *packageEntry) (*pkg.Package,
 func (c *PackageCataloger) assembleEggOrWheelMetadata(entry *packageEntry) (*pkg.PythonPackageMetadata, []source.Location, error) {
 	var sources = []source.Location{entry.Metadata.Location}
 
-	metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.Path, strings.NewReader(entry.Metadata.Contents))
+	metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.Path, entry.Metadata.Contents)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -153,7 +152,7 @@ func (c *PackageCataloger) processRecordFiles(entry *source.FileData) (files []p
 		sources = append(sources, entry.Location)
 
 		// parse the record contents
-		records, err := parseWheelOrEggRecord(strings.NewReader(entry.Contents))
+		records, err := parseWheelOrEggRecord(entry.Contents)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -171,7 +170,7 @@ func (c *PackageCataloger) processTopLevelPackages(entry *source.FileData) (pkgs
 
 	sources = append(sources, entry.Location)
 
-	scanner := bufio.NewScanner(strings.NewReader(entry.Contents))
+	scanner := bufio.NewScanner(entry.Contents)
 	for scanner.Scan() {
 		pkgs = append(pkgs, scanner.Text())
 	}

syft/cataloger/rpmdb/cataloger.go

@@ -5,7 +5,6 @@ package rpmdb
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
@@ -37,12 +36,12 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 
 	var pkgs []pkg.Package
 	for _, location := range fileMatches {
-		dbContents, err := resolver.FileContentsByLocation(location)
+		dbContentReader, err := resolver.FileContentsByLocation(location)
 		if err != nil {
 			return nil, err
 		}
 
-		pkgs, err = parseRpmDB(resolver, location, strings.NewReader(dbContents))
+		pkgs, err = parseRpmDB(resolver, location, dbContentReader)
 		if err != nil {
 			return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.Path, err)
 		}

syft/distro/identify.go

@@ -1,6 +1,7 @@
 package distro
 
 import (
+	"io/ioutil"
 	"regexp"
 	"strings"
 
@@ -16,28 +17,28 @@ type parseEntry struct {
 	fn   parseFunc
 }
 
+var identityFiles = []parseEntry{
+	{
+		// most distros provide a link at this location
+		path: "/etc/os-release",
+		fn:   parseOsRelease,
+	},
+	{
+		// standard location for rhel & debian distros
+		path: "/usr/lib/os-release",
+		fn:   parseOsRelease,
+	},
+	{
+		// check for busybox (important to check this last since other distros contain the busybox binary)
+		path: "/bin/busybox",
+		fn:   parseBusyBox,
+	},
+}
+
 // Identify parses distro-specific files to determine distro metadata like version and release.
 func Identify(resolver source.Resolver) *Distro {
 	var distro *Distro
 
-	identityFiles := []parseEntry{
-		{
-			// most distros provide a link at this location
-			path: "/etc/os-release",
-			fn:   parseOsRelease,
-		},
-		{
-			// standard location for rhel & debian distros
-			path: "/usr/lib/os-release",
-			fn:   parseOsRelease,
-		},
-		{
-			// check for busybox (important to check this last since other distros contain the busybox binary)
-			path: "/bin/busybox",
-			fn:   parseBusyBox,
-		},
-	}
-
 identifyLoop:
 	for _, entry := range identityFiles {
 		locations, err := resolver.FilesByPath(entry.path)
@@ -52,19 +53,25 @@ identifyLoop:
 		}
 
 		for _, location := range locations {
-			content, err := resolver.FileContentsByLocation(location)
+			contentReader, err := resolver.FileContentsByLocation(location)
 
 			if err != nil {
 				log.Debugf("unable to get contents from %s: %s", entry.path, err)
 				continue
 			}
 
-			if content == "" {
+			content, err := ioutil.ReadAll(contentReader)
+			if err != nil {
+				log.Errorf("unable to read %q: %+v", location.Path, err)
+				break
+			}
+
+			if len(content) == 0 {
 				log.Debugf("no contents in file, skipping: %s", entry.path)
 				continue
 			}
 
-			if candidateDistro := entry.fn(content); candidateDistro != nil {
+			if candidateDistro := entry.fn(string(content)); candidateDistro != nil {
 				distro = candidateDistro
 				break identifyLoop
 			}

syft/source/all_layers_resolver.go

@@ -3,6 +3,7 @@ package source
 import (
 	"archive/tar"
 	"fmt"
+	"io"
 
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/image"
@@ -150,7 +151,7 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *
 		return nil
 	}
 
-	relativeRef := entry.Source.SquashedTree.File(file.Path(path))
+	relativeRef := entry.Layer.SquashedTree.File(file.Path(path))
 	if relativeRef == nil {
 		return nil
 	}
@@ -162,22 +163,22 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
 // file.Reference is a path relative to a particular layer.
-func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
+func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
 	return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
 }
 
 // FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
 // If the path does not exist an error is returned.
-func (r *AllLayersResolver) FileContentsByLocation(location Location) (string, error) {
+func (r *AllLayersResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
 	return r.img.FileContentsByRef(location.ref)
 }
 
-type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]string, error)
+type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]io.ReadCloser, error)
 
-func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]string, error) {
+func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]io.ReadCloser, error) {
 	var fileRefs = make([]file.Reference, len(locations))
 	var locationByRefs = make(map[file.Reference]Location)
-	var results = make(map[Location]string)
+	var results = make(map[Location]io.ReadCloser)
 
 	for i, location := range locations {
 		locationByRefs[location.ref] = location

syft/source/all_layers_resolver_test.go

@@ -116,8 +116,8 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 					t.Fatalf("failed to get metadata: %+v", err)
 				}
 
-				if entry.Source.Metadata.Index != expected.layer {
+				if entry.Layer.Metadata.Index != expected.layer {
-					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, expected.layer)
 				}
 			}
 		})
@@ -229,8 +229,8 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 					t.Fatalf("failed to get metadata: %+v", err)
 				}
 
-				if entry.Source.Metadata.Index != expected.layer {
+				if entry.Layer.Metadata.Index != expected.layer {
-					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, expected.layer)
 				}
 			}
 		})

syft/source/content_requester_test.go

@@ -1,6 +1,7 @@
 package source
 
 import (
+	"io/ioutil"
 	"testing"
 
 	"github.com/anchore/stereoscope/pkg/imagetest"
@@ -54,10 +55,14 @@ func TestContentRequester(t *testing.T) {
 
 			for _, entry := range data {
 				if expected, ok := test.expectedContents[entry.Location.Path]; ok {
-					for expected != entry.Contents {
+					actualBytes, err := ioutil.ReadAll(entry.Contents)
+					if err != nil {
+						t.Fatalf("could not read %q: %+v", entry.Location.Path, err)
+					}
+					for expected != string(actualBytes) {
 						t.Errorf("mismatched contents for %q", entry.Location.Path)
 						dmp := diffmatchpatch.New()
-						diffs := dmp.DiffMain(expected, entry.Contents, true)
+						diffs := dmp.DiffMain(expected, string(actualBytes), true)
 						t.Errorf("diff: %s", dmp.DiffPrettyText(diffs))
 					}
 					continue

syft/source/directory_resolver.go

@@ -2,11 +2,13 @@ package source
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
 
+	"github.com/anchore/stereoscope/pkg/file"
+
 	"github.com/anchore/syft/internal/log"
 	"github.com/bmatcuk/doublestar"
 )
@@ -96,35 +98,16 @@ func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Locatio
 }
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative a directory.
-func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
+func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
-	refContents := make(map[Location]string)
+	refContents := make(map[Location]io.ReadCloser)
 	for _, location := range locations {
-		contents, err := fileContents(location.Path)
+		refContents[location] = file.NewDeferredReadCloser(location.Path)
-
-		if err != nil {
-			return nil, fmt.Errorf("could not read contents of file: %s", location.Path)
-		}
-		refContents[location] = string(contents)
 	}
 	return refContents, nil
 }
 
 // FileContentsByLocation fetches file contents for a single file reference relative to a directory.
 // If the path does not exist an error is returned.
-func (s DirectoryResolver) FileContentsByLocation(location Location) (string, error) {
+func (s DirectoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
-	contents, err := fileContents(location.Path)
+	return file.NewDeferredReadCloser(location.Path), nil
-	if err != nil {
-		return "", fmt.Errorf("could not read contents of file: %s", location.Path)
-	}
-
-	return string(contents), nil
-}
-
-func fileContents(path string) ([]byte, error) {
-	contents, err := ioutil.ReadFile(path)
-
-	if err != nil {
-		return nil, err
-	}
-	return contents, nil
 }

syft/source/file_data.go

@@ -1,6 +1,8 @@
 package source
 
+import "io"
+
 type FileData struct {
 	Location Location
-	Contents string
+	Contents io.ReadCloser
 }

syft/source/image_squash_resolver.go

@@ -2,6 +2,7 @@ package source
 
 import (
 	"fmt"
+	"io"
 
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/image"
@@ -121,12 +122,12 @@ func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Locat
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
 // file.Reference is a path relative to a particular layer, in this case only from the squashed representation.
-func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
+func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
 	return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
 }
 
 // FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
 // If the path does not exist an error is returned.
-func (r *ImageSquashResolver) FileContentsByLocation(location Location) (string, error) {
+func (r *ImageSquashResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
 	return r.img.FileContentsByRef(location.ref)
 }

syft/source/image_squash_resolver_test.go

@@ -89,8 +89,8 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 				t.Fatalf("failed to get metadata: %+v", err)
 			}
 
-			if entry.Source.Metadata.Index != c.resolveLayer {
+			if entry.Layer.Metadata.Index != c.resolveLayer {
-				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, c.resolveLayer)
 			}
 		})
 	}
@@ -179,8 +179,8 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 				t.Fatalf("failed to get metadata: %+v", err)
 			}
 
-			if entry.Source.Metadata.Index != c.resolveLayer {
+			if entry.Layer.Metadata.Index != c.resolveLayer {
-				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, c.resolveLayer)
 			}
 		})
 	}

syft/source/location.go

@@ -34,7 +34,7 @@ func NewLocationFromImage(ref file.Reference, img *image.Image) Location {
 
 	return Location{
 		Path:         string(ref.Path),
-		FileSystemID: entry.Source.Metadata.Digest,
+		FileSystemID: entry.Layer.Metadata.Digest,
 		ref:          ref,
 	}
 }

syft/source/resolver.go

@@ -2,6 +2,7 @@ package source
 
 import (
 	"fmt"
+	"io"
 
 	"github.com/anchore/stereoscope/pkg/image"
 )
@@ -14,8 +15,8 @@ type Resolver interface {
 
 // ContentResolver knows how to get file content for given file.References
 type ContentResolver interface {
-	FileContentsByLocation(Location) (string, error)
+	FileContentsByLocation(Location) (io.ReadCloser, error)
-	MultipleFileContentsByLocation([]Location) (map[Location]string, error)
+	MultipleFileContentsByLocation([]Location) (map[Location]io.ReadCloser, error)
 	// TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering).
 }
 

syft/source/source_test.go

@@ -1,6 +1,7 @@
 package source
 
 import (
+	"io/ioutil"
 	"os"
 	"testing"
 
@@ -137,9 +138,14 @@ func TestMultipleFileContentsByLocation(t *testing.T) {
 			location := locations[0]
 
 			contents, err := p.Resolver.MultipleFileContentsByLocation([]Location{location})
-			content := contents[location]
+			contentReader := contents[location]
 
-			if content != test.expected {
+			content, err := ioutil.ReadAll(contentReader)
+			if err != nil {
+				t.Fatalf("cannot read contents: %+v", err)
+			}
+
+			if string(content) != test.expected {
 				t.Errorf("unexpected contents from file: '%s' != '%s'", content, test.expected)
 			}