Resolve symlinks when fetching file contents (#782)

syft/file/all_regular_files.go

@@ -0,0 +1,30 @@
+package file
+
+import (
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/source"
+)
+
+func allRegularFiles(resolver source.FileResolver) (locations []source.Location) {
+	for location := range resolver.AllLocations() {
+		resolvedLocations, err := resolver.FilesByPath(location.RealPath)
+		if err != nil {
+			log.Warnf("unable to resolve %+v: %+v", location, err)
+			continue
+		}
+
+		for _, resolvedLocation := range resolvedLocations {
+			metadata, err := resolver.FileMetadataByLocation(resolvedLocation)
+			if err != nil {
+				log.Warnf("unable to get metadata for %+v: %+v", location, err)
+				continue
+			}
+
+			if metadata.Type != source.RegularFile {
+				continue
+			}
+			locations = append(locations, resolvedLocation)
+		}
+	}
+	return locations
+}

syft/file/all_regular_files_test.go

@@ -0,0 +1,74 @@
+package file
+
+import (
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/syft/source"
+	"github.com/scylladb/go-set/strset"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func Test_allRegularFiles(t *testing.T) {
+	type access struct {
+		realPath    string
+		virtualPath string
+	}
+	tests := []struct {
+		name             string
+		setup            func() source.FileResolver
+		wantRealPaths    *strset.Set
+		wantVirtualPaths *strset.Set
+	}{
+		{
+			name: "image",
+			setup: func() source.FileResolver {
+				testImage := "image-file-type-mix"
+
+				if *updateImageGoldenFiles {
+					imagetest.UpdateGoldenFixtureImage(t, testImage)
+				}
+
+				img := imagetest.GetGoldenFixtureImage(t, testImage)
+
+				s, err := source.NewFromImage(img, "---")
+				require.NoError(t, err)
+
+				r, err := s.FileResolver(source.SquashedScope)
+				require.NoError(t, err)
+
+				return r
+			},
+			wantRealPaths:    strset.New("/file-1.txt"),
+			wantVirtualPaths: strset.New("/file-1.txt", "/symlink-1", "/hardlink-1"),
+		},
+		{
+			name: "directory",
+			setup: func() source.FileResolver {
+				s, err := source.NewFromDirectory("test-fixtures/symlinked-root/nested/link-root")
+				require.NoError(t, err)
+				r, err := s.FileResolver(source.SquashedScope)
+				require.NoError(t, err)
+				return r
+			},
+			wantRealPaths:    strset.New("file1.txt", "nested/file2.txt"),
+			wantVirtualPaths: strset.New("nested/linked-file1.txt"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			resolver := tt.setup()
+			locations := allRegularFiles(resolver)
+			realLocations := strset.New()
+			virtualLocations := strset.New()
+			for _, l := range locations {
+				realLocations.Add(l.RealPath)
+				if l.VirtualPath != "" {
+					virtualLocations.Add(l.VirtualPath)
+				}
+			}
+			assert.ElementsMatch(t, tt.wantRealPaths.List(), realLocations.List(), "mismatched real paths")
+			assert.ElementsMatch(t, tt.wantVirtualPaths.List(), virtualLocations.List(), "mismatched virtual paths")
+		})
+	}
+}

syft/file/classification_cataloger.go

@@ -19,11 +19,12 @@ func (i *ClassificationCataloger) Catalog(resolver source.FileResolver) (map[sou
 	results := make(map[source.Coordinates][]Classification)
 
 	numResults := 0
-	for location := range resolver.AllLocations() {
+	for _, location := range allRegularFiles(resolver) {
 		for _, classifier := range i.classifiers {
 			result, err := classifier.Classify(resolver, location)
 			if err != nil {
-				return nil, err
+				log.Warnf("file classification cataloger failed with class=%q at location=%+v: %+v", classifier.Class, location, err)
+				continue
 			}
 			if result != nil {
 				results[location.Coordinates] = append(results[location.Coordinates], *result)

syft/file/classification_cataloger_test.go

@@ -1,6 +1,7 @@
 package file
 
 import (
+	"github.com/anchore/stereoscope/pkg/imagetest"
 	"testing"
 
 	"github.com/anchore/syft/syft/source"
@@ -88,7 +89,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 		{
 			name:       "positive-busybox",
 			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "busybox",
+			location:   "[", // note: busybox is a link to [
 			expected: []Classification{
 				{
 					Class: "busybox-binary",
@@ -116,13 +117,67 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 			actualResults, err := c.Catalog(resolver)
 			test.expectedErr(t, err)
 
-			loc := source.NewLocation(test.location)
+			ok := false
+			for actualLoc, actualClassification := range actualResults {
+				if test.location == actualLoc.RealPath {
+					ok = true
+					assert.Equal(t, test.expected, actualClassification)
+				}
+			}
+
+			if !ok {
+				t.Fatalf("could not find test location=%q", test.location)
+			}
+
+		})
+	}
+}
+
+func TestClassifierCataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T) {
+	tests := []struct {
+		name         string
+		fixtureImage string
+		location     string
+		expected     []Classification
+		expectedErr  func(assert.TestingT, error, ...interface{}) bool
+	}{
+		{
+			name:         "busybox-regression",
+			fixtureImage: "image-busybox",
+			location:     "/bin/[",
+			expected: []Classification{
+				{
+					Class: "busybox-binary",
+					Metadata: map[string]string{
+						"version": "1.35.0",
+					},
+				},
+			},
+			expectedErr: assert.NoError,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			c, err := NewClassificationCataloger(DefaultClassifiers)
+			test.expectedErr(t, err)
+
+			img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureImage)
+			src, err := source.NewFromImage(img, "test-img")
+			test.expectedErr(t, err)
+
+			resolver, err := src.FileResolver(source.SquashedScope)
+			test.expectedErr(t, err)
+
+			actualResults, err := c.Catalog(resolver)
+			test.expectedErr(t, err)
 
 			ok := false
-			for actual_loc, actual_classification := range actualResults {
-				if loc.RealPath == actual_loc.RealPath {
+			for actuaLoc, actualClassification := range actualResults {
+				if actuaLoc.RealPath == test.location {
 					ok = true
-					assert.Equal(t, test.expected, actual_classification)
+					assert.Equal(t, test.expected, actualClassification)
 				}
 			}
 

syft/file/contents_cataloger.go

@@ -3,6 +3,7 @@ package file
 import (
 	"bytes"
 	"encoding/base64"
+	"fmt"
 	"io"
 
 	"github.com/anchore/syft/internal"
@@ -66,9 +67,12 @@ func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, locati
 	buf := &bytes.Buffer{}
 	encoder := base64.NewEncoder(base64.StdEncoding, buf)
 	if _, err = io.Copy(encoder, contentReader); err != nil {
-		return "", internal.ErrPath{Path: location.RealPath, Err: err}
+		return "", internal.ErrPath{Context: "contents-cataloger", Path: location.RealPath, Err: err}
+	}
+	// note: it's important to close the reader before reading from the buffer since closing will flush the remaining bytes
+	if err := encoder.Close(); err != nil {
+		return "", fmt.Errorf("unable to close base64 encoder: %w", err)
 	}
-	encoder.Close()
 
 	return buf.String(), nil
 }

syft/file/digest_cataloger.go

@@ -2,6 +2,7 @@ package file
 
 import (
 	"crypto"
+	"errors"
 	"fmt"
 	"hash"
 	"io"
@@ -19,6 +20,8 @@ import (
 	"github.com/anchore/syft/syft/source"
 )
 
+var errUndigestableFile = errors.New("undigestable file")
+
 type DigestsCataloger struct {
 	hashes []crypto.Hash
 }
@@ -31,16 +34,18 @@ func NewDigestsCataloger(hashes []crypto.Hash) (*DigestsCataloger, error) {
 
 func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]Digest, error) {
 	results := make(map[source.Coordinates][]Digest)
-	var locations []source.Location
-	for location := range resolver.AllLocations() {
-		locations = append(locations, location)
-	}
+	locations := allRegularFiles(resolver)
 	stage, prog := digestsCatalogingProgress(int64(len(locations)))
 	for _, location := range locations {
 		stage.Current = location.RealPath
 		result, err := i.catalogLocation(resolver, location)
+
+		if errors.Is(err, errUndigestableFile) {
+			continue
+		}
+
 		if internal.IsErrPathPermission(err) {
-			log.Debugf("file digests cataloger skipping - %+v", err)
+			log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
 			continue
 		}
 
@@ -56,6 +61,16 @@ func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Coo
 }
 
 func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]Digest, error) {
+	meta, err := resolver.FileMetadataByLocation(location)
+	if err != nil {
+		return nil, err
+	}
+
+	// we should only attempt to report digests for files that are regular files (don't attempt to resolve links)
+	if meta.Type != source.RegularFile {
+		return nil, errUndigestableFile
+	}
+
 	contentReader, err := resolver.FileContentsByLocation(location)
 	if err != nil {
 		return nil, err
@@ -72,7 +87,7 @@ func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, locatio
 
 	size, err := io.Copy(io.MultiWriter(writers...), contentReader)
 	if err != nil {
-		return nil, internal.ErrPath{Path: location.RealPath, Err: err}
+		return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err}
 	}
 
 	if size == 0 {

syft/file/digest_cataloger_test.go

@@ -3,8 +3,10 @@ package file
 import (
 	"crypto"
 	"fmt"
+	"github.com/stretchr/testify/require"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/anchore/stereoscope/pkg/file"
@@ -16,11 +18,11 @@ import (
 	"github.com/anchore/syft/syft/source"
 )
 
-func testDigests(t testing.TB, files []string, hashes ...crypto.Hash) map[source.Coordinates][]Digest {
+func testDigests(t testing.TB, root string, files []string, hashes ...crypto.Hash) map[source.Coordinates][]Digest {
 	digests := make(map[source.Coordinates][]Digest)
 
 	for _, f := range files {
-		fh, err := os.Open(f)
+		fh, err := os.Open(filepath.Join(root, f))
 		if err != nil {
 			t.Fatalf("could not open %q : %+v", f, err)
 		}
@@ -29,6 +31,12 @@ func testDigests(t testing.TB, files []string, hashes ...crypto.Hash) map[source
 			t.Fatalf("could not read %q : %+v", f, err)
 		}
 
+		if len(b) == 0 {
+			// we don't keep digests for empty files
+			digests[source.NewLocation(f).Coordinates] = []Digest{}
+			continue
+		}
+
 		for _, hash := range hashes {
 			h := hash.New()
 			h.Write(b)
@@ -42,55 +50,43 @@ func testDigests(t testing.TB, files []string, hashes ...crypto.Hash) map[source
 	return digests
 }
 
-func TestDigestsCataloger_SimpleContents(t *testing.T) {
-	regularFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
+func TestDigestsCataloger(t *testing.T) {
 
 	tests := []struct {
-		name       string
-		digests    []crypto.Hash
-		files      []string
-		expected   map[source.Coordinates][]Digest
-		catalogErr bool
+		name     string
+		digests  []crypto.Hash
+		files    []string
+		expected map[source.Coordinates][]Digest
 	}{
 		{
 			name:     "md5",
 			digests:  []crypto.Hash{crypto.MD5},
-			files:    regularFiles,
-			expected: testDigests(t, regularFiles, crypto.MD5),
+			files:    []string{"test-fixtures/last/empty/empty", "test-fixtures/last/path.txt"},
+			expected: testDigests(t, "test-fixtures/last", []string{"empty/empty", "path.txt"}, crypto.MD5),
 		},
 		{
 			name:     "md5-sha1-sha256",
 			digests:  []crypto.Hash{crypto.MD5, crypto.SHA1, crypto.SHA256},
-			files:    regularFiles,
-			expected: testDigests(t, regularFiles, crypto.MD5, crypto.SHA1, crypto.SHA256),
-		},
-		{
-			name:       "directory returns error",
-			digests:    []crypto.Hash{crypto.MD5},
-			files:      []string{"test-fixtures/last"},
-			catalogErr: true,
+			files:    []string{"test-fixtures/last/empty/empty", "test-fixtures/last/path.txt"},
+			expected: testDigests(t, "test-fixtures/last", []string{"empty/empty", "path.txt"}, crypto.MD5, crypto.SHA1, crypto.SHA256),
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			c, err := NewDigestsCataloger(test.digests)
-			if err != nil {
-				t.Fatalf("could not create cataloger: %+v", err)
-			}
+			require.NoError(t, err)
+
+			src, err := source.NewFromDirectory("test-fixtures/last/")
+			require.NoError(t, err)
+
+			resolver, err := src.FileResolver(source.SquashedScope)
+			require.NoError(t, err)
 
-			resolver := source.NewMockResolverForPaths(test.files...)
 			actual, err := c.Catalog(resolver)
-			if err != nil && !test.catalogErr {
-				t.Fatalf("could not catalog (but should have been able to): %+v", err)
-			} else if err == nil && test.catalogErr {
-				t.Fatalf("expected catalog error but did not get one")
-			} else if test.catalogErr && err != nil {
-				return
-			}
-
-			assert.Equal(t, actual, test.expected, "mismatched digests")
+			require.NoError(t, err)
 
+			assert.Equal(t, test.expected, actual, "mismatched digests")
 		})
 	}
 }

syft/file/metadata_cataloger_test.go

@@ -66,7 +66,7 @@ func TestFileMetadataCataloger(t *testing.T) {
 				LinkDestination: "file-1.txt",
 				UserID:          1,
 				GroupID:         2,
-				MIMEType:        "text/plain",
+				MIMEType:        "",
 			},
 		},
 		{
@@ -78,7 +78,7 @@ func TestFileMetadataCataloger(t *testing.T) {
 				LinkDestination: "file-1.txt",
 				UserID:          0,
 				GroupID:         0,
-				MIMEType:        "text/plain",
+				MIMEType:        "",
 			},
 		},
 		{
@@ -89,7 +89,7 @@ func TestFileMetadataCataloger(t *testing.T) {
 				Type:     "CharacterDevice",
 				UserID:   0,
 				GroupID:  0,
-				MIMEType: "text/plain",
+				MIMEType: "",
 			},
 		},
 		{
@@ -100,7 +100,7 @@ func TestFileMetadataCataloger(t *testing.T) {
 				Type:     "BlockDevice",
 				UserID:   0,
 				GroupID:  0,
-				MIMEType: "text/plain",
+				MIMEType: "",
 			},
 		},
 		{
@@ -111,7 +111,7 @@ func TestFileMetadataCataloger(t *testing.T) {
 				Type:     "FIFONode",
 				UserID:   0,
 				GroupID:  0,
-				MIMEType: "text/plain",
+				MIMEType: "",
 			},
 		},
 		{
@@ -122,7 +122,7 @@ func TestFileMetadataCataloger(t *testing.T) {
 				Type:     "Directory",
 				UserID:   0,
 				GroupID:  0,
-				MIMEType: "text/plain",
+				MIMEType: "",
 			},
 		},
 	}

syft/file/secrets_cataloger.go

@@ -42,10 +42,7 @@ func NewSecretsCataloger(patterns map[string]*regexp.Regexp, revealValues bool,
 
 func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]SearchResult, error) {
 	results := make(map[source.Coordinates][]SearchResult)
-	var locations []source.Location
-	for location := range resolver.AllLocations() {
-		locations = append(locations, location)
-	}
+	locations := allRegularFiles(resolver)
 	stage, prog, secretsDiscovered := secretsCatalogingProgress(int64(len(locations)))
 	for _, location := range locations {
 		stage.Current = location.RealPath
@@ -75,6 +72,10 @@ func (i *SecretsCataloger) catalogLocation(resolver source.FileResolver, locatio
 		return nil, err
 	}
 
+	if metadata.Size == 0 {
+		return nil, nil
+	}
+
 	if i.skipFilesAboveSize > 0 && metadata.Size > i.skipFilesAboveSize {
 		return nil, nil
 	}
@@ -82,7 +83,7 @@ func (i *SecretsCataloger) catalogLocation(resolver source.FileResolver, locatio
 	// TODO: in the future we can swap out search strategies here
 	secrets, err := catalogLocationByLine(resolver, location, i.patterns)
 	if err != nil {
-		return nil, internal.ErrPath{Path: location.RealPath, Err: err}
+		return nil, internal.ErrPath{Context: "secrets-cataloger", Path: location.RealPath, Err: err}
 	}
 
 	if i.revealValues {

syft/file/test-fixtures/classifiers/positive/[

@@ -0,0 +1,3 @@
+# note: this SHOULD match as busybox 3.33.3
+
+noise!BusyBox v3.33.3!noise

syft/file/test-fixtures/classifiers/positive/busybox

@@ -1,3 +0,0 @@
-# note: this SHOULD match as busybox 3.33.3
-
-noise!BusyBox v3.33.3!noise

syft/file/test-fixtures/classifiers/positive/busybox

@@ -0,0 +1 @@
+./[

syft/file/test-fixtures/image-busybox/Dockerfile

@@ -0,0 +1 @@
+FROM busybox:1.35

syft/file/test-fixtures/image-file-type-mix/Dockerfile

@@ -9,3 +9,5 @@ RUN ln file-1.txt hardlink-1
 RUN mknod char-device-1 c 89 1
 RUN mknod block-device-1 b 0 1
 RUN mknod fifo-1 p
+RUN mkdir /dir
+RUN rm -rf home etc/group etc/localtime etc/mtab etc/network etc/passwd etc/shadow var usr bin/*

syft/file/test-fixtures/symlinked-root/nested/link-root

@@ -0,0 +1 @@
+../real-root

syft/file/test-fixtures/symlinked-root/real-root/file1.txt

@@ -0,0 +1 @@
+contents!

syft/file/test-fixtures/symlinked-root/real-root/nested/file2.txt

@@ -0,0 +1 @@
+more contents!

syft/file/test-fixtures/symlinked-root/real-root/nested/linked-file1.txt

@@ -0,0 +1 @@
+../file1.txt