allow file metadata digests to be optional + add link destination

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

internal/presenter/poweruser/json_file_metadata.go

@@ -18,9 +18,10 @@ type JSONFileMetadata struct {
 type JSONFileMetadataEntry struct {
 	Mode            int             `json:"mode"`
 	Type            source.FileType `json:"type"`
+	LinkDestination string          `json:"linkDestination,omitempty"`
 	UserID          int             `json:"userID"`
 	GroupID         int             `json:"groupID"`
-	Digests []file.Digest   `json:"digests"`
+	Digests         []file.Digest   `json:"digests,omitempty"`
 }
 
 func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests map[source.Location][]file.Digest) ([]JSONFileMetadata, error) {
@@ -31,7 +32,7 @@ func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests m
 			return nil, fmt.Errorf("invalid mode found in file catalog @ location=%+v mode=%q: %w", location, metadata.Mode, err)
 		}
 
-		digestResults := make([]file.Digest, 0)
+		var digestResults []file.Digest
 		if digestsForLocation, exists := digests[location]; exists {
 			digestResults = digestsForLocation
 		}
@@ -41,6 +42,7 @@ func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests m
 			Metadata: JSONFileMetadataEntry{
 				Mode:            mode,
 				Type:            metadata.Type,
+				LinkDestination: metadata.LinkDestination,
 				UserID:          metadata.UserID,
 				GroupID:         metadata.GroupID,
 				Digests:         digestResults,
@@ -50,10 +52,10 @@ func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests m
 
 	// sort by real path then virtual path to ensure the result is stable across multiple runs
 	sort.SliceStable(results, func(i, j int) bool {
-		if results[i].Location.RealPath != results[j].Location.RealPath {
+		if results[i].Location.RealPath == results[j].Location.RealPath {
 			return results[i].Location.VirtualPath < results[j].Location.VirtualPath
 		}
-		return false
+		return results[i].Location.RealPath < results[j].Location.RealPath
 	})
 	return results, nil
 }

internal/presenter/poweruser/json_presenter_test.go

@@ -0,0 +1,172 @@
+package poweruser
+
+import (
+	"bytes"
+	"flag"
+	"testing"
+
+	"github.com/sergi/go-diff/diffmatchpatch"
+
+	"github.com/anchore/syft/syft/file"
+
+	"github.com/anchore/go-testutils"
+	"github.com/anchore/syft/internal/config"
+	"github.com/anchore/syft/syft/distro"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+var updateJSONGoldenFiles = flag.Bool("update-json", false, "update the *.golden files for json presenters")
+
+func must(c pkg.CPE, e error) pkg.CPE {
+	if e != nil {
+		panic(e)
+	}
+	return c
+}
+
+func TestJSONPresenter(t *testing.T) {
+	var buffer bytes.Buffer
+
+	catalog := pkg.NewCatalog()
+
+	catalog.Add(pkg.Package{
+		ID:      "package-1-id",
+		Name:    "package-1",
+		Version: "1.0.1",
+		Locations: []source.Location{
+			{
+				RealPath: "/a/place/a",
+			},
+		},
+		Type:         pkg.PythonPkg,
+		FoundBy:      "the-cataloger-1",
+		Language:     pkg.Python,
+		MetadataType: pkg.PythonPackageMetadataType,
+		Licenses:     []string{"MIT"},
+		Metadata: pkg.PythonPackageMetadata{
+			Name:    "package-1",
+			Version: "1.0.1",
+		},
+		PURL: "a-purl-1",
+		CPEs: []pkg.CPE{
+			must(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")),
+		},
+	})
+	catalog.Add(pkg.Package{
+		ID:      "package-2-id",
+		Name:    "package-2",
+		Version: "2.0.1",
+		Locations: []source.Location{
+			{
+				RealPath: "/b/place/b",
+			},
+		},
+		Type:         pkg.DebPkg,
+		FoundBy:      "the-cataloger-2",
+		MetadataType: pkg.DpkgMetadataType,
+		Metadata: pkg.DpkgMetadata{
+			Package: "package-2",
+			Version: "2.0.1",
+		},
+		PURL: "a-purl-2",
+		CPEs: []pkg.CPE{
+			must(pkg.NewCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*")),
+		},
+	})
+
+	cfg := JSONDocumentConfig{
+		ApplicationConfig: config.Application{},
+		PackageCatalog:    catalog,
+		FileMetadata: map[source.Location]source.FileMetadata{
+			source.NewLocation("/a/place"): {
+				Mode:    0775,
+				Type:    "directory",
+				UserID:  0,
+				GroupID: 0,
+			},
+			source.NewLocation("/a/place/a"): {
+				Mode:    0775,
+				Type:    "regularFile",
+				UserID:  0,
+				GroupID: 0,
+			},
+			source.NewLocation("/b"): {
+				Mode:            0775,
+				Type:            "symbolicLink",
+				LinkDestination: "/c",
+				UserID:          0,
+				GroupID:         0,
+			},
+			source.NewLocation("/b/place/b"): {
+				Mode:    0644,
+				Type:    "regularFile",
+				UserID:  1,
+				GroupID: 2,
+			},
+		},
+		FileDigests: map[source.Location][]file.Digest{
+			source.NewLocation("/a/place/a"): {
+				{
+					Algorithm: "sha256",
+					Value:     "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
+				},
+			},
+			source.NewLocation("/b/place/b"): {
+				{
+					Algorithm: "sha256",
+					Value:     "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c",
+				},
+			},
+		},
+		Distro: &distro.Distro{
+			Type:       distro.RedHat,
+			RawVersion: "7",
+			IDLike:     "rhel",
+		},
+		SourceMetadata: source.Metadata{
+			Scheme: source.ImageScheme,
+			ImageMetadata: source.ImageMetadata{
+				UserInput:      "user-image-input",
+				ID:             "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
+				ManifestDigest: "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
+				MediaType:      "application/vnd.docker.distribution.manifest.v2+json",
+				Tags: []string{
+					"stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b",
+				},
+				Size: 38,
+				Layers: []source.LayerMetadata{
+					{
+						MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
+						Digest:    "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b",
+						Size:      22,
+					},
+					{
+						MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
+						Digest:    "sha256:366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
+						Size:      16,
+					},
+				},
+				RawManifest: []byte("eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJh..."),
+				RawConfig:   []byte("eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZp..."),
+			},
+		},
+	}
+
+	if err := NewJSONPresenter(cfg).Present(&buffer); err != nil {
+		t.Fatal(err)
+	}
+	actual := buffer.Bytes()
+
+	if *updateJSONGoldenFiles {
+		testutils.UpdateGoldenFileContents(t, actual)
+	}
+
+	var expected = testutils.GetGoldenFileContents(t)
+
+	if !bytes.Equal(expected, actual) {
+		dmp := diffmatchpatch.New()
+		diffs := dmp.DiffMain(string(expected), string(actual), true)
+		t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs))
+	}
+}

internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden

@@ -0,0 +1,198 @@
+{
+ "fileMetadata": [
+  {
+   "location": {
+    "path": "/a/place"
+   },
+   "metadata": {
+    "mode": 775,
+    "type": "directory",
+    "userID": 0,
+    "groupID": 0
+   }
+  },
+  {
+   "location": {
+    "path": "/a/place/a"
+   },
+   "metadata": {
+    "mode": 775,
+    "type": "regularFile",
+    "userID": 0,
+    "groupID": 0,
+    "digests": [
+     {
+      "algorithm": "sha256",
+      "value": "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703"
+     }
+    ]
+   }
+  },
+  {
+   "location": {
+    "path": "/b"
+   },
+   "metadata": {
+    "mode": 775,
+    "type": "symbolicLink",
+    "linkDestination": "/c",
+    "userID": 0,
+    "groupID": 0
+   }
+  },
+  {
+   "location": {
+    "path": "/b/place/b"
+   },
+   "metadata": {
+    "mode": 644,
+    "type": "regularFile",
+    "userID": 1,
+    "groupID": 2,
+    "digests": [
+     {
+      "algorithm": "sha256",
+      "value": "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c"
+     }
+    ]
+   }
+  }
+ ],
+ "artifacts": [
+  {
+   "id": "package-1-id",
+   "name": "package-1",
+   "version": "1.0.1",
+   "type": "python",
+   "foundBy": "the-cataloger-1",
+   "locations": [
+    {
+     "path": "/a/place/a"
+    }
+   ],
+   "licenses": [
+    "MIT"
+   ],
+   "language": "python",
+   "cpes": [
+    "cpe:2.3:*:some:package:1:*:*:*:*:*:*:*"
+   ],
+   "purl": "a-purl-1",
+   "metadataType": "PythonPackageMetadata",
+   "metadata": {
+    "name": "package-1",
+    "version": "1.0.1",
+    "license": "",
+    "author": "",
+    "authorEmail": "",
+    "platform": "",
+    "sitePackagesRootPath": ""
+   }
+  },
+  {
+   "id": "package-2-id",
+   "name": "package-2",
+   "version": "2.0.1",
+   "type": "deb",
+   "foundBy": "the-cataloger-2",
+   "locations": [
+    {
+     "path": "/b/place/b"
+    }
+   ],
+   "licenses": [],
+   "language": "",
+   "cpes": [
+    "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"
+   ],
+   "purl": "a-purl-2",
+   "metadataType": "DpkgMetadata",
+   "metadata": {
+    "package": "package-2",
+    "source": "",
+    "version": "2.0.1",
+    "sourceVersion": "",
+    "architecture": "",
+    "maintainer": "",
+    "installedSize": 0,
+    "files": null
+   }
+  }
+ ],
+ "artifactRelationships": [],
+ "source": {
+  "type": "image",
+  "target": {
+   "userInput": "user-image-input",
+   "imageID": "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
+   "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
+   "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+   "tags": [
+    "stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b"
+   ],
+   "imageSize": 38,
+   "layers": [
+    {
+     "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+     "digest": "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b",
+     "size": 22
+    },
+    {
+     "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+     "digest": "sha256:366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
+     "size": 16
+    }
+   ],
+   "manifest": "ZXlKelkyaGxiV0ZXWlhKemFXOXVJam95TENKdFpXUnBZVlI1Y0dVaU9pSmguLi4=",
+   "config": "ZXlKaGNtTm9hWFJsWTNSMWNtVWlPaUpoYldRMk5DSXNJbU52Ym1acC4uLg==",
+   "scope": ""
+  }
+ },
+ "distro": {
+  "name": "redhat",
+  "version": "7",
+  "idLike": "rhel"
+ },
+ "descriptor": {
+  "name": "syft",
+  "version": "[not provided]",
+  "configuration": {
+   "configPath": "",
+   "output": "",
+   "quiet": false,
+   "log": {
+    "structured": false,
+    "level": "",
+    "file-location": ""
+   },
+   "dev": {
+    "profile-cpu": false,
+    "profile-mem": false
+   },
+   "check-for-app-update": false,
+   "anchore": {
+    "host": "",
+    "path": "",
+    "dockerfile": "",
+    "overwrite-existing-image": false
+   },
+   "package": {
+    "cataloger": {
+     "enabled": false,
+     "scope": ""
+    }
+   },
+   "file-metadata": {
+    "cataloger": {
+     "enabled": false,
+     "scope": ""
+    },
+    "digests": null
+   }
+  }
+ },
+ "schema": {
+  "version": "1.0.4",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.4.json"
+ }
+}

schema/json/schema-1.0.4.json

@@ -345,8 +345,7 @@
         "mode",
         "type",
         "userID",
-        "groupID",
+        "groupID"
-        "digests"
       ],
       "properties": {
         "mode": {
@@ -355,6 +354,9 @@
         "type": {
           "type": "string"
         },
+        "linkDestination": {
+          "type": "string"
+        },
         "userID": {
           "type": "integer"
         },

syft/file/digest_cataloger.go

@@ -74,8 +74,11 @@ func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, locatio
 		return nil, fmt.Errorf("unable to observe contents of %+v: %+v", location.RealPath, err)
 	}
 
+	if size == 0 {
+		return make([]Digest, 0), nil
+	}
+
 	result := make([]Digest, len(i.hashes))
-	if size > 0 {
 	// only capture digests when there is content. It is important to do this based on SIZE and not
 	// FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only
 	// file type but a body is still allowed.
@@ -85,7 +88,6 @@ func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, locatio
 			Value:     fmt.Sprintf("%+x", hasher.Sum(nil)),
 		}
 	}
-	}
 
 	return result, nil
 }

syft/file/digest_cataloger_test.go

@@ -7,6 +7,10 @@ import (
 	"os"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/file"
+
+	"github.com/anchore/stereoscope/pkg/imagetest"
+
 	"github.com/stretchr/testify/assert"
 
 	"github.com/anchore/syft/syft/source"
@@ -38,12 +42,13 @@ func testDigests(t testing.TB, files []string, hashes ...crypto.Hash) map[source
 	return digests
 }
 
-func TestDigestsCataloger(t *testing.T) {
+func TestDigestsCataloger_SimpleContents(t *testing.T) {
-	files := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
+	regularFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
 
 	tests := []struct {
 		name           string
 		algorithms     []string
+		files          []string
 		expected       map[source.Location][]Digest
 		constructorErr bool
 		catalogErr     bool
@@ -51,22 +56,32 @@ func TestDigestsCataloger(t *testing.T) {
 		{
 			name:           "bad algorithm",
 			algorithms:     []string{"sha-nothing"},
+			files:          regularFiles,
 			constructorErr: true,
 		},
 		{
 			name:           "unsupported algorithm",
 			algorithms:     []string{"sha512"},
+			files:          regularFiles,
 			constructorErr: true,
 		},
 		{
-			name:       "md5-sha1-sha256",
+			name:       "md5",
 			algorithms: []string{"md5"},
-			expected:   testDigests(t, files, crypto.MD5),
+			files:      regularFiles,
+			expected:   testDigests(t, regularFiles, crypto.MD5),
 		},
 		{
 			name:       "md5-sha1-sha256",
 			algorithms: []string{"md5", "sha1", "sha256"},
-			expected:   testDigests(t, files, crypto.MD5, crypto.SHA1, crypto.SHA256),
+			files:      regularFiles,
+			expected:   testDigests(t, regularFiles, crypto.MD5, crypto.SHA1, crypto.SHA256),
+		},
+		{
+			name:       "directory returns error",
+			algorithms: []string{"md5"},
+			files:      []string{"test-fixtures/last"},
+			catalogErr: true,
 		},
 	}
 
@@ -81,7 +96,7 @@ func TestDigestsCataloger(t *testing.T) {
 				return
 			}
 
-			resolver := source.NewMockResolverForPaths(files...)
+			resolver := source.NewMockResolverForPaths(test.files...)
 			actual, err := c.Catalog(resolver)
 			if err != nil && !test.catalogErr {
 				t.Fatalf("could not catalog (but should have been able to): %+v", err)
@@ -96,3 +111,80 @@ func TestDigestsCataloger(t *testing.T) {
 		})
 	}
 }
+
+func TestDigestsCataloger_MixFileTypes(t *testing.T) {
+	testImage := "image-file-type-mix"
+
+	if *updateImageGoldenFiles {
+		imagetest.UpdateGoldenFixtureImage(t, testImage)
+	}
+
+	img := imagetest.GetGoldenFixtureImage(t, testImage)
+
+	src, err := source.NewFromImage(img, "---")
+	if err != nil {
+		t.Fatalf("could not create source: %+v", err)
+	}
+
+	resolver, err := src.FileResolver(source.SquashedScope)
+	if err != nil {
+		t.Fatalf("could not create resolver: %+v", err)
+	}
+
+	tests := []struct {
+		path     string
+		expected string
+	}{
+		{
+			path:     "/file-1.txt",
+			expected: "888c139e550867814eb7c33b84d76e4d",
+		},
+		{
+			path: "/hardlink-1",
+		},
+		{
+			path: "/symlink-1",
+		},
+		{
+			path: "/char-device-1",
+		},
+		{
+			path: "/block-device-1",
+		},
+		{
+			path: "/fifo-1",
+		},
+		{
+			path: "/bin",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.path, func(t *testing.T) {
+			c, err := NewDigestsCataloger([]string{"md5"})
+			if err != nil {
+				t.Fatalf("unable to get cataloger: %+v", err)
+			}
+
+			actual, err := c.Catalog(resolver)
+			if err != nil {
+				t.Fatalf("could not catalog: %+v", err)
+			}
+
+			_, ref, err := img.SquashedTree().File(file.Path(test.path))
+			if err != nil {
+				t.Fatalf("unable to get file=%q : %+v", test.path, err)
+			}
+			l := source.NewLocationFromImage(test.path, *ref, img)
+
+			if len(actual[l]) == 0 {
+				if test.expected != "" {
+					t.Fatalf("no digest found, but expected one")
+				}
+
+			} else {
+				assert.Equal(t, actual[l][0].Value, test.expected, "mismatched digests")
+			}
+		})
+	}
+}

syft/file/metadata_cataloger_test.go

@@ -50,7 +50,7 @@ func TestFileMetadataCataloger(t *testing.T) {
 			exists: true,
 			expected: source.FileMetadata{
 				Mode:    0644,
-				Type:    "regularFile",
+				Type:    "RegularFile",
 				UserID:  1,
 				GroupID: 2,
 			},
@@ -60,7 +60,8 @@ func TestFileMetadataCataloger(t *testing.T) {
 			exists: true,
 			expected: source.FileMetadata{
 				Mode:            0644,
-				Type:    "hardLink",
+				Type:            "HardLink",
+				LinkDestination: "file-1.txt",
 				UserID:          1,
 				GroupID:         2,
 			},
@@ -70,7 +71,8 @@ func TestFileMetadataCataloger(t *testing.T) {
 			exists: true,
 			expected: source.FileMetadata{
 				Mode:            0777 | os.ModeSymlink,
-				Type:    "symbolicLink",
+				Type:            "SymbolicLink",
+				LinkDestination: "file-1.txt",
 				UserID:          0,
 				GroupID:         0,
 			},
@@ -80,7 +82,7 @@ func TestFileMetadataCataloger(t *testing.T) {
 			exists: true,
 			expected: source.FileMetadata{
 				Mode:    0644 | os.ModeDevice | os.ModeCharDevice,
-				Type:    "characterDevice",
+				Type:    "CharacterDevice",
 				UserID:  0,
 				GroupID: 0,
 			},
@@ -90,7 +92,7 @@ func TestFileMetadataCataloger(t *testing.T) {
 			exists: true,
 			expected: source.FileMetadata{
 				Mode:    0644 | os.ModeDevice,
-				Type:    "blockDevice",
+				Type:    "BlockDevice",
 				UserID:  0,
 				GroupID: 0,
 			},
@@ -100,7 +102,7 @@ func TestFileMetadataCataloger(t *testing.T) {
 			exists: true,
 			expected: source.FileMetadata{
 				Mode:    0644 | os.ModeNamedPipe,
-				Type:    "fifoNode",
+				Type:    "FIFONode",
 				UserID:  0,
 				GroupID: 0,
 			},
@@ -110,7 +112,7 @@ func TestFileMetadataCataloger(t *testing.T) {
 			exists: true,
 			expected: source.FileMetadata{
 				Mode:    0755 | os.ModeDir,
-				Type:    "directory",
+				Type:    "Directory",
 				UserID:  0,
 				GroupID: 0,
 			},

syft/source/file_metadata.go

@@ -11,6 +11,7 @@ type FileMetadata struct {
 	Type            FileType
 	UserID          int
 	GroupID         int
+	LinkDestination string
 }
 
 func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata, error) {
@@ -24,5 +25,6 @@ func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata,
 		Type:            newFileTypeFromTarHeaderTypeFlag(entry.Metadata.TypeFlag),
 		UserID:          entry.Metadata.UserID,
 		GroupID:         entry.Metadata.GroupID,
+		LinkDestination: entry.Metadata.Linkname,
 	}, nil
 }