From db35186c7db018110e5571b475cae8375059337a Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 26 Mar 2021 17:32:37 -0400 Subject: [PATCH] allow file metadata digests to be optional + add link destination Signed-off-by: Alex Goodman --- .../presenter/poweruser/json_file_metadata.go | 28 +-- .../poweruser/json_presenter_test.go | 172 +++++++++++++++ .../snapshot/TestJSONPresenter.golden | 198 ++++++++++++++++++ schema/json/schema-1.0.4.json | 6 +- syft/file/digest_cataloger.go | 20 +- syft/file/digest_cataloger_test.go | 104 ++++++++- syft/file/metadata_cataloger_test.go | 28 +-- syft/source/file_metadata.go | 18 +- 8 files changed, 523 insertions(+), 51 deletions(-) create mode 100644 internal/presenter/poweruser/json_presenter_test.go create mode 100644 internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden diff --git a/internal/presenter/poweruser/json_file_metadata.go b/internal/presenter/poweruser/json_file_metadata.go index 1b0a8d08e..35d2209d6 100644 --- a/internal/presenter/poweruser/json_file_metadata.go +++ b/internal/presenter/poweruser/json_file_metadata.go @@ -16,11 +16,12 @@ type JSONFileMetadata struct { } type JSONFileMetadataEntry struct { - Mode int `json:"mode"` - Type source.FileType `json:"type"` - UserID int `json:"userID"` - GroupID int `json:"groupID"` - Digests []file.Digest `json:"digests"` + Mode int `json:"mode"` + Type source.FileType `json:"type"` + LinkDestination string `json:"linkDestination,omitempty"` + UserID int `json:"userID"` + GroupID int `json:"groupID"` + Digests []file.Digest `json:"digests,omitempty"` } func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests map[source.Location][]file.Digest) ([]JSONFileMetadata, error) { @@ -31,7 +32,7 @@ func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests m return nil, fmt.Errorf("invalid mode found in file catalog @ location=%+v mode=%q: %w", location, metadata.Mode, err) } - digestResults := make([]file.Digest, 0) + var digestResults []file.Digest if digestsForLocation, exists := digests[location]; exists { digestResults = digestsForLocation } @@ -39,21 +40,22 @@ func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests m results = append(results, JSONFileMetadata{ Location: location, Metadata: JSONFileMetadataEntry{ - Mode: mode, - Type: metadata.Type, - UserID: metadata.UserID, - GroupID: metadata.GroupID, - Digests: digestResults, + Mode: mode, + Type: metadata.Type, + LinkDestination: metadata.LinkDestination, + UserID: metadata.UserID, + GroupID: metadata.GroupID, + Digests: digestResults, }, }) } // sort by real path then virtual path to ensure the result is stable across multiple runs sort.SliceStable(results, func(i, j int) bool { - if results[i].Location.RealPath != results[j].Location.RealPath { + if results[i].Location.RealPath == results[j].Location.RealPath { return results[i].Location.VirtualPath < results[j].Location.VirtualPath } - return false + return results[i].Location.RealPath < results[j].Location.RealPath }) return results, nil } diff --git a/internal/presenter/poweruser/json_presenter_test.go b/internal/presenter/poweruser/json_presenter_test.go new file mode 100644 index 000000000..eacd59cc0 --- /dev/null +++ b/internal/presenter/poweruser/json_presenter_test.go @@ -0,0 +1,172 @@ +package poweruser + +import ( + "bytes" + "flag" + "testing" + + "github.com/sergi/go-diff/diffmatchpatch" + + "github.com/anchore/syft/syft/file" + + "github.com/anchore/go-testutils" + "github.com/anchore/syft/internal/config" + "github.com/anchore/syft/syft/distro" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +var updateJSONGoldenFiles = flag.Bool("update-json", false, "update the *.golden files for json presenters") + +func must(c pkg.CPE, e error) pkg.CPE { + if e != nil { + panic(e) + } + return c +} + +func TestJSONPresenter(t *testing.T) { + var buffer bytes.Buffer + + catalog := pkg.NewCatalog() + + catalog.Add(pkg.Package{ + ID: "package-1-id", + Name: "package-1", + Version: "1.0.1", + Locations: []source.Location{ + { + RealPath: "/a/place/a", + }, + }, + Type: pkg.PythonPkg, + FoundBy: "the-cataloger-1", + Language: pkg.Python, + MetadataType: pkg.PythonPackageMetadataType, + Licenses: []string{"MIT"}, + Metadata: pkg.PythonPackageMetadata{ + Name: "package-1", + Version: "1.0.1", + }, + PURL: "a-purl-1", + CPEs: []pkg.CPE{ + must(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")), + }, + }) + catalog.Add(pkg.Package{ + ID: "package-2-id", + Name: "package-2", + Version: "2.0.1", + Locations: []source.Location{ + { + RealPath: "/b/place/b", + }, + }, + Type: pkg.DebPkg, + FoundBy: "the-cataloger-2", + MetadataType: pkg.DpkgMetadataType, + Metadata: pkg.DpkgMetadata{ + Package: "package-2", + Version: "2.0.1", + }, + PURL: "a-purl-2", + CPEs: []pkg.CPE{ + must(pkg.NewCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*")), + }, + }) + + cfg := JSONDocumentConfig{ + ApplicationConfig: config.Application{}, + PackageCatalog: catalog, + FileMetadata: map[source.Location]source.FileMetadata{ + source.NewLocation("/a/place"): { + Mode: 0775, + Type: "directory", + UserID: 0, + GroupID: 0, + }, + source.NewLocation("/a/place/a"): { + Mode: 0775, + Type: "regularFile", + UserID: 0, + GroupID: 0, + }, + source.NewLocation("/b"): { + Mode: 0775, + Type: "symbolicLink", + LinkDestination: "/c", + UserID: 0, + GroupID: 0, + }, + source.NewLocation("/b/place/b"): { + Mode: 0644, + Type: "regularFile", + UserID: 1, + GroupID: 2, + }, + }, + FileDigests: map[source.Location][]file.Digest{ + source.NewLocation("/a/place/a"): { + { + Algorithm: "sha256", + Value: "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703", + }, + }, + source.NewLocation("/b/place/b"): { + { + Algorithm: "sha256", + Value: "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c", + }, + }, + }, + Distro: &distro.Distro{ + Type: distro.RedHat, + RawVersion: "7", + IDLike: "rhel", + }, + SourceMetadata: source.Metadata{ + Scheme: source.ImageScheme, + ImageMetadata: source.ImageMetadata{ + UserInput: "user-image-input", + ID: "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0", + ManifestDigest: "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", + MediaType: "application/vnd.docker.distribution.manifest.v2+json", + Tags: []string{ + "stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b", + }, + Size: 38, + Layers: []source.LayerMetadata{ + { + MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", + Digest: "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b", + Size: 22, + }, + { + MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", + Digest: "sha256:366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703", + Size: 16, + }, + }, + RawManifest: []byte("eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJh..."), + RawConfig: []byte("eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZp..."), + }, + }, + } + + if err := NewJSONPresenter(cfg).Present(&buffer); err != nil { + t.Fatal(err) + } + actual := buffer.Bytes() + + if *updateJSONGoldenFiles { + testutils.UpdateGoldenFileContents(t, actual) + } + + var expected = testutils.GetGoldenFileContents(t) + + if !bytes.Equal(expected, actual) { + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(string(expected), string(actual), true) + t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs)) + } +} diff --git a/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden b/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden new file mode 100644 index 000000000..9b428db23 --- /dev/null +++ b/internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden @@ -0,0 +1,198 @@ +{ + "fileMetadata": [ + { + "location": { + "path": "/a/place" + }, + "metadata": { + "mode": 775, + "type": "directory", + "userID": 0, + "groupID": 0 + } + }, + { + "location": { + "path": "/a/place/a" + }, + "metadata": { + "mode": 775, + "type": "regularFile", + "userID": 0, + "groupID": 0, + "digests": [ + { + "algorithm": "sha256", + "value": "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703" + } + ] + } + }, + { + "location": { + "path": "/b" + }, + "metadata": { + "mode": 775, + "type": "symbolicLink", + "linkDestination": "/c", + "userID": 0, + "groupID": 0 + } + }, + { + "location": { + "path": "/b/place/b" + }, + "metadata": { + "mode": 644, + "type": "regularFile", + "userID": 1, + "groupID": 2, + "digests": [ + { + "algorithm": "sha256", + "value": "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c" + } + ] + } + } + ], + "artifacts": [ + { + "id": "package-1-id", + "name": "package-1", + "version": "1.0.1", + "type": "python", + "foundBy": "the-cataloger-1", + "locations": [ + { + "path": "/a/place/a" + } + ], + "licenses": [ + "MIT" + ], + "language": "python", + "cpes": [ + "cpe:2.3:*:some:package:1:*:*:*:*:*:*:*" + ], + "purl": "a-purl-1", + "metadataType": "PythonPackageMetadata", + "metadata": { + "name": "package-1", + "version": "1.0.1", + "license": "", + "author": "", + "authorEmail": "", + "platform": "", + "sitePackagesRootPath": "" + } + }, + { + "id": "package-2-id", + "name": "package-2", + "version": "2.0.1", + "type": "deb", + "foundBy": "the-cataloger-2", + "locations": [ + { + "path": "/b/place/b" + } + ], + "licenses": [], + "language": "", + "cpes": [ + "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*" + ], + "purl": "a-purl-2", + "metadataType": "DpkgMetadata", + "metadata": { + "package": "package-2", + "source": "", + "version": "2.0.1", + "sourceVersion": "", + "architecture": "", + "maintainer": "", + "installedSize": 0, + "files": null + } + } + ], + "artifactRelationships": [], + "source": { + "type": "image", + "target": { + "userInput": "user-image-input", + "imageID": "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0", + "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "tags": [ + "stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b" + ], + "imageSize": 38, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b", + "size": 22 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703", + "size": 16 + } + ], + "manifest": "ZXlKelkyaGxiV0ZXWlhKemFXOXVJam95TENKdFpXUnBZVlI1Y0dVaU9pSmguLi4=", + "config": "ZXlKaGNtTm9hWFJsWTNSMWNtVWlPaUpoYldRMk5DSXNJbU52Ym1acC4uLg==", + "scope": "" + } + }, + "distro": { + "name": "redhat", + "version": "7", + "idLike": "rhel" + }, + "descriptor": { + "name": "syft", + "version": "[not provided]", + "configuration": { + "configPath": "", + "output": "", + "quiet": false, + "log": { + "structured": false, + "level": "", + "file-location": "" + }, + "dev": { + "profile-cpu": false, + "profile-mem": false + }, + "check-for-app-update": false, + "anchore": { + "host": "", + "path": "", + "dockerfile": "", + "overwrite-existing-image": false + }, + "package": { + "cataloger": { + "enabled": false, + "scope": "" + } + }, + "file-metadata": { + "cataloger": { + "enabled": false, + "scope": "" + }, + "digests": null + } + } + }, + "schema": { + "version": "1.0.4", + "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.4.json" + } +} diff --git a/schema/json/schema-1.0.4.json b/schema/json/schema-1.0.4.json index 04618439c..62719cdeb 100644 --- a/schema/json/schema-1.0.4.json +++ b/schema/json/schema-1.0.4.json @@ -345,8 +345,7 @@ "mode", "type", "userID", - "groupID", - "digests" + "groupID" ], "properties": { "mode": { @@ -355,6 +354,9 @@ "type": { "type": "string" }, + "linkDestination": { + "type": "string" + }, "userID": { "type": "integer" }, diff --git a/syft/file/digest_cataloger.go b/syft/file/digest_cataloger.go index 10a5bba42..a528d2a81 100644 --- a/syft/file/digest_cataloger.go +++ b/syft/file/digest_cataloger.go @@ -74,16 +74,18 @@ func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, locatio return nil, fmt.Errorf("unable to observe contents of %+v: %+v", location.RealPath, err) } + if size == 0 { + return make([]Digest, 0), nil + } + result := make([]Digest, len(i.hashes)) - if size > 0 { - // only capture digests when there is content. It is important to do this based on SIZE and not - // FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only - // file type but a body is still allowed. - for idx, hasher := range hashers { - result[idx] = Digest{ - Algorithm: cleanAlgorithmName(i.hashes[idx].String()), - Value: fmt.Sprintf("%+x", hasher.Sum(nil)), - } + // only capture digests when there is content. It is important to do this based on SIZE and not + // FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only + // file type but a body is still allowed. + for idx, hasher := range hashers { + result[idx] = Digest{ + Algorithm: cleanAlgorithmName(i.hashes[idx].String()), + Value: fmt.Sprintf("%+x", hasher.Sum(nil)), } } diff --git a/syft/file/digest_cataloger_test.go b/syft/file/digest_cataloger_test.go index ad21f9184..7f4516d43 100644 --- a/syft/file/digest_cataloger_test.go +++ b/syft/file/digest_cataloger_test.go @@ -7,6 +7,10 @@ import ( "os" "testing" + "github.com/anchore/stereoscope/pkg/file" + + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/stretchr/testify/assert" "github.com/anchore/syft/syft/source" @@ -38,12 +42,13 @@ func testDigests(t testing.TB, files []string, hashes ...crypto.Hash) map[source return digests } -func TestDigestsCataloger(t *testing.T) { - files := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"} +func TestDigestsCataloger_SimpleContents(t *testing.T) { + regularFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"} tests := []struct { name string algorithms []string + files []string expected map[source.Location][]Digest constructorErr bool catalogErr bool @@ -51,22 +56,32 @@ func TestDigestsCataloger(t *testing.T) { { name: "bad algorithm", algorithms: []string{"sha-nothing"}, + files: regularFiles, constructorErr: true, }, { name: "unsupported algorithm", algorithms: []string{"sha512"}, + files: regularFiles, constructorErr: true, }, { - name: "md5-sha1-sha256", + name: "md5", algorithms: []string{"md5"}, - expected: testDigests(t, files, crypto.MD5), + files: regularFiles, + expected: testDigests(t, regularFiles, crypto.MD5), }, { name: "md5-sha1-sha256", algorithms: []string{"md5", "sha1", "sha256"}, - expected: testDigests(t, files, crypto.MD5, crypto.SHA1, crypto.SHA256), + files: regularFiles, + expected: testDigests(t, regularFiles, crypto.MD5, crypto.SHA1, crypto.SHA256), + }, + { + name: "directory returns error", + algorithms: []string{"md5"}, + files: []string{"test-fixtures/last"}, + catalogErr: true, }, } @@ -81,7 +96,7 @@ func TestDigestsCataloger(t *testing.T) { return } - resolver := source.NewMockResolverForPaths(files...) + resolver := source.NewMockResolverForPaths(test.files...) actual, err := c.Catalog(resolver) if err != nil && !test.catalogErr { t.Fatalf("could not catalog (but should have been able to): %+v", err) @@ -96,3 +111,80 @@ func TestDigestsCataloger(t *testing.T) { }) } } + +func TestDigestsCataloger_MixFileTypes(t *testing.T) { + testImage := "image-file-type-mix" + + if *updateImageGoldenFiles { + imagetest.UpdateGoldenFixtureImage(t, testImage) + } + + img := imagetest.GetGoldenFixtureImage(t, testImage) + + src, err := source.NewFromImage(img, "---") + if err != nil { + t.Fatalf("could not create source: %+v", err) + } + + resolver, err := src.FileResolver(source.SquashedScope) + if err != nil { + t.Fatalf("could not create resolver: %+v", err) + } + + tests := []struct { + path string + expected string + }{ + { + path: "/file-1.txt", + expected: "888c139e550867814eb7c33b84d76e4d", + }, + { + path: "/hardlink-1", + }, + { + path: "/symlink-1", + }, + { + path: "/char-device-1", + }, + { + path: "/block-device-1", + }, + { + path: "/fifo-1", + }, + { + path: "/bin", + }, + } + + for _, test := range tests { + t.Run(test.path, func(t *testing.T) { + c, err := NewDigestsCataloger([]string{"md5"}) + if err != nil { + t.Fatalf("unable to get cataloger: %+v", err) + } + + actual, err := c.Catalog(resolver) + if err != nil { + t.Fatalf("could not catalog: %+v", err) + } + + _, ref, err := img.SquashedTree().File(file.Path(test.path)) + if err != nil { + t.Fatalf("unable to get file=%q : %+v", test.path, err) + } + l := source.NewLocationFromImage(test.path, *ref, img) + + if len(actual[l]) == 0 { + if test.expected != "" { + t.Fatalf("no digest found, but expected one") + } + + } else { + assert.Equal(t, actual[l][0].Value, test.expected, "mismatched digests") + } + }) + } +} diff --git a/syft/file/metadata_cataloger_test.go b/syft/file/metadata_cataloger_test.go index a6971d6b3..80b301ef7 100644 --- a/syft/file/metadata_cataloger_test.go +++ b/syft/file/metadata_cataloger_test.go @@ -50,7 +50,7 @@ func TestFileMetadataCataloger(t *testing.T) { exists: true, expected: source.FileMetadata{ Mode: 0644, - Type: "regularFile", + Type: "RegularFile", UserID: 1, GroupID: 2, }, @@ -59,20 +59,22 @@ func TestFileMetadataCataloger(t *testing.T) { path: "/hardlink-1", exists: true, expected: source.FileMetadata{ - Mode: 0644, - Type: "hardLink", - UserID: 1, - GroupID: 2, + Mode: 0644, + Type: "HardLink", + LinkDestination: "file-1.txt", + UserID: 1, + GroupID: 2, }, }, { path: "/symlink-1", exists: true, expected: source.FileMetadata{ - Mode: 0777 | os.ModeSymlink, - Type: "symbolicLink", - UserID: 0, - GroupID: 0, + Mode: 0777 | os.ModeSymlink, + Type: "SymbolicLink", + LinkDestination: "file-1.txt", + UserID: 0, + GroupID: 0, }, }, { @@ -80,7 +82,7 @@ func TestFileMetadataCataloger(t *testing.T) { exists: true, expected: source.FileMetadata{ Mode: 0644 | os.ModeDevice | os.ModeCharDevice, - Type: "characterDevice", + Type: "CharacterDevice", UserID: 0, GroupID: 0, }, @@ -90,7 +92,7 @@ func TestFileMetadataCataloger(t *testing.T) { exists: true, expected: source.FileMetadata{ Mode: 0644 | os.ModeDevice, - Type: "blockDevice", + Type: "BlockDevice", UserID: 0, GroupID: 0, }, @@ -100,7 +102,7 @@ func TestFileMetadataCataloger(t *testing.T) { exists: true, expected: source.FileMetadata{ Mode: 0644 | os.ModeNamedPipe, - Type: "fifoNode", + Type: "FIFONode", UserID: 0, GroupID: 0, }, @@ -110,7 +112,7 @@ func TestFileMetadataCataloger(t *testing.T) { exists: true, expected: source.FileMetadata{ Mode: 0755 | os.ModeDir, - Type: "directory", + Type: "Directory", UserID: 0, GroupID: 0, }, diff --git a/syft/source/file_metadata.go b/syft/source/file_metadata.go index 45c6a877a..9140810d2 100644 --- a/syft/source/file_metadata.go +++ b/syft/source/file_metadata.go @@ -7,10 +7,11 @@ import ( ) type FileMetadata struct { - Mode os.FileMode - Type FileType - UserID int - GroupID int + Mode os.FileMode + Type FileType + UserID int + GroupID int + LinkDestination string } func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata, error) { @@ -20,9 +21,10 @@ func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata, } return FileMetadata{ - Mode: entry.Metadata.Mode, - Type: newFileTypeFromTarHeaderTypeFlag(entry.Metadata.TypeFlag), - UserID: entry.Metadata.UserID, - GroupID: entry.Metadata.GroupID, + Mode: entry.Metadata.Mode, + Type: newFileTypeFromTarHeaderTypeFlag(entry.Metadata.TypeFlag), + UserID: entry.Metadata.UserID, + GroupID: entry.Metadata.GroupID, + LinkDestination: entry.Metadata.Linkname, }, nil }