Migrate SPDX-JSON relationships to SBOM model (#634)

* remove power-user document shape Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add power-user specific fields to syft-json format Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * port remaining spdx-json relationships to sbom model Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add coordinate set Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add SBOM file path helper Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * use internal mimetype helper in go binary cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add new package-of relationship Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update json schema to v2 Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * replace power-user presenter with syft-json format Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix tests and linting Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * remove "package-of" relationship (in favor of "contains") Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add tests for spdx22json format encoding enhancements Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update TODO and log entries Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * introduce sbom.Descriptor Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2026-11-02 15:36:41 +01:00 · 2021-11-23 14:54:17 -05:00
parent e3b34813d7
commit bd9007fc0e
41 changed files with 2238 additions and 718 deletions
@@ -8,8 +8,7 @@ import (
 )

 func encoder(output io.Writer, s sbom.SBOM) error {
-	// TODO: application config not available yet
-	doc := ToFormatModel(s, nil)
+	doc := ToFormatModel(s)

 	enc := json.NewEncoder(output)
 	// prevent > and < from being escaped in the payload
@@ -4,6 +4,15 @@ import (
 	"flag"
 	"testing"

+	"github.com/anchore/syft/syft/file"
+
+	"github.com/anchore/syft/syft/artifact"
+
+	"github.com/anchore/syft/syft/distro"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/sbom"
+	"github.com/anchore/syft/syft/source"
+
 	"github.com/anchore/syft/internal/formats/common/testutils"
 )

@@ -24,3 +33,167 @@ func TestImagePresenter(t *testing.T) {
 		*updateJson,
 	)
 }
+
+func TestEncodeFullJSONDocument(t *testing.T) {
+	catalog := pkg.NewCatalog()
+
+	p1 := pkg.Package{
+		Name:    "package-1",
+		Version: "1.0.1",
+		Locations: []source.Location{
+			{
+				Coordinates: source.Coordinates{
+					RealPath: "/a/place/a",
+				},
+			},
+		},
+		Type:         pkg.PythonPkg,
+		FoundBy:      "the-cataloger-1",
+		Language:     pkg.Python,
+		MetadataType: pkg.PythonPackageMetadataType,
+		Licenses:     []string{"MIT"},
+		Metadata: pkg.PythonPackageMetadata{
+			Name:    "package-1",
+			Version: "1.0.1",
+			Files:   []pkg.PythonFileRecord{},
+		},
+		PURL: "a-purl-1",
+		CPEs: []pkg.CPE{
+			pkg.MustCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*"),
+		},
+	}
+
+	p2 := pkg.Package{
+		Name:    "package-2",
+		Version: "2.0.1",
+		Locations: []source.Location{
+			{
+				Coordinates: source.Coordinates{
+					RealPath: "/b/place/b",
+				},
+			},
+		},
+		Type:         pkg.DebPkg,
+		FoundBy:      "the-cataloger-2",
+		MetadataType: pkg.DpkgMetadataType,
+		Metadata: pkg.DpkgMetadata{
+			Package: "package-2",
+			Version: "2.0.1",
+			Files:   []pkg.DpkgFileRecord{},
+		},
+		PURL: "a-purl-2",
+		CPEs: []pkg.CPE{
+			pkg.MustCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
+		},
+	}
+
+	catalog.Add(p1)
+	catalog.Add(p2)
+
+	s := sbom.SBOM{
+		Artifacts: sbom.Artifacts{
+			PackageCatalog: catalog,
+			FileMetadata: map[source.Coordinates]source.FileMetadata{
+				source.NewLocation("/a/place").Coordinates: {
+					Mode:    0775,
+					Type:    "directory",
+					UserID:  0,
+					GroupID: 0,
+				},
+				source.NewLocation("/a/place/a").Coordinates: {
+					Mode:    0775,
+					Type:    "regularFile",
+					UserID:  0,
+					GroupID: 0,
+				},
+				source.NewLocation("/b").Coordinates: {
+					Mode:            0775,
+					Type:            "symbolicLink",
+					LinkDestination: "/c",
+					UserID:          0,
+					GroupID:         0,
+				},
+				source.NewLocation("/b/place/b").Coordinates: {
+					Mode:    0644,
+					Type:    "regularFile",
+					UserID:  1,
+					GroupID: 2,
+				},
+			},
+			FileDigests: map[source.Coordinates][]file.Digest{
+				source.NewLocation("/a/place/a").Coordinates: {
+					{
+						Algorithm: "sha256",
+						Value:     "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
+					},
+				},
+				source.NewLocation("/b/place/b").Coordinates: {
+					{
+						Algorithm: "sha256",
+						Value:     "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c",
+					},
+				},
+			},
+			FileContents: map[source.Coordinates]string{
+				source.NewLocation("/a/place/a").Coordinates: "the-contents",
+			},
+			Distro: &distro.Distro{
+				Type:       distro.RedHat,
+				RawVersion: "7",
+				IDLike:     "rhel",
+			},
+		},
+		Relationships: []artifact.Relationship{
+			{
+				From: p1,
+				To:   p2,
+				Type: artifact.OwnershipByFileOverlapRelationship,
+				Data: map[string]string{
+					"file": "path",
+				},
+			},
+		},
+		Source: source.Metadata{
+			Scheme: source.ImageScheme,
+			ImageMetadata: source.ImageMetadata{
+				UserInput:      "user-image-input",
+				ID:             "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
+				ManifestDigest: "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
+				MediaType:      "application/vnd.docker.distribution.manifest.v2+json",
+				Tags: []string{
+					"stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b",
+				},
+				Size: 38,
+				Layers: []source.LayerMetadata{
+					{
+						MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
+						Digest:    "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b",
+						Size:      22,
+					},
+					{
+						MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
+						Digest:    "sha256:366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
+						Size:      16,
+					},
+				},
+				RawManifest: []byte("eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJh..."),
+				RawConfig:   []byte("eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZp..."),
+				RepoDigests: []string{},
+			},
+		},
+		Descriptor: sbom.Descriptor{
+			Name:    "syft",
+			Version: "v0.42.0-bogus",
+			// the application configuration should be persisted here, however, we do not want to import
+			// the application configuration in this package (it's reserved only for ingestion by the cmd package)
+			Configuration: map[string]string{
+				"config-key": "config-value",
+			},
+		},
+	}
+
+	testutils.AssertPresenterAgainstGoldenSnapshot(t,
+		Format().Presenter(s),
+		*updateJson,
+	)
+}
@@ -4,10 +4,12 @@ package model
 type Document struct {
 	Artifacts             []Package      `json:"artifacts"` // Artifacts is the list of packages discovered and placed into the catalog
 	ArtifactRelationships []Relationship `json:"artifactRelationships"`
-	Source                Source         `json:"source"`     // Source represents the original object that was cataloged
-	Distro                Distro         `json:"distro"`     // Distro represents the Linux distribution that was detected from the source
-	Descriptor            Descriptor     `json:"descriptor"` // Descriptor is a block containing self-describing information about syft
-	Schema                Schema         `json:"schema"`     // Schema is a block reserved for defining the version for the shape of this JSON document and where to find the schema document to validate the shape
+	Files                 []File         `json:"files,omitempty"`   // note: must have omitempty
+	Secrets               []Secrets      `json:"secrets,omitempty"` // note: must have omitempty
+	Source                Source         `json:"source"`            // Source represents the original object that was cataloged
+	Distro                Distro         `json:"distro"`            // Distro represents the Linux distribution that was detected from the source
+	Descriptor            Descriptor     `json:"descriptor"`        // Descriptor is a block containing self-describing information about syft
+	Schema                Schema         `json:"schema"`            // Schema is a block reserved for defining the version for the shape of this JSON document and where to find the schema document to validate the shape
 }

 // Descriptor describes what created the document as well as surrounding metadata
@@ -0,0 +1,25 @@
+package model
+
+import (
+	"github.com/anchore/syft/syft/file"
+
+	"github.com/anchore/syft/syft/source"
+)
+
+type File struct {
+	ID              string                `json:"id"`
+	Location        source.Coordinates    `json:"location"`
+	Metadata        *FileMetadataEntry    `json:"metadata,omitempty"`
+	Contents        string                `json:"contents,omitempty"`
+	Digests         []file.Digest         `json:"digests,omitempty"`
+	Classifications []file.Classification `json:"classifications,omitempty"`
+}
+
+type FileMetadataEntry struct {
+	Mode            int             `json:"mode"`
+	Type            source.FileType `json:"type"`
+	LinkDestination string          `json:"linkDestination,omitempty"`
+	UserID          int             `json:"userID"`
+	GroupID         int             `json:"groupID"`
+	MIMEType        string          `json:"mimeType"`
+}
@@ -9,7 +9,7 @@ import (
 	"github.com/anchore/syft/syft/source"
 )

-// Package represents a pkg.Package object specialized for JSON marshaling and unmarshaling.
+// Package represents a pkg.Package object specialized for JSON marshaling and unmarshalling.
 type Package struct {
 	PackageBasicData
 	PackageCustomData
@@ -4,5 +4,5 @@ type Relationship struct {
 	Parent   string      `json:"parent"`
 	Child    string      `json:"child"`
 	Type     string      `json:"type"`
-	Metadata interface{} `json:"metadata"`
+	Metadata interface{} `json:"metadata,omitempty"`
 }
@@ -0,0 +1,11 @@
+package model
+
+import (
+	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/source"
+)
+
+type Secrets struct {
+	Location source.Coordinates  `json:"location"`
+	Secrets  []file.SearchResult `json:"secrets"`
+}
@@ -77,10 +77,13 @@
 },
 "descriptor": {
  "name": "syft",
-  "version": "[not provided]"
+  "version": "v0.42.0-bogus",
+  "configuration": {
+   "config-key": "config-value"
+  }
 },
 "schema": {
-  "version": "1.1.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.1.0.json"
+  "version": "2.0.0",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.0.json"
 }
 }
@@ -0,0 +1,185 @@
+{
+ "artifacts": [
+  {
+   "id": "962403cfb7be50d7",
+   "name": "package-1",
+   "version": "1.0.1",
+   "type": "python",
+   "foundBy": "the-cataloger-1",
+   "locations": [
+    {
+     "path": "/a/place/a"
+    }
+   ],
+   "licenses": [
+    "MIT"
+   ],
+   "language": "python",
+   "cpes": [
+    "cpe:2.3:*:some:package:1:*:*:*:*:*:*:*"
+   ],
+   "purl": "a-purl-1",
+   "metadataType": "PythonPackageMetadata",
+   "metadata": {
+    "name": "package-1",
+    "version": "1.0.1",
+    "license": "",
+    "author": "",
+    "authorEmail": "",
+    "platform": "",
+    "sitePackagesRootPath": ""
+   }
+  },
+  {
+   "id": "b11f44847bba0ed1",
+   "name": "package-2",
+   "version": "2.0.1",
+   "type": "deb",
+   "foundBy": "the-cataloger-2",
+   "locations": [
+    {
+     "path": "/b/place/b"
+    }
+   ],
+   "licenses": [],
+   "language": "",
+   "cpes": [
+    "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"
+   ],
+   "purl": "a-purl-2",
+   "metadataType": "DpkgMetadata",
+   "metadata": {
+    "package": "package-2",
+    "source": "",
+    "version": "2.0.1",
+    "sourceVersion": "",
+    "architecture": "",
+    "maintainer": "",
+    "installedSize": 0,
+    "files": []
+   }
+  }
+ ],
+ "artifactRelationships": [
+  {
+   "parent": "962403cfb7be50d7",
+   "child": "b11f44847bba0ed1",
+   "type": "ownership-by-file-overlap",
+   "metadata": {
+    "file": "path"
+   }
+  }
+ ],
+ "files": [
+  {
+   "id": "913b4592e2c2ebdf",
+   "location": {
+    "path": "/a/place"
+   },
+   "metadata": {
+    "mode": 775,
+    "type": "directory",
+    "userID": 0,
+    "groupID": 0,
+    "mimeType": ""
+   }
+  },
+  {
+   "id": "e7c88bd18e11b0b",
+   "location": {
+    "path": "/a/place/a"
+   },
+   "metadata": {
+    "mode": 775,
+    "type": "regularFile",
+    "userID": 0,
+    "groupID": 0,
+    "mimeType": ""
+   },
+   "contents": "the-contents",
+   "digests": [
+    {
+     "algorithm": "sha256",
+     "value": "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703"
+    }
+   ]
+  },
+  {
+   "id": "5c3dc6885f48b5a1",
+   "location": {
+    "path": "/b"
+   },
+   "metadata": {
+    "mode": 775,
+    "type": "symbolicLink",
+    "linkDestination": "/c",
+    "userID": 0,
+    "groupID": 0,
+    "mimeType": ""
+   }
+  },
+  {
+   "id": "799d2f12da0bcec4",
+   "location": {
+    "path": "/b/place/b"
+   },
+   "metadata": {
+    "mode": 644,
+    "type": "regularFile",
+    "userID": 1,
+    "groupID": 2,
+    "mimeType": ""
+   },
+   "digests": [
+    {
+     "algorithm": "sha256",
+     "value": "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c"
+    }
+   ]
+  }
+ ],
+ "source": {
+  "type": "image",
+  "target": {
+   "userInput": "user-image-input",
+   "imageID": "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
+   "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
+   "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+   "tags": [
+    "stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b"
+   ],
+   "imageSize": 38,
+   "layers": [
+    {
+     "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+     "digest": "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b",
+     "size": 22
+    },
+    {
+     "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+     "digest": "sha256:366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
+     "size": 16
+    }
+   ],
+   "manifest": "ZXlKelkyaGxiV0ZXWlhKemFXOXVJam95TENKdFpXUnBZVlI1Y0dVaU9pSmguLi4=",
+   "config": "ZXlKaGNtTm9hWFJsWTNSMWNtVWlPaUpoYldRMk5DSXNJbU52Ym1acC4uLg==",
+   "repoDigests": []
+  }
+ },
+ "distro": {
+  "name": "redhat",
+  "version": "7",
+  "idLike": "rhel"
+ },
+ "descriptor": {
+  "name": "syft",
+  "version": "v0.42.0-bogus",
+  "configuration": {
+   "config-key": "config-value"
+  }
+ },
+ "schema": {
+  "version": "2.0.0",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.0.json"
+ }
+}
@@ -98,10 +98,13 @@
 },
 "descriptor": {
  "name": "syft",
-  "version": "[not provided]"
+  "version": "v0.42.0-bogus",
+  "configuration": {
+   "config-key": "config-value"
+  }
 },
 "schema": {
-  "version": "1.1.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.1.0.json"
+  "version": "2.0.0",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.0.json"
 }
 }
@@ -2,6 +2,10 @@ package syftjson

 import (
 	"fmt"
+	"sort"
+	"strconv"
+
+	"github.com/anchore/syft/syft/file"

 	"github.com/anchore/syft/syft/artifact"

@@ -10,14 +14,13 @@ import (
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/formats/syftjson/model"
 	"github.com/anchore/syft/internal/log"
-	"github.com/anchore/syft/internal/version"
 	"github.com/anchore/syft/syft/distro"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
 )

-// TODO: this is export4ed for the use of the power-user command (temp)
-func ToFormatModel(s sbom.SBOM, applicationConfig interface{}) model.Document {
+// TODO: this is exported for the use of the power-user command (temp)
+func ToFormatModel(s sbom.SBOM) model.Document {
 	src, err := toSourceModel(s.Source)
 	if err != nil {
 		log.Warnf("unable to create syft-json source object: %+v", err)
@@ -26,13 +29,11 @@ func ToFormatModel(s sbom.SBOM, applicationConfig interface{}) model.Document {
 	return model.Document{
 		Artifacts:             toPackageModels(s.Artifacts.PackageCatalog),
 		ArtifactRelationships: toRelationshipModel(s.Relationships),
+		Files:                 toFile(s),
+		Secrets:               toSecrets(s.Artifacts.Secrets),
 		Source:                src,
 		Distro:                toDistroModel(s.Artifacts.Distro),
-		Descriptor: model.Descriptor{
-			Name:          internal.ApplicationName,
-			Version:       version.FromBuild().Version,
-			Configuration: applicationConfig,
-		},
+		Descriptor:            toDescriptor(s.Descriptor),
 		Schema: model.Schema{
 			Version: internal.JSONSchemaVersion,
 			URL:     fmt.Sprintf("https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-%s.json", internal.JSONSchemaVersion),
@@ -40,6 +41,93 @@ func ToFormatModel(s sbom.SBOM, applicationConfig interface{}) model.Document {
 	}
 }

+func toDescriptor(d sbom.Descriptor) model.Descriptor {
+	return model.Descriptor{
+		Name:          d.Name,
+		Version:       d.Version,
+		Configuration: d.Configuration,
+	}
+}
+
+func toSecrets(data map[source.Coordinates][]file.SearchResult) []model.Secrets {
+	results := make([]model.Secrets, 0)
+	for coordinates, secrets := range data {
+		results = append(results, model.Secrets{
+			Location: coordinates,
+			Secrets:  secrets,
+		})
+	}
+
+	// sort by real path then virtual path to ensure the result is stable across multiple runs
+	sort.SliceStable(results, func(i, j int) bool {
+		return results[i].Location.RealPath < results[j].Location.RealPath
+	})
+	return results
+}
+
+func toFile(s sbom.SBOM) []model.File {
+	results := make([]model.File, 0)
+	artifacts := s.Artifacts
+
+	for _, coordinates := range sbom.AllCoordinates(s) {
+		var metadata *source.FileMetadata
+		if metadataForLocation, exists := artifacts.FileMetadata[coordinates]; exists {
+			metadata = &metadataForLocation
+		}
+
+		var digests []file.Digest
+		if digestsForLocation, exists := artifacts.FileDigests[coordinates]; exists {
+			digests = digestsForLocation
+		}
+
+		var classifications []file.Classification
+		if classificationsForLocation, exists := artifacts.FileClassifications[coordinates]; exists {
+			classifications = classificationsForLocation
+		}
+
+		var contents string
+		if contentsForLocation, exists := artifacts.FileContents[coordinates]; exists {
+			contents = contentsForLocation
+		}
+
+		results = append(results, model.File{
+			ID:              string(coordinates.ID()),
+			Location:        coordinates,
+			Metadata:        toFileMetadataEntry(coordinates, metadata),
+			Digests:         digests,
+			Classifications: classifications,
+			Contents:        contents,
+		})
+	}
+
+	// sort by real path then virtual path to ensure the result is stable across multiple runs
+	sort.SliceStable(results, func(i, j int) bool {
+		return results[i].Location.RealPath < results[j].Location.RealPath
+	})
+	return results
+}
+
+func toFileMetadataEntry(coordinates source.Coordinates, metadata *source.FileMetadata) *model.FileMetadataEntry {
+	if metadata == nil {
+		return nil
+	}
+
+	mode, err := strconv.Atoi(fmt.Sprintf("%o", metadata.Mode))
+	if err != nil {
+		log.Warnf("invalid mode found in file catalog @ location=%+v mode=%q: %+v", coordinates, metadata.Mode, err)
+		mode = 0
+	}
+
+	return &model.FileMetadataEntry{
+		Mode:            mode,
+		Type:            metadata.Type,
+		LinkDestination: metadata.LinkDestination,
+		UserID:          metadata.UserID,
+		GroupID:         metadata.GroupID,
+		MIMEType:        metadata.MIMEType,
+	}
+}
+
 func toPackageModels(catalog *pkg.Catalog) []model.Package {
 	artifacts := make([]model.Package, 0)
 	if catalog == nil {
@@ -20,10 +20,19 @@ func toSyftModel(doc model.Document) (*sbom.SBOM, error) {
 			PackageCatalog: toSyftCatalog(doc.Artifacts),
 			Distro:         &dist,
 		},
-		Source: *toSyftSourceData(doc.Source),
+		Source:     *toSyftSourceData(doc.Source),
+		Descriptor: toSyftDescriptor(doc.Descriptor),
 	}, nil
 }

+func toSyftDescriptor(d model.Descriptor) sbom.Descriptor {
+	return sbom.Descriptor{
+		Name:          d.Name,
+		Version:       d.Version,
+		Configuration: d.Configuration,
+	}
+}
+
 func toSyftSourceData(s model.Source) *source.Metadata {
 	switch s.Type {
 	case "directory":