Add package-to-file location evidence relationships (#1698)

* add evident-by relationship

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* wire up evident-by relationship geneation

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* handle evident-by relationship in spdx formats

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix decoding file info for syft json format

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* bump json schema to incorporate file size attribute

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* refactor to create relationships for primary evidence only

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove unused 7.0.2 json schema

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

---------

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

internal/constants.go

@@ -6,5 +6,5 @@ const (
 
 	// JSONSchemaVersion is the current schema version output by the JSON encoder
 	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
-	JSONSchemaVersion = "7.1.3"
+	JSONSchemaVersion = "7.1.4"
 )

syft/artifact/relationship.go

@@ -7,6 +7,12 @@ const (
 	// has been completed.
 	OwnershipByFileOverlapRelationship RelationshipType = "ownership-by-file-overlap"
 
+	// EvidentByRelationship is a package-to-file relationship indicating the that existence of this package is evident
+	// by the contents of a file. This does not necessarily mean that the package is contained within that file
+	// or that it is described by it (either or both may be true). This does NOT map to an existing specific SPDX
+	// relationship. Instead, this should be mapped to OTHER and the comment field be updated to show EVIDENT_BY.
+	EvidentByRelationship RelationshipType = "evident-by"
+
 	// ContainsRelationship (supports any-to-any linkages) is a proxy for the SPDX 2.2 CONTAINS relationship.
 	ContainsRelationship RelationshipType = "contains"
 

syft/formats/common/spdxhelpers/to_format_model.go

@@ -408,6 +408,8 @@ func lookupRelationship(ty artifact.RelationshipType) (bool, RelationshipType, s
 		return true, DependencyOfRelationship, ""
 	case artifact.OwnershipByFileOverlapRelationship:
 		return true, OtherRelationship, fmt.Sprintf("%s: indicates that the parent package claims ownership of a child package since the parent metadata indicates overlap with a location that a cataloger found the child package by", ty)
+	case artifact.EvidentByRelationship:
+		return true, OtherRelationship, fmt.Sprintf("%s: indicates the package's existence is evident by the given file", ty)
 	}
 	return false, "", ""
 }

syft/formats/common/spdxhelpers/to_format_model_test.go

@@ -209,6 +209,12 @@ func Test_lookupRelationship(t *testing.T) {
 			ty:      OtherRelationship,
 			comment: "ownership-by-file-overlap: indicates that the parent package claims ownership of a child package since the parent metadata indicates overlap with a location that a cataloger found the child package by",
 		},
+		{
+			input:   artifact.EvidentByRelationship,
+			exists:  true,
+			ty:      OtherRelationship,
+			comment: "evident-by: indicates the package's existence is evident by the given file",
+		},
 		{
 			input:  "made-up",
 			exists: false,

syft/formats/common/spdxhelpers/to_syft_model.go

@@ -176,9 +176,16 @@ func toSyftRelationships(spdxIDMap map[string]interface{}, doc *spdx.Document) [
 		var to artifact.Identifiable
 		var typ artifact.RelationshipType
 		if toLocationOk {
-			if r.Relationship == string(ContainsRelationship) {
+			switch RelationshipType(r.Relationship) {
+			case ContainsRelationship:
 				typ = artifact.ContainsRelationship
 				to = toLocation
+			case OtherRelationship:
+				// Encoding uses a specifically formatted comment...
+				if strings.Index(r.RelationshipComment, string(artifact.EvidentByRelationship)) == 0 {
+					typ = artifact.EvidentByRelationship
+					to = toLocation
+				}
 			}
 		} else {
 			switch RelationshipType(r.Relationship) {
@@ -188,7 +195,7 @@ func toSyftRelationships(spdxIDMap map[string]interface{}, doc *spdx.Document) [
 			case OtherRelationship:
 				// Encoding uses a specifically formatted comment...
 				if strings.Index(r.RelationshipComment, string(artifact.OwnershipByFileOverlapRelationship)) == 0 {
-					typ = artifact.DependencyOfRelationship
+					typ = artifact.OwnershipByFileOverlapRelationship
 					to = toPackage
 				}
 			}

syft/formats/common/spdxhelpers/to_syft_model_test.go

@@ -4,9 +4,11 @@ import (
 	"testing"
 
 	"github.com/spdx/tools-golang/spdx"
+	"github.com/spdx/tools-golang/spdx/v2/common"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
 )
@@ -307,3 +309,113 @@ func TestH1Digest(t *testing.T) {
 		})
 	}
 }
+
+func Test_toSyftRelationships(t *testing.T) {
+	type args struct {
+		spdxIDMap map[string]interface{}
+		doc       *spdx.Document
+	}
+
+	pkg1 := pkg.Package{
+		Name:    "github.com/googleapis/gnostic",
+		Version: "v0.5.5",
+	}
+	pkg1.SetID()
+
+	pkg2 := pkg.Package{
+		Name:    "rfc3339",
+		Version: "1.2",
+		Type:    pkg.RpmPkg,
+	}
+	pkg2.SetID()
+
+	pkg3 := pkg.Package{
+		Name:    "rfc3339",
+		Version: "1.2",
+		Type:    pkg.PythonPkg,
+	}
+	pkg3.SetID()
+
+	loc1 := source.NewLocationFromCoordinates(source.Coordinates{
+		RealPath:     "/somewhere/real",
+		FileSystemID: "abc",
+	})
+
+	tests := []struct {
+		name string
+		args args
+		want []artifact.Relationship
+	}{
+		{
+			name: "evident-by relationship",
+			args: args{
+				spdxIDMap: map[string]interface{}{
+					string(toSPDXID(pkg1)): &pkg1,
+					string(toSPDXID(loc1)): &loc1,
+				},
+				doc: &spdx.Document{
+					Relationships: []*spdx.Relationship{
+						{
+							RefA: common.DocElementID{
+								ElementRefID: toSPDXID(pkg1),
+							},
+							RefB: common.DocElementID{
+								ElementRefID: toSPDXID(loc1),
+							},
+							Relationship:        spdx.RelationshipOther,
+							RelationshipComment: "evident-by: indicates the package's existence is evident by the given file",
+						},
+					},
+				},
+			},
+			want: []artifact.Relationship{
+				{
+					From: pkg1,
+					To:   loc1,
+					Type: artifact.EvidentByRelationship,
+				},
+			},
+		},
+		{
+			name: "ownership-by-file-overlap relationship",
+			args: args{
+				spdxIDMap: map[string]interface{}{
+					string(toSPDXID(pkg2)): &pkg2,
+					string(toSPDXID(pkg3)): &pkg3,
+				},
+				doc: &spdx.Document{
+					Relationships: []*spdx.Relationship{
+						{
+							RefA: common.DocElementID{
+								ElementRefID: toSPDXID(pkg2),
+							},
+							RefB: common.DocElementID{
+								ElementRefID: toSPDXID(pkg3),
+							},
+							Relationship:        spdx.RelationshipOther,
+							RelationshipComment: "ownership-by-file-overlap: indicates that the parent package claims ownership of a child package since the parent metadata indicates overlap with a location that a cataloger found the child package by",
+						},
+					},
+				},
+			},
+			want: []artifact.Relationship{
+				{
+					From: pkg2,
+					To:   pkg3,
+					Type: artifact.OwnershipByFileOverlapRelationship,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := toSyftRelationships(tt.args.spdxIDMap, tt.args.doc)
+			require.Len(t, actual, len(tt.want))
+			for i := range actual {
+				require.Equal(t, tt.want[i].From.ID(), actual[i].From.ID())
+				require.Equal(t, tt.want[i].To.ID(), actual[i].To.ID())
+				require.Equal(t, tt.want[i].Type, actual[i].Type)
+			}
+		})
+	}
+}

syft/formats/syftjson/model/file.go

@@ -20,4 +20,5 @@ type FileMetadataEntry struct {
 	UserID          int    `json:"userID"`
 	GroupID         int    `json:"groupID"`
 	MIMEType        string `json:"mimeType"`
+	Size            int64  `json:"size"`
 }

syft/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden

@@ -89,7 +89,7 @@
   }
  },
  "schema": {
-  "version": "7.1.3",
+  "version": "7.1.4",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-7.1.3.json"
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-7.1.4.json"
  }
 }

syft/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden

@@ -81,7 +81,8 @@
     "type": "Directory",
     "userID": 0,
     "groupID": 0,
-    "mimeType": ""
+    "mimeType": "",
+    "size": 0
    }
   },
   {
@@ -94,7 +95,8 @@
     "type": "RegularFile",
     "userID": 0,
     "groupID": 0,
-    "mimeType": ""
+    "mimeType": "",
+    "size": 0
    },
    "contents": "the-contents",
    "digests": [
@@ -115,7 +117,8 @@
     "linkDestination": "/c",
     "userID": 0,
     "groupID": 0,
-    "mimeType": ""
+    "mimeType": "",
+    "size": 0
    }
   },
   {
@@ -128,7 +131,8 @@
     "type": "RegularFile",
     "userID": 1,
     "groupID": 2,
-    "mimeType": ""
+    "mimeType": "",
+    "size": 0
    },
    "digests": [
     {
@@ -185,7 +189,7 @@
   }
  },
  "schema": {
-  "version": "7.1.3",
+  "version": "7.1.4",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-7.1.3.json"
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-7.1.4.json"
  }
 }

syft/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden

@@ -112,7 +112,7 @@
   }
  },
  "schema": {
-  "version": "7.1.3",
+  "version": "7.1.4",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-7.1.3.json"
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-7.1.4.json"
  }
 }

syft/formats/syftjson/to_format_model.go

@@ -144,6 +144,7 @@ func toFileMetadataEntry(coordinates source.Coordinates, metadata *source.FileMe
 		UserID:          metadata.UserID,
 		GroupID:         metadata.GroupID,
 		MIMEType:        metadata.MIMEType,
+		Size:            metadata.Size,
 	}
 }
 

syft/formats/syftjson/to_syft_model.go

@@ -1,13 +1,17 @@
 package syftjson
 
 import (
+	"os"
+	"strconv"
 	"strings"
 
 	"github.com/google/go-cmp/cmp"
 
+	stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/cpe"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/formats/syftjson/model"
 	"github.com/anchore/syft/syft/linux"
 	"github.com/anchore/syft/syft/pkg"
@@ -20,9 +24,13 @@ func toSyftModel(doc model.Document) (*sbom.SBOM, error) {
 
 	catalog := toSyftCatalog(doc.Artifacts, idAliases)
 
+	fileArtifacts := toSyftFiles(doc.Files)
+
 	return &sbom.SBOM{
 		Artifacts: sbom.Artifacts{
 			PackageCatalog:    catalog,
+			FileMetadata:      fileArtifacts.FileMetadata,
+			FileDigests:       fileArtifacts.FileDigests,
 			LinuxDistribution: toSyftLinuxRelease(doc.Distro),
 		},
 		Source:        *toSyftSourceData(doc.Source),
@@ -31,6 +39,72 @@ func toSyftModel(doc model.Document) (*sbom.SBOM, error) {
 	}, nil
 }
 
+func toSyftFiles(files []model.File) sbom.Artifacts {
+	ret := sbom.Artifacts{
+		FileMetadata: make(map[source.Coordinates]source.FileMetadata),
+		FileDigests:  make(map[source.Coordinates][]file.Digest),
+	}
+
+	for _, f := range files {
+		coord := f.Location
+		if f.Metadata != nil {
+			mode, err := strconv.ParseInt(strconv.Itoa(f.Metadata.Mode), 8, 64)
+			if err != nil {
+				log.Warnf("invalid mode found in file catalog @ location=%+v mode=%q: %+v", coord, f.Metadata.Mode, err)
+				mode = 0
+			}
+
+			fm := os.FileMode(mode)
+
+			ret.FileMetadata[coord] = source.FileMetadata{
+				Path:            coord.RealPath,
+				LinkDestination: f.Metadata.LinkDestination,
+				Size:            f.Metadata.Size,
+				UserID:          f.Metadata.UserID,
+				GroupID:         f.Metadata.GroupID,
+				Type:            toSyftFileType(f.Metadata.Type),
+				IsDir:           fm.IsDir(),
+				Mode:            fm,
+				MIMEType:        f.Metadata.MIMEType,
+			}
+		}
+
+		for _, d := range f.Digests {
+			ret.FileDigests[coord] = append(ret.FileDigests[coord], file.Digest{
+				Algorithm: d.Algorithm,
+				Value:     d.Value,
+			})
+		}
+	}
+
+	return ret
+}
+
+func toSyftFileType(ty string) stereoscopeFile.Type {
+	switch ty {
+	case "SymbolicLink":
+		return stereoscopeFile.TypeSymLink
+	case "HardLink":
+		return stereoscopeFile.TypeHardLink
+	case "Directory":
+		return stereoscopeFile.TypeDirectory
+	case "Socket":
+		return stereoscopeFile.TypeSocket
+	case "BlockDevice":
+		return stereoscopeFile.TypeBlockDevice
+	case "CharacterDevice":
+		return stereoscopeFile.TypeCharacterDevice
+	case "FIFONode":
+		return stereoscopeFile.TypeFIFO
+	case "RegularFile":
+		return stereoscopeFile.TypeRegular
+	case "IrregularFile":
+		return stereoscopeFile.TypeIrregular
+	default:
+		return stereoscopeFile.TypeIrregular
+	}
+}
+
 func toSyftLinuxRelease(d model.LinuxRelease) *linux.Release {
 	if cmp.Equal(d, model.LinuxRelease{}) {
 		return nil
@@ -117,7 +191,7 @@ func toSyftRelationship(idMap map[string]interface{}, relationship model.Relatio
 	typ := artifact.RelationshipType(relationship.Type)
 
 	switch typ {
-	case artifact.OwnershipByFileOverlapRelationship, artifact.ContainsRelationship, artifact.DependencyOfRelationship:
+	case artifact.OwnershipByFileOverlapRelationship, artifact.ContainsRelationship, artifact.DependencyOfRelationship, artifact.EvidentByRelationship:
 	default:
 		if !strings.Contains(string(typ), "dependency-of") {
 			log.Warnf("unknown relationship type: %s", typ)

syft/formats/syftjson/to_syft_model_test.go

@@ -6,8 +6,11 @@ import (
 	"github.com/scylladb/go-set/strset"
 	"github.com/stretchr/testify/assert"
 
+	stereoFile "github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/formats/syftjson/model"
+	"github.com/anchore/syft/syft/sbom"
 	"github.com/anchore/syft/syft/source"
 )
 
@@ -124,3 +127,104 @@ func Test_idsHaveChanged(t *testing.T) {
 	assert.NotNil(t, to)
 	assert.Equal(t, "pkg-2", to.Name)
 }
+
+func Test_toSyftFiles(t *testing.T) {
+	coord := source.Coordinates{
+		RealPath:     "/somerwhere/place",
+		FileSystemID: "abc",
+	}
+
+	tests := []struct {
+		name  string
+		files []model.File
+		want  sbom.Artifacts
+	}{
+		{
+			name:  "empty",
+			files: []model.File{},
+			want: sbom.Artifacts{
+				FileMetadata: map[source.Coordinates]source.FileMetadata{},
+				FileDigests:  map[source.Coordinates][]file.Digest{},
+			},
+		},
+		{
+			name: "no metadata",
+			files: []model.File{
+				{
+					ID:       string(coord.ID()),
+					Location: coord,
+					Metadata: nil,
+					Digests: []file.Digest{
+						{
+							Algorithm: "sha256",
+							Value:     "123",
+						},
+					},
+				},
+			},
+			want: sbom.Artifacts{
+				FileMetadata: map[source.Coordinates]source.FileMetadata{},
+				FileDigests: map[source.Coordinates][]file.Digest{
+					coord: {
+						{
+							Algorithm: "sha256",
+							Value:     "123",
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "single file",
+			files: []model.File{
+				{
+					ID:       string(coord.ID()),
+					Location: coord,
+					Metadata: &model.FileMetadataEntry{
+						Mode:            777,
+						Type:            "RegularFile",
+						LinkDestination: "",
+						UserID:          42,
+						GroupID:         32,
+						MIMEType:        "text/plain",
+						Size:            92,
+					},
+					Digests: []file.Digest{
+						{
+							Algorithm: "sha256",
+							Value:     "123",
+						},
+					},
+				},
+			},
+			want: sbom.Artifacts{
+				FileMetadata: map[source.Coordinates]source.FileMetadata{
+					coord: {
+						Path:            coord.RealPath,
+						LinkDestination: "",
+						Size:            92,
+						UserID:          42,
+						GroupID:         32,
+						Type:            stereoFile.TypeRegular,
+						IsDir:           false,
+						Mode:            511, // 777 octal = 511 decimal
+						MIMEType:        "text/plain",
+					},
+				},
+				FileDigests: map[source.Coordinates][]file.Digest{
+					coord: {
+						{
+							Algorithm: "sha256",
+							Value:     "123",
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, toSyftFiles(tt.files))
+		})
+	}
+}

syft/pkg/relationships.go

@@ -2,7 +2,8 @@ package pkg
 
 import "github.com/anchore/syft/syft/artifact"
 
-// TODO: as more relationships are added, this function signature will probably accommodate selection
 func NewRelationships(catalog *Catalog) []artifact.Relationship {
-	return RelationshipsByFileOwnership(catalog)
+	rels := RelationshipsByFileOwnership(catalog)
+	rels = append(rels, RelationshipsEvidentBy(catalog)...)
+	return rels
 }

syft/pkg/relationships_evident_by.go

@@ -0,0 +1,25 @@
+package pkg
+
+import (
+	"github.com/anchore/syft/syft/artifact"
+)
+
+func RelationshipsEvidentBy(catalog *Catalog) []artifact.Relationship {
+	var edges []artifact.Relationship
+	for _, p := range catalog.Sorted() {
+		for _, l := range p.Locations.ToSlice() {
+			if v, exists := l.Annotations[EvidenceAnnotationKey]; !exists || v != PrimaryEvidenceAnnotation {
+				// skip non-primary evidence from being expressed as a relationship.
+				// note: this may be configurable in the future.
+				continue
+			}
+			edges = append(edges, artifact.Relationship{
+				From: p,
+				To:   l.Coordinates,
+				Type: artifact.EvidentByRelationship,
+			})
+		}
+	}
+
+	return edges
+}

syft/pkg/relationships_evident_by_test.go

@@ -0,0 +1,87 @@
+package pkg
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/source"
+)
+
+func TestRelationshipsEvidentBy(t *testing.T) {
+
+	c := NewCatalog()
+
+	coordA := source.Coordinates{
+		RealPath:     "/somewhere/real",
+		FileSystemID: "abc",
+	}
+	coordC := source.Coordinates{
+		RealPath:     "/somewhere/real",
+		FileSystemID: "abc",
+	}
+	coordD := source.Coordinates{
+		RealPath:     "/somewhere/real",
+		FileSystemID: "abc",
+	}
+	pkgA := Package{
+		Locations: source.NewLocationSet(
+			// added!
+			source.NewLocationFromCoordinates(coordA).WithAnnotation(EvidenceAnnotationKey, PrimaryEvidenceAnnotation),
+			// ignored...
+			source.NewLocationFromCoordinates(coordC).WithAnnotation(EvidenceAnnotationKey, SupportingEvidenceAnnotation),
+			source.NewLocationFromCoordinates(coordD),
+		),
+	}
+	pkgA.SetID()
+	c.Add(pkgA)
+
+	coordB := source.Coordinates{
+		RealPath:     "/somewhere-else/real",
+		FileSystemID: "def",
+	}
+	pkgB := Package{
+		Locations: source.NewLocationSet(
+			// added!
+			source.NewLocationFromCoordinates(coordB).WithAnnotation(EvidenceAnnotationKey, PrimaryEvidenceAnnotation),
+		),
+	}
+	pkgB.SetID()
+	c.Add(pkgB)
+
+	tests := []struct {
+		name    string
+		catalog *Catalog
+		want    []artifact.Relationship
+	}{
+		{
+			name:    "go case",
+			catalog: c,
+			want: []artifact.Relationship{
+				{
+					From: pkgB,
+					To:   coordB,
+					Type: artifact.EvidentByRelationship,
+				},
+				{
+					From: pkgA,
+					To:   coordA,
+					Type: artifact.EvidentByRelationship,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := RelationshipsEvidentBy(tt.catalog)
+			require.Len(t, actual, len(tt.want))
+			for i := range actual {
+				assert.Equal(t, tt.want[i].From.ID(), actual[i].From.ID(), "from mismatch at index %d", i)
+				assert.Equal(t, tt.want[i].To.ID(), actual[i].To.ID(), "to mismatch at index %d", i)
+				assert.Equal(t, tt.want[i].Type, actual[i].Type, "type mismatch at index %d", i)
+			}
+		})
+	}
+}