oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2025-11-17 16:33:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

Introduce minimal source coordinates (#623)

Browse Source 
* split source.Location and create source.Coordinates for minimal path addressing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* move coordinates into separate file

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* Update syft/source/coordinates.go

Co-authored-by: Dan Luhring <luhring@users.noreply.github.com>
This commit is contained in:
Alex Goodman 2021-11-18 13:13:22 -05:00 committed by GitHub
parent 9090c3a772
commit e38cde35ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 262 additions and 249 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
internal/anchore/import_package_sbom_test.go
View File

@ -59,8 +59,10 @@ func TestPackageSbomToModel(t *testing.T) {
FoundBy: "foundBy",
Locations: []source.Location{
{
RealPath: "path",
FileSystemID: "layerID",
Coordinates: source.Coordinates{
RealPath: "path",
FileSystemID: "layerID",
},
},
},
Licenses: []string{"license"},
@ -157,8 +159,10 @@ func TestPackageSbomImport(t *testing.T) {
FoundBy: "foundBy",
Locations: []source.Location{
{
RealPath: "path",
FileSystemID: "layerID",
Coordinates: source.Coordinates{
RealPath: "path",
FileSystemID: "layerID",
},
},
},
Licenses: []string{"license"},

10
internal/formats/common/spdxhelpers/source_info_test.go
View File

@ -19,14 +19,8 @@ func Test_SourceInfo(t *testing.T) {
input: pkg.Package{
// note: no type given
Locations: []source.Location{
{
RealPath: "/a-place",
VirtualPath: "/b-place",
},
{
RealPath: "/c-place",
VirtualPath: "/d-place",
},
source.NewVirtualLocation("/a-place", "/b-place"),
source.NewVirtualLocation("/c-place", "/d-place"),
},
},
expected: []string{

4
internal/formats/common/testutils/utils.go
View File

@ -200,7 +200,7 @@ func newDirectoryCatalog() *pkg.Catalog {
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Locations: []source.Location{
{RealPath: "/some/path/pkg1"},
source.NewLocation("/some/path/pkg1"),
},
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
@ -225,7 +225,7 @@ func newDirectoryCatalog() *pkg.Catalog {
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Locations: []source.Location{
{RealPath: "/some/path/pkg1"},
source.NewLocation("/some/path/pkg1"),
},
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{

20
internal/formats/syftjson/model/package.go
View File

@ -17,16 +17,16 @@ type Package struct {
// PackageBasicData contains non-ambiguous values (type-wise) from pkg.Package.
type PackageBasicData struct {
ID string `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
Type pkg.Type `json:"type"`
FoundBy string `json:"foundBy"`
Locations []source.Location `json:"locations"`
Licenses []string `json:"licenses"`
Language pkg.Language `json:"language"`
CPEs []string `json:"cpes"`
PURL string `json:"purl"`
ID string `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
Type pkg.Type `json:"type"`
FoundBy string `json:"foundBy"`
Locations []source.Coordinates `json:"locations"`
Licenses []string `json:"licenses"`
Language pkg.Language `json:"language"`
CPEs []string `json:"cpes"`
PURL string `json:"purl"`
}
// PackageCustomData contains ambiguous values (type-wise) from pkg.Package.

4
internal/formats/syftjson/test-fixtures/snapshot/TestDirectoryPresenter.golden
View File

@ -1,7 +1,7 @@
{
"artifacts": [
{
"id": "cbf4f3077fc7deee",
"id": "2a115ac97d018a0e",
"name": "package-1",
"version": "1.0.1",
"type": "python",
@ -36,7 +36,7 @@
}
},
{
"id": "1a39aadd9705c2b9",
"id": "5e920b2bece2c3ae",
"name": "package-2",
"version": "2.0.1",
"type": "deb",

4
internal/formats/syftjson/test-fixtures/snapshot/TestImagePresenter.golden
View File

@ -1,7 +1,7 @@
{
"artifacts": [
{
"id": "d1d433485a31ed07",
"id": "888661d4f0362f02",
"name": "package-1",
"version": "1.0.1",
"type": "python",
@ -32,7 +32,7 @@
}
},
{
"id": "2db629ca48fa6786",
"id": "4068ff5e8926b305",
"name": "package-2",
"version": "2.0.1",
"type": "deb",

13
internal/formats/syftjson/to_format_model.go
View File

@ -58,17 +58,16 @@ func toPackageModel(p pkg.Package) model.Package {
cpes[i] = c.BindToFmtString()
}
// ensure collections are never nil for presentation reasons
var locations = make([]source.Location, 0)
if p.Locations != nil {
locations = p.Locations
}
var licenses = make([]string, 0)
if p.Licenses != nil {
licenses = p.Licenses
}
var coordinates = make([]source.Coordinates, len(p.Locations))
for i, l := range p.Locations {
coordinates[i] = l.Coordinates
}
return model.Package{
PackageBasicData: model.PackageBasicData{
ID: string(p.ID()),
@ -76,7 +75,7 @@ func toPackageModel(p pkg.Package) model.Package {
Version: p.Version,
Type: p.Type,
FoundBy: p.FoundBy,
Locations: locations,
Locations: coordinates,
Licenses: licenses,
Language: p.Language,
CPEs: cpes,

7
internal/formats/syftjson/to_syft_model.go
View File

@ -60,11 +60,16 @@ func toSyftPackage(p model.Package) pkg.Package {
cpes = append(cpes, value)
}
var locations = make([]source.Location, len(p.Locations))
for i, c := range p.Locations {
locations[i] = source.NewLocationFromCoordinates(c)
}
return pkg.Package{
Name: p.Name,
Version: p.Version,
FoundBy: p.FoundBy,
Locations: p.Locations,
Locations: locations,
Licenses: p.Licenses,
Language: p.Language,
Type: p.Type,

11
internal/presenter/poweruser/json_file_classifications.go
View File

@ -8,16 +8,16 @@ import (
)
type JSONFileClassifications struct {
Location source.Location `json:"location"`
Location source.Coordinates `json:"location"`
Classification file.Classification `json:"classification"`
}
func NewJSONFileClassifications(data map[source.Location][]file.Classification) []JSONFileClassifications {
func NewJSONFileClassifications(data map[source.Coordinates][]file.Classification) []JSONFileClassifications {
results := make([]JSONFileClassifications, 0)
for location, classifications := range data {
for coordinates, classifications := range data {
for _, classification := range classifications {
results = append(results, JSONFileClassifications{
Location: location,
Location: coordinates,
Classification: classification,
})
}
@ -25,9 +25,6 @@ func NewJSONFileClassifications(data map[source.Location][]file.Classification)
// sort by real path then virtual path to ensure the result is stable across multiple runs
sort.SliceStable(results, func(i, j int) bool {
if results[i].Location.RealPath == results[j].Location.RealPath {
return results[i].Location.VirtualPath < results[j].Location.VirtualPath
}
return results[i].Location.RealPath < results[j].Location.RealPath
})
return results

13
internal/presenter/poweruser/json_file_contents.go
View File

@ -7,24 +7,21 @@ import (
)
type JSONFileContents struct {
Location source.Location `json:"location"`
Contents string `json:"contents"`
Location source.Coordinates `json:"location"`
Contents string `json:"contents"`
}
func NewJSONFileContents(data map[source.Location]string) []JSONFileContents {
func NewJSONFileContents(data map[source.Coordinates]string) []JSONFileContents {
results := make([]JSONFileContents, 0)
for location, contents := range data {
for coordinates, contents := range data {
results = append(results, JSONFileContents{
Location: location,
Location: coordinates,
Contents: contents,
})
}
// sort by real path then virtual path to ensure the result is stable across multiple runs
sort.SliceStable(results, func(i, j int) bool {
if results[i].Location.RealPath == results[j].Location.RealPath {
return results[i].Location.VirtualPath < results[j].Location.VirtualPath
}
return results[i].Location.RealPath < results[j].Location.RealPath
})
return results

15
internal/presenter/poweruser/json_file_metadata.go
View File

@ -11,7 +11,7 @@ import (
)
type JSONFileMetadata struct {
Location source.Location `json:"location"`
Location source.Coordinates `json:"location"`
Metadata JSONFileMetadataEntry `json:"metadata"`
}
@ -25,21 +25,21 @@ type JSONFileMetadataEntry struct {
MIMEType string `json:"mimeType"`
}
func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests map[source.Location][]file.Digest) ([]JSONFileMetadata, error) {
func NewJSONFileMetadata(data map[source.Coordinates]source.FileMetadata, digests map[source.Coordinates][]file.Digest) ([]JSONFileMetadata, error) {
results := make([]JSONFileMetadata, 0)
for location, metadata := range data {
for coordinates, metadata := range data {
mode, err := strconv.Atoi(fmt.Sprintf("%o", metadata.Mode))
if err != nil {
return nil, fmt.Errorf("invalid mode found in file catalog @ location=%+v mode=%q: %w", location, metadata.Mode, err)
return nil, fmt.Errorf("invalid mode found in file catalog @ location=%+v mode=%q: %w", coordinates, metadata.Mode, err)
}
var digestResults []file.Digest
if digestsForLocation, exists := digests[location]; exists {
if digestsForLocation, exists := digests[coordinates]; exists {
digestResults = digestsForLocation
}
results = append(results, JSONFileMetadata{
Location: location,
Location: coordinates,
Metadata: JSONFileMetadataEntry{
Mode: mode,
Type: metadata.Type,
@ -54,9 +54,6 @@ func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests m
// sort by real path then virtual path to ensure the result is stable across multiple runs
sort.SliceStable(results, func(i, j int) bool {
if results[i].Location.RealPath == results[j].Location.RealPath {
return results[i].Location.VirtualPath < results[j].Location.VirtualPath
}
return results[i].Location.RealPath < results[j].Location.RealPath
})
return results, nil

28
internal/presenter/poweruser/json_presenter_test.go
View File

@ -37,7 +37,9 @@ func TestJSONPresenter(t *testing.T) {
Version: "1.0.1",
Locations: []source.Location{
{
RealPath: "/a/place/a",
Coordinates: source.Coordinates{
RealPath: "/a/place/a",
},
},
},
Type: pkg.PythonPkg,
@ -60,7 +62,9 @@ func TestJSONPresenter(t *testing.T) {
Version: "2.0.1",
Locations: []source.Location{
{
RealPath: "/b/place/b",
Coordinates: source.Coordinates{
RealPath: "/b/place/b",
},
},
},
Type: pkg.DebPkg,
@ -86,49 +90,49 @@ func TestJSONPresenter(t *testing.T) {
cfg := sbom.SBOM{
Artifacts: sbom.Artifacts{
PackageCatalog: catalog,
FileMetadata: map[source.Location]source.FileMetadata{
source.NewLocation("/a/place"): {
FileMetadata: map[source.Coordinates]source.FileMetadata{
source.NewLocation("/a/place").Coordinates: {
Mode: 0775,
Type: "directory",
UserID: 0,
GroupID: 0,
},
source.NewLocation("/a/place/a"): {
source.NewLocation("/a/place/a").Coordinates: {
Mode: 0775,
Type: "regularFile",
UserID: 0,
GroupID: 0,
},
source.NewLocation("/b"): {
source.NewLocation("/b").Coordinates: {
Mode: 0775,
Type: "symbolicLink",
LinkDestination: "/c",
UserID: 0,
GroupID: 0,
},
source.NewLocation("/b/place/b"): {
source.NewLocation("/b/place/b").Coordinates: {
Mode: 0644,
Type: "regularFile",
UserID: 1,
GroupID: 2,
},
},
FileDigests: map[source.Location][]file.Digest{
source.NewLocation("/a/place/a"): {
FileDigests: map[source.Coordinates][]file.Digest{
source.NewLocation("/a/place/a").Coordinates: {
{
Algorithm: "sha256",
Value: "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
},
},
source.NewLocation("/b/place/b"): {
source.NewLocation("/b/place/b").Coordinates: {
{
Algorithm: "sha256",
Value: "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c",
},
},
},
FileContents: map[source.Location]string{
source.NewLocation("/a/place/a"): "the-contents",
FileContents: map[source.Coordinates]string{
source.NewLocation("/a/place/a").Coordinates: "the-contents",
},
Distro: &distro.Distro{
Type: distro.RedHat,

13
internal/presenter/poweruser/json_secrets.go
View File

@ -8,25 +8,22 @@ import (
)
type JSONSecrets struct {
Location source.Location `json:"location"`
Location source.Coordinates `json:"location"`
Secrets []file.SearchResult `json:"secrets"`
}
func NewJSONSecrets(data map[source.Location][]file.SearchResult) []JSONSecrets {
func NewJSONSecrets(data map[source.Coordinates][]file.SearchResult) []JSONSecrets {
results := make([]JSONSecrets, 0)
for location, secrets := range data {
for coordinates, secrets := range data {
results = append(results, JSONSecrets{
Location: location,
Location: coordinates,
Secrets: secrets,
})
}
// sort by real path then virtual path to ensure the result is stable across multiple runs
sort.SliceStable(results, func(i, j int) bool {
if results[i].Location.RealPath != results[j].Location.RealPath {
return results[i].Location.VirtualPath < results[j].Location.VirtualPath
}
return false
return results[i].Location.RealPath < results[j].Location.RealPath
})
return results
}

4
internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden
View File

@ -72,7 +72,7 @@
],
"artifacts": [
{
"id": "b84dfe0eb2c5670f",
"id": "962403cfb7be50d7",
"name": "package-1",
"version": "1.0.1",
"type": "python",
@ -102,7 +102,7 @@
}
},
{
"id": "6619226d6979963f",
"id": "b11f44847bba0ed1",
"name": "package-2",
"version": "2.0.1",
"type": "deb",

6
syft/file/classification_cataloger.go
View File

@ -15,8 +15,8 @@ func NewClassificationCataloger(classifiers []Classifier) (*ClassificationCatalo
}, nil
}
func (i *ClassificationCataloger) Catalog(resolver source.FileResolver) (map[source.Location][]Classification, error) {
results := make(map[source.Location][]Classification)
func (i *ClassificationCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]Classification, error) {
results := make(map[source.Coordinates][]Classification)
numResults := 0
for location := range resolver.AllLocations() {
@ -26,7 +26,7 @@ func (i *ClassificationCataloger) Catalog(resolver source.FileResolver) (map[sou
return nil, err
}
if result != nil {
results[location] = append(results[location], *result)
results[location.Coordinates] = append(results[location.Coordinates], *result)
numResults++
}
}

12
syft/file/classifier_test.go
View File

@ -19,7 +19,9 @@ func TestFilepathMatches(t *testing.T) {
{
name: "simple-filename-match",
location: source.Location{
RealPath: "python2.7",
Coordinates: source.Coordinates{
RealPath: "python2.7",
},
},
patterns: []string{
`python([0-9]+\.[0-9]+)$`,
@ -29,7 +31,9 @@ func TestFilepathMatches(t *testing.T) {
{
name: "filepath-match",
location: source.Location{
RealPath: "/usr/bin/python2.7",
Coordinates: source.Coordinates{
RealPath: "/usr/bin/python2.7",
},
},
patterns: []string{
`python([0-9]+\.[0-9]+)$`,
@ -59,7 +63,9 @@ func TestFilepathMatches(t *testing.T) {
{
name: "anchored-filename-match-FAILS",
location: source.Location{
RealPath: "/usr/bin/python2.7",
Coordinates: source.Coordinates{
RealPath: "/usr/bin/python2.7",
},
},
patterns: []string{
`^python([0-9]+\.[0-9]+)$`,

6
syft/file/contents_cataloger.go
View File

@ -23,8 +23,8 @@ func NewContentsCataloger(globs []string, skipFilesAboveSize int64) (*ContentsCa
}, nil
}
func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Location]string, error) {
results := make(map[source.Location]string)
func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates]string, error) {
results := make(map[source.Coordinates]string)
var locations []source.Location
locations, err := resolver.FilesByGlob(i.globs...)
@ -49,7 +49,7 @@ func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Lo
if err != nil {
return nil, err
}
results[location] = result
results[location.Coordinates] = result
}
log.Debugf("file contents cataloger processed %d files", len(results))

32
syft/file/contents_cataloger_test.go
View File

@ -15,41 +15,41 @@ func TestContentsCataloger(t *testing.T) {
globs []string
maxSize int64
files []string
expected map[source.Location]string
expected map[source.Coordinates]string
}{
{
name: "multi-pattern",
globs: []string{"test-fixtures/last/*.txt", "test-fixtures/*.txt"},
files: allFiles,
expected: map[source.Location]string{
source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
expected: map[source.Coordinates]string{
source.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "no-patterns",
globs: []string{},
files: []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"},
expected: map[source.Location]string{},
expected: map[source.Coordinates]string{},
},
{
name: "all-txt",
globs: []string{"**/*.txt"},
files: allFiles,
expected: map[source.Location]string{
source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
expected: map[source.Coordinates]string{
source.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
name: "subpath",
globs: []string{"test-fixtures/*.txt"},
files: allFiles,
expected: map[source.Location]string{
source.NewLocation("test-fixtures/another-path.txt"): "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
expected: map[source.Coordinates]string{
source.NewLocation("test-fixtures/another-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hbm90aGVyLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
{
@ -57,9 +57,9 @@ func TestContentsCataloger(t *testing.T) {
maxSize: 42,
globs: []string{"**/*.txt"},
files: allFiles,
expected: map[source.Location]string{
source.NewLocation("test-fixtures/last/path.txt"): "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt"): "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
expected: map[source.Coordinates]string{
source.NewLocation("test-fixtures/last/path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9sYXN0L3BhdGgudHh0IGZpbGUgY29udGVudHMh",
source.NewLocation("test-fixtures/a-path.txt").Coordinates: "dGVzdC1maXh0dXJlcy9hLXBhdGgudHh0IGZpbGUgY29udGVudHMh",
},
},
}

6
syft/file/digest_cataloger.go
View File

@ -29,8 +29,8 @@ func NewDigestsCataloger(hashes []crypto.Hash) (*DigestsCataloger, error) {
}, nil
}
func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Location][]Digest, error) {
results := make(map[source.Location][]Digest)
func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]Digest, error) {
results := make(map[source.Coordinates][]Digest)
var locations []source.Location
for location := range resolver.AllLocations() {
locations = append(locations, location)
@ -48,7 +48,7 @@ func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Loc
return nil, err
}
prog.N++
results[location] = result
results[location.Coordinates] = result
}
log.Debugf("file digests cataloger processed %d files", prog.N)
prog.SetCompleted()

12
syft/file/digest_cataloger_test.go
View File

@ -16,8 +16,8 @@ import (
"github.com/anchore/syft/syft/source"
)
func testDigests(t testing.TB, files []string, hashes ...crypto.Hash) map[source.Location][]Digest {
digests := make(map[source.Location][]Digest)
func testDigests(t testing.TB, files []string, hashes ...crypto.Hash) map[source.Coordinates][]Digest {
digests := make(map[source.Coordinates][]Digest)
for _, f := range files {
fh, err := os.Open(f)
@ -32,7 +32,7 @@ func testDigests(t testing.TB, files []string, hashes ...crypto.Hash) map[source
for _, hash := range hashes {
h := hash.New()
h.Write(b)
digests[source.NewLocation(f)] = append(digests[source.NewLocation(f)], Digest{
digests[source.NewLocation(f).Coordinates] = append(digests[source.NewLocation(f).Coordinates], Digest{
Algorithm: CleanDigestAlgorithmName(hash.String()),
Value: fmt.Sprintf("%x", h.Sum(nil)),
})
@ -49,7 +49,7 @@ func TestDigestsCataloger_SimpleContents(t *testing.T) {
name string
digests []crypto.Hash
files []string
expected map[source.Location][]Digest
expected map[source.Coordinates][]Digest
catalogErr bool
}{
{
@ -160,13 +160,13 @@ func TestDigestsCataloger_MixFileTypes(t *testing.T) {
}
l := source.NewLocationFromImage(test.path, *ref, img)
if len(actual[l]) == 0 {
if len(actual[l.Coordinates]) == 0 {
if test.expected != "" {
t.Fatalf("no digest found, but expected one")
}
} else {
assert.Equal(t, actual[l][0].Value, test.expected, "mismatched digests")
assert.Equal(t, actual[l.Coordinates][0].Value, test.expected, "mismatched digests")
}
})
}

6
syft/file/metadata_cataloger.go
View File

@ -16,8 +16,8 @@ func NewMetadataCataloger() *MetadataCataloger {
return &MetadataCataloger{}
}
func (i *MetadataCataloger) Catalog(resolver source.FileResolver) (map[source.Location]source.FileMetadata, error) {
results := make(map[source.Location]source.FileMetadata)
func (i *MetadataCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates]source.FileMetadata, error) {
results := make(map[source.Coordinates]source.FileMetadata)
var locations []source.Location
for location := range resolver.AllLocations() {
locations = append(locations, location)
@ -30,7 +30,7 @@ func (i *MetadataCataloger) Catalog(resolver source.FileResolver) (map[source.Lo
return nil, err
}
results[location] = metadata
results[location.Coordinates] = metadata
prog.N++
}
log.Debugf("file metadata cataloger processed %d files", prog.N)

2
syft/file/metadata_cataloger_test.go
View File

@ -136,7 +136,7 @@ func TestFileMetadataCataloger(t *testing.T) {
l := source.NewLocationFromImage(test.path, *ref, img)
assert.Equal(t, test.expected, actual[l], "mismatched metadata")
assert.Equal(t, test.expected, actual[l.Coordinates], "mismatched metadata")
})
}

6
syft/file/secrets_cataloger.go
View File

@ -40,8 +40,8 @@ func NewSecretsCataloger(patterns map[string]*regexp.Regexp, revealValues bool,
}, nil
}
func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Location][]SearchResult, error) {
results := make(map[source.Location][]SearchResult)
func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]SearchResult, error) {
results := make(map[source.Coordinates][]SearchResult)
var locations []source.Location
for location := range resolver.AllLocations() {
locations = append(locations, location)
@ -60,7 +60,7 @@ func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Loc
}
if len(result) > 0 {
secretsDiscovered.N += int64(len(result))
results[location] = result
results[location.Coordinates] = result
}
prog.N++
}

8
syft/file/secrets_cataloger_test.go
View File

@ -198,11 +198,11 @@ func TestSecretsCataloger(t *testing.T) {
}
loc := source.NewLocation(test.fixture)
if _, exists := actualResults[loc]; !exists {
if _, exists := actualResults[loc.Coordinates]; !exists {
t.Fatalf("could not find location=%q in results", loc)
}
assert.Equal(t, test.expected, actualResults[loc], "mismatched secrets")
assert.Equal(t, test.expected, actualResults[loc.Coordinates], "mismatched secrets")
})
}
}
@ -432,13 +432,13 @@ j4f668YfhUbKdRF6S6734856
}
loc := source.NewLocation(test.fixture)
if _, exists := actualResults[loc]; !exists && test.expected != nil {
if _, exists := actualResults[loc.Coordinates]; !exists && test.expected != nil {
t.Fatalf("could not find location=%q in results", loc)
} else if !exists && test.expected == nil {
return
}
assert.Equal(t, test.expected, actualResults[loc], "mismatched secrets")
assert.Equal(t, test.expected, actualResults[loc.Coordinates], "mismatched secrets")
})
}
}

35
syft/pkg/catalog_test.go
View File

@ -11,27 +11,15 @@ import (
var catalogAddAndRemoveTestPkgs = []Package{
{
Locations: []source.Location{
{
RealPath: "/a/path",
VirtualPath: "/another/path",
},
{
RealPath: "/b/path",
VirtualPath: "/bee/path",
},
source.NewVirtualLocation("/a/path", "/another/path"),
source.NewVirtualLocation("/b/path", "/bee/path"),
},
Type: RpmPkg,
},
{
Locations: []source.Location{
{
RealPath: "/c/path",
VirtualPath: "/another/path",
},
{
RealPath: "/d/path",
VirtualPath: "/another/path",
},
source.NewVirtualLocation("/c/path", "/another/path"),
source.NewVirtualLocation("/d/path", "/another/path"),
},
Type: NpmPkg,
},
@ -118,14 +106,8 @@ func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) {
func TestCatalog_PathIndexDeduplicatesRealVsVirtualPaths(t *testing.T) {
p1 := Package{
Locations: []source.Location{
{
RealPath: "/b/path",
VirtualPath: "/another/path",
},
{
RealPath: "/b/path",
VirtualPath: "/b/path",
},
source.NewVirtualLocation("/b/path", "/another/path"),
source.NewVirtualLocation("/b/path", "/b/path"),
},
Type: RpmPkg,
Name: "Package-1",
@ -133,10 +115,7 @@ func TestCatalog_PathIndexDeduplicatesRealVsVirtualPaths(t *testing.T) {
p2 := Package{
Locations: []source.Location{
{
RealPath: "/b/path",
VirtualPath: "/b/path",
},
source.NewVirtualLocation("/b/path", "/b/path"),
},
Type: RpmPkg,
Name: "Package-2",

37
syft/pkg/cataloger/golang/parse_go_bin_test.go
View File

@ -34,8 +34,10 @@ func TestBuildGoPkgInfo(t *testing.T) {
Type: pkg.GoModulePkg,
Locations: []source.Location{
{
RealPath: "/a-path",
FileSystemID: "layer-id",
Coordinates: source.Coordinates{
RealPath: "/a-path",
FileSystemID: "layer-id",
},
},
},
MetadataType: pkg.GolangBinMetadataType,
@ -51,8 +53,10 @@ func TestBuildGoPkgInfo(t *testing.T) {
Type: pkg.GoModulePkg,
Locations: []source.Location{
{
RealPath: "/a-path",
FileSystemID: "layer-id",
Coordinates: source.Coordinates{
RealPath: "/a-path",
FileSystemID: "layer-id",
},
},
},
MetadataType: pkg.GolangBinMetadataType,
@ -79,8 +83,10 @@ func TestBuildGoPkgInfo(t *testing.T) {
Type: pkg.GoModulePkg,
Locations: []source.Location{
{
RealPath: "/a-path",
FileSystemID: "layer-id",
Coordinates: source.Coordinates{
RealPath: "/a-path",
FileSystemID: "layer-id",
},
},
},
MetadataType: pkg.GolangBinMetadataType,
@ -96,8 +102,10 @@ func TestBuildGoPkgInfo(t *testing.T) {
Type: pkg.GoModulePkg,
Locations: []source.Location{
{
RealPath: "/a-path",
FileSystemID: "layer-id",
Coordinates: source.Coordinates{
RealPath: "/a-path",
FileSystemID: "layer-id",
},
},
},
MetadataType: pkg.GolangBinMetadataType,
@ -113,8 +121,10 @@ func TestBuildGoPkgInfo(t *testing.T) {
Type: pkg.GoModulePkg,
Locations: []source.Location{
{
RealPath: "/a-path",
FileSystemID: "layer-id",
Coordinates: source.Coordinates{
RealPath: "/a-path",
FileSystemID: "layer-id",
},
},
},
MetadataType: pkg.GolangBinMetadataType,
@ -130,7 +140,12 @@ func TestBuildGoPkgInfo(t *testing.T) {
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
location := source.Location{RealPath: "/a-path", FileSystemID: "layer-id"}
location := source.Location{
Coordinates: source.Coordinates{
RealPath: "/a-path",
FileSystemID: "layer-id",
},
}
pkgs := buildGoPkgInfo(location, tt.mod, goCompiledVersion)
assert.Equal(t, tt.expected, pkgs)
})

8
syft/pkg/package_test.go
View File

@ -14,9 +14,11 @@ func TestFingerprint(t *testing.T) {
FoundBy: "Archimedes",
Locations: []source.Location{
{
RealPath: "39.0742° N, 21.8243° E",
VirtualPath: "/Ancient-Greece",
FileSystemID: "Earth",
Coordinates: source.Coordinates{
RealPath: "39.0742° N, 21.8243° E",
FileSystemID: "Earth",
},
VirtualPath: "/Ancient-Greece",
},
},
Licenses: []string{

60
syft/pkg/relationships_by_file_ownership_test.go
View File

@ -19,14 +19,8 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
setup: func(t testing.TB) ([]Package, []artifact.Relationship) {
parent := Package{
Locations: []source.Location{
{
RealPath: "/a/path",
VirtualPath: "/another/path",
},
{
RealPath: "/b/path",
VirtualPath: "/bee/path",
},
source.NewVirtualLocation("/a/path", "/another/path"),
source.NewVirtualLocation("/b/path", "/bee/path"),
},
Type: RpmPkg,
MetadataType: RpmdbMetadataType,
@ -41,14 +35,8 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
child := Package{
Locations: []source.Location{
{
RealPath: "/c/path",
VirtualPath: "/another/path",
},
{
RealPath: "/d/path",
VirtualPath: "/another/path",
},
source.NewVirtualLocation("/c/path", "/another/path"),
source.NewVirtualLocation("/d/path", "/another/path"),
},
Type: NpmPkg,
}
@ -72,14 +60,8 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
setup: func(t testing.TB) ([]Package, []artifact.Relationship) {
parent := Package{
Locations: []source.Location{
{
RealPath: "/a/path",
VirtualPath: "/some/other/path",
},
{
RealPath: "/b/path",
VirtualPath: "/bee/path",
},
source.NewVirtualLocation("/a/path", "/some/other/path"),
source.NewVirtualLocation("/b/path", "/bee/path"),
},
Type: RpmPkg,
MetadataType: RpmdbMetadataType,
@ -94,14 +76,8 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
child := Package{
Locations: []source.Location{
{
RealPath: "/c/path",
VirtualPath: "/another/path",
},
{
RealPath: "/d/path",
VirtualPath: "",
},
source.NewVirtualLocation("/c/path", "/another/path"),
source.NewLocation("/d/path"),
},
Type: NpmPkg,
}
@ -124,14 +100,8 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
setup: func(t testing.TB) ([]Package, []artifact.Relationship) {
parent := Package{
Locations: []source.Location{
{
RealPath: "/a/path",
VirtualPath: "/some/other/path",
},
{
RealPath: "/b/path",
VirtualPath: "/bee/path",
},
source.NewVirtualLocation("/a/path", "/some/other/path"),
source.NewVirtualLocation("/b/path", "/bee/path"),
},
Type: RpmPkg,
MetadataType: RpmdbMetadataType,
@ -146,14 +116,8 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
child := Package{
Locations: []source.Location{
{
RealPath: "/c/path",
VirtualPath: "/another/path",
},
{
RealPath: "/d/path",
VirtualPath: "",
},
source.NewVirtualLocation("/c/path", "/another/path"),
source.NewLocation("/d/path"),
},
Type: NpmPkg,
}

10
syft/sbom/sbom.go
View File

@ -16,10 +16,10 @@ type SBOM struct {
type Artifacts struct {
PackageCatalog *pkg.Catalog
FileMetadata map[source.Location]source.FileMetadata
FileDigests map[source.Location][]file.Digest
FileClassifications map[source.Location][]file.Classification
FileContents map[source.Location]string
Secrets map[source.Location][]file.SearchResult
FileMetadata map[source.Coordinates]source.FileMetadata
FileDigests map[source.Coordinates][]file.Digest
FileClassifications map[source.Coordinates][]file.Classification
FileContents map[source.Coordinates]string
Secrets map[source.Coordinates][]file.SearchResult
Distro *distro.Distro
}

34
syft/source/coordinates.go Normal file
View File

@ -0,0 +1,34 @@
package source
import (
"fmt"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
)
// Coordinates contains the minimal information needed to describe how to find a file within any possible source object (e.g. image and directory sources)
type Coordinates struct {
RealPath string `json:"path"` // The path where all path ancestors have no hardlinks / symlinks
FileSystemID string `json:"layerID,omitempty"` // An ID representing the filesystem. For container images, this is a layer digest. For directories or a root filesystem, this is blank.
}
func (c Coordinates) ID() artifact.ID {
f, err := artifact.IDFromHash(c)
if err != nil {
// TODO: what to do in this case?
log.Warnf("unable to get fingerprint of location coordinate=%+v: %+v", c, err)
return ""
}
return f
}
func (c Coordinates) String() string {
str := fmt.Sprintf("RealPath=%q", c.RealPath)
if c.FileSystemID != "" {
str += fmt.Sprintf(" Layer=%q", c.FileSystemID)
}
return fmt.Sprintf("Location<%s>", str)
}

73
syft/source/location.go
View File

@ -3,28 +3,46 @@ package source
import (
"fmt"
"github.com/mitchellh/hashstructure/v2"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
)
var _ hashstructure.Hashable = (*Location)(nil)
// Location represents a path relative to a particular filesystem resolved to a specific file.Reference. This struct is used as a key
// in content fetching to uniquely identify a file relative to a request (the VirtualPath). Note that the VirtualPath
// and ref are ignored fields when using github.com/mitchellh/hashstructure. The reason for this is to ensure that
// only the minimally expressible fields of a location are baked into the uniqueness of a Location. Since VirutalPath
// and ref are not captured in JSON output they cannot be included in this minimal definition.
// in content fetching to uniquely identify a file relative to a request (the VirtualPath).
type Location struct {
RealPath string `json:"path"` // The path where all path ancestors have no hardlinks / symlinks
VirtualPath string `hash:"ignore" json:"-"` // The path to the file which may or may not have hardlinks / symlinks
FileSystemID string `json:"layerID,omitempty"` // An ID representing the filesystem. For container images this is a layer digest, directories or root filesystem this is blank.
ref file.Reference `hash:"ignore"` // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
Coordinates
VirtualPath string // The path to the file which may or may not have hardlinks / symlinks
ref file.Reference // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
}
// NewLocation creates a new Location representing a path without denoting a filesystem or FileCatalog reference.
func NewLocation(path string) Location {
func NewLocation(realPath string) Location {
return Location{
RealPath: path,
Coordinates: Coordinates{
RealPath: realPath,
},
}
}
// NewVirtualLocation creates a new location for a path accessed by a virtual path (a path with a symlink or hardlink somewhere in the path)
func NewVirtualLocation(realPath, virtualPath string) Location {
return Location{
Coordinates: Coordinates{
RealPath: realPath,
},
VirtualPath: virtualPath,
}
}
// NewLocationFromCoordinates creates a new location for the given Coordinates.
func NewLocationFromCoordinates(coordinates Coordinates) Location {
return Location{
Coordinates: coordinates,
}
}
@ -34,25 +52,31 @@ func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Ima
if err != nil {
log.Warnf("unable to find file catalog entry for ref=%+v", ref)
return Location{
Coordinates: Coordinates{
RealPath: string(ref.RealPath),
},
VirtualPath: virtualPath,
RealPath: string(ref.RealPath),
ref: ref,
}
}
return Location{
VirtualPath: virtualPath,
RealPath: string(ref.RealPath),
FileSystemID: entry.Layer.Metadata.Digest,
ref: ref,
Coordinates: Coordinates{
RealPath: string(ref.RealPath),
FileSystemID: entry.Layer.Metadata.Digest,
},
VirtualPath: virtualPath,
ref: ref,
}
}
// NewLocationFromDirectory creates a new Location representing the given path (extracted from the ref) relative to the given directory.
func NewLocationFromDirectory(responsePath string, ref file.Reference) Location {
return Location{
RealPath: responsePath,
ref: ref,
Coordinates: Coordinates{
RealPath: responsePath,
},
ref: ref,
}
}
@ -74,13 +98,8 @@ func (l Location) String() string {
return fmt.Sprintf("Location<%s>", str)
}
func (l Location) ID() artifact.ID {
f, err := artifact.IDFromHash(l)
if err != nil {
// TODO: what to do in this case?
log.Warnf("unable to get fingerprint of location=%+v: %+v", l, err)
return ""
}
return f
func (l Location) Hash() (uint64, error) {
// since location is part of the package definition it is important that only coordinates are used during object
// hashing. (Location hash should be a pass-through for the coordinates and not include ref or VirtualPath.)
return hashstructure.Hash(l.ID(), hashstructure.FormatV2, nil)
}