mirror of
https://github.com/anchore/syft.git
synced 2026-11-04 08:26:38 +01:00
Introduce relationships as first-class objects (#607)
* migrate pkg.ID and pkg.Relationship to artifact package Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * return relationships from tasks Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix more tests Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add artifact.Identifiable by Identity() method Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix linting Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * remove catalog ID assignment Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * adjust spdx helpers to use copy of packages Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * stabilize package ID relative to encode-decode format cycles Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * rename Identity() to ID() Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * use zero value for nils in ID generation Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * enable source.Location to be identifiable Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * hoist up package relationship discovery to analysis stage Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update ownership-by-file-overlap relationship description Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add test reminders to put new relationships under test Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * adjust PHP composer.lock parser function to return relationships Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
+31
-66
@@ -4,6 +4,9 @@ import (
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/jinzhu/copier"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
@@ -11,18 +14,18 @@ import (
|
||||
|
||||
// Catalog represents a collection of Packages.
|
||||
type Catalog struct {
|
||||
byID map[ID]*Package
|
||||
idsByType map[Type][]ID
|
||||
idsByPath map[string][]ID // note: this is real path or virtual path
|
||||
byID map[artifact.ID]Package
|
||||
idsByType map[Type][]artifact.ID
|
||||
idsByPath map[string][]artifact.ID // note: this is real path or virtual path
|
||||
lock sync.RWMutex
|
||||
}
|
||||
|
||||
// NewCatalog returns a new empty Catalog
|
||||
func NewCatalog(pkgs ...Package) *Catalog {
|
||||
catalog := Catalog{
|
||||
byID: make(map[ID]*Package),
|
||||
idsByType: make(map[Type][]ID),
|
||||
idsByPath: make(map[string][]ID),
|
||||
byID: make(map[artifact.ID]Package),
|
||||
idsByType: make(map[Type][]artifact.ID),
|
||||
idsByPath: make(map[string][]artifact.ID),
|
||||
}
|
||||
|
||||
for _, p := range pkgs {
|
||||
@@ -38,21 +41,26 @@ func (c *Catalog) PackageCount() int {
|
||||
}
|
||||
|
||||
// Package returns the package with the given ID.
|
||||
func (c *Catalog) Package(id ID) *Package {
|
||||
func (c *Catalog) Package(id artifact.ID) *Package {
|
||||
v, exists := c.byID[id]
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
return v
|
||||
var p Package
|
||||
if err := copier.Copy(&p, &v); err != nil {
|
||||
log.Warnf("unable to copy package id=%q name=%q: %+v", id, v.Name, err)
|
||||
return nil
|
||||
}
|
||||
return &p
|
||||
}
|
||||
|
||||
// PackagesByPath returns all packages that were discovered from the given path.
|
||||
func (c *Catalog) PackagesByPath(path string) []*Package {
|
||||
func (c *Catalog) PackagesByPath(path string) []Package {
|
||||
return c.Packages(c.idsByPath[path])
|
||||
}
|
||||
|
||||
// Packages returns all packages for the given ID.
|
||||
func (c *Catalog) Packages(ids []ID) (result []*Package) {
|
||||
func (c *Catalog) Packages(ids []artifact.ID) (result []Package) {
|
||||
for _, i := range ids {
|
||||
p, exists := c.byID[i]
|
||||
if exists {
|
||||
@@ -67,68 +75,32 @@ func (c *Catalog) Add(p Package) {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
|
||||
if p.ID == "" {
|
||||
fingerprint, err := p.Fingerprint()
|
||||
if err != nil {
|
||||
log.Warnf("failed to add package to catalog: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
p.ID = ID(fingerprint)
|
||||
}
|
||||
// note: since we are capturing the ID, we cannot modify the package being added from this point forward
|
||||
id := p.ID()
|
||||
|
||||
// store by package ID
|
||||
c.byID[p.ID] = &p
|
||||
c.byID[id] = p
|
||||
|
||||
// store by package type
|
||||
c.idsByType[p.Type] = append(c.idsByType[p.Type], p.ID)
|
||||
c.idsByType[p.Type] = append(c.idsByType[p.Type], id)
|
||||
|
||||
// store by file location paths
|
||||
observedPaths := internal.NewStringSet()
|
||||
for _, l := range p.Locations {
|
||||
if l.RealPath != "" && !observedPaths.Contains(l.RealPath) {
|
||||
c.idsByPath[l.RealPath] = append(c.idsByPath[l.RealPath], p.ID)
|
||||
c.idsByPath[l.RealPath] = append(c.idsByPath[l.RealPath], id)
|
||||
observedPaths.Add(l.RealPath)
|
||||
}
|
||||
if l.VirtualPath != "" && l.RealPath != l.VirtualPath && !observedPaths.Contains(l.VirtualPath) {
|
||||
c.idsByPath[l.VirtualPath] = append(c.idsByPath[l.VirtualPath], p.ID)
|
||||
c.idsByPath[l.VirtualPath] = append(c.idsByPath[l.VirtualPath], id)
|
||||
observedPaths.Add(l.VirtualPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Catalog) Remove(id ID) {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
|
||||
_, exists := c.byID[id]
|
||||
if !exists {
|
||||
log.Errorf("package ID does not exist in the catalog : id=%+v", id)
|
||||
return
|
||||
}
|
||||
|
||||
// Remove all index references to this package ID
|
||||
for t, ids := range c.idsByType {
|
||||
c.idsByType[t] = removeID(id, ids)
|
||||
if len(c.idsByType[t]) == 0 {
|
||||
delete(c.idsByType, t)
|
||||
}
|
||||
}
|
||||
|
||||
for p, ids := range c.idsByPath {
|
||||
c.idsByPath[p] = removeID(id, ids)
|
||||
if len(c.idsByPath[p]) == 0 {
|
||||
delete(c.idsByPath, p)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove package
|
||||
delete(c.byID, id)
|
||||
}
|
||||
|
||||
// Enumerate all packages for the given type(s), enumerating all packages if no type is specified.
|
||||
func (c *Catalog) Enumerate(types ...Type) <-chan *Package {
|
||||
channel := make(chan *Package)
|
||||
func (c *Catalog) Enumerate(types ...Type) <-chan Package {
|
||||
channel := make(chan Package)
|
||||
go func() {
|
||||
defer close(channel)
|
||||
for ty, ids := range c.idsByType {
|
||||
@@ -146,7 +118,10 @@ func (c *Catalog) Enumerate(types ...Type) <-chan *Package {
|
||||
}
|
||||
}
|
||||
for _, id := range ids {
|
||||
channel <- c.Package(id)
|
||||
p := c.Package(id)
|
||||
if p != nil {
|
||||
channel <- *p
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -155,8 +130,7 @@ func (c *Catalog) Enumerate(types ...Type) <-chan *Package {
|
||||
|
||||
// Sorted enumerates all packages for the given types sorted by package name. Enumerates all packages if no type
|
||||
// is specified.
|
||||
func (c *Catalog) Sorted(types ...Type) []*Package {
|
||||
pkgs := make([]*Package, 0)
|
||||
func (c *Catalog) Sorted(types ...Type) (pkgs []Package) {
|
||||
for p := range c.Enumerate(types...) {
|
||||
pkgs = append(pkgs, p)
|
||||
}
|
||||
@@ -176,12 +150,3 @@ func (c *Catalog) Sorted(types ...Type) []*Package {
|
||||
|
||||
return pkgs
|
||||
}
|
||||
|
||||
func removeID(id ID, target []ID) (result []ID) {
|
||||
for _, value := range target {
|
||||
if value != id {
|
||||
result = append(result, value)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
+42
-80
@@ -10,7 +10,6 @@ import (
|
||||
|
||||
var catalogAddAndRemoveTestPkgs = []Package{
|
||||
{
|
||||
ID: "my-id",
|
||||
Locations: []source.Location{
|
||||
{
|
||||
RealPath: "/a/path",
|
||||
@@ -24,7 +23,6 @@ var catalogAddAndRemoveTestPkgs = []Package{
|
||||
Type: RpmPkg,
|
||||
},
|
||||
{
|
||||
ID: "my-other-id",
|
||||
Locations: []source.Location{
|
||||
{
|
||||
RealPath: "/c/path",
|
||||
@@ -45,6 +43,11 @@ type expectedIndexes struct {
|
||||
}
|
||||
|
||||
func TestCatalogAddPopulatesIndex(t *testing.T) {
|
||||
|
||||
fixtureID := func(i int) string {
|
||||
return string(catalogAddAndRemoveTestPkgs[i].ID())
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
pkgs []Package
|
||||
@@ -55,16 +58,16 @@ func TestCatalogAddPopulatesIndex(t *testing.T) {
|
||||
pkgs: catalogAddAndRemoveTestPkgs,
|
||||
expectedIndexes: expectedIndexes{
|
||||
byType: map[Type]*strset.Set{
|
||||
RpmPkg: strset.New("my-id"),
|
||||
NpmPkg: strset.New("my-other-id"),
|
||||
RpmPkg: strset.New(fixtureID(0)),
|
||||
NpmPkg: strset.New(fixtureID(1)),
|
||||
},
|
||||
byPath: map[string]*strset.Set{
|
||||
"/another/path": strset.New("my-id", "my-other-id"),
|
||||
"/a/path": strset.New("my-id"),
|
||||
"/b/path": strset.New("my-id"),
|
||||
"/bee/path": strset.New("my-id"),
|
||||
"/c/path": strset.New("my-other-id"),
|
||||
"/d/path": strset.New("my-other-id"),
|
||||
"/another/path": strset.New(fixtureID(0), fixtureID(1)),
|
||||
"/a/path": strset.New(fixtureID(0)),
|
||||
"/b/path": strset.New(fixtureID(0)),
|
||||
"/bee/path": strset.New(fixtureID(0)),
|
||||
"/c/path": strset.New(fixtureID(1)),
|
||||
"/d/path": strset.New(fixtureID(1)),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -80,50 +83,6 @@ func TestCatalogAddPopulatesIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatalogRemove(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pkgs []Package
|
||||
removeId ID
|
||||
expectedIndexes expectedIndexes
|
||||
}{
|
||||
{
|
||||
name: "vanilla-add",
|
||||
removeId: "my-other-id",
|
||||
pkgs: catalogAddAndRemoveTestPkgs,
|
||||
expectedIndexes: expectedIndexes{
|
||||
byType: map[Type]*strset.Set{
|
||||
RpmPkg: strset.New("my-id"),
|
||||
},
|
||||
byPath: map[string]*strset.Set{
|
||||
"/another/path": strset.New("my-id"),
|
||||
"/a/path": strset.New("my-id"),
|
||||
"/b/path": strset.New("my-id"),
|
||||
"/bee/path": strset.New("my-id"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
c := NewCatalog(test.pkgs...)
|
||||
c.Remove(test.removeId)
|
||||
|
||||
assertIndexes(t, c, test.expectedIndexes)
|
||||
|
||||
if c.Package(test.removeId) != nil {
|
||||
t.Errorf("expected package to be removed, but was found!")
|
||||
}
|
||||
|
||||
if c.PackageCount() != len(test.pkgs)-1 {
|
||||
t.Errorf("expected count to be affected but was not")
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) {
|
||||
// assert path index
|
||||
if len(c.idsByPath) != len(expectedIndexes.byPath) {
|
||||
@@ -132,7 +91,7 @@ func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) {
|
||||
for path, expectedIds := range expectedIndexes.byPath {
|
||||
actualIds := strset.New()
|
||||
for _, p := range c.PackagesByPath(path) {
|
||||
actualIds.Add(string(p.ID))
|
||||
actualIds.Add(string(p.ID()))
|
||||
}
|
||||
|
||||
if !expectedIds.IsEqual(actualIds) {
|
||||
@@ -147,7 +106,7 @@ func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) {
|
||||
for ty, expectedIds := range expectedIndexes.byType {
|
||||
actualIds := strset.New()
|
||||
for p := range c.Enumerate(ty) {
|
||||
actualIds.Add(string(p.ID))
|
||||
actualIds.Add(string(p.ID()))
|
||||
}
|
||||
|
||||
if !expectedIds.IsEqual(actualIds) {
|
||||
@@ -157,39 +116,42 @@ func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) {
|
||||
}
|
||||
|
||||
func TestCatalog_PathIndexDeduplicatesRealVsVirtualPaths(t *testing.T) {
|
||||
p1 := Package{
|
||||
Locations: []source.Location{
|
||||
{
|
||||
RealPath: "/b/path",
|
||||
VirtualPath: "/another/path",
|
||||
},
|
||||
{
|
||||
RealPath: "/b/path",
|
||||
VirtualPath: "/b/path",
|
||||
},
|
||||
},
|
||||
Type: RpmPkg,
|
||||
Name: "Package-1",
|
||||
}
|
||||
|
||||
p2 := Package{
|
||||
Locations: []source.Location{
|
||||
{
|
||||
RealPath: "/b/path",
|
||||
VirtualPath: "/b/path",
|
||||
},
|
||||
},
|
||||
Type: RpmPkg,
|
||||
Name: "Package-2",
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
pkg Package
|
||||
}{
|
||||
{
|
||||
name: "multiple locations with shared path",
|
||||
pkg: Package{
|
||||
ID: "my-id",
|
||||
Locations: []source.Location{
|
||||
{
|
||||
RealPath: "/b/path",
|
||||
VirtualPath: "/another/path",
|
||||
},
|
||||
{
|
||||
RealPath: "/b/path",
|
||||
VirtualPath: "/b/path",
|
||||
},
|
||||
},
|
||||
Type: RpmPkg,
|
||||
},
|
||||
pkg: p1,
|
||||
},
|
||||
{
|
||||
name: "one location with shared path",
|
||||
pkg: Package{
|
||||
ID: "my-id",
|
||||
Locations: []source.Location{
|
||||
{
|
||||
RealPath: "/b/path",
|
||||
VirtualPath: "/b/path",
|
||||
},
|
||||
},
|
||||
Type: RpmPkg,
|
||||
},
|
||||
pkg: p2,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
@@ -21,7 +23,7 @@ var _ common.ParserFn = parseApkDB
|
||||
|
||||
// parseApkDb parses individual packages from a given Alpine DB file. For more information on specific fields
|
||||
// see https://wiki.alpinelinux.org/wiki/Apk_spec .
|
||||
func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
// larger capacity for the scanner.
|
||||
const maxScannerCapacity = 1024 * 1024
|
||||
// a new larger buffer for the scanner
|
||||
@@ -47,7 +49,7 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
for scanner.Scan() {
|
||||
metadata, err := parseApkDBEntry(strings.NewReader(scanner.Text()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
if metadata != nil {
|
||||
packages = append(packages, pkg.Package{
|
||||
@@ -62,10 +64,10 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse APK DB file: %w", err)
|
||||
return nil, nil, fmt.Errorf("failed to parse APK DB file: %w", err)
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
return packages, nil, nil
|
||||
}
|
||||
|
||||
// nolint:funlen
|
||||
|
||||
@@ -775,7 +775,8 @@ func TestMultiplePackages(t *testing.T) {
|
||||
}
|
||||
}()
|
||||
|
||||
pkgs, err := parseApkDB(file.Name(), file)
|
||||
// TODO: no relationships are under test yet
|
||||
pkgs, _, err := parseApkDB(file.Name(), file)
|
||||
if err != nil {
|
||||
t.Fatal("Unable to read file contents: ", err)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package cataloger
|
||||
import (
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/distro"
|
||||
"github.com/anchore/syft/syft/event"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
@@ -38,8 +39,9 @@ func newMonitor() (*progress.Manual, *progress.Manual) {
|
||||
// In order to efficiently retrieve contents from a underlying container image the content fetch requests are
|
||||
// done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single
|
||||
// request.
|
||||
func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers ...Cataloger) (*pkg.Catalog, error) {
|
||||
func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers ...Cataloger) (*pkg.Catalog, []artifact.Relationship, error) {
|
||||
catalog := pkg.NewCatalog()
|
||||
var allRelationships []artifact.Relationship
|
||||
|
||||
filesProcessed, packagesDiscovered := newMonitor()
|
||||
|
||||
@@ -47,7 +49,7 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers
|
||||
var errs error
|
||||
for _, theCataloger := range catalogers {
|
||||
// find packages from the underlying raw data
|
||||
packages, err := theCataloger.Catalog(resolver)
|
||||
packages, relationships, err := theCataloger.Catalog(resolver)
|
||||
if err != nil {
|
||||
errs = multierror.Append(errs, err)
|
||||
continue
|
||||
@@ -68,14 +70,18 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers
|
||||
// add to catalog
|
||||
catalog.Add(p)
|
||||
}
|
||||
|
||||
allRelationships = append(allRelationships, relationships...)
|
||||
}
|
||||
|
||||
allRelationships = append(allRelationships, pkg.NewRelationships(catalog)...)
|
||||
|
||||
if errs != nil {
|
||||
return nil, errs
|
||||
return nil, nil, errs
|
||||
}
|
||||
|
||||
filesProcessed.SetCompleted()
|
||||
packagesDiscovered.SetCompleted()
|
||||
|
||||
return catalog, nil
|
||||
return catalog, allRelationships, nil
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ catalogers defined in child packages as well as the interface definition to impl
|
||||
package cataloger
|
||||
|
||||
import (
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/apkdb"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/deb"
|
||||
@@ -27,7 +28,7 @@ type Cataloger interface {
|
||||
// Name returns a string that uniquely describes a cataloger
|
||||
Name() string
|
||||
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
|
||||
Catalog(resolver source.FileResolver) ([]pkg.Package, error)
|
||||
Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error)
|
||||
}
|
||||
|
||||
// ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
|
||||
|
||||
@@ -83,7 +83,7 @@ func candidateVendors(p pkg.Package) []string {
|
||||
// allow * as a candidate. Note: do NOT allow Java packages to have * vendors.
|
||||
switch p.Language {
|
||||
case pkg.Ruby, pkg.JavaScript:
|
||||
vendors.addValue("*")
|
||||
vendors.addValue(wfn.Any)
|
||||
}
|
||||
|
||||
switch p.MetadataType {
|
||||
|
||||
@@ -637,7 +637,7 @@ func TestCandidateProducts(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("%+v %+v", test.p, test.expected), func(t *testing.T) {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assert.ElementsMatch(t, test.expected, candidateProducts(test.p))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ package common
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
@@ -35,18 +37,18 @@ func (c *GenericCataloger) Name() string {
|
||||
}
|
||||
|
||||
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
|
||||
func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) {
|
||||
func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
var packages []pkg.Package
|
||||
parserByLocation := c.selectFiles(resolver)
|
||||
var relationships []artifact.Relationship
|
||||
|
||||
for location, parser := range parserByLocation {
|
||||
for location, parser := range c.selectFiles(resolver) {
|
||||
contentReader, err := resolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
// TODO: fail or log?
|
||||
return nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err)
|
||||
return nil, nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err)
|
||||
}
|
||||
|
||||
entries, err := parser(location.RealPath, contentReader)
|
||||
discoveredPackages, discoveredRelationships, err := parser(location.RealPath, contentReader)
|
||||
internal.CloseAndLogError(contentReader, location.VirtualPath)
|
||||
if err != nil {
|
||||
// TODO: should we fail? or only log?
|
||||
@@ -54,14 +56,16 @@ func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package,
|
||||
continue
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
entry.FoundBy = c.upstreamCataloger
|
||||
entry.Locations = []source.Location{location}
|
||||
for _, p := range discoveredPackages {
|
||||
p.FoundBy = c.upstreamCataloger
|
||||
p.Locations = append(p.Locations, location)
|
||||
|
||||
packages = append(packages, entry)
|
||||
packages = append(packages, p)
|
||||
}
|
||||
|
||||
relationships = append(relationships, discoveredRelationships...)
|
||||
}
|
||||
return packages, nil
|
||||
return packages, relationships, nil
|
||||
}
|
||||
|
||||
// SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging
|
||||
|
||||
@@ -8,11 +8,12 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
func parser(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parser(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
contents, err := ioutil.ReadAll(reader)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
@@ -21,7 +22,7 @@ func parser(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
{
|
||||
Name: string(contents),
|
||||
},
|
||||
}, nil
|
||||
}, nil, nil
|
||||
}
|
||||
|
||||
func TestGenericCataloger(t *testing.T) {
|
||||
@@ -47,7 +48,7 @@ func TestGenericCataloger(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
actualPkgs, err := cataloger.Catalog(resolver)
|
||||
actualPkgs, _, err := cataloger.Catalog(resolver)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, actualPkgs, len(expectedPkgs))
|
||||
|
||||
|
||||
@@ -3,8 +3,9 @@ package common
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
// ParserFn standardizes a function signature for parser functions that accept the virtual file path (not usable for file reads) and contents and return any discovered packages from that file
|
||||
type ParserFn func(string, io.Reader) ([]pkg.Package, error)
|
||||
type ParserFn func(string, io.Reader) ([]pkg.Package, []artifact.Relationship, error)
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/anchore/syft/internal"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
@@ -36,24 +37,23 @@ func (c *Cataloger) Name() string {
|
||||
}
|
||||
|
||||
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files.
|
||||
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) {
|
||||
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
dbFileMatches, err := resolver.FilesByGlob(pkg.DpkgDBGlob)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err)
|
||||
return nil, nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err)
|
||||
}
|
||||
|
||||
var results []pkg.Package
|
||||
var pkgs []pkg.Package
|
||||
var allPackages []pkg.Package
|
||||
for _, dbLocation := range dbFileMatches {
|
||||
dbContents, err := resolver.FileContentsByLocation(dbLocation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
pkgs, err = parseDpkgStatus(dbContents)
|
||||
pkgs, err := parseDpkgStatus(dbContents)
|
||||
internal.CloseAndLogError(dbContents, dbLocation.VirtualPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err)
|
||||
return nil, nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err)
|
||||
}
|
||||
|
||||
for i := range pkgs {
|
||||
@@ -70,9 +70,9 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error)
|
||||
addLicenses(resolver, dbLocation, p)
|
||||
}
|
||||
|
||||
results = append(results, pkgs...)
|
||||
allPackages = append(allPackages, pkgs...)
|
||||
}
|
||||
return results, nil
|
||||
return allPackages, nil, nil
|
||||
}
|
||||
|
||||
func addLicenses(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) {
|
||||
|
||||
@@ -100,7 +100,7 @@ func TestDpkgCataloger(t *testing.T) {
|
||||
t.Errorf("could not get resolver error: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := c.Catalog(resolver)
|
||||
actual, _, err := c.Catalog(resolver)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to catalog: %+v", err)
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ var (
|
||||
// parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed.
|
||||
func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
|
||||
buffedReader := bufio.NewReader(reader)
|
||||
var packages = make([]pkg.Package, 0)
|
||||
var packages []pkg.Package
|
||||
|
||||
continueProcessing := true
|
||||
for continueProcessing {
|
||||
|
||||
+10
-6
@@ -6,7 +6,10 @@ package golang
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
@@ -35,17 +38,18 @@ func (c *Cataloger) Name() string {
|
||||
}
|
||||
|
||||
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
|
||||
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) {
|
||||
pkgs := make([]pkg.Package, 0)
|
||||
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
var pkgs []pkg.Package
|
||||
|
||||
fileMatches, err := resolver.FilesByMIMEType(mimeTypes...)
|
||||
if err != nil {
|
||||
return pkgs, fmt.Errorf("failed to find bin by mime types: %w", err)
|
||||
return pkgs, nil, fmt.Errorf("failed to find bin by mime types: %w", err)
|
||||
}
|
||||
|
||||
for _, location := range fileMatches {
|
||||
r, err := resolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return pkgs, fmt.Errorf("failed to resolve file contents by location: %w", err)
|
||||
return pkgs, nil, fmt.Errorf("failed to resolve file contents by location: %w", err)
|
||||
}
|
||||
|
||||
goPkgs, err := parseGoBin(location, r)
|
||||
@@ -53,9 +57,9 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error)
|
||||
log.Warnf("could not parse possible go binary: %+v", err)
|
||||
}
|
||||
|
||||
r.Close()
|
||||
internal.CloseAndLogError(r, location.RealPath)
|
||||
pkgs = append(pkgs, goPkgs...)
|
||||
}
|
||||
|
||||
return pkgs, nil
|
||||
return pkgs, nil, nil
|
||||
}
|
||||
@@ -23,9 +23,7 @@ func parseGoBin(location source.Location, reader io.ReadCloser) ([]pkg.Package,
|
||||
|
||||
goVersion, mod := findVers(x)
|
||||
|
||||
pkgs := buildGoPkgInfo(location, mod, goVersion)
|
||||
|
||||
return pkgs, nil
|
||||
return buildGoPkgInfo(location, mod, goVersion), nil
|
||||
}
|
||||
|
||||
func buildGoPkgInfo(location source.Location, mod, goVersion string) []pkg.Package {
|
||||
|
||||
@@ -6,22 +6,23 @@ import (
|
||||
"io/ioutil"
|
||||
"sort"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"golang.org/x/mod/modfile"
|
||||
)
|
||||
|
||||
// parseGoMod takes a go.mod and lists all packages discovered.
|
||||
func parseGoMod(path string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parseGoMod(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
packages := make(map[string]pkg.Package)
|
||||
|
||||
contents, err := ioutil.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read go module: %w", err)
|
||||
return nil, nil, fmt.Errorf("failed to read go module: %w", err)
|
||||
}
|
||||
|
||||
file, err := modfile.Parse(path, contents, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse go module: %w", err)
|
||||
return nil, nil, fmt.Errorf("failed to parse go module: %w", err)
|
||||
}
|
||||
|
||||
for _, m := range file.Require {
|
||||
@@ -59,5 +60,5 @@ func parseGoMod(path string, reader io.Reader) ([]pkg.Package, error) {
|
||||
return pkgsSlice[i].Name < pkgsSlice[j].Name
|
||||
})
|
||||
|
||||
return pkgsSlice, nil
|
||||
return pkgsSlice, nil, nil
|
||||
}
|
||||
|
||||
@@ -70,7 +70,8 @@ func TestParseGoMod(t *testing.T) {
|
||||
t.Fatalf(err.Error())
|
||||
}
|
||||
|
||||
actual, err := parseGoMod(test.fixture, f)
|
||||
// TODO: no relationships are under test yet
|
||||
actual, _, err := parseGoMod(test.fixture, f)
|
||||
if err != nil {
|
||||
t.Fatalf(err.Error())
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/anchore/syft/internal/log"
|
||||
|
||||
"github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common"
|
||||
)
|
||||
@@ -34,12 +35,12 @@ type archiveParser struct {
|
||||
}
|
||||
|
||||
// parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives.
|
||||
func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true)
|
||||
// note: even on error, we should always run cleanup functions
|
||||
defer cleanupFn()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
return parser.parse()
|
||||
}
|
||||
@@ -80,29 +81,31 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo
|
||||
}
|
||||
|
||||
// parse the loaded archive and return all packages found.
|
||||
func (j *archiveParser) parse() ([]pkg.Package, error) {
|
||||
var pkgs = make([]pkg.Package, 0)
|
||||
func (j *archiveParser) parse() ([]pkg.Package, []artifact.Relationship, error) {
|
||||
var pkgs []pkg.Package
|
||||
var relationships []artifact.Relationship
|
||||
|
||||
// find the parent package from the java manifest
|
||||
parentPkg, err := j.discoverMainPackage()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err)
|
||||
return nil, nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err)
|
||||
}
|
||||
|
||||
// find aux packages from pom.properties/pom.xml and potentially modify the existing parentPkg
|
||||
auxPkgs, err := j.discoverPkgsFromAllMavenFiles(parentPkg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
pkgs = append(pkgs, auxPkgs...)
|
||||
|
||||
if j.detectNested {
|
||||
// find nested java archive packages
|
||||
nestedPkgs, err := j.discoverPkgsFromNestedArchives(parentPkg)
|
||||
nestedPkgs, nestedRelationships, err := j.discoverPkgsFromNestedArchives(parentPkg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
pkgs = append(pkgs, nestedPkgs...)
|
||||
relationships = append(relationships, nestedRelationships...)
|
||||
}
|
||||
|
||||
// lastly, add the parent package to the list (assuming the parent exists)
|
||||
@@ -110,7 +113,7 @@ func (j *archiveParser) parse() ([]pkg.Package, error) {
|
||||
pkgs = append([]pkg.Package{*parentPkg}, pkgs...)
|
||||
}
|
||||
|
||||
return pkgs, nil
|
||||
return pkgs, relationships, nil
|
||||
}
|
||||
|
||||
// discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages.
|
||||
@@ -189,31 +192,32 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(parentPkg *pkg.Package) ([
|
||||
|
||||
// discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and
|
||||
// associating each discovered package to the given parent package.
|
||||
func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, error) {
|
||||
var pkgs = make([]pkg.Package, 0)
|
||||
func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
var pkgs []pkg.Package
|
||||
var relationships []artifact.Relationship
|
||||
|
||||
// search and parse pom.properties files & fetch the contents
|
||||
openers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to extract files from zip: %w", err)
|
||||
return nil, nil, fmt.Errorf("unable to extract files from zip: %w", err)
|
||||
}
|
||||
|
||||
// discover nested artifacts
|
||||
for archivePath, archiveOpener := range openers {
|
||||
archiveReadCloser, err := archiveOpener.Open()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to open archived file from tempdir: %w", err)
|
||||
return nil, nil, fmt.Errorf("unable to open archived file from tempdir: %w", err)
|
||||
}
|
||||
nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
|
||||
nestedPkgs, err := parseJavaArchive(nestedPath, archiveReadCloser)
|
||||
nestedPkgs, nestedRelationships, err := parseJavaArchive(nestedPath, archiveReadCloser)
|
||||
if err != nil {
|
||||
if closeErr := archiveReadCloser.Close(); closeErr != nil {
|
||||
log.Warnf("unable to close archived file from tempdir: %+v", closeErr)
|
||||
}
|
||||
return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
|
||||
return nil, nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
|
||||
}
|
||||
if err = archiveReadCloser.Close(); err != nil {
|
||||
return nil, fmt.Errorf("unable to close archived file from tempdir: %w", err)
|
||||
return nil, nil, fmt.Errorf("unable to close archived file from tempdir: %w", err)
|
||||
}
|
||||
|
||||
// attach the parent package to all discovered packages that are not already associated with a java archive
|
||||
@@ -226,9 +230,11 @@ func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) (
|
||||
}
|
||||
pkgs = append(pkgs, p)
|
||||
}
|
||||
|
||||
relationships = append(relationships, nestedRelationships...)
|
||||
}
|
||||
|
||||
return pkgs, nil
|
||||
return pkgs, relationships, nil
|
||||
}
|
||||
|
||||
func pomPropertiesByParentPath(archivePath string, extractPaths []string, virtualPath string) (map[string]pkg.PomProperties, error) {
|
||||
|
||||
@@ -242,7 +242,7 @@ func TestParseJar(t *testing.T) {
|
||||
t.Fatalf("should not have filed... %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parser.parse()
|
||||
actual, _, err := parser.parse()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse java archive: %+v", err)
|
||||
}
|
||||
@@ -507,7 +507,7 @@ func TestParseNestedJar(t *testing.T) {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parseJavaArchive(fixture.Name(), fixture)
|
||||
actual, _, err := parseJavaArchive(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse java archive: %+v", err)
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/mitchellh/mapstructure"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common"
|
||||
)
|
||||
@@ -162,8 +163,8 @@ func licensesFromJSON(p PackageJSON) ([]string, error) {
|
||||
}
|
||||
|
||||
// parsePackageJSON parses a package.json and returns the discovered JavaScript packages.
|
||||
func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
packages := make([]pkg.Package, 0)
|
||||
func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
var packages []pkg.Package
|
||||
dec := json.NewDecoder(reader)
|
||||
|
||||
for {
|
||||
@@ -171,17 +172,17 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
if err := dec.Decode(&p); err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse package.json file: %w", err)
|
||||
return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err)
|
||||
}
|
||||
|
||||
if !p.hasNameAndVersionValues() {
|
||||
log.Debug("encountered package.json file without a name and/or version field, ignoring this file")
|
||||
return nil, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
licenses, err := licensesFromJSON(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse package.json file: %w", err)
|
||||
return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err)
|
||||
}
|
||||
|
||||
packages = append(packages, pkg.Package{
|
||||
@@ -200,7 +201,7 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
})
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
return packages, nil, nil
|
||||
}
|
||||
|
||||
func (p PackageJSON) hasNameAndVersionValues() bool {
|
||||
|
||||
@@ -124,7 +124,7 @@ func TestParsePackageJSON(t *testing.T) {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parsePackageJSON("", fixture)
|
||||
actual, _, err := parsePackageJSON("", fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse package-lock.json: %+v", err)
|
||||
}
|
||||
@@ -150,7 +150,8 @@ func TestParsePackageJSON_Partial(t *testing.T) { // see https://github.com/anch
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parsePackageJSON("", fixture)
|
||||
// TODO: no relationships are under test yet
|
||||
actual, _, err := parsePackageJSON("", fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse package-lock.json: %+v", err)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common"
|
||||
)
|
||||
@@ -28,11 +29,11 @@ type Dependency struct {
|
||||
}
|
||||
|
||||
// parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages.
|
||||
func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
// in the case we find package-lock.json files in the node_modules directories, skip those
|
||||
// as the whole purpose of the lock file is for the specific dependencies of the root project
|
||||
if pathContainsNodeModulesDirectory(path) {
|
||||
return nil, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
var packages []pkg.Package
|
||||
@@ -43,7 +44,7 @@ func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, error) {
|
||||
if err := dec.Decode(&lock); err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse package-lock.json file: %w", err)
|
||||
return nil, nil, fmt.Errorf("failed to parse package-lock.json file: %w", err)
|
||||
}
|
||||
for name, pkgMeta := range lock.Dependencies {
|
||||
packages = append(packages, pkg.Package{
|
||||
@@ -55,5 +56,5 @@ func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, error) {
|
||||
}
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
return packages, nil, nil
|
||||
}
|
||||
|
||||
@@ -109,7 +109,8 @@ func TestParsePackageLock(t *testing.T) {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parsePackageLock(fixture.Name(), fixture)
|
||||
// TODO: no relationships are under test yet
|
||||
actual, _, err := parsePackageLock(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse package-lock.json: %+v", err)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"regexp"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common"
|
||||
)
|
||||
@@ -34,11 +35,11 @@ const (
|
||||
noVersion = ""
|
||||
)
|
||||
|
||||
func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
// in the case we find yarn.lock files in the node_modules directories, skip those
|
||||
// as the whole purpose of the lock file is for the specific dependencies of the project
|
||||
if pathContainsNodeModulesDirectory(path) {
|
||||
return nil, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
var packages []pkg.Package
|
||||
@@ -79,10 +80,10 @@ func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, error) {
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse yarn.lock file: %w", err)
|
||||
return nil, nil, fmt.Errorf("failed to parse yarn.lock file: %w", err)
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
return packages, nil, nil
|
||||
}
|
||||
|
||||
func findPackageName(line string) string {
|
||||
|
||||
@@ -70,7 +70,8 @@ func TestParseYarnLock(t *testing.T) {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parseYarnLock(fixture.Name(), fixture)
|
||||
// TODO: no relationships are under test yet
|
||||
actual, _, err := parseYarnLock(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse yarn.lock: %+v", err)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common"
|
||||
)
|
||||
@@ -23,7 +25,7 @@ type Dependency struct {
|
||||
var _ common.ParserFn = parseComposerLock
|
||||
|
||||
// parseComposerLock is a parser function for Composer.lock contents, returning "Default" php packages discovered.
|
||||
func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
packages := make([]pkg.Package, 0)
|
||||
dec := json.NewDecoder(reader)
|
||||
|
||||
@@ -32,7 +34,7 @@ func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
if err := dec.Decode(&lock); err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse composer.lock file: %w", err)
|
||||
return nil, nil, fmt.Errorf("failed to parse composer.lock file: %w", err)
|
||||
}
|
||||
for _, pkgMeta := range lock.Packages {
|
||||
version := pkgMeta.Version
|
||||
@@ -46,5 +48,5 @@ func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
}
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
return packages, nil, nil
|
||||
}
|
||||
|
||||
@@ -28,7 +28,8 @@ func TestParseComposerFileLock(t *testing.T) {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parseComposerLock(fixture.Name(), fixture)
|
||||
// TODO: no relationships are under test yet
|
||||
actual, _, err := parseComposerLock(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse requirements: %+v", err)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
|
||||
"github.com/anchore/syft/syft/source"
|
||||
@@ -31,13 +32,13 @@ func (c *PackageCataloger) Name() string {
|
||||
}
|
||||
|
||||
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations.
|
||||
func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) {
|
||||
func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
var fileMatches []source.Location
|
||||
|
||||
for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob, eggFileMetadataGlob} {
|
||||
matches, err := resolver.FilesByGlob(glob)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find files by glob: %s", glob)
|
||||
return nil, nil, fmt.Errorf("failed to find files by glob: %s", glob)
|
||||
}
|
||||
fileMatches = append(fileMatches, matches...)
|
||||
}
|
||||
@@ -46,13 +47,13 @@ func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package,
|
||||
for _, location := range fileMatches {
|
||||
p, err := c.catalogEggOrWheel(resolver, location)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.RealPath, err)
|
||||
return nil, nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.RealPath, err)
|
||||
}
|
||||
if p != nil {
|
||||
pkgs = append(pkgs, *p)
|
||||
}
|
||||
}
|
||||
return pkgs, nil
|
||||
return pkgs, nil, nil
|
||||
}
|
||||
|
||||
// catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents.
|
||||
|
||||
@@ -144,7 +144,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
||||
|
||||
test.expectedPackage.Locations = locations
|
||||
|
||||
actual, err := NewPythonPackageCataloger().Catalog(resolver)
|
||||
actual, _, err := NewPythonPackageCataloger().Catalog(resolver)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to catalog python package: %+v", err)
|
||||
}
|
||||
@@ -173,7 +173,7 @@ func TestIgnorePackage(t *testing.T) {
|
||||
t.Run(test.MetadataFixture, func(t *testing.T) {
|
||||
resolver := source.NewMockResolverForPaths(test.MetadataFixture)
|
||||
|
||||
actual, err := NewPythonPackageCataloger().Catalog(resolver)
|
||||
actual, _, err := NewPythonPackageCataloger().Catalog(resolver)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to catalog python package: %+v", err)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common"
|
||||
)
|
||||
@@ -37,7 +38,7 @@ type Dependency struct {
|
||||
var _ common.ParserFn = parsePipfileLock
|
||||
|
||||
// parsePipfileLock is a parser function for Pipfile.lock contents, returning "Default" python packages discovered.
|
||||
func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
packages := make([]pkg.Package, 0)
|
||||
dec := json.NewDecoder(reader)
|
||||
|
||||
@@ -46,7 +47,7 @@ func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
if err := dec.Decode(&lock); err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse Pipfile.lock file: %w", err)
|
||||
return nil, nil, fmt.Errorf("failed to parse Pipfile.lock file: %w", err)
|
||||
}
|
||||
for name, pkgMeta := range lock.Default {
|
||||
version := strings.TrimPrefix(pkgMeta.Version, "==")
|
||||
@@ -59,5 +60,5 @@ func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
}
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
return packages, nil, nil
|
||||
}
|
||||
|
||||
@@ -39,7 +39,8 @@ func TestParsePipFileLock(t *testing.T) {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parsePipfileLock(fixture.Name(), fixture)
|
||||
// TODO: no relationships are under test yet
|
||||
actual, _, err := parsePipfileLock(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse requirements: %+v", err)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common"
|
||||
"github.com/pelletier/go-toml"
|
||||
@@ -13,17 +14,17 @@ import (
|
||||
var _ common.ParserFn = parsePoetryLock
|
||||
|
||||
// parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered.
|
||||
func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
tree, err := toml.LoadReader(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err)
|
||||
return nil, nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err)
|
||||
}
|
||||
|
||||
metadata := PoetryMetadata{}
|
||||
err = tree.Unmarshal(&metadata)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse poetry.lock: %v", err)
|
||||
return nil, nil, fmt.Errorf("unable to parse poetry.lock: %v", err)
|
||||
}
|
||||
|
||||
return metadata.Pkgs(), nil
|
||||
return metadata.Pkgs(), nil, nil
|
||||
}
|
||||
|
||||
@@ -45,7 +45,8 @@ func TestParsePoetryLock(t *testing.T) {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parsePoetryLock(fixture.Name(), fixture)
|
||||
// TODO: no relationships are under test yet
|
||||
actual, _, err := parsePoetryLock(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common"
|
||||
)
|
||||
@@ -15,7 +16,7 @@ var _ common.ParserFn = parseRequirementsTxt
|
||||
|
||||
// parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a
|
||||
// specific version.
|
||||
func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
packages := make([]pkg.Package, 0)
|
||||
|
||||
scanner := bufio.NewScanner(reader)
|
||||
@@ -55,10 +56,10 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse python requirements file: %w", err)
|
||||
return nil, nil, fmt.Errorf("failed to parse python requirements file: %w", err)
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
return packages, nil, nil
|
||||
}
|
||||
|
||||
// removeTrailingComment takes a requirements.txt line and strips off comment strings.
|
||||
|
||||
@@ -50,7 +50,8 @@ func TestParseRequirementsTxt(t *testing.T) {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parseRequirementsTxt(fixture.Name(), fixture)
|
||||
// TODO: no relationships are under test yet
|
||||
actual, _, err := parseRequirementsTxt(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse requirements: %+v", err)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common"
|
||||
)
|
||||
@@ -19,7 +20,7 @@ var _ common.ParserFn = parseSetup
|
||||
// " mypy2 == v0.770", ' mypy3== v0.770', --> match(name=mypy2 version=v0.770), match(name=mypy3, version=v0.770)
|
||||
var pinnedDependency = regexp.MustCompile(`['"]\W?(\w+\W?==\W?[\w\.]*)`)
|
||||
|
||||
func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parseSetup(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
packages := make([]pkg.Package, 0)
|
||||
|
||||
scanner := bufio.NewScanner(reader)
|
||||
@@ -46,5 +47,5 @@ func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
}
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
return packages, nil, nil
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestParseSetup(t *testing.T) {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parseSetup(fixture.Name(), fixture)
|
||||
actual, _, err := parseSetup(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse requirements: %+v", err)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
@@ -27,24 +28,26 @@ func (c *Cataloger) Name() string {
|
||||
}
|
||||
|
||||
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
|
||||
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) {
|
||||
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
fileMatches, err := resolver.FilesByGlob(pkg.RpmDBGlob)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err)
|
||||
return nil, nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err)
|
||||
}
|
||||
|
||||
var pkgs []pkg.Package
|
||||
for _, location := range fileMatches {
|
||||
dbContentReader, err := resolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
pkgs, err = parseRpmDB(resolver, location, dbContentReader)
|
||||
discoveredPkgs, err := parseRpmDB(resolver, location, dbContentReader)
|
||||
internal.CloseAndLogError(dbContentReader, location.VirtualPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err)
|
||||
return nil, nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err)
|
||||
}
|
||||
|
||||
pkgs = append(pkgs, discoveredPkgs...)
|
||||
}
|
||||
return pkgs, nil
|
||||
return pkgs, nil, nil
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common"
|
||||
)
|
||||
@@ -16,7 +17,7 @@ var _ common.ParserFn = parseGemFileLockEntries
|
||||
var sectionsOfInterest = internal.NewStringSetFromSlice([]string{"GEM"})
|
||||
|
||||
// parseGemFileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered.
|
||||
func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
pkgs := make([]pkg.Package, 0)
|
||||
scanner := bufio.NewScanner(reader)
|
||||
|
||||
@@ -49,9 +50,9 @@ func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, error)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
return pkgs, nil
|
||||
return pkgs, nil, nil
|
||||
}
|
||||
|
||||
func isDependencyLine(line string) bool {
|
||||
|
||||
@@ -68,7 +68,8 @@ func TestParseGemfileLockEntries(t *testing.T) {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parseGemFileLockEntries(fixture.Name(), fixture)
|
||||
// TODO: no relationships are under test yet
|
||||
actual, _, err := parseGemFileLockEntries(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse gemfile lock: %+v", err)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/mitchellh/mapstructure"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common"
|
||||
)
|
||||
@@ -60,7 +61,7 @@ func processList(s string) []string {
|
||||
return results
|
||||
}
|
||||
|
||||
func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
var pkgs []pkg.Package
|
||||
var fields = make(map[string]interface{})
|
||||
scanner := bufio.NewScanner(reader)
|
||||
@@ -93,7 +94,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
if fields["name"] != "" && fields["version"] != "" {
|
||||
var metadata pkg.GemMetadata
|
||||
if err := mapstructure.Decode(fields, &metadata); err != nil {
|
||||
return nil, fmt.Errorf("unable to decode gem metadata: %w", err)
|
||||
return nil, nil, fmt.Errorf("unable to decode gem metadata: %w", err)
|
||||
}
|
||||
|
||||
pkgs = append(pkgs, pkg.Package{
|
||||
@@ -107,7 +108,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
})
|
||||
}
|
||||
|
||||
return pkgs, nil
|
||||
return pkgs, nil, nil
|
||||
}
|
||||
|
||||
// renderUtf8 takes any string escaped string sub-sections from the ruby string and replaces those sections with the UTF8 runes.
|
||||
|
||||
@@ -31,7 +31,8 @@ func TestParseGemspec(t *testing.T) {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parseGemSpecEntries(fixture.Name(), fixture)
|
||||
// TODO: no relationships are under test yet
|
||||
actual, _, err := parseGemSpecEntries(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse gemspec: %+v", err)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/common"
|
||||
"github.com/pelletier/go-toml"
|
||||
@@ -13,17 +14,17 @@ import (
|
||||
var _ common.ParserFn = parseCargoLock
|
||||
|
||||
// parseCargoLock is a parser function for Cargo.lock contents, returning all rust cargo crates discovered.
|
||||
func parseCargoLock(_ string, reader io.Reader) ([]pkg.Package, error) {
|
||||
func parseCargoLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
tree, err := toml.LoadReader(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to load Cargo.lock for parsing: %v", err)
|
||||
return nil, nil, fmt.Errorf("unable to load Cargo.lock for parsing: %v", err)
|
||||
}
|
||||
|
||||
metadata := CargoMetadata{}
|
||||
err = tree.Unmarshal(&metadata)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse Cargo.lock: %v", err)
|
||||
return nil, nil, fmt.Errorf("unable to parse Cargo.lock: %v", err)
|
||||
}
|
||||
|
||||
return metadata.Pkgs(), nil
|
||||
return metadata.Pkgs(), nil, nil
|
||||
}
|
||||
|
||||
@@ -177,7 +177,8 @@ func TestParseCargoLock(t *testing.T) {
|
||||
t.Fatalf("failed to open fixture: %+v", err)
|
||||
}
|
||||
|
||||
actual, err := parseCargoLock(fixture.Name(), fixture)
|
||||
// TODO: no relationships are under test yet
|
||||
actual, _, err := parseCargoLock(fixture.Name(), fixture)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
@@ -45,5 +45,8 @@ func MustCPE(cpeStr string) CPE {
|
||||
|
||||
func normalizeCpeField(field string) string {
|
||||
// keep dashes and forward slashes unescaped
|
||||
if field == "*" {
|
||||
return wfn.Any
|
||||
}
|
||||
return strings.ReplaceAll(wfn.StripSlashes(field), `\/`, "/")
|
||||
}
|
||||
|
||||
+35
-1
@@ -1,6 +1,10 @@
|
||||
package pkg
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func must(c CPE, e error) CPE {
|
||||
if e != nil {
|
||||
@@ -46,3 +50,33 @@ func TestNewCPE(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_normalizeCpeField(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
field string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
field: "something",
|
||||
expected: "something",
|
||||
},
|
||||
{
|
||||
field: "some\\thing",
|
||||
expected: `some\thing`,
|
||||
},
|
||||
{
|
||||
field: "*",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
field: "",
|
||||
expected: "",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.field, func(t *testing.T) {
|
||||
assert.Equal(t, test.expected, normalizeCpeField(test.field))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
package pkg
|
||||
|
||||
// ID represents a unique value for each package added to a package catalog.
|
||||
type ID string
|
||||
@@ -21,7 +21,7 @@ type JavaMetadata struct {
|
||||
Manifest *JavaManifest `mapstructure:"Manifest" json:"manifest,omitempty"`
|
||||
PomProperties *PomProperties `mapstructure:"PomProperties" json:"pomProperties,omitempty"`
|
||||
PomProject *PomProject `mapstructure:"PomProject" json:"pomProject,omitempty"`
|
||||
Parent *Package `json:"-"`
|
||||
Parent *Package `hash:"ignore" json:"-"` // note: the parent cannot be included in the minimal definition of uniqueness since this field is not reproducible in an encode-decode cycle (is lossy).
|
||||
}
|
||||
|
||||
// PomProperties represents the fields of interest extracted from a Java archive's pom.properties file.
|
||||
|
||||
+24
-26
@@ -6,41 +6,39 @@ package pkg
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"github.com/mitchellh/hashstructure"
|
||||
)
|
||||
|
||||
// Package represents an application or library that has been bundled into a distributable format.
|
||||
// TODO: if we ignore FoundBy for ID generation should we merge the field to show it was found in two places?
|
||||
type Package struct {
|
||||
ID ID `hash:"ignore"` // uniquely identifies a package, set by the cataloger
|
||||
Name string // the package name
|
||||
Version string // the version of the package
|
||||
FoundBy string // the specific cataloger that discovered this package
|
||||
Locations []source.Location // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package)
|
||||
// TODO: should we move licenses into metadata?
|
||||
Licenses []string // licenses discovered with the package metadata
|
||||
Language Language // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc)
|
||||
Type Type // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc)
|
||||
CPEs []CPE // all possible Common Platform Enumerators
|
||||
PURL string // the Package URL (see https://github.com/package-url/purl-spec)
|
||||
MetadataType MetadataType // the shape of the additional data in the "metadata" field
|
||||
Metadata interface{} // additional data found while parsing the package source
|
||||
Name string // the package name
|
||||
Version string // the version of the package
|
||||
FoundBy string // the specific cataloger that discovered this package
|
||||
Locations []source.Location // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package)
|
||||
Licenses []string // licenses discovered with the package metadata
|
||||
Language Language // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc)
|
||||
Type Type // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc)
|
||||
CPEs []CPE // all possible Common Platform Enumerators
|
||||
PURL string // the Package URL (see https://github.com/package-url/purl-spec)
|
||||
MetadataType MetadataType // the shape of the additional data in the "metadata" field
|
||||
Metadata interface{} // additional data found while parsing the package source
|
||||
}
|
||||
|
||||
func (p Package) ID() artifact.ID {
|
||||
f, err := artifact.IDFromHash(p)
|
||||
if err != nil {
|
||||
// TODO: what to do in this case?
|
||||
log.Warnf("unable to get fingerprint of package=%s@%s: %+v", p.Name, p.Version, err)
|
||||
return ""
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
// Stringer to represent a package.
|
||||
func (p Package) String() string {
|
||||
return fmt.Sprintf("Pkg(type=%s, name=%s, version=%s)", p.Type, p.Name, p.Version)
|
||||
}
|
||||
|
||||
func (p Package) Fingerprint() (string, error) {
|
||||
f, err := hashstructure.Hash(p, &hashstructure.HashOptions{
|
||||
ZeroNil: true,
|
||||
SlicesAsSets: true,
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not build package fingerprint for: %s version: %s", p.Name, p.Version)
|
||||
}
|
||||
|
||||
return fmt.Sprint(f), nil
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
|
||||
func TestFingerprint(t *testing.T) {
|
||||
originalPkg := Package{
|
||||
ID: "π",
|
||||
Name: "pi",
|
||||
Version: "3.14",
|
||||
FoundBy: "Archimedes",
|
||||
@@ -190,10 +189,10 @@ func TestFingerprint(t *testing.T) {
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
transformedPkg := test.transform(originalPkg)
|
||||
originalFingerprint, err := originalPkg.Fingerprint()
|
||||
assert.NoError(t, err, "expected no error on package fingerprint")
|
||||
transformedFingerprint, err := transformedPkg.Fingerprint()
|
||||
assert.NoError(t, err, "expected no error on package fingerprint")
|
||||
originalFingerprint := originalPkg.ID()
|
||||
assert.NotEmpty(t, originalFingerprint)
|
||||
transformedFingerprint := transformedPkg.ID()
|
||||
assert.NotEmpty(t, transformedFingerprint)
|
||||
|
||||
if test.expectIdentical {
|
||||
assert.Equal(t, originalFingerprint, transformedFingerprint)
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
package pkg
|
||||
|
||||
const (
|
||||
// OwnershipByFileOverlapRelationship indicates that the parent package owns the child package made evident by the set of provided files
|
||||
OwnershipByFileOverlapRelationship RelationshipType = "ownership-by-file-overlap"
|
||||
)
|
||||
|
||||
type RelationshipType string
|
||||
|
||||
type Relationship struct {
|
||||
Parent ID
|
||||
Child ID
|
||||
Type RelationshipType
|
||||
Metadata interface{}
|
||||
}
|
||||
|
||||
// TODO: as more relationships are added, this function signature will probably accommodate selection
|
||||
func NewRelationships(catalog *Catalog) []Relationship {
|
||||
return ownershipByFilesRelationships(catalog)
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package pkg
|
||||
|
||||
import "github.com/anchore/syft/syft/artifact"
|
||||
|
||||
// TODO: as more relationships are added, this function signature will probably accommodate selection
|
||||
func NewRelationships(catalog *Catalog) []artifact.Relationship {
|
||||
return RelationshipsByFileOwnership(catalog)
|
||||
}
|
||||
+20
-15
@@ -2,6 +2,7 @@ package pkg
|
||||
|
||||
import (
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/bmatcuk/doublestar/v2"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
)
|
||||
@@ -20,17 +21,19 @@ type ownershipByFilesMetadata struct {
|
||||
Files []string `json:"files"`
|
||||
}
|
||||
|
||||
func ownershipByFilesRelationships(catalog *Catalog) []Relationship {
|
||||
// RelationshipsByFileOwnership creates a package-to-package relationship based on discovering which packages have
|
||||
// evidence locations that overlap with ownership claim from another package's package manager metadata.
|
||||
func RelationshipsByFileOwnership(catalog *Catalog) []artifact.Relationship {
|
||||
var relationships = findOwnershipByFilesRelationships(catalog)
|
||||
|
||||
var edges []Relationship
|
||||
var edges []artifact.Relationship
|
||||
for parent, children := range relationships {
|
||||
for child, files := range children {
|
||||
edges = append(edges, Relationship{
|
||||
Parent: parent,
|
||||
Child: child,
|
||||
Type: OwnershipByFileOverlapRelationship,
|
||||
Metadata: ownershipByFilesMetadata{
|
||||
edges = append(edges, artifact.Relationship{
|
||||
From: catalog.byID[parent],
|
||||
To: catalog.byID[child],
|
||||
Type: artifact.OwnershipByFileOverlapRelationship,
|
||||
Data: ownershipByFilesMetadata{
|
||||
Files: files.List(),
|
||||
},
|
||||
})
|
||||
@@ -42,14 +45,15 @@ func ownershipByFilesRelationships(catalog *Catalog) []Relationship {
|
||||
|
||||
// findOwnershipByFilesRelationships find overlaps in file ownership with a file that defines another package. Specifically, a .Location.Path of
|
||||
// a package is found to be owned by another (from the owner's .Metadata.Files[]).
|
||||
func findOwnershipByFilesRelationships(catalog *Catalog) map[ID]map[ID]*strset.Set {
|
||||
var relationships = make(map[ID]map[ID]*strset.Set)
|
||||
func findOwnershipByFilesRelationships(catalog *Catalog) map[artifact.ID]map[artifact.ID]*strset.Set {
|
||||
var relationships = make(map[artifact.ID]map[artifact.ID]*strset.Set)
|
||||
|
||||
if catalog == nil {
|
||||
return relationships
|
||||
}
|
||||
|
||||
for _, candidateOwnerPkg := range catalog.Sorted() {
|
||||
id := candidateOwnerPkg.ID()
|
||||
if candidateOwnerPkg.Metadata == nil {
|
||||
continue
|
||||
}
|
||||
@@ -68,17 +72,18 @@ func findOwnershipByFilesRelationships(catalog *Catalog) map[ID]map[ID]*strset.S
|
||||
|
||||
// look for package(s) in the catalog that may be owned by this package and mark the relationship
|
||||
for _, subPackage := range catalog.PackagesByPath(ownedFilePath) {
|
||||
if subPackage.ID == candidateOwnerPkg.ID {
|
||||
subID := subPackage.ID()
|
||||
if subID == id {
|
||||
continue
|
||||
}
|
||||
if _, exists := relationships[candidateOwnerPkg.ID]; !exists {
|
||||
relationships[candidateOwnerPkg.ID] = make(map[ID]*strset.Set)
|
||||
if _, exists := relationships[id]; !exists {
|
||||
relationships[id] = make(map[artifact.ID]*strset.Set)
|
||||
}
|
||||
|
||||
if _, exists := relationships[candidateOwnerPkg.ID][subPackage.ID]; !exists {
|
||||
relationships[candidateOwnerPkg.ID][subPackage.ID] = strset.New()
|
||||
if _, exists := relationships[id][subID]; !exists {
|
||||
relationships[id][subID] = strset.New()
|
||||
}
|
||||
relationships[candidateOwnerPkg.ID][subPackage.ID].Add(ownedFilePath)
|
||||
relationships[id][subID].Add(ownedFilePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
+52
-45
@@ -3,21 +3,21 @@ package pkg
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"github.com/go-test/deep"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestOwnershipByFilesRelationship(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
pkgs []Package
|
||||
expectedRelations []Relationship
|
||||
name string
|
||||
setup func(t testing.TB) ([]Package, []artifact.Relationship)
|
||||
}{
|
||||
{
|
||||
name: "owns-by-real-path",
|
||||
pkgs: []Package{
|
||||
{
|
||||
ID: "parent",
|
||||
setup: func(t testing.TB) ([]Package, []artifact.Relationship) {
|
||||
parent := Package{
|
||||
Locations: []source.Location{
|
||||
{
|
||||
RealPath: "/a/path",
|
||||
@@ -37,9 +37,9 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
|
||||
{Path: "/d/path"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "child",
|
||||
}
|
||||
|
||||
child := Package{
|
||||
Locations: []source.Location{
|
||||
{
|
||||
RealPath: "/c/path",
|
||||
@@ -51,26 +51,26 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Type: NpmPkg,
|
||||
},
|
||||
},
|
||||
expectedRelations: []Relationship{
|
||||
{
|
||||
Parent: "parent",
|
||||
Child: "child",
|
||||
Type: OwnershipByFileOverlapRelationship,
|
||||
Metadata: ownershipByFilesMetadata{
|
||||
}
|
||||
|
||||
relationship := artifact.Relationship{
|
||||
From: parent,
|
||||
To: child,
|
||||
Type: artifact.OwnershipByFileOverlapRelationship,
|
||||
Data: ownershipByFilesMetadata{
|
||||
Files: []string{
|
||||
"/d/path",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return []Package{parent, child}, []artifact.Relationship{relationship}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "owns-by-virtual-path",
|
||||
pkgs: []Package{
|
||||
{
|
||||
ID: "parent",
|
||||
setup: func(t testing.TB) ([]Package, []artifact.Relationship) {
|
||||
parent := Package{
|
||||
Locations: []source.Location{
|
||||
{
|
||||
RealPath: "/a/path",
|
||||
@@ -90,9 +90,9 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
|
||||
{Path: "/another/path"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "child",
|
||||
}
|
||||
|
||||
child := Package{
|
||||
Locations: []source.Location{
|
||||
{
|
||||
RealPath: "/c/path",
|
||||
@@ -104,26 +104,25 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Type: NpmPkg,
|
||||
},
|
||||
},
|
||||
expectedRelations: []Relationship{
|
||||
{
|
||||
Parent: "parent",
|
||||
Child: "child",
|
||||
Type: OwnershipByFileOverlapRelationship,
|
||||
Metadata: ownershipByFilesMetadata{
|
||||
}
|
||||
|
||||
relationship := artifact.Relationship{
|
||||
From: parent,
|
||||
To: child,
|
||||
Type: artifact.OwnershipByFileOverlapRelationship,
|
||||
Data: ownershipByFilesMetadata{
|
||||
Files: []string{
|
||||
"/another/path",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return []Package{parent, child}, []artifact.Relationship{relationship}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ignore-empty-path",
|
||||
pkgs: []Package{
|
||||
{
|
||||
ID: "parent",
|
||||
setup: func(t testing.TB) ([]Package, []artifact.Relationship) {
|
||||
parent := Package{
|
||||
Locations: []source.Location{
|
||||
{
|
||||
RealPath: "/a/path",
|
||||
@@ -143,9 +142,9 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
|
||||
{Path: ""},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "child",
|
||||
}
|
||||
|
||||
child := Package{
|
||||
Locations: []source.Location{
|
||||
{
|
||||
RealPath: "/c/path",
|
||||
@@ -157,18 +156,26 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Type: NpmPkg,
|
||||
},
|
||||
}
|
||||
|
||||
return []Package{parent, child}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
c := NewCatalog(test.pkgs...)
|
||||
relationships := ownershipByFilesRelationships(c)
|
||||
pkgs, expectedRelations := test.setup(t)
|
||||
c := NewCatalog(pkgs...)
|
||||
relationships := RelationshipsByFileOwnership(c)
|
||||
|
||||
for _, d := range deep.Equal(test.expectedRelations, relationships) {
|
||||
t.Errorf("diff: %+v", d)
|
||||
assert.Len(t, relationships, len(expectedRelations))
|
||||
for idx, expectedRelationship := range expectedRelations {
|
||||
actualRelationship := relationships[idx]
|
||||
assert.Equal(t, expectedRelationship.From.ID(), actualRelationship.From.ID())
|
||||
assert.Equal(t, expectedRelationship.To.ID(), actualRelationship.To.ID())
|
||||
assert.Equal(t, expectedRelationship.Type, actualRelationship.Type)
|
||||
assert.Equal(t, expectedRelationship.Data, actualRelationship.Data)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user