Introduce relationships as first-class objects (#607)

* migrate pkg.ID and pkg.Relationship to artifact package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* return relationships from tasks

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix more tests

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add artifact.Identifiable by Identity() method

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove catalog ID assignment

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* adjust spdx helpers to use copy of packages

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* stabilize package ID relative to encode-decode format cycles

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* rename Identity() to ID()

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* use zero value for nils in ID generation

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* enable source.Location to be identifiable

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* hoist up package relationship discovery to analysis stage

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update ownership-by-file-overlap relationship description

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add test reminders to put new relationships under test

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* adjust PHP composer.lock parser function to return relationships

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
2021-11-16 14:14:13 -05:00
committed by GitHub
parent 45ea4177e8
commit ef627d82ef
100 changed files with 663 additions and 633 deletions
+6 -4
View File
@@ -8,6 +8,8 @@ import (
"strconv"
"strings"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/internal/log"
@@ -21,7 +23,7 @@ var _ common.ParserFn = parseApkDB
// parseApkDb parses individual packages from a given Alpine DB file. For more information on specific fields
// see https://wiki.alpinelinux.org/wiki/Apk_spec .
func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
// larger capacity for the scanner.
const maxScannerCapacity = 1024 * 1024
// a new larger buffer for the scanner
@@ -47,7 +49,7 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
for scanner.Scan() {
metadata, err := parseApkDBEntry(strings.NewReader(scanner.Text()))
if err != nil {
return nil, err
return nil, nil, err
}
if metadata != nil {
packages = append(packages, pkg.Package{
@@ -62,10 +64,10 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("failed to parse APK DB file: %w", err)
return nil, nil, fmt.Errorf("failed to parse APK DB file: %w", err)
}
return packages, nil
return packages, nil, nil
}
// nolint:funlen
@@ -775,7 +775,8 @@ func TestMultiplePackages(t *testing.T) {
}
}()
pkgs, err := parseApkDB(file.Name(), file)
// TODO: no relationships are under test yet
pkgs, _, err := parseApkDB(file.Name(), file)
if err != nil {
t.Fatal("Unable to read file contents: ", err)
}
+10 -4
View File
@@ -3,6 +3,7 @@ package cataloger
import (
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/pkg"
@@ -38,8 +39,9 @@ func newMonitor() (*progress.Manual, *progress.Manual) {
// In order to efficiently retrieve contents from a underlying container image the content fetch requests are
// done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single
// request.
func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers ...Cataloger) (*pkg.Catalog, error) {
func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers ...Cataloger) (*pkg.Catalog, []artifact.Relationship, error) {
catalog := pkg.NewCatalog()
var allRelationships []artifact.Relationship
filesProcessed, packagesDiscovered := newMonitor()
@@ -47,7 +49,7 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers
var errs error
for _, theCataloger := range catalogers {
// find packages from the underlying raw data
packages, err := theCataloger.Catalog(resolver)
packages, relationships, err := theCataloger.Catalog(resolver)
if err != nil {
errs = multierror.Append(errs, err)
continue
@@ -68,14 +70,18 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers
// add to catalog
catalog.Add(p)
}
allRelationships = append(allRelationships, relationships...)
}
allRelationships = append(allRelationships, pkg.NewRelationships(catalog)...)
if errs != nil {
return nil, errs
return nil, nil, errs
}
filesProcessed.SetCompleted()
packagesDiscovered.SetCompleted()
return catalog, nil
return catalog, allRelationships, nil
}
+2 -1
View File
@@ -6,6 +6,7 @@ catalogers defined in child packages as well as the interface definition to impl
package cataloger
import (
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/apkdb"
"github.com/anchore/syft/syft/pkg/cataloger/deb"
@@ -27,7 +28,7 @@ type Cataloger interface {
// Name returns a string that uniquely describes a cataloger
Name() string
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
Catalog(resolver source.FileResolver) ([]pkg.Package, error)
Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error)
}
// ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
+1 -1
View File
@@ -83,7 +83,7 @@ func candidateVendors(p pkg.Package) []string {
// allow * as a candidate. Note: do NOT allow Java packages to have * vendors.
switch p.Language {
case pkg.Ruby, pkg.JavaScript:
vendors.addValue("*")
vendors.addValue(wfn.Any)
}
switch p.MetadataType {
@@ -637,7 +637,7 @@ func TestCandidateProducts(t *testing.T) {
}
for _, test := range tests {
t.Run(fmt.Sprintf("%+v %+v", test.p, test.expected), func(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
assert.ElementsMatch(t, test.expected, candidateProducts(test.p))
})
}
+14 -10
View File
@@ -6,6 +6,8 @@ package common
import (
"fmt"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
@@ -35,18 +37,18 @@ func (c *GenericCataloger) Name() string {
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) {
func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
var packages []pkg.Package
parserByLocation := c.selectFiles(resolver)
var relationships []artifact.Relationship
for location, parser := range parserByLocation {
for location, parser := range c.selectFiles(resolver) {
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
// TODO: fail or log?
return nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err)
return nil, nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err)
}
entries, err := parser(location.RealPath, contentReader)
discoveredPackages, discoveredRelationships, err := parser(location.RealPath, contentReader)
internal.CloseAndLogError(contentReader, location.VirtualPath)
if err != nil {
// TODO: should we fail? or only log?
@@ -54,14 +56,16 @@ func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package,
continue
}
for _, entry := range entries {
entry.FoundBy = c.upstreamCataloger
entry.Locations = []source.Location{location}
for _, p := range discoveredPackages {
p.FoundBy = c.upstreamCataloger
p.Locations = append(p.Locations, location)
packages = append(packages, entry)
packages = append(packages, p)
}
relationships = append(relationships, discoveredRelationships...)
}
return packages, nil
return packages, relationships, nil
}
// SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging
@@ -8,11 +8,12 @@ import (
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func parser(_ string, reader io.Reader) ([]pkg.Package, error) {
func parser(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
contents, err := ioutil.ReadAll(reader)
if err != nil {
panic(err)
@@ -21,7 +22,7 @@ func parser(_ string, reader io.Reader) ([]pkg.Package, error) {
{
Name: string(contents),
},
}, nil
}, nil, nil
}
func TestGenericCataloger(t *testing.T) {
@@ -47,7 +48,7 @@ func TestGenericCataloger(t *testing.T) {
}
}
actualPkgs, err := cataloger.Catalog(resolver)
actualPkgs, _, err := cataloger.Catalog(resolver)
assert.NoError(t, err)
assert.Len(t, actualPkgs, len(expectedPkgs))
+2 -1
View File
@@ -3,8 +3,9 @@ package common
import (
"io"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
)
// ParserFn standardizes a function signature for parser functions that accept the virtual file path (not usable for file reads) and contents and return any discovered packages from that file
type ParserFn func(string, io.Reader) ([]pkg.Package, error)
type ParserFn func(string, io.Reader) ([]pkg.Package, []artifact.Relationship, error)
+9 -9
View File
@@ -13,6 +13,7 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
@@ -36,24 +37,23 @@ func (c *Cataloger) Name() string {
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files.
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) {
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
dbFileMatches, err := resolver.FilesByGlob(pkg.DpkgDBGlob)
if err != nil {
return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err)
return nil, nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err)
}
var results []pkg.Package
var pkgs []pkg.Package
var allPackages []pkg.Package
for _, dbLocation := range dbFileMatches {
dbContents, err := resolver.FileContentsByLocation(dbLocation)
if err != nil {
return nil, err
return nil, nil, err
}
pkgs, err = parseDpkgStatus(dbContents)
pkgs, err := parseDpkgStatus(dbContents)
internal.CloseAndLogError(dbContents, dbLocation.VirtualPath)
if err != nil {
return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err)
return nil, nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err)
}
for i := range pkgs {
@@ -70,9 +70,9 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error)
addLicenses(resolver, dbLocation, p)
}
results = append(results, pkgs...)
allPackages = append(allPackages, pkgs...)
}
return results, nil
return allPackages, nil, nil
}
func addLicenses(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) {
+1 -1
View File
@@ -100,7 +100,7 @@ func TestDpkgCataloger(t *testing.T) {
t.Errorf("could not get resolver error: %+v", err)
}
actual, err := c.Catalog(resolver)
actual, _, err := c.Catalog(resolver)
if err != nil {
t.Fatalf("failed to catalog: %+v", err)
}
+1 -1
View File
@@ -23,7 +23,7 @@ var (
// parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed.
func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
buffedReader := bufio.NewReader(reader)
var packages = make([]pkg.Package, 0)
var packages []pkg.Package
continueProcessing := true
for continueProcessing {
@@ -6,7 +6,10 @@ package golang
import (
"fmt"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
@@ -35,17 +38,18 @@ func (c *Cataloger) Name() string {
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) {
pkgs := make([]pkg.Package, 0)
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
var pkgs []pkg.Package
fileMatches, err := resolver.FilesByMIMEType(mimeTypes...)
if err != nil {
return pkgs, fmt.Errorf("failed to find bin by mime types: %w", err)
return pkgs, nil, fmt.Errorf("failed to find bin by mime types: %w", err)
}
for _, location := range fileMatches {
r, err := resolver.FileContentsByLocation(location)
if err != nil {
return pkgs, fmt.Errorf("failed to resolve file contents by location: %w", err)
return pkgs, nil, fmt.Errorf("failed to resolve file contents by location: %w", err)
}
goPkgs, err := parseGoBin(location, r)
@@ -53,9 +57,9 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error)
log.Warnf("could not parse possible go binary: %+v", err)
}
r.Close()
internal.CloseAndLogError(r, location.RealPath)
pkgs = append(pkgs, goPkgs...)
}
return pkgs, nil
return pkgs, nil, nil
}
+1 -3
View File
@@ -23,9 +23,7 @@ func parseGoBin(location source.Location, reader io.ReadCloser) ([]pkg.Package,
goVersion, mod := findVers(x)
pkgs := buildGoPkgInfo(location, mod, goVersion)
return pkgs, nil
return buildGoPkgInfo(location, mod, goVersion), nil
}
func buildGoPkgInfo(location source.Location, mod, goVersion string) []pkg.Package {
+5 -4
View File
@@ -6,22 +6,23 @@ import (
"io/ioutil"
"sort"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"golang.org/x/mod/modfile"
)
// parseGoMod takes a go.mod and lists all packages discovered.
func parseGoMod(path string, reader io.Reader) ([]pkg.Package, error) {
func parseGoMod(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
packages := make(map[string]pkg.Package)
contents, err := ioutil.ReadAll(reader)
if err != nil {
return nil, fmt.Errorf("failed to read go module: %w", err)
return nil, nil, fmt.Errorf("failed to read go module: %w", err)
}
file, err := modfile.Parse(path, contents, nil)
if err != nil {
return nil, fmt.Errorf("failed to parse go module: %w", err)
return nil, nil, fmt.Errorf("failed to parse go module: %w", err)
}
for _, m := range file.Require {
@@ -59,5 +60,5 @@ func parseGoMod(path string, reader io.Reader) ([]pkg.Package, error) {
return pkgsSlice[i].Name < pkgsSlice[j].Name
})
return pkgsSlice, nil
return pkgsSlice, nil, nil
}
@@ -70,7 +70,8 @@ func TestParseGoMod(t *testing.T) {
t.Fatalf(err.Error())
}
actual, err := parseGoMod(test.fixture, f)
// TODO: no relationships are under test yet
actual, _, err := parseGoMod(test.fixture, f)
if err != nil {
t.Fatalf(err.Error())
}
+23 -17
View File
@@ -9,6 +9,7 @@ import (
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/file"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
@@ -34,12 +35,12 @@ type archiveParser struct {
}
// parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives.
func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, error) {
func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true)
// note: even on error, we should always run cleanup functions
defer cleanupFn()
if err != nil {
return nil, err
return nil, nil, err
}
return parser.parse()
}
@@ -80,29 +81,31 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo
}
// parse the loaded archive and return all packages found.
func (j *archiveParser) parse() ([]pkg.Package, error) {
var pkgs = make([]pkg.Package, 0)
func (j *archiveParser) parse() ([]pkg.Package, []artifact.Relationship, error) {
var pkgs []pkg.Package
var relationships []artifact.Relationship
// find the parent package from the java manifest
parentPkg, err := j.discoverMainPackage()
if err != nil {
return nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err)
return nil, nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err)
}
// find aux packages from pom.properties/pom.xml and potentially modify the existing parentPkg
auxPkgs, err := j.discoverPkgsFromAllMavenFiles(parentPkg)
if err != nil {
return nil, err
return nil, nil, err
}
pkgs = append(pkgs, auxPkgs...)
if j.detectNested {
// find nested java archive packages
nestedPkgs, err := j.discoverPkgsFromNestedArchives(parentPkg)
nestedPkgs, nestedRelationships, err := j.discoverPkgsFromNestedArchives(parentPkg)
if err != nil {
return nil, err
return nil, nil, err
}
pkgs = append(pkgs, nestedPkgs...)
relationships = append(relationships, nestedRelationships...)
}
// lastly, add the parent package to the list (assuming the parent exists)
@@ -110,7 +113,7 @@ func (j *archiveParser) parse() ([]pkg.Package, error) {
pkgs = append([]pkg.Package{*parentPkg}, pkgs...)
}
return pkgs, nil
return pkgs, relationships, nil
}
// discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages.
@@ -189,31 +192,32 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(parentPkg *pkg.Package) ([
// discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and
// associating each discovered package to the given parent package.
func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, error) {
var pkgs = make([]pkg.Package, 0)
func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) {
var pkgs []pkg.Package
var relationships []artifact.Relationship
// search and parse pom.properties files & fetch the contents
openers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
if err != nil {
return nil, fmt.Errorf("unable to extract files from zip: %w", err)
return nil, nil, fmt.Errorf("unable to extract files from zip: %w", err)
}
// discover nested artifacts
for archivePath, archiveOpener := range openers {
archiveReadCloser, err := archiveOpener.Open()
if err != nil {
return nil, fmt.Errorf("unable to open archived file from tempdir: %w", err)
return nil, nil, fmt.Errorf("unable to open archived file from tempdir: %w", err)
}
nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
nestedPkgs, err := parseJavaArchive(nestedPath, archiveReadCloser)
nestedPkgs, nestedRelationships, err := parseJavaArchive(nestedPath, archiveReadCloser)
if err != nil {
if closeErr := archiveReadCloser.Close(); closeErr != nil {
log.Warnf("unable to close archived file from tempdir: %+v", closeErr)
}
return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
return nil, nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
}
if err = archiveReadCloser.Close(); err != nil {
return nil, fmt.Errorf("unable to close archived file from tempdir: %w", err)
return nil, nil, fmt.Errorf("unable to close archived file from tempdir: %w", err)
}
// attach the parent package to all discovered packages that are not already associated with a java archive
@@ -226,9 +230,11 @@ func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) (
}
pkgs = append(pkgs, p)
}
relationships = append(relationships, nestedRelationships...)
}
return pkgs, nil
return pkgs, relationships, nil
}
func pomPropertiesByParentPath(archivePath string, extractPaths []string, virtualPath string) (map[string]pkg.PomProperties, error) {
@@ -242,7 +242,7 @@ func TestParseJar(t *testing.T) {
t.Fatalf("should not have filed... %+v", err)
}
actual, err := parser.parse()
actual, _, err := parser.parse()
if err != nil {
t.Fatalf("failed to parse java archive: %+v", err)
}
@@ -507,7 +507,7 @@ func TestParseNestedJar(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseJavaArchive(fixture.Name(), fixture)
actual, _, err := parseJavaArchive(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse java archive: %+v", err)
}
@@ -13,6 +13,7 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
@@ -162,8 +163,8 @@ func licensesFromJSON(p PackageJSON) ([]string, error) {
}
// parsePackageJSON parses a package.json and returns the discovered JavaScript packages.
func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
packages := make([]pkg.Package, 0)
func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
var packages []pkg.Package
dec := json.NewDecoder(reader)
for {
@@ -171,17 +172,17 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
if err := dec.Decode(&p); err == io.EOF {
break
} else if err != nil {
return nil, fmt.Errorf("failed to parse package.json file: %w", err)
return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err)
}
if !p.hasNameAndVersionValues() {
log.Debug("encountered package.json file without a name and/or version field, ignoring this file")
return nil, nil
return nil, nil, nil
}
licenses, err := licensesFromJSON(p)
if err != nil {
return nil, fmt.Errorf("failed to parse package.json file: %w", err)
return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err)
}
packages = append(packages, pkg.Package{
@@ -200,7 +201,7 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
})
}
return packages, nil
return packages, nil, nil
}
func (p PackageJSON) hasNameAndVersionValues() bool {
@@ -124,7 +124,7 @@ func TestParsePackageJSON(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parsePackageJSON("", fixture)
actual, _, err := parsePackageJSON("", fixture)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}
@@ -150,7 +150,8 @@ func TestParsePackageJSON_Partial(t *testing.T) { // see https://github.com/anch
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parsePackageJSON("", fixture)
// TODO: no relationships are under test yet
actual, _, err := parsePackageJSON("", fixture)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}
@@ -5,6 +5,7 @@ import (
"fmt"
"io"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
@@ -28,11 +29,11 @@ type Dependency struct {
}
// parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages.
func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, error) {
func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
// in the case we find package-lock.json files in the node_modules directories, skip those
// as the whole purpose of the lock file is for the specific dependencies of the root project
if pathContainsNodeModulesDirectory(path) {
return nil, nil
return nil, nil, nil
}
var packages []pkg.Package
@@ -43,7 +44,7 @@ func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, error) {
if err := dec.Decode(&lock); err == io.EOF {
break
} else if err != nil {
return nil, fmt.Errorf("failed to parse package-lock.json file: %w", err)
return nil, nil, fmt.Errorf("failed to parse package-lock.json file: %w", err)
}
for name, pkgMeta := range lock.Dependencies {
packages = append(packages, pkg.Package{
@@ -55,5 +56,5 @@ func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, error) {
}
}
return packages, nil
return packages, nil, nil
}
@@ -109,7 +109,8 @@ func TestParsePackageLock(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parsePackageLock(fixture.Name(), fixture)
// TODO: no relationships are under test yet
actual, _, err := parsePackageLock(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}
@@ -7,6 +7,7 @@ import (
"regexp"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
@@ -34,11 +35,11 @@ const (
noVersion = ""
)
func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, error) {
func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
// in the case we find yarn.lock files in the node_modules directories, skip those
// as the whole purpose of the lock file is for the specific dependencies of the project
if pathContainsNodeModulesDirectory(path) {
return nil, nil
return nil, nil, nil
}
var packages []pkg.Package
@@ -79,10 +80,10 @@ func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, error) {
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("failed to parse yarn.lock file: %w", err)
return nil, nil, fmt.Errorf("failed to parse yarn.lock file: %w", err)
}
return packages, nil
return packages, nil, nil
}
func findPackageName(line string) string {
@@ -70,7 +70,8 @@ func TestParseYarnLock(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseYarnLock(fixture.Name(), fixture)
// TODO: no relationships are under test yet
actual, _, err := parseYarnLock(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse yarn.lock: %+v", err)
}
@@ -5,6 +5,8 @@ import (
"fmt"
"io"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
@@ -23,7 +25,7 @@ type Dependency struct {
var _ common.ParserFn = parseComposerLock
// parseComposerLock is a parser function for Composer.lock contents, returning "Default" php packages discovered.
func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, error) {
func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
packages := make([]pkg.Package, 0)
dec := json.NewDecoder(reader)
@@ -32,7 +34,7 @@ func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, error) {
if err := dec.Decode(&lock); err == io.EOF {
break
} else if err != nil {
return nil, fmt.Errorf("failed to parse composer.lock file: %w", err)
return nil, nil, fmt.Errorf("failed to parse composer.lock file: %w", err)
}
for _, pkgMeta := range lock.Packages {
version := pkgMeta.Version
@@ -46,5 +48,5 @@ func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, error) {
}
}
return packages, nil
return packages, nil, nil
}
@@ -28,7 +28,8 @@ func TestParseComposerFileLock(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseComposerLock(fixture.Name(), fixture)
// TODO: no relationships are under test yet
actual, _, err := parseComposerLock(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse requirements: %+v", err)
}
@@ -7,6 +7,7 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
@@ -31,13 +32,13 @@ func (c *PackageCataloger) Name() string {
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations.
func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) {
func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
var fileMatches []source.Location
for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob, eggFileMetadataGlob} {
matches, err := resolver.FilesByGlob(glob)
if err != nil {
return nil, fmt.Errorf("failed to find files by glob: %s", glob)
return nil, nil, fmt.Errorf("failed to find files by glob: %s", glob)
}
fileMatches = append(fileMatches, matches...)
}
@@ -46,13 +47,13 @@ func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package,
for _, location := range fileMatches {
p, err := c.catalogEggOrWheel(resolver, location)
if err != nil {
return nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.RealPath, err)
return nil, nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.RealPath, err)
}
if p != nil {
pkgs = append(pkgs, *p)
}
}
return pkgs, nil
return pkgs, nil, nil
}
// catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents.
@@ -144,7 +144,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
test.expectedPackage.Locations = locations
actual, err := NewPythonPackageCataloger().Catalog(resolver)
actual, _, err := NewPythonPackageCataloger().Catalog(resolver)
if err != nil {
t.Fatalf("failed to catalog python package: %+v", err)
}
@@ -173,7 +173,7 @@ func TestIgnorePackage(t *testing.T) {
t.Run(test.MetadataFixture, func(t *testing.T) {
resolver := source.NewMockResolverForPaths(test.MetadataFixture)
actual, err := NewPythonPackageCataloger().Catalog(resolver)
actual, _, err := NewPythonPackageCataloger().Catalog(resolver)
if err != nil {
t.Fatalf("failed to catalog python package: %+v", err)
}
@@ -6,6 +6,7 @@ import (
"io"
"strings"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
@@ -37,7 +38,7 @@ type Dependency struct {
var _ common.ParserFn = parsePipfileLock
// parsePipfileLock is a parser function for Pipfile.lock contents, returning "Default" python packages discovered.
func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, error) {
func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
packages := make([]pkg.Package, 0)
dec := json.NewDecoder(reader)
@@ -46,7 +47,7 @@ func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, error) {
if err := dec.Decode(&lock); err == io.EOF {
break
} else if err != nil {
return nil, fmt.Errorf("failed to parse Pipfile.lock file: %w", err)
return nil, nil, fmt.Errorf("failed to parse Pipfile.lock file: %w", err)
}
for name, pkgMeta := range lock.Default {
version := strings.TrimPrefix(pkgMeta.Version, "==")
@@ -59,5 +60,5 @@ func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, error) {
}
}
return packages, nil
return packages, nil, nil
}
@@ -39,7 +39,8 @@ func TestParsePipFileLock(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parsePipfileLock(fixture.Name(), fixture)
// TODO: no relationships are under test yet
actual, _, err := parsePipfileLock(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse requirements: %+v", err)
}
@@ -4,6 +4,7 @@ import (
"fmt"
"io"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/pelletier/go-toml"
@@ -13,17 +14,17 @@ import (
var _ common.ParserFn = parsePoetryLock
// parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered.
func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, error) {
func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
tree, err := toml.LoadReader(reader)
if err != nil {
return nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err)
return nil, nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err)
}
metadata := PoetryMetadata{}
err = tree.Unmarshal(&metadata)
if err != nil {
return nil, fmt.Errorf("unable to parse poetry.lock: %v", err)
return nil, nil, fmt.Errorf("unable to parse poetry.lock: %v", err)
}
return metadata.Pkgs(), nil
return metadata.Pkgs(), nil, nil
}
@@ -45,7 +45,8 @@ func TestParsePoetryLock(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parsePoetryLock(fixture.Name(), fixture)
// TODO: no relationships are under test yet
actual, _, err := parsePoetryLock(fixture.Name(), fixture)
if err != nil {
t.Error(err)
}
@@ -6,6 +6,7 @@ import (
"io"
"strings"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
@@ -15,7 +16,7 @@ var _ common.ParserFn = parseRequirementsTxt
// parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a
// specific version.
func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) {
func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
packages := make([]pkg.Package, 0)
scanner := bufio.NewScanner(reader)
@@ -55,10 +56,10 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) {
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("failed to parse python requirements file: %w", err)
return nil, nil, fmt.Errorf("failed to parse python requirements file: %w", err)
}
return packages, nil
return packages, nil, nil
}
// removeTrailingComment takes a requirements.txt line and strips off comment strings.
@@ -50,7 +50,8 @@ func TestParseRequirementsTxt(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseRequirementsTxt(fixture.Name(), fixture)
// TODO: no relationships are under test yet
actual, _, err := parseRequirementsTxt(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse requirements: %+v", err)
}
+3 -2
View File
@@ -6,6 +6,7 @@ import (
"regexp"
"strings"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
@@ -19,7 +20,7 @@ var _ common.ParserFn = parseSetup
// " mypy2 == v0.770", ' mypy3== v0.770', --> match(name=mypy2 version=v0.770), match(name=mypy3, version=v0.770)
var pinnedDependency = regexp.MustCompile(`['"]\W?(\w+\W?==\W?[\w\.]*)`)
func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) {
func parseSetup(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
packages := make([]pkg.Package, 0)
scanner := bufio.NewScanner(reader)
@@ -46,5 +47,5 @@ func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) {
}
}
return packages, nil
return packages, nil, nil
}
@@ -45,7 +45,7 @@ func TestParseSetup(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseSetup(fixture.Name(), fixture)
actual, _, err := parseSetup(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse requirements: %+v", err)
}
+9 -6
View File
@@ -8,6 +8,7 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
@@ -27,24 +28,26 @@ func (c *Cataloger) Name() string {
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) {
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
fileMatches, err := resolver.FilesByGlob(pkg.RpmDBGlob)
if err != nil {
return nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err)
return nil, nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err)
}
var pkgs []pkg.Package
for _, location := range fileMatches {
dbContentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, err
return nil, nil, err
}
pkgs, err = parseRpmDB(resolver, location, dbContentReader)
discoveredPkgs, err := parseRpmDB(resolver, location, dbContentReader)
internal.CloseAndLogError(dbContentReader, location.VirtualPath)
if err != nil {
return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err)
return nil, nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err)
}
pkgs = append(pkgs, discoveredPkgs...)
}
return pkgs, nil
return pkgs, nil, nil
}
@@ -6,6 +6,7 @@ import (
"strings"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
@@ -16,7 +17,7 @@ var _ common.ParserFn = parseGemFileLockEntries
var sectionsOfInterest = internal.NewStringSetFromSlice([]string{"GEM"})
// parseGemFileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered.
func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
pkgs := make([]pkg.Package, 0)
scanner := bufio.NewScanner(reader)
@@ -49,9 +50,9 @@ func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, error)
}
}
if err := scanner.Err(); err != nil {
return nil, err
return nil, nil, err
}
return pkgs, nil
return pkgs, nil, nil
}
func isDependencyLine(line string) bool {
@@ -68,7 +68,8 @@ func TestParseGemfileLockEntries(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseGemFileLockEntries(fixture.Name(), fixture)
// TODO: no relationships are under test yet
actual, _, err := parseGemFileLockEntries(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse gemfile lock: %+v", err)
}
+4 -3
View File
@@ -12,6 +12,7 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
@@ -60,7 +61,7 @@ func processList(s string) []string {
return results
}
func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
var pkgs []pkg.Package
var fields = make(map[string]interface{})
scanner := bufio.NewScanner(reader)
@@ -93,7 +94,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
if fields["name"] != "" && fields["version"] != "" {
var metadata pkg.GemMetadata
if err := mapstructure.Decode(fields, &metadata); err != nil {
return nil, fmt.Errorf("unable to decode gem metadata: %w", err)
return nil, nil, fmt.Errorf("unable to decode gem metadata: %w", err)
}
pkgs = append(pkgs, pkg.Package{
@@ -107,7 +108,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
})
}
return pkgs, nil
return pkgs, nil, nil
}
// renderUtf8 takes any string escaped string sub-sections from the ruby string and replaces those sections with the UTF8 runes.
@@ -31,7 +31,8 @@ func TestParseGemspec(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseGemSpecEntries(fixture.Name(), fixture)
// TODO: no relationships are under test yet
actual, _, err := parseGemSpecEntries(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse gemspec: %+v", err)
}
+5 -4
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"io"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/pelletier/go-toml"
@@ -13,17 +14,17 @@ import (
var _ common.ParserFn = parseCargoLock
// parseCargoLock is a parser function for Cargo.lock contents, returning all rust cargo crates discovered.
func parseCargoLock(_ string, reader io.Reader) ([]pkg.Package, error) {
func parseCargoLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
tree, err := toml.LoadReader(reader)
if err != nil {
return nil, fmt.Errorf("unable to load Cargo.lock for parsing: %v", err)
return nil, nil, fmt.Errorf("unable to load Cargo.lock for parsing: %v", err)
}
metadata := CargoMetadata{}
err = tree.Unmarshal(&metadata)
if err != nil {
return nil, fmt.Errorf("unable to parse Cargo.lock: %v", err)
return nil, nil, fmt.Errorf("unable to parse Cargo.lock: %v", err)
}
return metadata.Pkgs(), nil
return metadata.Pkgs(), nil, nil
}
@@ -177,7 +177,8 @@ func TestParseCargoLock(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseCargoLock(fixture.Name(), fixture)
// TODO: no relationships are under test yet
actual, _, err := parseCargoLock(fixture.Name(), fixture)
if err != nil {
t.Error(err)
}