oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-05 08:56:41 +01:00
Code Issues Packages Projects Releases Wiki Activity

Performance improvements around package ID (#698)

Browse Source 
* set package ID in catalogers and improve hashing performance

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update setting ID + tests

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
2021-12-16 08:55:53 -05:00
committed by GitHub
parent 727b84ce0d
commit a27907659d
59 changed files with 319 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/pkg/cataloger/javascript/parse_package_json.go
+26 -23
View File
@@ -50,7 +50,6 @@ type Repository struct {
// ---> name: "Isaac Z. Schlueter" email: "i@izs.me" url: "http://blog.izs.me"
var authorPattern = regexp.MustCompile(`^\s*(?P<name>[^<(]*)(\s+<(?P<email>.*)>)?(\s\((?P<url>.*)\))?\s*$`)
// Exports Author.UnmarshalJSON interface to help normalize the json structure.
func (a *Author) UnmarshalJSON(b []byte) error {
var authorStr string
var fields map[string]string
@@ -135,7 +134,7 @@ func licenseFromJSON(b []byte) (string, error) {
return "", errors.New("unable to unmarshal license field as either string or object")
}
func licensesFromJSON(p PackageJSON) ([]string, error) {
func (p PackageJSON) licensesFromJSON() ([]string, error) {
if p.License == nil && p.Licenses == nil {
// This package.json doesn't specify any licenses whatsoever
return []string{}, nil
@@ -163,8 +162,8 @@ func licensesFromJSON(p PackageJSON) ([]string, error) {
}
// parsePackageJSON parses a package.json and returns the discovered JavaScript packages.
func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
var packages []pkg.Package
func parsePackageJSON(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
var packages []*pkg.Package
dec := json.NewDecoder(reader)
for {
@@ -180,30 +179,34 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.Rel
return nil, nil, nil
}
licenses, err := licensesFromJSON(p)
if err != nil {
return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err)
}
packages = append(packages, pkg.Package{
Name: p.Name,
Version: p.Version,
Licenses: licenses,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageJSONMetadataType,
Metadata: pkg.NpmPackageJSONMetadata{
Author: p.Author.AuthorString(),
Homepage: p.Homepage,
URL: p.Repository.URL,
Licenses: licenses,
},
})
packages = append(packages, newPackageJSONPackage(p))
}
return packages, nil, nil
}
func newPackageJSONPackage(p PackageJSON) *pkg.Package {
licenses, err := p.licensesFromJSON()
if err != nil {
log.Warnf("unable to extract licenses from javascript package.json: %+v", err)
}
return &pkg.Package{
Name: p.Name,
Version: p.Version,
Licenses: licenses,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageJSONMetadataType,
Metadata: pkg.NpmPackageJSONMetadata{
Author: p.Author.AuthorString(),
Homepage: p.Homepage,
URL: p.Repository.URL,
Licenses: licenses,
},
}
}
func (p PackageJSON) hasNameAndVersionValues() bool {
return p.Name != "" && p.Version != ""
}
syft/pkg/cataloger/javascript/parse_package_json_test.go
+1 -1
View File
@@ -136,7 +136,7 @@ func TestParsePackageJSON(t *testing.T) {
t.Fatalf("unexpected package count: %d!=1", len(actual))
}
for _, d := range deep.Equal(actual[0], test.ExpectedPkg) {
for _, d := range deep.Equal(actual[0], &test.ExpectedPkg) {
t.Errorf("diff: %+v", d)
}
syft/pkg/cataloger/javascript/parse_package_lock.go
+3 -3
View File
@@ -29,14 +29,14 @@ type Dependency struct {
}
// parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages.
func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
func parsePackageLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
// in the case we find package-lock.json files in the node_modules directories, skip those
// as the whole purpose of the lock file is for the specific dependencies of the root project
if pathContainsNodeModulesDirectory(path) {
return nil, nil, nil
}
var packages []pkg.Package
var packages []*pkg.Package
dec := json.NewDecoder(reader)
for {
@@ -47,7 +47,7 @@ func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, []artifact.
return nil, nil, fmt.Errorf("failed to parse package-lock.json file: %w", err)
}
for name, pkgMeta := range lock.Dependencies {
packages = append(packages, pkg.Package{
packages = append(packages, &pkg.Package{
Name: name,
Version: pkgMeta.Version,
Language: pkg.JavaScript,
syft/pkg/cataloger/javascript/parse_package_lock_test.go
+8 -21
View File
@@ -4,10 +4,12 @@ import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/pkg"
)
func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
func assertPkgsEqual(t *testing.T, actual []*pkg.Package, expected map[string]pkg.Package) {
t.Helper()
if len(actual) != len(expected) {
for _, a := range actual {
@@ -18,26 +20,11 @@ func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg
for _, a := range actual {
expectedPkg, ok := expected[a.Name]
if !ok {
t.Errorf("unexpected package found: '%s'", a.Name)
}
if expectedPkg.Version != a.Version {
t.Errorf("%s : unexpected package version: '%s', expected: '%s'", a.Name, a.Version, expectedPkg.Version)
}
if a.Language != expectedPkg.Language {
t.Errorf("%s : bad language: '%+v', expected: '%+v'", a.Name, a.Language, expectedPkg.Language)
}
if a.Type != expectedPkg.Type {
t.Errorf("%s : bad package type: %+v, expected: %+v", a.Name, a.Type, expectedPkg.Type)
}
if len(a.Licenses) < len(expectedPkg.Licenses) {
t.Errorf("%s : bad package licenses count: '%+v'", a.Name, a.Licenses)
}
assert.True(t, ok)
assert.Equal(t, expectedPkg.Version, a.Version, "bad version")
assert.Equal(t, expectedPkg.Language, a.Language, "bad language")
assert.Equal(t, expectedPkg.Type, a.Type, "bad type")
assert.Equal(t, expectedPkg.Licenses, a.Licenses, "bad license count")
}
}
syft/pkg/cataloger/javascript/parse_yarn_lock.go
+4 -4
View File
@@ -35,14 +35,14 @@ const (
noVersion = ""
)
func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
// in the case we find yarn.lock files in the node_modules directories, skip those
// as the whole purpose of the lock file is for the specific dependencies of the project
if pathContainsNodeModulesDirectory(path) {
return nil, nil, nil
}
var packages []pkg.Package
var packages []*pkg.Package
scanner := bufio.NewScanner(reader)
parsedPackages := internal.NewStringSet()
currentPackage := noPackage
@@ -106,8 +106,8 @@ func findPackageVersion(line string) string {
return noVersion
}
func newYarnLockPackage(name, version string) pkg.Package {
return pkg.Package{
func newYarnLockPackage(name, version string) *pkg.Package {
return &pkg.Package{
Name: name,
Version: version,
Language: pkg.JavaScript,