Compare commits

...
5 Commits
23 changed files with 856 additions and 706 deletions
+6
View File
@@ -141,13 +141,19 @@ jobs:
- name: Build & publish release artifacts
run: make release
env:
# for mac signing and notarization...
QUILL_SIGN_P12: ${{ secrets.ANCHORE_APPLE_DEVELOPER_ID_CERT_CHAIN }}
QUILL_SIGN_PASSWORD: ${{ secrets.ANCHORE_APPLE_DEVELOPER_ID_CERT_PASS }}
QUILL_NOTARY_ISSUER: ${{ secrets.APPLE_NOTARY_ISSUER }}
QUILL_NOTARY_KEY_ID: ${{ secrets.APPLE_NOTARY_KEY_ID }}
QUILL_NOTARY_KEY: ${{ secrets.APPLE_NOTARY_KEY }}
# for creating the release (requires write access to packages and content)
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# for updating brew formula in anchore/homebrew-syft
GITHUB_BREW_TOKEN: ${{ secrets.ANCHORE_GIT_READ_TOKEN }}
# for updating the VERSION file in S3...
AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_AWS_SECRET_ACCESS_KEY }}
- uses: anchore/sbom-action@v0
continue-on-error: true
+1 -19
View File
@@ -1,7 +1,6 @@
package pkg
import (
"sort"
"sync"
"github.com/jinzhu/copier"
@@ -199,24 +198,7 @@ func (c *Catalog) Sorted(types ...Type) (pkgs []Package) {
pkgs = append(pkgs, p)
}
sort.SliceStable(pkgs, func(i, j int) bool {
if pkgs[i].Name == pkgs[j].Name {
if pkgs[i].Version == pkgs[j].Version {
iLocations := pkgs[i].Locations.ToSlice()
jLocations := pkgs[j].Locations.ToSlice()
if pkgs[i].Type == pkgs[j].Type && len(iLocations) > 0 && len(jLocations) > 0 {
if iLocations[0].String() == jLocations[0].String() {
// compare IDs as a final fallback
return pkgs[i].ID() < pkgs[j].ID()
}
return iLocations[0].String() < jLocations[0].String()
}
return pkgs[i].Type < pkgs[j].Type
}
return pkgs[i].Version < pkgs[j].Version
}
return pkgs[i].Name < pkgs[j].Name
})
Sort(pkgs)
return pkgs
}
+3 -4
View File
@@ -45,9 +45,7 @@ func newMonitor() (*progress.Manual, *progress.Manual) {
func Catalog(resolver source.FileResolver, release *linux.Release, catalogers ...pkg.Cataloger) (*pkg.Catalog, []artifact.Relationship, error) {
catalog := pkg.NewCatalog()
var allRelationships []artifact.Relationship
filesProcessed, packagesDiscovered := newMonitor()
// perform analysis, accumulating errors for each failed analysis
var errs error
for _, c := range catalogers {
@@ -70,7 +68,9 @@ func Catalog(resolver source.FileResolver, release *linux.Release, catalogers ..
p.CPEs = append(p.CPEs, cpe.Generate(p)...)
// generate PURL (note: this is excluded from package ID, so is safe to mutate)
p.PURL = pkg.URL(p, release)
if p.PURL == "" {
p.PURL = pkg.URL(p, release)
}
// if we were not able to identify the language we have an opportunity
// to try and get this value from the PURL. Worst case we assert that
@@ -86,7 +86,6 @@ func Catalog(resolver source.FileResolver, release *linux.Release, catalogers ..
} else {
allRelationships = append(allRelationships, owningRelationships...)
}
// add to catalog
catalog.Add(p)
}
+34 -21
View File
@@ -7,6 +7,7 @@ import (
"regexp"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -22,10 +23,10 @@ type Classifier struct {
// source location. If any of the patterns match, the file will be considered a candidate for parsing.
// If no patterns are provided, the reader is automatically considered a candidate.
FilepathPatterns []*regexp.Regexp
// EvidencePattern is a list of regular expressions that will be used to match against the file contents of a
// EvidencePatterns is a list of regular expressions that will be used to match against the file contents of a
// given file in the source location. If any of the patterns match, the file will be considered a candidate for parsing.
EvidencePatterns []*regexp.Regexp
// CPE is the CPE we want to match against
// CPEs are the specific CPEs we want to include for this binary with updated version information
CPEs []pkg.CPE
}
@@ -45,29 +46,41 @@ func (c Classifier) Examine(reader source.LocationReadCloser) (p *pkg.Package, r
}
var classifiedPackage *pkg.Package
for _, patternTemplate := range c.EvidencePatterns {
if !patternTemplate.Match(contents) {
for _, evidencePattern := range c.EvidencePatterns {
if !evidencePattern.Match(contents) {
continue
}
matchMetadata := internal.MatchNamedCaptureGroups(patternTemplate, string(contents))
if classifiedPackage == nil {
classifiedPackage = &pkg.Package{
Name: path.Base(reader.VirtualPath),
Version: matchMetadata["version"],
Language: pkg.Binary,
Locations: source.NewLocationSet(reader.Location),
Type: pkg.BinaryPkg,
CPEs: c.CPEs,
MetadataType: pkg.BinaryMetadataType,
Metadata: pkg.BinaryMetadata{
Classifier: c.Package,
RealPath: reader.RealPath,
VirtualPath: reader.VirtualPath,
},
}
break
matchMetadata := internal.MatchNamedCaptureGroups(evidencePattern, string(contents))
version, ok := matchMetadata["version"]
if !ok {
log.Debugf("no version found in binary from pattern %v", evidencePattern)
continue
}
var cpes []pkg.CPE
for _, cpe := range c.CPEs {
cpe.Version = version
if err == nil {
cpes = append(cpes, cpe)
}
}
classifiedPackage = &pkg.Package{
Name: path.Base(reader.VirtualPath),
Version: version,
Language: pkg.Binary,
Locations: source.NewLocationSet(reader.Location),
Type: pkg.BinaryPkg,
CPEs: cpes,
MetadataType: pkg.BinaryMetadataType,
Metadata: pkg.BinaryMetadata{
Classifier: c.Package,
RealPath: reader.RealPath,
VirtualPath: reader.VirtualPath,
},
}
break
}
return classifiedPackage, nil, nil
}
@@ -0,0 +1,96 @@
package generic
import (
"regexp"
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func Test_ClassifierCPEs(t *testing.T) {
tests := []struct {
name string
fixture string
classifier Classifier
cpes []string
}{
{
name: "no CPEs",
fixture: "test-fixtures/version.txt",
classifier: Classifier{
Package: "some-app",
FilepathPatterns: []*regexp.Regexp{
regexp.MustCompile(".*/version.txt"),
},
EvidencePatterns: []*regexp.Regexp{
regexp.MustCompile(`(?m)my-verison:(?P<version>[0-9.]+)`),
},
CPEs: []pkg.CPE{},
},
cpes: nil,
},
{
name: "one CPE",
fixture: "test-fixtures/version.txt",
classifier: Classifier{
Package: "some-app",
FilepathPatterns: []*regexp.Regexp{
regexp.MustCompile(".*/version.txt"),
},
EvidencePatterns: []*regexp.Regexp{
regexp.MustCompile(`(?m)my-verison:(?P<version>[0-9.]+)`),
},
CPEs: []pkg.CPE{
pkg.MustCPE("cpe:2.3:a:some:app:*:*:*:*:*:*:*:*"),
},
},
cpes: []string{
"cpe:2.3:a:some:app:1.8:*:*:*:*:*:*:*",
},
},
{
name: "multiple CPEs",
fixture: "test-fixtures/version.txt",
classifier: Classifier{
Package: "some-app",
FilepathPatterns: []*regexp.Regexp{
regexp.MustCompile(".*/version.txt"),
},
EvidencePatterns: []*regexp.Regexp{
regexp.MustCompile(`(?m)my-verison:(?P<version>[0-9.]+)`),
},
CPEs: []pkg.CPE{
pkg.MustCPE("cpe:2.3:a:some:app:*:*:*:*:*:*:*:*"),
pkg.MustCPE("cpe:2.3:a:some:apps:*:*:*:*:*:*:*:*"),
},
},
cpes: []string{
"cpe:2.3:a:some:app:1.8:*:*:*:*:*:*:*",
"cpe:2.3:a:some:apps:1.8:*:*:*:*:*:*:*",
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resolver := source.NewMockResolverForPaths(test.fixture)
locations, err := resolver.FilesByPath(test.fixture)
require.NoError(t, err)
require.Len(t, locations, 1)
location := locations[0]
readCloser, err := resolver.FileContentsByLocation(location)
require.NoError(t, err)
p, _, err := test.classifier.Examine(source.NewLocationReadCloser(location, readCloser))
require.NoError(t, err)
var cpes []string
for _, c := range p.CPEs {
cpes = append(cpes, pkg.CPEString(c))
}
require.Equal(t, test.cpes, cpes)
})
}
}
@@ -0,0 +1 @@
my-verison:1.8
@@ -40,6 +40,19 @@ func NewCatalogTester() *CatalogTester {
}
}
func (p *CatalogTester) FromDirectory(t *testing.T, path string) *CatalogTester {
t.Helper()
s, err := source.NewFromDirectory(path)
require.NoError(t, err)
resolver, err := s.FileResolver(source.AllLayersScope)
require.NoError(t, err)
p.resolver = resolver
return p
}
func (p *CatalogTester) FromFile(t *testing.T, path string) *CatalogTester {
t.Helper()
+8 -72
View File
@@ -4,88 +4,24 @@ Package javascript provides a concrete Cataloger implementation for JavaScript e
package javascript
import (
"encoding/json"
"io"
"path"
"strings"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)
// NewJavascriptPackageCataloger returns a new JavaScript cataloger object based on detection of npm based packages.
func NewJavascriptPackageCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/package.json": parsePackageJSON,
}
return common.NewGenericCataloger(nil, globParsers, "javascript-package-cataloger")
func NewJavascriptPackageCataloger() *generic.Cataloger {
return generic.NewCataloger("javascript-package-cataloger").
WithParserByGlobs(parsePackageJSON, "**/package.json")
}
// NewJavascriptLockCataloger returns a new Javascript cataloger object base on package lock files.
func NewJavascriptLockCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/package-lock.json": parsePackageLock,
"**/yarn.lock": parseYarnLock,
"**/pnpm-lock.yaml": parsePnpmLock,
}
return common.NewGenericCataloger(nil, globParsers, "javascript-lock-cataloger", addLicenses)
func NewJavascriptLockCataloger() *generic.Cataloger {
return generic.NewCataloger("javascript-lock-cataloger").
WithParserByGlobs(parsePackageLock, "**/package-lock.json").
WithParserByGlobs(parseYarnLock, "**/yarn.lock").
WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml")
}
func NewNodeBinaryCataloger() *generic.Cataloger {
return generic.NewCataloger("node-binary-cataloger").
WithParserByMimeTypes(parseNodeBinary, internal.ExecutableMIMETypeSet.List()...)
}
func addLicenses(resolver source.FileResolver, location source.Location, p *pkg.Package) error {
dir := path.Dir(location.RealPath)
pkgPath := []string{dir, "node_modules"}
pkgPath = append(pkgPath, strings.Split(p.Name, "/")...)
pkgPath = append(pkgPath, "package.json")
pkgFile := path.Join(pkgPath...)
locations, err := resolver.FilesByPath(pkgFile)
if err != nil {
log.Debugf("an error occurred attempting to read: %s - %+v", pkgFile, err)
return nil
}
if len(locations) == 0 {
return nil
}
for _, location := range locations {
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
log.Debugf("error getting file content reader for %s: %v", pkgFile, err)
return nil
}
contents, err := io.ReadAll(contentReader)
if err != nil {
log.Debugf("error reading file contents for %s: %v", pkgFile, err)
return nil
}
var pkgJSON packageJSON
err = json.Unmarshal(contents, &pkgJSON)
if err != nil {
log.Debugf("error parsing %s: %v", pkgFile, err)
return nil
}
licenses, err := pkgJSON.licensesFromJSON()
if err != nil {
log.Debugf("error getting licenses from %s: %v", pkgFile, err)
return nil
}
p.Licenses = append(p.Licenses, licenses...)
}
return nil
}
+97 -77
View File
@@ -3,100 +3,120 @@ package javascript
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
"github.com/anchore/syft/syft/source"
)
func Test_JavascriptCataloger(t *testing.T) {
expected := map[string]pkg.Package{
"@actions/core": {
Name: "@actions/core",
Version: "1.6.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
locationSet := source.NewLocationSet(source.NewLocation("package-lock.json"))
expectedPkgs := []pkg.Package{
{
Name: "@actions/core",
Version: "1.6.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/%40actions/core@1.6.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
},
"wordwrap": {
Name: "wordwrap",
Version: "0.0.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "ansi-regex",
Version: "3.0.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/ansi-regex@3.0.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"get-stdin": {
Name: "get-stdin",
Version: "5.0.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "cowsay",
Version: "1.4.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/cowsay@1.4.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
},
"minimist": {
Name: "minimist",
Version: "0.0.10",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "get-stdin",
Version: "5.0.1",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/get-stdin@5.0.1",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"optimist": {
Name: "optimist",
Version: "0.6.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "is-fullwidth-code-point",
Version: "2.0.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/is-fullwidth-code-point@2.0.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"string-width": {
Name: "string-width",
Version: "2.1.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "minimist",
Version: "0.0.10",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/minimist@0.0.10",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"strip-ansi": {
Name: "strip-ansi",
Version: "4.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "optimist",
Version: "0.6.1",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/optimist@0.6.1",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"strip-eof": {
Name: "wordwrap",
Version: "1.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "string-width",
Version: "2.1.1",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/string-width@2.1.1",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"ansi-regex": {
Name: "ansi-regex",
Version: "3.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "strip-ansi",
Version: "4.0.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/strip-ansi@4.0.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"is-fullwidth-code-point": {
Name: "is-fullwidth-code-point",
Version: "2.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "strip-eof",
Version: "1.0.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/strip-eof@1.0.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"cowsay": {
Name: "cowsay",
Version: "1.4.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
{
Name: "wordwrap",
Version: "0.0.3",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/wordwrap@0.0.3",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
}
s, err := source.NewFromDirectory("test-fixtures/pkg-lock")
require.NoError(t, err)
pkgtest.NewCatalogTester().
FromDirectory(t, "test-fixtures/pkg-lock").
Expects(expectedPkgs, nil).
TestCataloger(t, NewJavascriptLockCataloger())
resolver, err := s.FileResolver(source.AllLayersScope)
require.NoError(t, err)
actual, _, err := NewJavascriptLockCataloger().Catalog(resolver)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}
var pkgs []*pkg.Package
for _, p := range actual {
p2 := p
pkgs = append(pkgs, &p2)
}
assertPkgsEqual(t, pkgs, expected)
}
+176
View File
@@ -0,0 +1,176 @@
package javascript
import (
"encoding/json"
"io"
"path"
"strings"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func newPackageJSONPackage(u packageJSON, locations ...source.Location) pkg.Package {
licenses, err := u.licensesFromJSON()
if err != nil {
log.Warnf("unable to extract licenses from javascript package.json: %+v", err)
}
p := pkg.Package{
Name: u.Name,
Version: u.Version,
Licenses: licenses,
PURL: packageURL(u.Name, u.Version),
Locations: source.NewLocationSet(locations...),
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageJSONMetadataType,
Metadata: pkg.NpmPackageJSONMetadata{
Name: u.Name,
Version: u.Version,
Author: u.Author.AuthorString(),
Homepage: u.Homepage,
URL: u.Repository.URL,
Licenses: licenses,
Private: u.Private,
},
}
p.SetID()
return p
}
func newPackageLockPackage(resolver source.FileResolver, location source.Location, name string, u lockDependency, licenseMap map[string]string) pkg.Package {
var sb strings.Builder
sb.WriteString(u.Resolved)
sb.WriteString(u.Integrity)
var licenses []string
if l, exists := licenseMap[sb.String()]; exists {
licenses = append(licenses, l)
}
return finalizeLockPkg(
resolver,
location,
pkg.Package{
Name: name,
Version: u.Version,
Locations: source.NewLocationSet(location),
PURL: packageURL(name, u.Version),
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: licenses,
},
)
}
func newPnpmPackage(resolver source.FileResolver, location source.Location, name, version string) pkg.Package {
return finalizeLockPkg(
resolver,
location,
pkg.Package{
Name: name,
Version: version,
Locations: source.NewLocationSet(location),
PURL: packageURL(name, version),
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
)
}
func newYarnLockPackage(resolver source.FileResolver, location source.Location, name, version string) pkg.Package {
return finalizeLockPkg(
resolver,
location,
pkg.Package{
Name: name,
Version: version,
Locations: source.NewLocationSet(location),
PURL: packageURL(name, version),
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
)
}
func finalizeLockPkg(resolver source.FileResolver, location source.Location, p pkg.Package) pkg.Package {
p.Licenses = append(p.Licenses, addLicenses(p.Name, resolver, location)...)
p.SetID()
return p
}
func addLicenses(name string, resolver source.FileResolver, location source.Location) (allLicenses []string) {
if resolver == nil {
return allLicenses
}
dir := path.Dir(location.RealPath)
pkgPath := []string{dir, "node_modules"}
pkgPath = append(pkgPath, strings.Split(name, "/")...)
pkgPath = append(pkgPath, "package.json")
pkgFile := path.Join(pkgPath...)
locations, err := resolver.FilesByPath(pkgFile)
if err != nil {
log.Debugf("an error occurred attempting to read: %s - %+v", pkgFile, err)
return allLicenses
}
if len(locations) == 0 {
return allLicenses
}
for _, l := range locations {
contentReader, err := resolver.FileContentsByLocation(l)
if err != nil {
log.Debugf("error getting file content reader for %s: %v", pkgFile, err)
return allLicenses
}
contents, err := io.ReadAll(contentReader)
if err != nil {
log.Debugf("error reading file contents for %s: %v", pkgFile, err)
return allLicenses
}
var pkgJSON packageJSON
err = json.Unmarshal(contents, &pkgJSON)
if err != nil {
log.Debugf("error parsing %s: %v", pkgFile, err)
return allLicenses
}
licenses, err := pkgJSON.licensesFromJSON()
if err != nil {
log.Debugf("error getting licenses from %s: %v", pkgFile, err)
return allLicenses
}
allLicenses = append(allLicenses, licenses...)
}
return allLicenses
}
// packageURL returns the PURL for the specific NPM package (see https://github.com/package-url/purl-spec)
func packageURL(name, version string) string {
var namespace string
fields := strings.SplitN(name, "/", 2)
if len(fields) > 1 {
namespace = fields[0]
name = fields[1]
}
return packageurl.NewPackageURL(
packageurl.TypeNPM,
namespace,
name,
version,
nil,
"",
).ToString()
}
@@ -0,0 +1,58 @@
package javascript
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/packageurl-go"
)
func Test_packageURL(t *testing.T) {
tests := []struct {
testName string
name string
version string
expected string
namespace string
}{
{
testName: "no namespace",
name: "arborist",
version: "2.6.2",
expected: "pkg:npm/arborist@2.6.2",
},
{
testName: "split by namespace",
name: "npmcli/arborist",
version: "2.6.2",
expected: "pkg:npm/npmcli/arborist@2.6.2",
namespace: "npmcli",
},
{
testName: "encoding @ symobl",
name: "@npmcli/arborist",
version: "2.6.2",
expected: "pkg:npm/%40npmcli/arborist@2.6.2",
namespace: "@npmcli",
},
}
for _, tt := range tests {
t.Run(tt.testName, func(t *testing.T) {
actual := packageURL(tt.name, tt.version)
assert.Equal(t, tt.expected, actual)
decoded, err := packageurl.FromString(actual)
require.NoError(t, err)
assert.Equal(t, tt.namespace, decoded.Namespace)
if decoded.Namespace != "" {
assert.Equal(t, tt.name, fmt.Sprintf("%s/%s", decoded.Namespace, decoded.Name))
} else {
assert.Equal(t, tt.name, decoded.Name)
}
assert.Equal(t, tt.version, decoded.Version)
})
}
}
@@ -36,8 +36,8 @@ func parseNodeBinary(_ source.FileResolver, _ *generic.Environment, reader sourc
// TODO add node specific metadata to the packages to help with vulnerability matching
if p != nil {
p.Language = pkg.JavaScript
p.SetID()
return []pkg.Package{*p}, nil, nil
}
p.SetID()
return nil, nil, nil
}
@@ -13,11 +13,12 @@ import (
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)
// integrity check
var _ common.ParserFn = parsePackageJSON
var _ generic.Parser = parsePackageJSON
// packageJSON represents a JavaScript package.json file
type packageJSON struct {
@@ -49,6 +50,32 @@ type repository struct {
// ---> name: "Isaac Z. Schlueter" email: "i@izs.me" url: "http://blog.izs.me"
var authorPattern = regexp.MustCompile(`^\s*(?P<name>[^<(]*)(\s+<(?P<email>.*)>)?(\s\((?P<url>.*)\))?\s*$`)
// parsePackageJSON parses a package.json and returns the discovered JavaScript packages.
func parsePackageJSON(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
var pkgs []pkg.Package
dec := json.NewDecoder(reader)
for {
var p packageJSON
if err := dec.Decode(&p); err == io.EOF {
break
} else if err != nil {
return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err)
}
if !p.hasNameAndVersionValues() {
log.Debugf("encountered package.json file without a name and/or version field, ignoring (path=%q)", reader.AccessPath())
return nil, nil, nil
}
pkgs = append(pkgs, newPackageJSONPackage(p, reader.Location))
}
pkg.Sort(pkgs)
return pkgs, nil, nil
}
func (a *author) UnmarshalJSON(b []byte) error {
var authorStr string
var fields map[string]string
@@ -172,55 +199,6 @@ func licensesFromJSON(b []byte) ([]license, error) {
return nil, errors.New("unmarshal failed")
}
// parsePackageJSON parses a package.json and returns the discovered JavaScript packages.
func parsePackageJSON(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
var packages []*pkg.Package
dec := json.NewDecoder(reader)
for {
var p packageJSON
if err := dec.Decode(&p); err == io.EOF {
break
} else if err != nil {
return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err)
}
if !p.hasNameAndVersionValues() {
log.Debugf("encountered package.json file without a name and/or version field, ignoring (path=%q)", path)
return nil, nil, nil
}
packages = append(packages, newPackageJSONPackage(p))
}
return packages, nil, nil
}
func newPackageJSONPackage(p packageJSON) *pkg.Package {
licenses, err := p.licensesFromJSON()
if err != nil {
log.Warnf("unable to extract licenses from javascript package.json: %+v", err)
}
return &pkg.Package{
Name: p.Name,
Version: p.Version,
Licenses: licenses,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageJSONMetadataType,
Metadata: pkg.NpmPackageJSONMetadata{
Name: p.Name,
Version: p.Version,
Author: p.Author.AuthorString(),
Homepage: p.Homepage,
URL: p.Repository.URL,
Licenses: licenses,
Private: p.Private,
},
}
}
func (p packageJSON) hasNameAndVersionValues() bool {
return p.Name != "" && p.Version != ""
}
@@ -1,13 +1,13 @@
package javascript
import (
"os"
"testing"
"github.com/go-test/deep"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
"github.com/anchore/syft/syft/source"
)
func TestParsePackageJSON(t *testing.T) {
@@ -20,6 +20,7 @@ func TestParsePackageJSON(t *testing.T) {
ExpectedPkg: pkg.Package{
Name: "npm",
Version: "6.14.6",
PURL: "pkg:npm/npm@6.14.6",
Type: pkg.NpmPkg,
Licenses: []string{"Artistic-2.0"},
Language: pkg.JavaScript,
@@ -39,6 +40,7 @@ func TestParsePackageJSON(t *testing.T) {
ExpectedPkg: pkg.Package{
Name: "npm",
Version: "6.14.6",
PURL: "pkg:npm/npm@6.14.6",
Type: pkg.NpmPkg,
Licenses: []string{"ISC"},
Language: pkg.JavaScript,
@@ -58,6 +60,7 @@ func TestParsePackageJSON(t *testing.T) {
ExpectedPkg: pkg.Package{
Name: "npm",
Version: "6.14.6",
PURL: "pkg:npm/npm@6.14.6",
Type: pkg.NpmPkg,
Licenses: []string{"MIT", "Apache-2.0"},
Language: pkg.JavaScript,
@@ -77,6 +80,7 @@ func TestParsePackageJSON(t *testing.T) {
ExpectedPkg: pkg.Package{
Name: "npm",
Version: "6.14.6",
PURL: "pkg:npm/npm@6.14.6",
Type: pkg.NpmPkg,
Licenses: nil,
Language: pkg.JavaScript,
@@ -96,6 +100,7 @@ func TestParsePackageJSON(t *testing.T) {
ExpectedPkg: pkg.Package{
Name: "npm",
Version: "6.14.6",
PURL: "pkg:npm/npm@6.14.6",
Type: pkg.NpmPkg,
Licenses: []string{},
Language: pkg.JavaScript,
@@ -115,6 +120,7 @@ func TestParsePackageJSON(t *testing.T) {
ExpectedPkg: pkg.Package{
Name: "npm",
Version: "6.14.6",
PURL: "pkg:npm/npm@6.14.6",
Type: pkg.NpmPkg,
Licenses: []string{"Artistic-2.0"},
Language: pkg.JavaScript,
@@ -134,6 +140,7 @@ func TestParsePackageJSON(t *testing.T) {
ExpectedPkg: pkg.Package{
Name: "function-bind",
Version: "1.1.1",
PURL: "pkg:npm/function-bind@1.1.1",
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
Language: pkg.JavaScript,
@@ -153,6 +160,7 @@ func TestParsePackageJSON(t *testing.T) {
ExpectedPkg: pkg.Package{
Name: "npm",
Version: "6.14.6",
PURL: "pkg:npm/npm@6.14.6",
Type: pkg.NpmPkg,
Licenses: []string{"Artistic-2.0"},
Language: pkg.JavaScript,
@@ -172,46 +180,16 @@ func TestParsePackageJSON(t *testing.T) {
for _, test := range tests {
t.Run(test.Fixture, func(t *testing.T) {
fixture, err := os.Open(test.Fixture)
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, _, err := parsePackageJSON("", fixture)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}
if len(actual) != 1 {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=1", len(actual))
}
for _, d := range deep.Equal(actual[0], &test.ExpectedPkg) {
t.Errorf("diff: %+v", d)
}
test.ExpectedPkg.Locations.Add(source.NewLocation(test.Fixture))
pkgtest.TestFileParser(t, test.Fixture, parsePackageJSON, []pkg.Package{test.ExpectedPkg}, nil)
})
}
}
func TestParsePackageJSON_Partial(t *testing.T) { // see https://github.com/anchore/syft/issues/311
const fixtureFile = "test-fixtures/pkg-json/package-partial.json"
fixture, err := os.Open(fixtureFile)
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
// TODO: no relationships are under test yet
actual, _, err := parsePackageJSON("", fixture)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}
if actualCount := len(actual); actualCount != 0 {
t.Errorf("no packages should've been returned (but got %d packages)", actualCount)
}
pkgtest.TestFileParser(t, fixtureFile, parsePackageJSON, nil, nil)
}
func Test_pathContainsNodeModulesDirectory(t *testing.T) {
@@ -8,28 +8,29 @@ import (
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)
// integrity check
var _ common.ParserFn = parsePackageLock
var _ generic.Parser = parsePackageLock
// PackageLock represents a JavaScript package.lock json file
type PackageLock struct {
// packageLock represents a JavaScript package.lock json file
type packageLock struct {
Requires bool `json:"requires"`
LockfileVersion int `json:"lockfileVersion"`
Dependencies map[string]Dependency
Packages map[string]Package
Dependencies map[string]lockDependency
Packages map[string]lockPackage
}
// Dependency represents a single package dependency listed in the package.lock json file
type Dependency struct {
// lockDependency represents a single package dependency listed in the package.lock json file
type lockDependency struct {
Version string `json:"version"`
Resolved string `json:"resolved"`
Integrity string `json:"integrity"`
}
type Package struct {
type lockPackage struct {
Version string `json:"version"`
Resolved string `json:"resolved"`
Integrity string `json:"integrity"`
@@ -37,18 +38,18 @@ type Package struct {
}
// parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages.
func parsePackageLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
func parsePackageLock(resolver source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
// in the case we find package-lock.json files in the node_modules directories, skip those
// as the whole purpose of the lock file is for the specific dependencies of the root project
if pathContainsNodeModulesDirectory(path) {
if pathContainsNodeModulesDirectory(reader.AccessPath()) {
return nil, nil, nil
}
var packages []*pkg.Package
var pkgs []pkg.Package
dec := json.NewDecoder(reader)
for {
var lock PackageLock
var lock packageLock
if err := dec.Decode(&lock); err == io.EOF {
break
} else if err != nil {
@@ -63,22 +64,11 @@ func parsePackageLock(path string, reader io.Reader) ([]*pkg.Package, []artifact
}
for name, pkgMeta := range lock.Dependencies {
var sb strings.Builder
sb.WriteString(pkgMeta.Resolved)
sb.WriteString(pkgMeta.Integrity)
var licenses []string
if license, exists := licenseMap[sb.String()]; exists {
licenses = append(licenses, license)
}
packages = append(packages, &pkg.Package{
Name: name,
Version: pkgMeta.Version,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: licenses,
})
pkgs = append(pkgs, newPackageLockPackage(resolver, reader.Location, name, pkgMeta, licenseMap))
}
}
return packages, nil, nil
pkg.Sort(pkgs)
return pkgs, nil, nil
}
@@ -1,157 +1,142 @@
package javascript
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
"github.com/anchore/syft/syft/source"
)
func assertPkgsEqual(t *testing.T, actual []*pkg.Package, expected map[string]pkg.Package) {
t.Helper()
if len(actual) != len(expected) {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
}
for _, a := range actual {
expectedPkg, ok := expected[a.Name]
assert.True(t, ok)
assert.Equal(t, expectedPkg.Version, a.Version, "bad version")
assert.Equal(t, expectedPkg.Language, a.Language, "bad language")
assert.Equal(t, expectedPkg.Type, a.Type, "bad type")
assert.Equal(t, expectedPkg.Licenses, a.Licenses, "bad license count")
}
}
func TestParsePackageLock(t *testing.T) {
expected := map[string]pkg.Package{
"@actions/core": {
var expectedRelationships []artifact.Relationship
expectedPkgs := []pkg.Package{
{
Name: "@actions/core",
Version: "1.6.0",
PURL: "pkg:npm/%40actions/core@1.6.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"wordwrap": {
Name: "wordwrap",
Version: "0.0.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"get-stdin": {
Name: "get-stdin",
Version: "5.0.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"minimist": {
Name: "minimist",
Version: "0.0.10",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"optimist": {
Name: "optimist",
Version: "0.6.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"string-width": {
Name: "string-width",
Version: "2.1.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"strip-ansi": {
Name: "strip-ansi",
Version: "4.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"strip-eof": {
Name: "wordwrap",
Version: "1.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"ansi-regex": {
{
Name: "ansi-regex",
Version: "3.0.0",
PURL: "pkg:npm/ansi-regex@3.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"is-fullwidth-code-point": {
Name: "is-fullwidth-code-point",
Version: "2.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"cowsay": {
{
Name: "cowsay",
Version: "1.4.0",
PURL: "pkg:npm/cowsay@1.4.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "get-stdin",
Version: "5.0.1",
PURL: "pkg:npm/get-stdin@5.0.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "is-fullwidth-code-point",
Version: "2.0.0",
PURL: "pkg:npm/is-fullwidth-code-point@2.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "minimist",
Version: "0.0.10",
PURL: "pkg:npm/minimist@0.0.10",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "optimist",
Version: "0.6.1",
PURL: "pkg:npm/optimist@0.6.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "string-width",
Version: "2.1.1",
PURL: "pkg:npm/string-width@2.1.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "strip-ansi",
Version: "4.0.0",
PURL: "pkg:npm/strip-ansi@4.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "strip-eof",
Version: "1.0.0",
PURL: "pkg:npm/strip-eof@1.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "wordwrap",
Version: "0.0.3",
PURL: "pkg:npm/wordwrap@0.0.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
}
fixture, err := os.Open("test-fixtures/pkg-lock/package-lock.json")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
fixture := "test-fixtures/pkg-lock/package-lock.json"
for i := range expectedPkgs {
expectedPkgs[i].Locations.Add(source.NewLocation(fixture))
}
// TODO: no relationships are under test yet
actual, _, err := parsePackageLock(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}
assertPkgsEqual(t, actual, expected)
pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
}
func TestParsePackageLockV2(t *testing.T) {
expected := map[string]pkg.Package{
"@types/prop-types": {
fixture := "test-fixtures/pkg-lock/package-lock-2.json"
var expectedRelationships []artifact.Relationship
expectedPkgs := []pkg.Package{
{
Name: "@types/prop-types",
Version: "15.7.5",
PURL: "pkg:npm/%40types/prop-types@15.7.5",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
},
"@types/react": {
Name: "@types/prop-types",
{
Name: "@types/react",
Version: "18.0.17",
PURL: "pkg:npm/%40types/react@18.0.17",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
},
"@types/scheduler": {
{
Name: "@types/scheduler",
Version: "0.16.2",
PURL: "pkg:npm/%40types/scheduler@0.16.2",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
},
"csstype": {
{
Name: "csstype",
Version: "3.1.0",
PURL: "pkg:npm/csstype@3.1.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
},
}
fixture, err := os.Open("test-fixtures/pkg-lock/package-lock-2.json")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
for i := range expectedPkgs {
expectedPkgs[i].Locations.Add(source.NewLocation(fixture))
}
actual, _, err := parsePackageLock(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}
assertPkgsEqual(t, actual, expected)
pkgtest.TestFileParser(t, fixture, parsePackageLock, expectedPkgs, expectedRelationships)
}
@@ -8,23 +8,24 @@ import (
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)
// integrity check
var _ common.ParserFn = parsePnpmLock
var _ generic.Parser = parsePnpmLock
type pnpmLockYaml struct {
Dependencies map[string]string `json:"dependencies"`
}
func parsePnpmLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
func parsePnpmLock(resolver source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
bytes, err := io.ReadAll(reader)
if err != nil {
return nil, nil, fmt.Errorf("failed to load pnpm-lock.yaml file: %w", err)
}
var pkgs []*pkg.Package
var pkgs []pkg.Package
var lockFile pnpmLockYaml
if err := yaml.Unmarshal(bytes, &lockFile); err != nil {
@@ -32,13 +33,10 @@ func parsePnpmLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re
}
for name, version := range lockFile.Dependencies {
pkgs = append(pkgs, &pkg.Package{
Name: name,
Version: version,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
})
pkgs = append(pkgs, newPnpmPackage(resolver, reader.Location, name, version))
}
pkg.Sort(pkgs)
return pkgs, nil, nil
}
@@ -1,59 +1,46 @@
package javascript
import (
"os"
"sort"
"testing"
"github.com/go-test/deep"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
"github.com/anchore/syft/syft/source"
)
func fixtureP(str string) *string {
return &str
}
func TestParsePnpmLock(t *testing.T) {
expected := []*pkg.Package{
var expectedRelationships []artifact.Relationship
fixture := "test-fixtures/pnpm/pnpm-lock.yaml"
locationSet := source.NewLocationSet(source.NewLocation(fixture))
expectedPkgs := []pkg.Package{
{
Name: "nanoid",
Version: "3.3.4",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Name: "nanoid",
Version: "3.3.4",
PURL: "pkg:npm/nanoid@3.3.4",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "picocolors",
Version: "1.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Name: "picocolors",
Version: "1.0.0",
PURL: "pkg:npm/picocolors@1.0.0",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "source-map-js",
Version: "1.0.2",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Name: "source-map-js",
Version: "1.0.2",
PURL: "pkg:npm/source-map-js@1.0.2",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
}
fixture, err := os.Open("test-fixtures/pnpm/pnpm-lock.yaml")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
// TODO: no relationships are under test yet
actual, _, err := parsePnpmLock(fixture.Name(), fixture)
if err != nil {
t.Error(err)
}
// we have to sort this for expected to match actual since yaml maps are unordered
sort.Slice(actual, func(p, q int) bool {
return actual[p].Name < actual[q].Name
})
differences := deep.Equal(expected, actual)
if differences != nil {
t.Errorf("returned package list differed from expectation: %+v", differences)
}
pkgtest.TestFileParser(t, fixture, parsePnpmLock, expectedPkgs, expectedRelationships)
}
@@ -3,17 +3,17 @@ package javascript
import (
"bufio"
"fmt"
"io"
"regexp"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)
// integrity check
var _ common.ParserFn = parseYarnLock
var _ generic.Parser = parseYarnLock
var (
// packageNameExp matches the name of the dependency in yarn.lock
@@ -42,14 +42,14 @@ const (
noVersion = ""
)
func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
func parseYarnLock(resolver source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
// in the case we find yarn.lock files in the node_modules directories, skip those
// as the whole purpose of the lock file is for the specific dependencies of the project
if pathContainsNodeModulesDirectory(path) {
if pathContainsNodeModulesDirectory(reader.AccessPath()) {
return nil, nil, nil
}
var packages []*pkg.Package
var pkgs []pkg.Package
scanner := bufio.NewScanner(reader)
parsedPackages := internal.NewStringSet()
currentPackage := noPackage
@@ -61,7 +61,7 @@ func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re
if packageName := findPackageName(line); packageName != noPackage {
// When we find a new package, check if we have unsaved identifiers
if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Contains(currentPackage+"@"+currentVersion) {
packages = append(packages, newYarnLockPackage(currentPackage, currentVersion))
pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, currentPackage, currentVersion))
parsedPackages.Add(currentPackage + "@" + currentVersion)
}
@@ -69,7 +69,7 @@ func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re
} else if version := findPackageVersion(line); version != noVersion {
currentVersion = version
} else if packageName, version := findPackageAndVersion(line); packageName != noPackage && version != noVersion && !parsedPackages.Contains(packageName+"@"+version) {
packages = append(packages, newYarnLockPackage(packageName, version))
pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, packageName, version))
parsedPackages.Add(packageName + "@" + version)
// Cleanup to indicate no unsaved identifiers
@@ -80,7 +80,7 @@ func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re
// check if we have valid unsaved data after end-of-file has reached
if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Contains(currentPackage+"@"+currentVersion) {
packages = append(packages, newYarnLockPackage(currentPackage, currentVersion))
pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, currentPackage, currentVersion))
parsedPackages.Add(currentPackage + "@" + currentVersion)
}
@@ -88,7 +88,9 @@ func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Re
return nil, nil, fmt.Errorf("failed to parse yarn.lock file: %w", err)
}
return packages, nil, nil
pkg.Sort(pkgs)
return pkgs, nil, nil
}
func findPackageName(line string) string {
@@ -114,12 +116,3 @@ func findPackageAndVersion(line string) (string, string) {
return noPackage, noVersion
}
func newYarnLockPackage(name, version string) *pkg.Package {
return &pkg.Package{
Name: name,
Version: version,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
}
}
@@ -1,170 +1,184 @@
package javascript
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
"github.com/anchore/syft/syft/source"
)
func TestParseYarnBerry(t *testing.T) {
expected := map[string]pkg.Package{
"@babel/code-frame": {
Name: "@babel/code-frame",
Version: "7.10.4",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
var expectedRelationships []artifact.Relationship
fixture := "test-fixtures/yarn-berry/yarn.lock"
locations := source.NewLocationSet(source.NewLocation(fixture))
expectedPkgs := []pkg.Package{
{
Name: "@babel/code-frame",
Version: "7.10.4",
Locations: locations,
PURL: "pkg:npm/%40babel/code-frame@7.10.4",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"@types/minimatch": {
Name: "@types/minimatch",
Version: "3.0.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "@types/minimatch",
Version: "3.0.3",
Locations: locations,
PURL: "pkg:npm/%40types/minimatch@3.0.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"@types/qs": {
Name: "@types/qs",
Version: "6.9.4",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "@types/qs",
Version: "6.9.4",
Locations: locations,
PURL: "pkg:npm/%40types/qs@6.9.4",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"ajv": {
Name: "ajv",
Version: "6.12.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "ajv",
Version: "6.12.3",
Locations: locations,
PURL: "pkg:npm/ajv@6.12.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"atob": {
Name: "atob",
Version: "2.1.2",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "asn1.js",
Version: "4.10.1",
Locations: locations,
PURL: "pkg:npm/asn1.js@4.10.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"aws-sdk": {
Name: "aws-sdk",
Version: "2.706.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "atob",
Version: "2.1.2",
Locations: locations,
PURL: "pkg:npm/atob@2.1.2",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"jhipster-core": {
Name: "jhipster-core",
Version: "7.3.4",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "aws-sdk",
Version: "2.706.0",
PURL: "pkg:npm/aws-sdk@2.706.0",
Locations: locations,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"asn1.js": {
Name: "asn1.js",
Version: "4.10.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "c0n-fab_u.laTION",
Version: "7.7.7",
Locations: locations,
PURL: "pkg:npm/c0n-fab_u.laTION@7.7.7",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"c0n-fab_u.laTION": {
Name: "c0n-fab_u.laTION",
Version: "7.7.7",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "jhipster-core",
Version: "7.3.4",
Locations: locations,
PURL: "pkg:npm/jhipster-core@7.3.4",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
}
testFixtures := []string{
"test-fixtures/yarn-berry/yarn.lock",
}
for _, file := range testFixtures {
file := file
t.Run(file, func(t *testing.T) {
t.Parallel()
pkgtest.TestFileParser(t, fixture, parseYarnLock, expectedPkgs, expectedRelationships)
fixture, err := os.Open(file)
require.NoError(t, err)
// TODO: no relationships are under test yet
actual, _, err := parseYarnLock(fixture.Name(), fixture)
require.NoError(t, err)
assertPkgsEqual(t, actual, expected)
})
}
}
func TestParseYarnLock(t *testing.T) {
expected := map[string]pkg.Package{
"@babel/code-frame": {
Name: "@babel/code-frame",
Version: "7.10.4",
var expectedRelationships []artifact.Relationship
fixture := "test-fixtures/yarn/yarn.lock"
locations := source.NewLocationSet(source.NewLocation(fixture))
expectedPkgs := []pkg.Package{
{
Name: "@babel/code-frame",
Version: "7.10.4",
Locations: locations,
PURL: "pkg:npm/%40babel/code-frame@7.10.4",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "@types/minimatch",
Version: "3.0.3",
Locations: locations,
PURL: "pkg:npm/%40types/minimatch@3.0.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "@types/qs",
Version: "6.9.4",
Locations: locations,
PURL: "pkg:npm/%40types/qs@6.9.4",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "ajv",
Version: "6.12.3",
Locations: locations,
PURL: "pkg:npm/ajv@6.12.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "asn1.js",
Version: "4.10.1",
Locations: locations,
PURL: "pkg:npm/asn1.js@4.10.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
{
Name: "atob",
Version: "2.1.2",
Locations: locations,
PURL: "pkg:npm/atob@2.1.2",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"@types/minimatch": {
Name: "@types/minimatch",
Version: "3.0.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "aws-sdk",
Version: "2.706.0",
Locations: locations,
PURL: "pkg:npm/aws-sdk@2.706.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"@types/qs": {
Name: "@types/qs",
Version: "6.9.4",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "jhipster-core",
Version: "7.3.4",
Locations: locations,
PURL: "pkg:npm/jhipster-core@7.3.4",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"ajv": {
Name: "ajv",
Version: "6.12.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"atob": {
Name: "atob",
Version: "2.1.2",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"aws-sdk": {
Name: "aws-sdk",
Version: "2.706.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"jhipster-core": {
Name: "jhipster-core",
Version: "7.3.4",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"asn1.js": {
Name: "asn1.js",
Version: "4.10.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"something-i-made-up": {
Name: "something-i-made-up",
Version: "7.7.7",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "something-i-made-up",
Version: "7.7.7",
Locations: locations,
PURL: "pkg:npm/something-i-made-up@7.7.7",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
}
testFixtures := []string{
"test-fixtures/yarn/yarn.lock",
}
pkgtest.TestFileParser(t, fixture, parseYarnLock, expectedPkgs, expectedRelationships)
for _, file := range testFixtures {
file := file
t.Run(file, func(t *testing.T) {
t.Parallel()
fixture, err := os.Open(file)
require.NoError(t, err)
// TODO: no relationships are under test yet
actual, _, err := parseYarnLock(fixture.Name(), fixture)
require.NoError(t, err)
assertPkgsEqual(t, actual, expected)
})
}
}
func TestParseYarnFindPackageNames(t *testing.T) {
@@ -227,7 +241,6 @@ func TestParseYarnFindPackageNames(t *testing.T) {
}
for _, test := range tests {
test := test
t.Run(test.expected, func(t *testing.T) {
t.Parallel()
actual := findPackageName(test.line)
@@ -316,7 +329,6 @@ func TestParseYarnFindPackageVersions(t *testing.T) {
}
for _, test := range tests {
test := test
t.Run(test.expected, func(t *testing.T) {
t.Parallel()
actual := findPackageVersion(test.line)
-30
View File
@@ -1,14 +1,5 @@
package pkg
import (
"strings"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/linux"
)
var _ urlIdentifier = (*NpmPackageJSONMetadata)(nil)
// NpmPackageJSONMetadata holds extra information that is used in pkg.Package
type NpmPackageJSONMetadata struct {
Name string `mapstructure:"name" json:"name"`
@@ -21,24 +12,3 @@ type NpmPackageJSONMetadata struct {
URL string `mapstructure:"url" json:"url"`
Private bool `mapstructure:"private" json:"private"`
}
// PackageURL returns the PURL for the specific NPM package (see https://github.com/package-url/purl-spec)
func (p NpmPackageJSONMetadata) PackageURL(_ *linux.Release) string {
var namespace string
name := p.Name
fields := strings.SplitN(p.Name, "/", 2)
if len(fields) > 1 {
namespace = fields[0]
name = fields[1]
}
return packageurl.NewPackageURL(
packageurl.TypeNPM,
namespace,
name,
p.Version,
nil,
"",
).ToString()
}
@@ -1,63 +0,0 @@
package pkg
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/packageurl-go"
)
func TestNpmPackageJSONMetadata_PackageURL(t *testing.T) {
tests := []struct {
name string
metadata NpmPackageJSONMetadata
expected string
namespace string
}{
{
name: "no namespace",
metadata: NpmPackageJSONMetadata{
Name: "arborist",
Version: "2.6.2",
},
expected: "pkg:npm/arborist@2.6.2",
},
{
name: "split by namespace",
metadata: NpmPackageJSONMetadata{
Name: "npmcli/arborist",
Version: "2.6.2",
},
expected: "pkg:npm/npmcli/arborist@2.6.2",
namespace: "npmcli",
},
{
name: "encoding @ symobl",
metadata: NpmPackageJSONMetadata{
Name: "@npmcli/arborist",
Version: "2.6.2",
},
expected: "pkg:npm/%40npmcli/arborist@2.6.2",
namespace: "@npmcli",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
actual := tt.metadata.PackageURL(nil)
assert.Equal(t, tt.expected, actual)
decoded, err := packageurl.FromString(actual)
require.NoError(t, err)
assert.Equal(t, tt.namespace, decoded.Namespace)
if decoded.Namespace != "" {
assert.Equal(t, tt.metadata.Name, fmt.Sprintf("%s/%s", decoded.Namespace, decoded.Name))
} else {
assert.Equal(t, tt.metadata.Name, decoded.Name)
}
assert.Equal(t, tt.metadata.Version, decoded.Version)
})
}
}
+22
View File
@@ -5,6 +5,7 @@ package pkg
import (
"fmt"
"sort"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
@@ -80,3 +81,24 @@ func (p *Package) merge(other Package) error {
func IsValid(p *Package) bool {
return p != nil && p.Name != ""
}
func Sort(pkgs []Package) {
sort.SliceStable(pkgs, func(i, j int) bool {
if pkgs[i].Name == pkgs[j].Name {
if pkgs[i].Version == pkgs[j].Version {
iLocations := pkgs[i].Locations.ToSlice()
jLocations := pkgs[j].Locations.ToSlice()
if pkgs[i].Type == pkgs[j].Type && len(iLocations) > 0 && len(jLocations) > 0 {
if iLocations[0].String() == jLocations[0].String() {
// compare IDs as a final fallback
return pkgs[i].ID() < pkgs[j].ID()
}
return iLocations[0].String() < jLocations[0].String()
}
return pkgs[i].Type < pkgs[j].Type
}
return pkgs[i].Version < pkgs[j].Version
}
return pkgs[i].Name < pkgs[j].Name
})
}