mirror of
https://github.com/anchore/syft.git
synced 2026-10-13 14:47:22 +02:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7c8aad9e1b | ||
|
|
987ba83674 | ||
|
|
f11377fe30 | ||
|
|
12f36420dd | ||
|
|
f851085668 | ||
|
|
da62a82413 |
+2
-2
@@ -26,7 +26,7 @@ tools:
|
||||
# used for linting
|
||||
- name: golangci-lint
|
||||
version:
|
||||
want: v1.64.8
|
||||
want: v2.0.2
|
||||
method: github-release
|
||||
with:
|
||||
repo: golangci/golangci-lint
|
||||
@@ -58,7 +58,7 @@ tools:
|
||||
# used to release all artifacts
|
||||
- name: goreleaser
|
||||
version:
|
||||
want: v2.8.1
|
||||
want: v2.8.2
|
||||
method: github-release
|
||||
with:
|
||||
repo: goreleaser/goreleaser
|
||||
|
||||
+49
-41
@@ -1,15 +1,8 @@
|
||||
issues:
|
||||
max-same-issues: 25
|
||||
uniq-by-line: false
|
||||
|
||||
# TODO: enable this when we have coverage on docstring comments
|
||||
# # The list of ids of default excludes to include or disable.
|
||||
# include:
|
||||
# - EXC0002 # disable excluding of issues about comments from golint
|
||||
|
||||
version: "2"
|
||||
run:
|
||||
tests: false
|
||||
linters:
|
||||
# inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
|
||||
disable-all: true
|
||||
default: none
|
||||
enable:
|
||||
- asciicheck
|
||||
- bodyclose
|
||||
@@ -22,11 +15,8 @@ linters:
|
||||
- goconst
|
||||
- gocritic
|
||||
- gocyclo
|
||||
- gofmt
|
||||
- goimports
|
||||
- goprintffuncname
|
||||
- gosec
|
||||
- gosimple
|
||||
- govet
|
||||
- ineffassign
|
||||
- misspell
|
||||
@@ -34,37 +24,35 @@ linters:
|
||||
- nolintlint
|
||||
- revive
|
||||
- staticcheck
|
||||
- stylecheck
|
||||
- typecheck
|
||||
- unconvert
|
||||
- unparam
|
||||
- unused
|
||||
- whitespace
|
||||
|
||||
linters-settings:
|
||||
funlen:
|
||||
# Checks the number of lines in a function.
|
||||
# If lower than 0, disable the check.
|
||||
# Default: 60
|
||||
lines: 70
|
||||
# Checks the number of statements in a function.
|
||||
# If lower than 0, disable the check.
|
||||
# Default: 40
|
||||
statements: 50
|
||||
gocritic:
|
||||
enabled-checks:
|
||||
- deferInLoop
|
||||
- ruleguard
|
||||
settings:
|
||||
ruleguard:
|
||||
rules: "test/rules/rules.go"
|
||||
gosec:
|
||||
excludes:
|
||||
- G115
|
||||
|
||||
run:
|
||||
timeout: 10m
|
||||
tests: false
|
||||
settings:
|
||||
funlen:
|
||||
lines: 70
|
||||
statements: 50
|
||||
gocritic:
|
||||
enabled-checks:
|
||||
- deferInLoop
|
||||
- ruleguard
|
||||
settings:
|
||||
ruleguard:
|
||||
rules: test/rules/rules.go
|
||||
gosec:
|
||||
excludes:
|
||||
- G115
|
||||
exclusions:
|
||||
generated: lax
|
||||
presets:
|
||||
- comments
|
||||
- common-false-positives
|
||||
- legacy
|
||||
- std-error-handling
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
|
||||
# do not enable...
|
||||
# - deadcode # The owner seems to have abandoned the linter. Replaced by "unused".
|
||||
@@ -91,3 +79,23 @@ run:
|
||||
# - testpackage
|
||||
# - varcheck # The owner seems to have abandoned the linter. Replaced by "unused".
|
||||
# - wsl # this doens't have an auto-fixer yet and is pretty noisy (https://github.com/bombsimon/wsl/issues/90)
|
||||
|
||||
issues:
|
||||
max-same-issues: 25
|
||||
uniq-by-line: false
|
||||
|
||||
# TODO: enable this when we have coverage on docstring comments
|
||||
# # The list of ids of default excludes to include or disable.
|
||||
# include:
|
||||
# - EXC0002 # disable excluding of issues about comments from golint
|
||||
|
||||
formatters:
|
||||
enable:
|
||||
- gofmt
|
||||
- goimports
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
|
||||
@@ -52,7 +52,7 @@ func Attest(app clio.Application) *cobra.Command {
|
||||
opts := defaultAttestOptions()
|
||||
|
||||
// template format explicitly not allowed
|
||||
opts.Format.Template.Enabled = false
|
||||
opts.Template.Enabled = false
|
||||
|
||||
return app.SetupCommand(&cobra.Command{
|
||||
Use: "attest --output [FORMAT] <IMAGE>",
|
||||
@@ -136,7 +136,7 @@ func writeSBOMToFormattedFile(s *sbom.SBOM, sbomFile io.Writer, opts *attestOpti
|
||||
return fmt.Errorf("no output file provided")
|
||||
}
|
||||
|
||||
encs, err := opts.Format.Encoders()
|
||||
encs, err := opts.Encoders()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create encoders: %w", err)
|
||||
}
|
||||
|
||||
@@ -110,7 +110,7 @@ func (o *scanOptions) PostLoad() error {
|
||||
}
|
||||
|
||||
func (o *scanOptions) validateLegacyOptionsNotUsed() error {
|
||||
if len(fangs.Flatten(o.Config.ConfigFile)) == 0 {
|
||||
if len(fangs.Flatten(o.ConfigFile)) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -122,7 +122,7 @@ func (o *scanOptions) validateLegacyOptionsNotUsed() error {
|
||||
File any `yaml:"file" json:"file" mapstructure:"file"`
|
||||
}
|
||||
|
||||
for _, f := range fangs.Flatten(o.Config.ConfigFile) {
|
||||
for _, f := range fangs.Flatten(o.ConfigFile) {
|
||||
by, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read config file during validations %q: %w", f, err)
|
||||
|
||||
@@ -89,7 +89,7 @@ func (o Output) SBOMWriter() (sbom.Writer, error) {
|
||||
|
||||
usesTemplateOutput := names.Has(string(template.ID))
|
||||
|
||||
if usesTemplateOutput && o.Format.Template.Path == "" {
|
||||
if usesTemplateOutput && o.Template.Path == "" {
|
||||
return nil, fmt.Errorf(`must specify path to template file when using "template" output format`)
|
||||
}
|
||||
|
||||
|
||||
@@ -36,11 +36,7 @@ func capture(target **os.File, writer io.Writer, bufSize int) func() {
|
||||
}()
|
||||
|
||||
buf := make([]byte, bufSize)
|
||||
for {
|
||||
if original == nil {
|
||||
break
|
||||
}
|
||||
|
||||
for original != nil {
|
||||
n, err := r.Read(buf)
|
||||
if n > 0 {
|
||||
_, _ = writer.Write(buf[0:n])
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"path"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
@@ -76,7 +77,6 @@ func newAlpineConfiguration(resolver file.Resolver) (*AlpineConfiguration, []fil
|
||||
return &AlpineConfiguration{
|
||||
APKKeys: keys,
|
||||
}, locations, nil
|
||||
|
||||
}
|
||||
|
||||
func getVersion(resolver file.Resolver) (string, []file.Location, error) {
|
||||
@@ -92,6 +92,7 @@ func getVersion(resolver file.Resolver) (string, []file.Location, error) {
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("unable to read alpine version: %w", err)
|
||||
}
|
||||
defer internal.CloseAndLogError(reader, locations[0].RealPath)
|
||||
|
||||
version, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
@@ -111,7 +112,11 @@ func getAPKKeys(resolver file.Resolver) (map[string]string, []file.Location, err
|
||||
}
|
||||
for _, location := range locations {
|
||||
basename := path.Base(location.RealPath)
|
||||
//nolint:gocritic
|
||||
reader, err := resolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to resolve file contents by location at %s: %w", location.RealPath, err)
|
||||
}
|
||||
content, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to read apk key content at %s: %w", location.RealPath, err)
|
||||
|
||||
@@ -124,5 +124,4 @@ func showAlpineConfiguration(s sbom.SBOM) {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(string(meta))
|
||||
|
||||
}
|
||||
|
||||
@@ -30,7 +30,6 @@ func main() {
|
||||
if err := enc.Encode(sbom.Descriptor.Configuration); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func imageReference() string {
|
||||
|
||||
@@ -13,5 +13,5 @@ func DefaultLocationComparer(x, y file.Location) bool {
|
||||
}
|
||||
|
||||
func LocationComparerWithoutLayer(x, y file.Location) bool {
|
||||
return cmp.Equal(x.Coordinates.RealPath, y.Coordinates.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath)
|
||||
return cmp.Equal(x.RealPath, y.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath)
|
||||
}
|
||||
|
||||
@@ -3,5 +3,5 @@ package internal
|
||||
const (
|
||||
// JSONSchemaVersion is the current schema version output by the JSON encoder
|
||||
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
|
||||
JSONSchemaVersion = "16.0.24"
|
||||
JSONSchemaVersion = "16.0.25"
|
||||
)
|
||||
|
||||
@@ -23,7 +23,7 @@ func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...strin
|
||||
defer file.Close()
|
||||
|
||||
// ignore directories
|
||||
if file.FileInfo.IsDir() {
|
||||
if file.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
|
||||
}
|
||||
}()
|
||||
|
||||
for _, file := range zipReader.Reader.File {
|
||||
for _, file := range zipReader.File {
|
||||
manifest.Add(file.Name, file.FileInfo())
|
||||
}
|
||||
return manifest, nil
|
||||
|
||||
@@ -53,7 +53,7 @@ func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths
|
||||
}
|
||||
}()
|
||||
|
||||
for _, file := range zipReader.Reader.File {
|
||||
for _, file := range zipReader.File {
|
||||
// if no paths are given then assume that all files should be traversed
|
||||
if len(paths) > 0 {
|
||||
if _, ok := request[file.Name]; !ok {
|
||||
|
||||
@@ -22,7 +22,7 @@ func less(i, j artifact.Relationship) bool {
|
||||
jFrom, ok3 := j.From.(pkg.Package)
|
||||
jTo, ok4 := j.To.(pkg.Package)
|
||||
|
||||
if !(ok1 && ok2 && ok3 && ok4) {
|
||||
if !ok1 && !ok2 && !ok3 && !ok4 {
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -245,10 +245,10 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
|
||||
}
|
||||
|
||||
for _, ref := range pathRefs {
|
||||
if oldRef, ok := locations[ref.Coordinates.ID()]; ok {
|
||||
if oldRef, ok := locations[ref.ID()]; ok {
|
||||
log.Debugf("found path duplicate of %s", oldRef.RealPath)
|
||||
}
|
||||
locations[ref.Coordinates.ID()] = ref
|
||||
locations[ref.ID()] = ref
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -69,7 +69,7 @@ func DefaultPackageTaskFactories() Factories {
|
||||
newSimplePackageTaskFactory(cpp.NewConanInfoCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "cpp", "conan"),
|
||||
newSimplePackageTaskFactory(javascript.NewPackageCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, JavaScript, Node),
|
||||
newSimplePackageTaskFactory(php.NewComposerInstalledCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "php", "composer"),
|
||||
newSimplePackageTaskFactory(r.NewPackageCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "r"),
|
||||
newSimplePackageTaskFactory(r.NewPackageCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, pkgcataloging.DirectoryTag, "r"),
|
||||
newSimplePackageTaskFactory(ruby.NewInstalledGemSpecCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "ruby", "gem", "gemspec"),
|
||||
newSimplePackageTaskFactory(rust.NewAuditBinaryCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "rust", "binary"),
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "anchore.io/schema/syft/json/16.0.24/document",
|
||||
"$id": "anchore.io/schema/syft/json/16.0.25/document",
|
||||
"$ref": "#/$defs/Document",
|
||||
"$defs": {
|
||||
"AlpmDbEntry": {
|
||||
@@ -995,6 +995,20 @@
|
||||
"size"
|
||||
]
|
||||
},
|
||||
"GithubActionsUseStatement": {
|
||||
"properties": {
|
||||
"value": {
|
||||
"type": "string"
|
||||
},
|
||||
"comment": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
]
|
||||
},
|
||||
"GoModuleBuildinfoEntry": {
|
||||
"properties": {
|
||||
"goBuildSettings": {
|
||||
@@ -1804,6 +1818,9 @@
|
||||
{
|
||||
"$ref": "#/$defs/ErlangRebarLockEntry"
|
||||
},
|
||||
{
|
||||
"$ref": "#/$defs/GithubActionsUseStatement"
|
||||
},
|
||||
{
|
||||
"$ref": "#/$defs/GoModuleBuildinfoEntry"
|
||||
},
|
||||
|
||||
@@ -48,15 +48,15 @@ func (m *LocationMetadata) merge(other LocationMetadata) error {
|
||||
}
|
||||
|
||||
func (l Location) WithAnnotation(key, value string) Location {
|
||||
if l.LocationMetadata.Annotations == nil {
|
||||
l.LocationMetadata.Annotations = map[string]string{}
|
||||
if l.Annotations == nil {
|
||||
l.Annotations = map[string]string{}
|
||||
}
|
||||
l.LocationMetadata.Annotations[key] = value
|
||||
l.Annotations[key] = value
|
||||
return l
|
||||
}
|
||||
|
||||
func (l Location) WithoutAnnotations() Location {
|
||||
l.LocationMetadata.Annotations = map[string]string{}
|
||||
l.Annotations = map[string]string{}
|
||||
|
||||
return l
|
||||
}
|
||||
|
||||
@@ -361,6 +361,7 @@ func collectDocRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out
|
||||
from, fromOk := a.(pkg.Package)
|
||||
toPackage, toPackageOk := b.(pkg.Package)
|
||||
toLocation, toLocationOk := b.(file.Location)
|
||||
//nolint:staticcheck
|
||||
if !fromOk || !(toPackageOk || toLocationOk) {
|
||||
log.Debugf("unable to find valid relationship mapping from SPDX, ignoring: (from: %+v) (to: %+v)", a, b)
|
||||
continue
|
||||
|
||||
@@ -213,13 +213,15 @@ func reduceOuter(expression string) string {
|
||||
func isBalanced(expression string) bool {
|
||||
count := 0
|
||||
for _, c := range expression {
|
||||
if c == '(' {
|
||||
switch c {
|
||||
case '(':
|
||||
count++
|
||||
} else if c == ')' {
|
||||
case ')':
|
||||
count--
|
||||
if count < 0 {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
}
|
||||
}
|
||||
return count == 0
|
||||
|
||||
@@ -34,7 +34,7 @@ const (
|
||||
//
|
||||
// Available options are: <omit>, NOASSERTION, Person: <person>, Organization: <org>
|
||||
// return values are: <type>, <value>
|
||||
func Originator(p pkg.Package) (typ string, author string) { //nolint: funlen
|
||||
func Originator(p pkg.Package) (typ string, author string) { //nolint: gocyclo,funlen
|
||||
if !hasMetadata(p) {
|
||||
return typ, author
|
||||
}
|
||||
@@ -57,6 +57,15 @@ func Originator(p pkg.Package) (typ string, author string) { //nolint: funlen
|
||||
case pkg.DpkgArchiveEntry:
|
||||
author = metadata.Maintainer
|
||||
|
||||
case pkg.GitHubActionsUseStatement:
|
||||
typ = orgType
|
||||
org := strings.Split(metadata.Value, "/")[0]
|
||||
if org == "actions" {
|
||||
// this is a GitHub action, so the org is GitHub
|
||||
org = "GitHub"
|
||||
}
|
||||
author = org
|
||||
|
||||
case pkg.JavaArchive:
|
||||
if metadata.Manifest != nil {
|
||||
author = metadata.Manifest.Main.MustGet("Specification-Vendor")
|
||||
|
||||
@@ -45,6 +45,7 @@ func Test_OriginatorSupplier(t *testing.T) {
|
||||
pkg.SwiplPackEntry{},
|
||||
pkg.OpamPackage{},
|
||||
pkg.YarnLockEntry{},
|
||||
pkg.TerraformLockProviderEntry{},
|
||||
)
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -384,20 +385,24 @@ func Test_OriginatorSupplier(t *testing.T) {
|
||||
supplier: "Person: me (me@auth.com)",
|
||||
},
|
||||
{
|
||||
name: "from ocaml opam",
|
||||
name: "from github actions workflow/action",
|
||||
input: pkg.Package{
|
||||
Metadata: pkg.OpamPackage{},
|
||||
Metadata: pkg.GitHubActionsUseStatement{
|
||||
Value: "actions/checkout@v4",
|
||||
},
|
||||
},
|
||||
originator: "",
|
||||
supplier: "",
|
||||
originator: "Organization: GitHub",
|
||||
supplier: "Organization: GitHub",
|
||||
},
|
||||
{
|
||||
name: "from terraform lock",
|
||||
name: "from github actions workflow/action",
|
||||
input: pkg.Package{
|
||||
Metadata: pkg.TerraformLockProviderEntry{},
|
||||
Metadata: pkg.GitHubActionsUseStatement{
|
||||
Value: "google/something@v6",
|
||||
},
|
||||
},
|
||||
originator: "",
|
||||
supplier: "",
|
||||
originator: "Organization: google",
|
||||
supplier: "Organization: google",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
|
||||
@@ -228,7 +228,7 @@ func toSyftRelationships(doc *model.Document, catalog *pkg.Collection, relations
|
||||
idMap[string(p.ID())] = p
|
||||
locations := p.Locations.ToSlice()
|
||||
for _, l := range locations {
|
||||
idMap[string(l.Coordinates.ID())] = l.Coordinates
|
||||
idMap[string(l.ID())] = l.Coordinates
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ func (r *ContainerImageAllLayers) fileByRef(ref stereoscopeFile.Reference, uniqu
|
||||
return nil, fmt.Errorf("unable to fetch metadata (ref=%+v): %w", ref, err)
|
||||
}
|
||||
|
||||
if entry.Metadata.Type == stereoscopeFile.TypeHardLink || entry.Metadata.Type == stereoscopeFile.TypeSymLink {
|
||||
if entry.Type == stereoscopeFile.TypeHardLink || entry.Type == stereoscopeFile.TypeSymLink {
|
||||
// a link may resolve in this layer or higher, assuming a squashed tree is used to search
|
||||
// we should search all possible resolutions within the valid source
|
||||
for _, subLayerIdx := range r.layers[layerIdx:] {
|
||||
@@ -102,7 +102,7 @@ func (r *ContainerImageAllLayers) FilesByPath(paths ...string) ([]file.Location,
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
|
||||
}
|
||||
if metadata.Metadata.IsDir() {
|
||||
if metadata.IsDir() {
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -146,7 +146,7 @@ func (r *ContainerImageAllLayers) FilesByGlob(patterns ...string) ([]file.Locati
|
||||
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.RequestPath, err)
|
||||
}
|
||||
// don't consider directories
|
||||
if metadata.Metadata.IsDir() {
|
||||
if metadata.IsDir() {
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -192,7 +192,7 @@ func (r *ContainerImageAllLayers) FileContentsByLocation(location file.Location)
|
||||
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err)
|
||||
}
|
||||
|
||||
switch entry.Metadata.Type {
|
||||
switch entry.Type {
|
||||
case stereoscopeFile.TypeSymLink, stereoscopeFile.TypeHardLink:
|
||||
// the location we are searching may be a symlink, we should always work with the resolved file
|
||||
newLocation := r.RelativeFileByPath(location, location.AccessPath)
|
||||
|
||||
@@ -58,7 +58,7 @@ func (r *ContainerImageSquash) FilesByPath(paths ...string) ([]file.Location, er
|
||||
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
|
||||
}
|
||||
// don't consider directories
|
||||
if metadata.Metadata.IsDir() {
|
||||
if metadata.IsDir() {
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -106,7 +106,7 @@ func (r *ContainerImageSquash) FilesByGlob(patterns ...string) ([]file.Location,
|
||||
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.RequestPath, err)
|
||||
}
|
||||
// don't consider directories
|
||||
if metadata.Metadata.IsDir() {
|
||||
if metadata.IsDir() {
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -151,7 +151,7 @@ func (r *ContainerImageSquash) FileContentsByLocation(location file.Location) (i
|
||||
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err)
|
||||
}
|
||||
|
||||
switch entry.Metadata.Type {
|
||||
switch entry.Type {
|
||||
case stereoscopeFile.TypeSymLink, stereoscopeFile.TypeHardLink:
|
||||
// the location we are searching may be a symlink, we should always work with the resolved file
|
||||
locations, err := r.FilesByPath(location.RealPath)
|
||||
|
||||
@@ -59,7 +59,7 @@ func (r *Directory) buildIndex() error {
|
||||
|
||||
r.tree = tree
|
||||
r.index = index
|
||||
r.filetreeResolver.searchContext = filetree.NewSearchContext(tree, index)
|
||||
r.searchContext = filetree.NewSearchContext(tree, index)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ func (r *File) buildIndex() error {
|
||||
|
||||
r.tree = tree
|
||||
r.index = index
|
||||
r.filetreeResolver.searchContext = filetree.NewSearchContext(tree, index)
|
||||
r.searchContext = filetree.NewSearchContext(tree, index)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -68,7 +68,7 @@ func (r filetreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err
|
||||
}
|
||||
|
||||
// don't consider directories
|
||||
if entry.Metadata.IsDir() {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -114,17 +114,17 @@ func (r filetreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro
|
||||
}
|
||||
entry, err := r.index.Get(*refVia.Reference)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to get file metadata for reference %s: %w", refVia.Reference.RealPath, err)
|
||||
return nil, fmt.Errorf("unable to get file metadata for reference %s: %w", refVia.RealPath, err)
|
||||
}
|
||||
|
||||
// don't consider directories
|
||||
if entry.Metadata.IsDir() {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
loc := file.NewVirtualLocationFromDirectory(
|
||||
r.responsePath(string(refVia.Reference.RealPath)), // the actual path relative to the resolver root
|
||||
r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
|
||||
r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root
|
||||
r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
|
||||
*refVia.Reference,
|
||||
)
|
||||
uniqueFileIDs.Add(*refVia.Reference)
|
||||
@@ -217,7 +217,7 @@ func (r *filetreeResolver) FilesByMIMEType(types ...string) ([]file.Location, er
|
||||
continue
|
||||
}
|
||||
location := file.NewVirtualLocationFromDirectory(
|
||||
r.responsePath(string(refVia.Reference.RealPath)),
|
||||
r.responsePath(string(refVia.RealPath)),
|
||||
r.responsePath(string(refVia.RequestPath)),
|
||||
*refVia.Reference,
|
||||
)
|
||||
|
||||
@@ -25,6 +25,7 @@ func AllTypes() []any {
|
||||
pkg.ELFBinaryPackageNoteJSONPayload{},
|
||||
pkg.ElixirMixLockEntry{},
|
||||
pkg.ErlangRebarLockEntry{},
|
||||
pkg.GitHubActionsUseStatement{},
|
||||
pkg.GolangBinaryBuildinfoEntry{},
|
||||
pkg.GolangModuleEntry{},
|
||||
pkg.HackageStackYamlEntry{},
|
||||
|
||||
@@ -78,6 +78,7 @@ var jsonTypes = makeJSONTypes(
|
||||
jsonNames(pkg.DpkgDBEntry{}, "dpkg-db-entry", "DpkgMetadata"),
|
||||
jsonNames(pkg.ELFBinaryPackageNoteJSONPayload{}, "elf-binary-package-note-json-payload"),
|
||||
jsonNames(pkg.RubyGemspec{}, "ruby-gemspec", "GemMetadata"),
|
||||
jsonNames(pkg.GitHubActionsUseStatement{}, "github-actions-use-statement"),
|
||||
jsonNames(pkg.GolangBinaryBuildinfoEntry{}, "go-module-buildinfo-entry", "GolangBinMetadata", "GolangMetadata"),
|
||||
jsonNames(pkg.GolangModuleEntry{}, "go-module-entry", "GolangModMetadata"),
|
||||
jsonNames(pkg.HackageStackYamlLockEntry{}, "haskell-hackage-stack-lock-entry", "HackageMetadataType"),
|
||||
|
||||
@@ -123,7 +123,7 @@ func parseApkDB(_ context.Context, resolver file.Resolver, env *generic.Environm
|
||||
// This should get fixed with https://gitlab.alpinelinux.org/alpine/apk-tools/-/issues/10875
|
||||
if r == nil {
|
||||
// find the repositories file from the relative directory of the DB file
|
||||
releases := findReleases(resolver, reader.Location.RealPath)
|
||||
releases := findReleases(resolver, reader.RealPath)
|
||||
|
||||
if len(releases) > 0 {
|
||||
r = &releases[0]
|
||||
@@ -173,7 +173,7 @@ func parseReleasesFromAPKRepository(reader file.LocationReadCloser) []linux.Rele
|
||||
|
||||
reposB, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
log.Tracef("unable to read APK repositories file %q: %+v", reader.Location.RealPath, err)
|
||||
log.Tracef("unable to read APK repositories file %q: %+v", reader.RealPath, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -80,7 +80,7 @@ func parseAlpmDB(_ context.Context, resolver file.Resolver, env *generic.Environ
|
||||
newPackage(
|
||||
data,
|
||||
env.LinuxRelease,
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
locs...,
|
||||
),
|
||||
}, nil, errs
|
||||
|
||||
@@ -33,13 +33,13 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
|
||||
}
|
||||
|
||||
if s == nil {
|
||||
log.WithFields("path", reader.Location.RealPath).Trace("file is not an SBOM")
|
||||
log.WithFields("path", reader.RealPath).Trace("file is not an SBOM")
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
// Bitnami exclusively uses SPDX JSON SBOMs
|
||||
if sFormat != "spdx-json" {
|
||||
log.WithFields("path", reader.Location.RealPath).Trace("file is not an SPDX JSON SBOM")
|
||||
log.WithFields("path", reader.RealPath).Trace("file is not an SPDX JSON SBOM")
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
|
||||
// where there is evidence of this file, and the catalogers have not run against any file other than,
|
||||
// the SBOM, this is the only location that is relevant for this cataloger.
|
||||
p.Locations = file.NewLocationSet(
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
)
|
||||
|
||||
// Parse the Bitnami-specific metadata
|
||||
@@ -70,7 +70,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
|
||||
|
||||
// Bitnami packages reported in a SPDX file are shipped under the same directory
|
||||
// as the SPDX file itself.
|
||||
metadata.Path = filepath.Dir(reader.Location.RealPath)
|
||||
metadata.Path = filepath.Dir(reader.RealPath)
|
||||
if p.ID() != mainPkgID {
|
||||
metadata.Files = packageFiles(s.Relationships, p, metadata.Path)
|
||||
secondaryPkgsFiles = append(secondaryPkgsFiles, metadata.Files...)
|
||||
@@ -86,12 +86,12 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
|
||||
}
|
||||
|
||||
// Resolve all files owned by the main package in the SBOM and update the metadata
|
||||
if mainPkgFiles, err := mainPkgFiles(resolver, reader.Location.RealPath, secondaryPkgsFiles); err == nil {
|
||||
if mainPkgFiles, err := mainPkgFiles(resolver, reader.RealPath, secondaryPkgsFiles); err == nil {
|
||||
for i, p := range pkgs {
|
||||
if p.ID() == mainPkgID {
|
||||
metadata, ok := p.Metadata.(*pkg.BitnamiSBOMEntry)
|
||||
if !ok {
|
||||
log.WithFields("spdx-filepath", reader.Location.RealPath).Trace("main package in SBOM does not have Bitnami metadata")
|
||||
log.WithFields("spdx-filepath", reader.RealPath).Trace("main package in SBOM does not have Bitnami metadata")
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -100,7 +100,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.WithFields("spdx-filepath", reader.Location.RealPath, "error", err).Trace("unable to resolve owned files for main package in SBOM")
|
||||
log.WithFields("spdx-filepath", reader.RealPath, "error", err).Trace("unable to resolve owned files for main package in SBOM")
|
||||
}
|
||||
|
||||
return pkgs, filterRelationships(s.Relationships, pkgs), nil
|
||||
|
||||
@@ -48,7 +48,7 @@ func parseConanfile(_ context.Context, _ file.Resolver, _ *generic.Environment,
|
||||
|
||||
p := newConanfilePackage(
|
||||
m,
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
)
|
||||
if p == nil {
|
||||
continue
|
||||
|
||||
@@ -60,7 +60,7 @@ func parseFullRequiresLine(line string, reader file.LocationReadCloser, pkgs *[]
|
||||
|
||||
p := newConaninfoPackage(
|
||||
meta,
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
)
|
||||
if p != nil {
|
||||
*pkgs = append(*pkgs, *p)
|
||||
@@ -83,7 +83,7 @@ func parseConaninfo(_ context.Context, _ file.Resolver, _ *generic.Environment,
|
||||
// First set the base package info by checking the relative path
|
||||
fullFilePath := string(reader.Location.LocationData.Reference().RealPath)
|
||||
if len(fullFilePath) == 0 {
|
||||
fullFilePath = reader.Location.LocationData.RealPath
|
||||
fullFilePath = reader.RealPath
|
||||
}
|
||||
|
||||
mainMetadata, err := parseConanMetadataFromFilePath(fullFilePath)
|
||||
@@ -102,7 +102,7 @@ func parseConaninfo(_ context.Context, _ file.Resolver, _ *generic.Environment,
|
||||
case errors.Is(err, io.EOF):
|
||||
mainPackage := newConaninfoPackage(
|
||||
mainMetadata,
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
)
|
||||
|
||||
mainPackageRef := *mainPackage
|
||||
|
||||
@@ -91,7 +91,7 @@ func handleConanLockV1(cl conanLock, reader file.LocationReadCloser, parsedPkgRe
|
||||
|
||||
p := newConanlockPackage(
|
||||
metadata,
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
)
|
||||
|
||||
if p != nil {
|
||||
@@ -115,7 +115,7 @@ func handleConanLockV2(cl conanLock, reader file.LocationReadCloser, indexToPkgM
|
||||
|
||||
p := newConanReferencePackage(
|
||||
reference,
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
)
|
||||
|
||||
if p != nil {
|
||||
|
||||
@@ -102,7 +102,7 @@ func parsePubspecLock(_ context.Context, _ file.Resolver, _ *generic.Environment
|
||||
newPubspecLockPackage(
|
||||
name,
|
||||
pubPkg,
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ func parseDpkgDB(ctx context.Context, resolver file.Resolver, env *generic.Envir
|
||||
return nil, nil, fmt.Errorf("unable to catalog dpkg DB=%q: %w", reader.RealPath, err)
|
||||
}
|
||||
|
||||
dbLoc := reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)
|
||||
dbLoc := reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)
|
||||
var pkgs []pkg.Package
|
||||
_ = sync.CollectSlice(&ctx, cataloging.ExecutorFile, sync.ToSeq(metadata), func(m pkg.DpkgDBEntry) (pkg.Package, error) {
|
||||
return newDpkgPackage(m, dbLoc, resolver, env.LinuxRelease, findDpkgInfoFiles(m.Package, resolver, reader.Location)...), nil
|
||||
|
||||
@@ -69,7 +69,7 @@ func parseDotnetPackagesLock(_ context.Context, _ file.Resolver, _ *generic.Envi
|
||||
name, _ := extractNameAndVersion(nameVersion)
|
||||
|
||||
dep := allDependencies[nameVersion]
|
||||
dotnetPkg := newDotnetPackagesLockPackage(name, dep, reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
|
||||
dotnetPkg := newDotnetPackagesLockPackage(name, dep, reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
|
||||
if dotnetPkg != nil {
|
||||
pkgs = append(pkgs, *dotnetPkg)
|
||||
pkgMap[nameVersion] = *dotnetPkg
|
||||
|
||||
@@ -61,7 +61,7 @@ func parseMixLock(_ context.Context, _ file.Resolver, _ *generic.Environment, re
|
||||
PkgHash: hash,
|
||||
PkgHashExt: hashExt,
|
||||
},
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ func parseOTPApp(_ context.Context, _ file.Resolver, _ *generic.Environment, rea
|
||||
|
||||
p := newPackageFromOTP(
|
||||
name, version,
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
)
|
||||
|
||||
packages = append(packages, p)
|
||||
|
||||
@@ -56,7 +56,7 @@ func parseRebarLock(_ context.Context, _ file.Resolver, _ *generic.Environment,
|
||||
Name: name,
|
||||
Version: version,
|
||||
},
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
)
|
||||
|
||||
pkgMap[name] = &p
|
||||
|
||||
@@ -27,14 +27,14 @@ var (
|
||||
|
||||
// parses individual CONTENTS files from the portage flat-file store (e.g. /var/db/pkg/*/*/CONTENTS).
|
||||
func parsePortageContents(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
cpvMatch := cpvRe.FindStringSubmatch(reader.Location.RealPath)
|
||||
cpvMatch := cpvRe.FindStringSubmatch(reader.RealPath)
|
||||
if cpvMatch == nil {
|
||||
return nil, nil, fmt.Errorf("failed to match package and version in %s", reader.Location.RealPath)
|
||||
return nil, nil, fmt.Errorf("failed to match package and version in %s", reader.RealPath)
|
||||
}
|
||||
|
||||
name, version := cpvMatch[1], cpvMatch[2]
|
||||
if name == "" || version == "" {
|
||||
log.WithFields("path", reader.Location.RealPath).Debug("failed to parse portage name and version")
|
||||
log.WithFields("path", reader.RealPath).Debug("failed to parse portage name and version")
|
||||
return nil, nil, fmt.Errorf("failed to parse portage name and version")
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ func parsePortageContents(_ context.Context, resolver file.Resolver, _ *generic.
|
||||
Version: version,
|
||||
PURL: packageURL(name, version),
|
||||
Locations: file.NewLocationSet(
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
),
|
||||
Type: pkg.PortagePkg,
|
||||
Metadata: pkg.PortageEntry{
|
||||
|
||||
@@ -2,6 +2,7 @@ package githubactions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/packageurl-go"
|
||||
@@ -10,8 +11,8 @@ import (
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
func newPackageFromUsageStatement(use string, location file.Location) (*pkg.Package, error) {
|
||||
name, version := parseStepUsageStatement(use)
|
||||
func newPackageFromUsageStatement(use, comment string, location file.Location) (*pkg.Package, error) {
|
||||
name, version := parseStepUsageStatement(use, comment)
|
||||
|
||||
if name == "" {
|
||||
log.WithFields("file", location.RealPath, "statement", use).Trace("unable to parse github action usage statement")
|
||||
@@ -19,19 +20,20 @@ func newPackageFromUsageStatement(use string, location file.Location) (*pkg.Pack
|
||||
}
|
||||
|
||||
if strings.Contains(name, ".github/workflows/") {
|
||||
return newGithubActionWorkflowPackageUsage(name, version, location), nil
|
||||
return newGithubActionWorkflowPackageUsage(name, version, location, pkg.GitHubActionsUseStatement{Value: use, Comment: comment}), nil
|
||||
}
|
||||
|
||||
return newGithubActionPackageUsage(name, version, location), nil
|
||||
return newGithubActionPackageUsage(name, version, location, pkg.GitHubActionsUseStatement{Value: use, Comment: comment}), nil
|
||||
}
|
||||
|
||||
func newGithubActionWorkflowPackageUsage(name, version string, workflowLocation file.Location) *pkg.Package {
|
||||
func newGithubActionWorkflowPackageUsage(name, version string, workflowLocation file.Location, m pkg.GitHubActionsUseStatement) *pkg.Package {
|
||||
p := &pkg.Package{
|
||||
Name: name,
|
||||
Version: version,
|
||||
Locations: file.NewLocationSet(workflowLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
|
||||
PURL: packageURL(name, version),
|
||||
Type: pkg.GithubActionWorkflowPkg,
|
||||
Metadata: m,
|
||||
}
|
||||
|
||||
p.SetID()
|
||||
@@ -39,13 +41,14 @@ func newGithubActionWorkflowPackageUsage(name, version string, workflowLocation
|
||||
return p
|
||||
}
|
||||
|
||||
func newGithubActionPackageUsage(name, version string, workflowLocation file.Location) *pkg.Package {
|
||||
func newGithubActionPackageUsage(name, version string, workflowLocation file.Location, m pkg.GitHubActionsUseStatement) *pkg.Package {
|
||||
p := &pkg.Package{
|
||||
Name: name,
|
||||
Version: version,
|
||||
Locations: file.NewLocationSet(workflowLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
|
||||
PURL: packageURL(name, version),
|
||||
Type: pkg.GithubActionPkg,
|
||||
Metadata: m,
|
||||
}
|
||||
|
||||
p.SetID()
|
||||
@@ -53,20 +56,32 @@ func newGithubActionPackageUsage(name, version string, workflowLocation file.Loc
|
||||
return p
|
||||
}
|
||||
|
||||
func parseStepUsageStatement(use string) (string, string) {
|
||||
// from octo-org/another-repo/.github/workflows/workflow.yml@v1 get octo-org/another-repo/.github/workflows/workflow.yml and v1
|
||||
// from ./.github/workflows/workflow-2.yml interpret as only the name
|
||||
|
||||
// from actions/cache@v3 get actions/cache and v3
|
||||
func parseStepUsageStatement(use, comment string) (string, string) {
|
||||
// from "octo-org/another-repo/.github/workflows/workflow.yml@v1" get octo-org/another-repo/.github/workflows/workflow.yml and v1
|
||||
// from "./.github/workflows/workflow-2.yml" interpret as only the name
|
||||
// from "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2" get actions/checkout and v4.2.2
|
||||
// from "actions/cache@v3" get actions/cache and v3
|
||||
|
||||
fields := strings.Split(use, "@")
|
||||
switch len(fields) {
|
||||
case 1:
|
||||
return use, ""
|
||||
case 2:
|
||||
return fields[0], fields[1]
|
||||
name := use
|
||||
version := ""
|
||||
|
||||
if len(fields) == 2 {
|
||||
name = fields[0]
|
||||
version = fields[1]
|
||||
}
|
||||
return "", ""
|
||||
|
||||
// if version looks like a commit hash and we have a comment, try to extract version from comment
|
||||
if version != "" && regexp.MustCompile(`^[0-9a-f]{7,}$`).MatchString(version) && comment != "" {
|
||||
versionRegex := regexp.MustCompile(`v?\d+\.\d+\.\d+`)
|
||||
matches := versionRegex.FindStringSubmatch(comment)
|
||||
|
||||
if len(matches) >= 1 {
|
||||
return name, matches[0]
|
||||
}
|
||||
}
|
||||
|
||||
return name, version
|
||||
}
|
||||
|
||||
func packageURL(name, version string) string {
|
||||
|
||||
@@ -43,7 +43,7 @@ func parseCompositeActionForActionUsage(_ context.Context, _ file.Resolver, _ *g
|
||||
continue
|
||||
}
|
||||
|
||||
p, err := newPackageFromUsageStatement(step.Uses, reader.Location)
|
||||
p, err := newPackageFromUsageStatement(step.Uses, step.UsesComment, reader.Location)
|
||||
if err != nil {
|
||||
errs = unknown.Append(errs, reader, err)
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ func Test_parseCompositeActionForActionUsage(t *testing.T) {
|
||||
Type: pkg.GithubActionPkg,
|
||||
Locations: fixtureLocationSet,
|
||||
PURL: "pkg:github/actions/setup-go@v4",
|
||||
Metadata: pkg.GitHubActionsUseStatement{Value: "actions/setup-go@v4"},
|
||||
},
|
||||
{
|
||||
Name: "actions/cache",
|
||||
@@ -27,6 +28,7 @@ func Test_parseCompositeActionForActionUsage(t *testing.T) {
|
||||
Type: pkg.GithubActionPkg,
|
||||
Locations: fixtureLocationSet,
|
||||
PURL: "pkg:github/actions/cache@v3",
|
||||
Metadata: pkg.GitHubActionsUseStatement{Value: "actions/cache@v3"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
@@ -24,14 +25,16 @@ type workflowDef struct {
|
||||
}
|
||||
|
||||
type workflowJobDef struct {
|
||||
Uses string `yaml:"uses"`
|
||||
Steps []stepDef `yaml:"steps"`
|
||||
Uses string `yaml:"uses"`
|
||||
UsesComment string `yaml:"-"`
|
||||
Steps []stepDef `yaml:"steps"`
|
||||
}
|
||||
|
||||
type stepDef struct {
|
||||
Name string `yaml:"name"`
|
||||
Uses string `yaml:"uses"`
|
||||
With struct {
|
||||
Name string `yaml:"name"`
|
||||
Uses string `yaml:"uses"`
|
||||
UsesComment string `yaml:"-"`
|
||||
With struct {
|
||||
Path string `yaml:"path"`
|
||||
Key string `yaml:"key"`
|
||||
} `yaml:"with"`
|
||||
@@ -43,17 +46,26 @@ func parseWorkflowForWorkflowUsage(_ context.Context, _ file.Resolver, _ *generi
|
||||
return nil, nil, fmt.Errorf("unable to read yaml workflow file: %w", errs)
|
||||
}
|
||||
|
||||
var wf workflowDef
|
||||
if errs = yaml.Unmarshal(contents, &wf); errs != nil {
|
||||
// parse the yaml file into a generic node to preserve comments
|
||||
var node yaml.Node
|
||||
if errs = yaml.Unmarshal(contents, &node); errs != nil {
|
||||
return nil, nil, fmt.Errorf("unable to parse yaml workflow file: %w", errs)
|
||||
}
|
||||
|
||||
// unmarshal the node into a workflowDef struct
|
||||
var wf workflowDef
|
||||
if errs = node.Decode(&wf); errs != nil {
|
||||
return nil, nil, fmt.Errorf("unable to decode workflow: %w", errs)
|
||||
}
|
||||
|
||||
attachUsageComments(&node, &wf)
|
||||
|
||||
// we use a collection to help with deduplication before raising to higher level processing
|
||||
pkgs := pkg.NewCollection()
|
||||
|
||||
for _, job := range wf.Jobs {
|
||||
if job.Uses != "" {
|
||||
p, err := newPackageFromUsageStatement(job.Uses, reader.Location)
|
||||
p, err := newPackageFromUsageStatement(job.Uses, job.UsesComment, reader.Location)
|
||||
if err != nil {
|
||||
errs = unknown.Append(errs, reader, err)
|
||||
}
|
||||
@@ -72,11 +84,20 @@ func parseWorkflowForActionUsage(_ context.Context, _ file.Resolver, _ *generic.
|
||||
return nil, nil, fmt.Errorf("unable to read yaml workflow file: %w", errs)
|
||||
}
|
||||
|
||||
var wf workflowDef
|
||||
if errs = yaml.Unmarshal(contents, &wf); errs != nil {
|
||||
// parse the yaml file into a generic node to preserve comments
|
||||
var node yaml.Node
|
||||
if errs = yaml.Unmarshal(contents, &node); errs != nil {
|
||||
return nil, nil, fmt.Errorf("unable to parse yaml workflow file: %w", errs)
|
||||
}
|
||||
|
||||
// unmarshal the node into a workflowDef struct
|
||||
var wf workflowDef
|
||||
if errs = node.Decode(&wf); errs != nil {
|
||||
return nil, nil, fmt.Errorf("unable to decode workflow: %w", errs)
|
||||
}
|
||||
|
||||
attachUsageComments(&node, &wf)
|
||||
|
||||
// we use a collection to help with deduplication before raising to higher level processing
|
||||
pkgs := pkg.NewCollection()
|
||||
|
||||
@@ -85,7 +106,7 @@ func parseWorkflowForActionUsage(_ context.Context, _ file.Resolver, _ *generic.
|
||||
if step.Uses == "" {
|
||||
continue
|
||||
}
|
||||
p, err := newPackageFromUsageStatement(step.Uses, reader.Location)
|
||||
p, err := newPackageFromUsageStatement(step.Uses, step.UsesComment, reader.Location)
|
||||
if err != nil {
|
||||
errs = unknown.Append(errs, reader, err)
|
||||
}
|
||||
@@ -97,3 +118,101 @@ func parseWorkflowForActionUsage(_ context.Context, _ file.Resolver, _ *generic.
|
||||
|
||||
return pkgs.Sorted(), nil, errs
|
||||
}
|
||||
|
||||
// attachUsageComments traverses the yaml node tree and attaches usage comments to the workflowDef job strcuts and step structs.
|
||||
// This is a best-effort approach to attach comments to the correct job or step.
|
||||
func attachUsageComments(node *yaml.Node, wf *workflowDef) {
|
||||
// for a document node, process its content (usually a single mapping node)
|
||||
if node.Kind == yaml.DocumentNode && len(node.Content) > 0 {
|
||||
processNode(node.Content[0], wf, nil, nil, nil)
|
||||
} else {
|
||||
processNode(node, wf, nil, nil, nil)
|
||||
}
|
||||
}
|
||||
|
||||
func processNode(node *yaml.Node, wf *workflowDef, currentJob *string, currentStep *int, inJobsSection *bool) {
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
for i := 0; i < len(node.Content); i += 2 {
|
||||
key := node.Content[i]
|
||||
value := node.Content[i+1]
|
||||
|
||||
// track if we're in the jobs section...
|
||||
if key.Value == "jobs" && inJobsSection == nil {
|
||||
inJobs := true
|
||||
inJobsSection = &inJobs
|
||||
processNode(value, wf, nil, nil, inJobsSection)
|
||||
continue
|
||||
}
|
||||
|
||||
// if we're in jobs section, and this is a job key...
|
||||
if inJobsSection != nil && *inJobsSection && currentJob == nil {
|
||||
job := key.Value
|
||||
currentJob = &job
|
||||
processNode(value, wf, currentJob, nil, inJobsSection)
|
||||
currentJob = nil
|
||||
continue
|
||||
}
|
||||
|
||||
// if this is a "uses" key...
|
||||
if key.Value == "uses" {
|
||||
processUsesNode(value, wf, currentJob, currentStep)
|
||||
}
|
||||
|
||||
// if this is a "steps" key inside a job...
|
||||
if key.Value == "steps" && currentJob != nil {
|
||||
for j, stepNode := range value.Content {
|
||||
stepIndex := j
|
||||
processNode(stepNode, wf, currentJob, &stepIndex, inJobsSection)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
processNode(key, wf, currentJob, currentStep, inJobsSection)
|
||||
processNode(value, wf, currentJob, currentStep, inJobsSection)
|
||||
}
|
||||
|
||||
case yaml.SequenceNode:
|
||||
for i, item := range node.Content {
|
||||
idx := i
|
||||
processNode(item, wf, currentJob, &idx, inJobsSection)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func processUsesNode(node *yaml.Node, wf *workflowDef, currentJob *string, currentStep *int) {
|
||||
if node.Kind != yaml.ScalarNode {
|
||||
return
|
||||
}
|
||||
|
||||
comment := node.LineComment
|
||||
if comment == "" {
|
||||
comment = node.HeadComment
|
||||
}
|
||||
if comment == "" {
|
||||
comment = node.FootComment
|
||||
}
|
||||
|
||||
if comment != "" {
|
||||
versionRegex := regexp.MustCompile(`v?\d+(\.\d+)*`)
|
||||
versionMatch := versionRegex.FindString(comment)
|
||||
|
||||
if versionMatch != "" {
|
||||
if currentJob != nil && currentStep == nil {
|
||||
// this is a job level "uses"
|
||||
if job, ok := wf.Jobs[*currentJob]; ok {
|
||||
job.UsesComment = versionMatch
|
||||
wf.Jobs[*currentJob] = job
|
||||
}
|
||||
} else if currentJob != nil && currentStep != nil {
|
||||
// this is a step level "uses"
|
||||
if job, ok := wf.Jobs[*currentJob]; ok {
|
||||
if *currentStep < len(job.Steps) {
|
||||
job.Steps[*currentStep].UsesComment = versionMatch
|
||||
wf.Jobs[*currentJob] = job
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ func Test_parseWorkflowForActionUsage(t *testing.T) {
|
||||
Type: pkg.GithubActionPkg,
|
||||
Locations: fixtureLocationSet,
|
||||
PURL: "", // don't have enough context without parsing the git origin, which still may not be accurate
|
||||
Metadata: pkg.GitHubActionsUseStatement{Value: "./.github/actions/bootstrap"},
|
||||
},
|
||||
{
|
||||
Name: "actions/cache",
|
||||
@@ -27,6 +28,7 @@ func Test_parseWorkflowForActionUsage(t *testing.T) {
|
||||
Type: pkg.GithubActionPkg,
|
||||
Locations: fixtureLocationSet,
|
||||
PURL: "pkg:github/actions/cache@v3",
|
||||
Metadata: pkg.GitHubActionsUseStatement{Value: "actions/cache@v3"},
|
||||
},
|
||||
{
|
||||
Name: "actions/cache/restore",
|
||||
@@ -34,6 +36,7 @@ func Test_parseWorkflowForActionUsage(t *testing.T) {
|
||||
Type: pkg.GithubActionPkg,
|
||||
Locations: fixtureLocationSet,
|
||||
PURL: "pkg:github/actions/cache@v3#restore",
|
||||
Metadata: pkg.GitHubActionsUseStatement{Value: "actions/cache/restore@v3"},
|
||||
},
|
||||
{
|
||||
Name: "actions/cache/save",
|
||||
@@ -41,6 +44,7 @@ func Test_parseWorkflowForActionUsage(t *testing.T) {
|
||||
Type: pkg.GithubActionPkg,
|
||||
Locations: fixtureLocationSet,
|
||||
PURL: "pkg:github/actions/cache@v3#save",
|
||||
Metadata: pkg.GitHubActionsUseStatement{Value: "actions/cache/save@v3"},
|
||||
},
|
||||
{
|
||||
Name: "actions/checkout",
|
||||
@@ -48,6 +52,7 @@ func Test_parseWorkflowForActionUsage(t *testing.T) {
|
||||
Type: pkg.GithubActionPkg,
|
||||
Locations: fixtureLocationSet,
|
||||
PURL: "pkg:github/actions/checkout@v4",
|
||||
Metadata: pkg.GitHubActionsUseStatement{Value: "actions/checkout@v4"},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -66,6 +71,9 @@ func Test_parseWorkflowForWorkflowUsage(t *testing.T) {
|
||||
Type: pkg.GithubActionWorkflowPkg,
|
||||
Locations: fixtureLocationSet,
|
||||
PURL: "pkg:github/octo-org/this-repo@172239021f7ba04fe7327647b213799853a9eb89#.github/workflows/workflow-1.yml",
|
||||
Metadata: pkg.GitHubActionsUseStatement{
|
||||
Value: "octo-org/this-repo/.github/workflows/workflow-1.yml@172239021f7ba04fe7327647b213799853a9eb89",
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "./.github/workflows/workflow-2.yml",
|
||||
@@ -73,6 +81,7 @@ func Test_parseWorkflowForWorkflowUsage(t *testing.T) {
|
||||
Type: pkg.GithubActionWorkflowPkg,
|
||||
Locations: fixtureLocationSet,
|
||||
PURL: "", // don't have enough context without parsing the git origin, which still may not be accurate
|
||||
Metadata: pkg.GitHubActionsUseStatement{Value: "./.github/workflows/workflow-2.yml"},
|
||||
},
|
||||
{
|
||||
Name: "octo-org/another-repo/.github/workflows/workflow.yml",
|
||||
@@ -80,6 +89,7 @@ func Test_parseWorkflowForWorkflowUsage(t *testing.T) {
|
||||
Type: pkg.GithubActionWorkflowPkg,
|
||||
Locations: fixtureLocationSet,
|
||||
PURL: "pkg:github/octo-org/another-repo@v1#.github/workflows/workflow.yml",
|
||||
Metadata: pkg.GitHubActionsUseStatement{Value: "octo-org/another-repo/.github/workflows/workflow.yml@v1"},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -87,6 +97,38 @@ func Test_parseWorkflowForWorkflowUsage(t *testing.T) {
|
||||
pkgtest.TestFileParser(t, fixture, parseWorkflowForWorkflowUsage, expected, expectedRelationships)
|
||||
}
|
||||
|
||||
func Test_parseWorkflowForVersionComments(t *testing.T) {
|
||||
fixture := "test-fixtures/workflow-with-version-comments.yaml"
|
||||
fixtureLocationSet := file.NewLocationSet(file.NewLocation(fixture).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
|
||||
|
||||
expected := []pkg.Package{
|
||||
{
|
||||
Name: "./.github/actions/bootstrap",
|
||||
Version: "",
|
||||
Type: pkg.GithubActionPkg,
|
||||
Locations: fixtureLocationSet,
|
||||
PURL: "", // don't have enough context without parsing the git origin, which still may not be accurate
|
||||
Metadata: pkg.GitHubActionsUseStatement{
|
||||
Value: "./.github/actions/bootstrap",
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "actions/checkout",
|
||||
Version: "v4.2.2",
|
||||
Type: pkg.GithubActionPkg,
|
||||
Locations: fixtureLocationSet,
|
||||
PURL: "pkg:github/actions/checkout@v4.2.2",
|
||||
Metadata: pkg.GitHubActionsUseStatement{
|
||||
Value: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683",
|
||||
Comment: "v4.2.2",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var expectedRelationships []artifact.Relationship
|
||||
pkgtest.TestFileParser(t, fixture, parseWorkflowForActionUsage, expected, expectedRelationships)
|
||||
}
|
||||
|
||||
func Test_corruptActionWorkflow(t *testing.T) {
|
||||
pkgtest.NewCatalogTester().
|
||||
FromFile(t, "test-fixtures/corrupt/workflow-multi-job.yaml").
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
name: "Validations"
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
call-workflow-1-in-local-repo:
|
||||
uses: octo-org/this-repo/.github/workflows/workflow-1.yml@172239021f7ba04fe7327647b213799853a9eb89 #v1.0.0
|
||||
|
||||
Static-Analysis:
|
||||
name: "Static analysis"
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
|
||||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
|
||||
- name: Run static analysis
|
||||
run: make static-analysis
|
||||
@@ -343,7 +343,7 @@ func getGOARCHFromBin(r io.ReaderAt) (string, error) {
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unrecognized file format: %w", err)
|
||||
}
|
||||
arch = fmt.Sprintf("%d", f.FileHeader.TargetMachine)
|
||||
arch = fmt.Sprintf("%d", f.TargetMachine)
|
||||
default:
|
||||
return "", errUnrecognizedFormat
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resol
|
||||
Name: m.Mod.Path,
|
||||
Version: m.Mod.Version,
|
||||
Licenses: pkg.NewLicenseSet(lics...),
|
||||
Locations: file.NewLocationSet(reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
|
||||
Locations: file.NewLocationSet(reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
|
||||
PURL: packageURL(m.Mod.Path, m.Mod.Version),
|
||||
Language: pkg.Go,
|
||||
Type: pkg.GoModulePkg,
|
||||
@@ -83,7 +83,7 @@ func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resol
|
||||
Name: m.New.Path,
|
||||
Version: m.New.Version,
|
||||
Licenses: pkg.NewLicenseSet(lics...),
|
||||
Locations: file.NewLocationSet(reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
|
||||
Locations: file.NewLocationSet(reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
|
||||
PURL: packageURL(m.New.Path, m.New.Version),
|
||||
Language: pkg.Go,
|
||||
Type: pkg.GoModulePkg,
|
||||
@@ -120,7 +120,7 @@ func parseGoSumFile(resolver file.Resolver, reader file.LocationReadCloser) (map
|
||||
return out, fmt.Errorf("no resolver provided")
|
||||
}
|
||||
|
||||
goSumPath := strings.TrimSuffix(reader.Location.RealPath, ".mod") + ".sum"
|
||||
goSumPath := strings.TrimSuffix(reader.RealPath, ".mod") + ".sum"
|
||||
goSumLocation := resolver.RelativeFileByPath(reader.Location, goSumPath)
|
||||
if goSumLocation == nil {
|
||||
return nil, fmt.Errorf("unable to resolve: %s", goSumPath)
|
||||
|
||||
@@ -10732,7 +10732,7 @@
|
||||
"cpe:2.3:a:infolific:enhanced_plugin_admin:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"enhanced-text-widget": [
|
||||
"cpe:2.3:a:themecheck:enhanced_text_widget:*:*:*:*:*:wordpress:*:*"
|
||||
"cpe:2.3:a:inisev:enhanced_text_widget:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"enhanced-tooltipglossary": [
|
||||
"cpe:2.3:a:cminds:tooltip_glossary:*:*:*:*:*:wordpress:*:*"
|
||||
@@ -11570,6 +11570,9 @@
|
||||
"gdpr-compliance-cookie-consent": [
|
||||
"cpe:2.3:a:stylemixthemes:gdpr_compliance_\\\u0026_cookie_consent:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"gdpr-cookie-compliance": [
|
||||
"cpe:2.3:a:mooveagency:gdpr_cookie_compliance:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"gdpr-cookie-consent": [
|
||||
"cpe:2.3:a:wpeka:wp_cookie_consent:*:*:*:*:-:wordpress:*:*"
|
||||
],
|
||||
@@ -13058,6 +13061,9 @@
|
||||
"login-as-customer-or-user": [
|
||||
"cpe:2.3:a:wp-buy:login_as_user_or_customer_\\(user_switching\\):*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"login-as-users": [
|
||||
"cpe:2.3:a:geekcodelab:login_as_users:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"login-block-ips": [
|
||||
"cpe:2.3:a:gunkastudios:login_block_ips:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@@ -13784,6 +13790,9 @@
|
||||
"my-favorites": [
|
||||
"cpe:2.3:a:takashimatsuyama:my_favorites:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"my-geo-posts-free": [
|
||||
"cpe:2.3:a:mindstien:my_geo_posts_free:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"my-tickets": [
|
||||
"cpe:2.3:a:my_tickets_project:my_tickets:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@@ -17489,7 +17498,7 @@
|
||||
"cpe:2.3:a:wpxpo:postx:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"ultimate-posts-widget": [
|
||||
"cpe:2.3:a:themecheck:ultimate_posts_widget:*:*:*:*:*:wordpress:*:*"
|
||||
"cpe:2.3:a:inisev:ultimate_posts_widget:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"ultimate-product-catalogue": [
|
||||
"cpe:2.3:a:etoilewebdesign:ultimate_product_catalog:*:*:*:*:*:wordpress:*:*"
|
||||
@@ -18909,6 +18918,7 @@
|
||||
"cpe:2.3:a:wpeasycart:wp_easycart:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wp-ecommerce-paypal": [
|
||||
"cpe:2.3:a:wpplugin:easy_paypal_\\\u0026_stripe_buy_now_button:*:*:*:*:*:wordpress:*:*",
|
||||
"cpe:2.3:a:wpplugin:easy_paypal_buy_now_button:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wp-edit-menu": [
|
||||
@@ -19936,6 +19946,9 @@
|
||||
"wpbulky-wp-bulk-edit-post-types": [
|
||||
"cpe:2.3:a:villatheme:wpbulky:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wpc-badge-management": [
|
||||
"cpe:2.3:a:wpclever:wpc_badge_management_for_woocommerce:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"wpc-shop-as-customer": [
|
||||
"cpe:2.3:a:wpclever:wpc_shop_as_a_customer_for_woocommerce:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
@@ -20743,6 +20756,9 @@
|
||||
"yuki": [
|
||||
"cpe:2.3:a:wpmoose:yuki:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"zbench": [
|
||||
"cpe:2.3:a:zww:zbench:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
"zigcy-baby": [
|
||||
"cpe:2.3:a:accesspressthemes:zigcy_baby:*:*:*:*:*:wordpress:*:*"
|
||||
],
|
||||
|
||||
@@ -7,15 +7,18 @@ import (
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/google/licensecheck"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/imagetest"
|
||||
"github.com/anchore/syft/internal/cmptest"
|
||||
"github.com/anchore/syft/internal/licenses"
|
||||
"github.com/anchore/syft/internal/relationship"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
@@ -27,6 +30,11 @@ import (
|
||||
"github.com/anchore/syft/syft/source/stereoscopesource"
|
||||
)
|
||||
|
||||
var (
|
||||
once sync.Once
|
||||
licenseScanner *licenses.Scanner
|
||||
)
|
||||
|
||||
type CatalogTester struct {
|
||||
expectedPkgs []pkg.Package
|
||||
expectedRelationships []artifact.Relationship
|
||||
@@ -44,10 +52,26 @@ type CatalogTester struct {
|
||||
licenseComparer cmptest.LicenseComparer
|
||||
packageStringer func(pkg.Package) string
|
||||
customAssertions []func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship)
|
||||
context context.Context
|
||||
}
|
||||
|
||||
func Context() context.Context {
|
||||
once.Do(func() {
|
||||
// most of the time in testing is initializing the scanner. Let's do that just once
|
||||
sc := &licenses.ScannerConfig{Scanner: licensecheck.Scan, CoverageThreshold: 75}
|
||||
scanner, err := licenses.NewScanner(sc)
|
||||
if err != nil {
|
||||
panic("unable to setup licences scanner for testing")
|
||||
}
|
||||
licenseScanner = &scanner
|
||||
})
|
||||
|
||||
return licenses.SetContextLicenseScanner(context.Background(), *licenseScanner)
|
||||
}
|
||||
|
||||
func NewCatalogTester() *CatalogTester {
|
||||
return &CatalogTester{
|
||||
context: Context(),
|
||||
locationComparer: cmptest.DefaultLocationComparer,
|
||||
licenseComparer: cmptest.DefaultLicenseComparer,
|
||||
packageStringer: stringPackage,
|
||||
@@ -64,6 +88,11 @@ func NewCatalogTester() *CatalogTester {
|
||||
}
|
||||
}
|
||||
|
||||
func (p *CatalogTester) WithContext(ctx context.Context) *CatalogTester {
|
||||
p.context = ctx
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *CatalogTester) FromDirectory(t *testing.T, path string) *CatalogTester {
|
||||
t.Helper()
|
||||
|
||||
@@ -202,7 +231,7 @@ func (p *CatalogTester) IgnoreUnfulfilledPathResponses(paths ...string) *Catalog
|
||||
|
||||
func (p *CatalogTester) TestParser(t *testing.T, parser generic.Parser) {
|
||||
t.Helper()
|
||||
pkgs, relationships, err := parser(context.Background(), p.resolver, p.env, p.reader)
|
||||
pkgs, relationships, err := parser(p.context, p.resolver, p.env, p.reader)
|
||||
// only test for errors if explicitly requested
|
||||
if p.wantErr != nil {
|
||||
p.wantErr(t, err)
|
||||
@@ -215,7 +244,7 @@ func (p *CatalogTester) TestCataloger(t *testing.T, cataloger pkg.Cataloger) {
|
||||
|
||||
resolver := NewObservingResolver(p.resolver)
|
||||
|
||||
pkgs, relationships, err := cataloger.Catalog(context.Background(), resolver)
|
||||
pkgs, relationships, err := cataloger.Catalog(p.context, resolver)
|
||||
|
||||
// this is a minimum set, the resolver may return more that just this list
|
||||
for _, path := range p.expectedPathResponses {
|
||||
|
||||
@@ -275,7 +275,7 @@ func (ni nativeImageElf) fetchPkgs() (pkgs []pkg.Package, relationships []artifa
|
||||
if dataSection == nil {
|
||||
return nil, nil, fmt.Errorf("no .data section found in binary: %w", err)
|
||||
}
|
||||
dataSectionBase := dataSection.SectionHeader.Addr
|
||||
dataSectionBase := dataSection.Addr
|
||||
data, err := dataSection.Data()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot read the .data section: %w", err)
|
||||
|
||||
@@ -63,7 +63,7 @@ func parseGradleLockfile(_ context.Context, _ file.Resolver, _ *generic.Environm
|
||||
Name: dep.Name,
|
||||
Version: dep.Version,
|
||||
Locations: file.NewLocationSet(
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
),
|
||||
Language: pkg.Java,
|
||||
Type: pkg.JavaPkg,
|
||||
|
||||
@@ -67,7 +67,7 @@ func parsePackageJSON(_ context.Context, _ file.Resolver, _ *generic.Environment
|
||||
// a compliance filter later will remove these packages based on compliance rules
|
||||
pkgs = append(
|
||||
pkgs,
|
||||
newPackageJSONPackage(p, reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
|
||||
newPackageJSONPackage(p, reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ func parseLinuxKernelModuleFile(_ context.Context, _ file.Resolver, _ *generic.E
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
metadata.Path = reader.Location.RealPath
|
||||
metadata.Path = reader.RealPath
|
||||
|
||||
return []pkg.Package{
|
||||
newLinuxKernelModulePackage(
|
||||
|
||||
@@ -74,7 +74,7 @@ func parseRockspec(_ context.Context, _ file.Resolver, _ *generic.Environment, r
|
||||
Homepage: homepage,
|
||||
Description: description,
|
||||
},
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
)
|
||||
|
||||
return []pkg.Package{p}, nil, nil
|
||||
|
||||
@@ -6,6 +6,9 @@ package nix
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
|
||||
@@ -28,6 +31,30 @@ func (c *storeCataloger) Name() string {
|
||||
return catalogerName
|
||||
}
|
||||
|
||||
// Find the parent Nix store path for a given path
|
||||
func findParentNixStorePath(path string) string {
|
||||
// Handle standard /nix/store/ paths
|
||||
parts := strings.Split(path, "/")
|
||||
for i := 0; i < len(parts)-2; i++ {
|
||||
if parts[i] == "nix" && parts[i+1] == "store" && i+2 < len(parts) {
|
||||
// Check if it matches the hash-name pattern
|
||||
storeItem := parts[i+2]
|
||||
if matched, _ := regexp.MatchString(`^[a-z0-9]{32}-.*`, storeItem); matched {
|
||||
return filepath.Join("/", filepath.Join(parts[:i+3]...))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle short-form Nix paths (without the /nix/store/ prefix)
|
||||
// These would be paths like /[hash]-[package-name]/...
|
||||
shortPathPattern := regexp.MustCompile(`^/([a-z0-9]{32}-[^/]+)`)
|
||||
if matches := shortPathPattern.FindStringSubmatch(path); len(matches) == 2 {
|
||||
return "/" + matches[1]
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func (c *storeCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
// we want to search for only directories, which isn't possible via the stereoscope API, so we need to apply the glob manually on all returned paths
|
||||
var pkgs []pkg.Package
|
||||
@@ -37,9 +64,23 @@ func (c *storeCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([
|
||||
for location := range resolver.AllLocations(ctx) {
|
||||
matchesStorePath, err := doublestar.Match("**/nix/store/*", location.RealPath)
|
||||
if err != nil {
|
||||
log.Debugf("Error matching path %s: %v", location.RealPath, err)
|
||||
return nil, nil, fmt.Errorf("failed to match nix store path: %w", err)
|
||||
}
|
||||
if !matchesStorePath {
|
||||
// Check if this is a "short-form" Nix store path (without the /nix/store/ prefix)
|
||||
// This pattern matches paths like /[hash]-[package-name]/...
|
||||
shortPathPattern := regexp.MustCompile(`^/([a-z0-9]{32})-([^/]+)`)
|
||||
if shortPathPattern.MatchString(location.RealPath) {
|
||||
matchesStorePath = true
|
||||
log.Debugf("Matched short-form Nix path: %s", location.RealPath)
|
||||
}
|
||||
}
|
||||
|
||||
if !matchesStorePath {
|
||||
log.Debugf("Not a Nix store path: %s", location.RealPath)
|
||||
continue
|
||||
}
|
||||
parentStorePath := findParentNixStorePath(location.RealPath)
|
||||
if parentStorePath != "" {
|
||||
if _, ok := filesByPath[parentStorePath]; !ok {
|
||||
@@ -50,17 +91,24 @@ func (c *storeCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([
|
||||
}
|
||||
|
||||
if !matchesStorePath {
|
||||
log.Debugf("Not a Nix store path: %s", location.RealPath)
|
||||
continue
|
||||
}
|
||||
|
||||
storePath := parseNixStorePath(location.RealPath)
|
||||
|
||||
if storePath == nil || !storePath.isValidPackage() {
|
||||
if storePath == nil {
|
||||
log.Debugf("Failed to parse Nix path: %s", location.RealPath)
|
||||
continue
|
||||
}
|
||||
|
||||
p := newNixStorePackage(*storePath, location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
|
||||
pkgs = append(pkgs, p)
|
||||
// Only create packages for non-derivation/source paths
|
||||
if storePath.shouldIncludeAsPackage() {
|
||||
p := newNixStorePackage(*storePath, location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
|
||||
pkgs = append(pkgs, p)
|
||||
} else {
|
||||
log.Debugf("Skipping non-package path: %s (type: %s)", location.RealPath, storePath.pathType)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// add file sets to packages
|
||||
@@ -80,7 +128,64 @@ func (c *storeCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([
|
||||
appendFiles(p, files.ToSlice()...)
|
||||
}
|
||||
|
||||
return pkgs, nil, nil
|
||||
// After all packages are created, enrich them with derivation data
|
||||
for i := range pkgs {
|
||||
p := &pkgs[i]
|
||||
metadata, ok := p.Metadata.(pkg.NixStoreEntry)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// Find and parse the derivation file
|
||||
derivPath := findDeriverPath(metadata.Path)
|
||||
if derivPath != "" {
|
||||
if deriv, err := ParseDerivation(derivPath); err == nil {
|
||||
// Enrich package metadata with derivation data
|
||||
if deriv.PName != "" {
|
||||
p.Name = deriv.PName
|
||||
}
|
||||
if deriv.Version != "" {
|
||||
p.Version = deriv.Version
|
||||
}
|
||||
|
||||
// Add license, homepage, etc. to metadata
|
||||
metadata.License = deriv.License
|
||||
metadata.Homepage = deriv.Homepage
|
||||
metadata.Description = deriv.Description
|
||||
metadata.DeriverPath = derivPath
|
||||
|
||||
// Update the package metadata
|
||||
p.Metadata = metadata
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Find and add relationships between packages
|
||||
relationships, err := findDependencies(pkgs)
|
||||
if err != nil {
|
||||
log.Warnf("failed to resolve nix dependencies: %v", err)
|
||||
}
|
||||
// Deduplicate packages based on name and version
|
||||
dedupedPkgs := deduplicateNixPackages(pkgs)
|
||||
pkgs = dedupedPkgs
|
||||
return pkgs, relationships, nil
|
||||
}
|
||||
|
||||
// ensureNixStorePrefix makes sure the path has the /nix/store/ prefix
|
||||
func ensureNixStorePrefix(path string) string {
|
||||
// Check if the path already has the /nix/store/ prefix
|
||||
if strings.Contains(path, "/nix/store/") {
|
||||
return path
|
||||
}
|
||||
|
||||
// Check if this is a short-form path starting with /[hash]-[name]
|
||||
shortPathPattern := regexp.MustCompile(`^/([a-z0-9]{32}-[^/]+)(.*)$`)
|
||||
if matches := shortPathPattern.FindStringSubmatch(path); len(matches) >= 3 {
|
||||
// Convert short form path to full path with /nix/store/ prefix
|
||||
return "/nix/store/" + matches[1] + matches[2]
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
func appendFiles(p *pkg.Package, location ...file.Location) {
|
||||
@@ -91,7 +196,9 @@ func appendFiles(p *pkg.Package, location ...file.Location) {
|
||||
}
|
||||
|
||||
for _, l := range location {
|
||||
metadata.Files = append(metadata.Files, l.RealPath)
|
||||
// Normalize the path to ensure it has the /nix/store/ prefix
|
||||
normalizedPath := ensureNixStorePrefix(l.RealPath)
|
||||
metadata.Files = append(metadata.Files, normalizedPath)
|
||||
}
|
||||
|
||||
if metadata.Files == nil {
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
package nix
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
// findDependencies attempts to find relationships between Nix packages
|
||||
// without using external commands
|
||||
func findDependencies(pkgs []pkg.Package) ([]artifact.Relationship, error) {
|
||||
var relationships []artifact.Relationship
|
||||
packageByPath := make(map[string]*pkg.Package)
|
||||
packageByStorePath := make(map[string]*pkg.Package)
|
||||
|
||||
// Index packages by store path and path for quick lookups
|
||||
for i := range pkgs {
|
||||
p := &pkgs[i]
|
||||
metadata, ok := p.Metadata.(pkg.NixStoreEntry)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
packageByStorePath[metadata.Path] = p
|
||||
|
||||
// Also index by the package files
|
||||
for _, filePath := range metadata.Files {
|
||||
packageByPath[filePath] = p
|
||||
}
|
||||
}
|
||||
|
||||
// For each package, try to find references to other packages
|
||||
for i := range pkgs {
|
||||
p := &pkgs[i]
|
||||
metadata, ok := p.Metadata.(pkg.NixStoreEntry)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// Scan for references using the derivation file if available
|
||||
if metadata.DeriverPath != "" {
|
||||
refs, err := findReferencesInDerivation(metadata.DeriverPath, packageByStorePath)
|
||||
if err == nil {
|
||||
relationships = append(relationships, refs...)
|
||||
}
|
||||
}
|
||||
|
||||
// Scan for references within the package files
|
||||
if len(metadata.Files) > 0 {
|
||||
refs, err := findReferencesInFiles(metadata.Files, p, packageByStorePath)
|
||||
if err == nil {
|
||||
relationships = append(relationships, refs...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return relationships, nil
|
||||
}
|
||||
|
||||
// findReferencesInDerivation tries to extract dependencies from a derivation file
|
||||
func findReferencesInDerivation(drvPath string, packageByPath map[string]*pkg.Package) ([]artifact.Relationship, error) {
|
||||
var relationships []artifact.Relationship
|
||||
|
||||
// Read the derivation file contents
|
||||
content, err := os.ReadFile(drvPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Look for store paths in the derivation file
|
||||
storePathPattern := regexp.MustCompile(`/nix/store/[a-z0-9]{32}-[^"'\s]+`)
|
||||
matches := storePathPattern.FindAllString(string(content), -1)
|
||||
|
||||
// Unique matches
|
||||
uniquePaths := make(map[string]struct{})
|
||||
for _, match := range matches {
|
||||
uniquePaths[match] = struct{}{}
|
||||
}
|
||||
|
||||
// Create relationships for found dependencies
|
||||
for path := range uniquePaths {
|
||||
depPkg, exists := packageByPath[path]
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip self-references
|
||||
fromMeta, ok := depPkg.Metadata.(pkg.NixStoreEntry)
|
||||
if !ok || fromMeta.Path == drvPath {
|
||||
continue
|
||||
}
|
||||
|
||||
// Add the dependency relationship
|
||||
pkg, exists := packageByPath[drvPath]
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
|
||||
relationships = append(relationships, artifact.Relationship{
|
||||
From: *pkg,
|
||||
To: *depPkg,
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
Data: map[string]string{"dependencyType": "buildtime"},
|
||||
})
|
||||
}
|
||||
|
||||
return relationships, nil
|
||||
}
|
||||
|
||||
// findReferencesInFiles looks for references to other packages within file contents
|
||||
func findReferencesInFiles(files []string, fromPkg *pkg.Package, packageByPath map[string]*pkg.Package) ([]artifact.Relationship, error) {
|
||||
var relationships []artifact.Relationship
|
||||
uniqueRefs := make(map[string]struct{})
|
||||
|
||||
// Limit search to a reasonable number of files to avoid performance issues
|
||||
maxFiles := 10
|
||||
fileCount := 0
|
||||
|
||||
for _, file := range files {
|
||||
if fileCount >= maxFiles {
|
||||
break
|
||||
}
|
||||
|
||||
// Skip files that are too large
|
||||
info, err := os.Stat(file)
|
||||
if err != nil || info.Size() > 1024*1024 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip directories
|
||||
if info.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
// Look for binary files that might have references
|
||||
if isBinary(file) {
|
||||
fileCount++
|
||||
refs, err := findReferencesInBinary(file)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, ref := range refs {
|
||||
uniqueRefs[ref] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create relationships for found dependencies
|
||||
for path := range uniqueRefs {
|
||||
depPkg, exists := packageByPath[path]
|
||||
if !exists {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip self-references
|
||||
fromMeta, ok := fromPkg.Metadata.(pkg.NixStoreEntry)
|
||||
if !ok || fromMeta.Path == path {
|
||||
continue
|
||||
}
|
||||
|
||||
relationships = append(relationships, artifact.Relationship{
|
||||
From: *fromPkg,
|
||||
To: *depPkg,
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
Data: map[string]string{"dependencyType": "runtime"},
|
||||
})
|
||||
}
|
||||
|
||||
return relationships, nil
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
// isBinary returns true if the file appears to be a binary
|
||||
func isBinary(path string) bool {
|
||||
ext := strings.ToLower(filepath.Ext(path))
|
||||
return ext == ".so" || ext == ".dylib" || ext == ".a" ||
|
||||
strings.HasSuffix(path, ".so.1") ||
|
||||
strings.Contains(path, ".so.")
|
||||
}
|
||||
|
||||
// findReferencesInBinary extracts Nix store references from binary files
|
||||
func findReferencesInBinary(path string) ([]string, error) {
|
||||
var references []string
|
||||
|
||||
// Use string search instead of executing nix-store
|
||||
data, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Search for store paths in the binary
|
||||
storePathPattern := regexp.MustCompile(`/nix/store/[a-z0-9]{32}-[^\\:\*\?"<>\|\x00-\x1F]+`)
|
||||
matches := storePathPattern.FindAll(data, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
references = append(references, string(match))
|
||||
}
|
||||
|
||||
return references, nil
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package nix
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
)
|
||||
|
||||
// NixDerivation represents a parsed Nix derivation file
|
||||
type NixDerivation struct {
|
||||
Path string
|
||||
Name string
|
||||
PName string
|
||||
Version string
|
||||
System string
|
||||
Builder string
|
||||
Args []string
|
||||
EnvVars map[string]string
|
||||
Outputs map[string]string
|
||||
InputDrvs map[string][]string
|
||||
InputSrcs []string
|
||||
License string
|
||||
Homepage string
|
||||
Description string
|
||||
OutputHash string
|
||||
}
|
||||
|
||||
// ParseDerivation attempts to parse a Nix derivation file without external commands
|
||||
func ParseDerivation(path string) (*NixDerivation, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
derivation := &NixDerivation{
|
||||
Path: path,
|
||||
EnvVars: make(map[string]string),
|
||||
Outputs: make(map[string]string),
|
||||
InputDrvs: make(map[string][]string),
|
||||
}
|
||||
|
||||
// Simple parser for .drv files
|
||||
scanner := bufio.NewScanner(file)
|
||||
inEnvVars := false
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
|
||||
// Process environment variables section
|
||||
if strings.Contains(line, "envVars = {") {
|
||||
inEnvVars = true
|
||||
continue
|
||||
} else if inEnvVars && strings.Contains(line, "};") {
|
||||
inEnvVars = false
|
||||
continue
|
||||
} else if inEnvVars && strings.Contains(line, " = ") {
|
||||
// Parse environment variables
|
||||
parts := strings.SplitN(line, " = ", 2)
|
||||
if len(parts) == 2 {
|
||||
key := strings.Trim(parts[0], " \"")
|
||||
value := strings.Trim(parts[1], " \";")
|
||||
derivation.EnvVars[key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract package information from environment variables
|
||||
derivation.Name = derivation.EnvVars["name"]
|
||||
derivation.PName = derivation.EnvVars["pname"]
|
||||
derivation.Version = derivation.EnvVars["version"]
|
||||
derivation.System = derivation.EnvVars["system"]
|
||||
derivation.License = derivation.EnvVars["license"]
|
||||
derivation.Homepage = derivation.EnvVars["homepage"]
|
||||
derivation.Description = derivation.EnvVars["description"]
|
||||
|
||||
// If pname is not set but name follows pname-version pattern, extract it
|
||||
if derivation.PName == "" && derivation.Name != "" {
|
||||
re := regexp.MustCompile(`^(.*?)-([0-9].*)$`)
|
||||
if matches := re.FindStringSubmatch(derivation.Name); len(matches) == 3 {
|
||||
derivation.PName = matches[1]
|
||||
if derivation.Version == "" {
|
||||
derivation.Version = matches[2]
|
||||
}
|
||||
} else {
|
||||
derivation.PName = derivation.Name
|
||||
}
|
||||
}
|
||||
|
||||
// Check if this is a patched package with security fixes
|
||||
for key := range derivation.EnvVars {
|
||||
if strings.HasPrefix(key, "patches") {
|
||||
value := derivation.EnvVars[key]
|
||||
if strings.Contains(value, "CVE-") {
|
||||
log.Debugf("Found security patch in %s: %s", derivation.Name, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return derivation, nil
|
||||
}
|
||||
|
||||
// findDeriverPath attempts to find the derivation file for a store path
|
||||
func findDeriverPath(storePath string) string {
|
||||
// If the path is already a derivation, return it
|
||||
if strings.HasSuffix(storePath, ".drv") {
|
||||
return storePath
|
||||
}
|
||||
|
||||
// Look for a .drv file with a similar name
|
||||
drvPathGuess := storePath + ".drv"
|
||||
if _, err := os.Stat(drvPathGuess); err == nil {
|
||||
return drvPathGuess
|
||||
}
|
||||
|
||||
// Fall back strategy using string manipulation
|
||||
// This is an approximation without running nix-store commands
|
||||
parts := strings.Split(storePath, "-")
|
||||
if len(parts) >= 2 {
|
||||
// Try to find the derivation file using the hash part
|
||||
hash := strings.Split(filepath.Base(storePath), "-")[0]
|
||||
if matched, _ := regexp.MatchString(`^[a-z0-9]{32}$`, hash); matched {
|
||||
drvDir := filepath.Join("/nix/store", hash+"-*.drv")
|
||||
matches, err := filepath.Glob(drvDir)
|
||||
if err == nil && len(matches) > 0 {
|
||||
return matches[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
@@ -56,3 +56,67 @@ func packageURL(storePath nixStorePath) string {
|
||||
"")
|
||||
return pURL.ToString()
|
||||
}
|
||||
|
||||
// deduplicateNixPackages combines information from packages with the same name and version
|
||||
func deduplicateNixPackages(pkgs []pkg.Package) []pkg.Package {
|
||||
if len(pkgs) == 0 {
|
||||
return pkgs
|
||||
}
|
||||
|
||||
// Use map to group by name+version
|
||||
packageMap := make(map[string]*pkg.Package)
|
||||
|
||||
for i := range pkgs {
|
||||
p := &pkgs[i]
|
||||
key := p.Name + ":" + p.Version
|
||||
|
||||
existing, exists := packageMap[key]
|
||||
if !exists {
|
||||
// First time seeing this name+version
|
||||
packageMap[key] = p
|
||||
continue
|
||||
}
|
||||
|
||||
// Merge information from this package into the existing one
|
||||
mergePackageInfo(existing, p)
|
||||
}
|
||||
|
||||
// Convert map back to slice
|
||||
result := make([]pkg.Package, 0, len(packageMap))
|
||||
for _, p := range packageMap {
|
||||
result = append(result, *p)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// mergePackageInfo combines information from src into dest
|
||||
func mergePackageInfo(dest, src *pkg.Package) {
|
||||
// Merge locations
|
||||
for _, loc := range src.Locations.ToSlice() {
|
||||
dest.Locations.Add(loc)
|
||||
}
|
||||
|
||||
// Merge metadata if both have NixStoreEntry type
|
||||
destMeta, destOk := dest.Metadata.(pkg.NixStoreEntry)
|
||||
srcMeta, srcOk := src.Metadata.(pkg.NixStoreEntry)
|
||||
|
||||
if destOk && srcOk {
|
||||
// Combine files lists
|
||||
destMeta.Files = append(destMeta.Files, srcMeta.Files...)
|
||||
|
||||
// Prefer non-empty fields from src
|
||||
if destMeta.License == "" && srcMeta.License != "" {
|
||||
destMeta.License = srcMeta.License
|
||||
}
|
||||
if destMeta.Homepage == "" && srcMeta.Homepage != "" {
|
||||
destMeta.Homepage = srcMeta.Homepage
|
||||
}
|
||||
if destMeta.Description == "" && srcMeta.Description != "" {
|
||||
destMeta.Description = srcMeta.Description
|
||||
}
|
||||
|
||||
// Update the metadata
|
||||
dest.Metadata = destMeta
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,134 +1,155 @@
|
||||
package nix
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
numericPattern = regexp.MustCompile(`\d`)
|
||||
|
||||
// attempts to find the right-most example of something that appears to be a version (semver or otherwise)
|
||||
// example input: h0cnbmfcn93xm5dg2x27ixhag1cwndga-glibc-2.34-210-bin
|
||||
// example output:
|
||||
// version: "2.34-210"
|
||||
// major: "2"
|
||||
// minor: "34"
|
||||
// patch: "210"
|
||||
// (there are other capture groups, but they can be ignored)
|
||||
rightMostVersionIshPattern = regexp.MustCompile(`-(?P<version>(?P<major>[0-9][a-zA-Z0-9]*)(\.(?P<minor>[0-9][a-zA-Z0-9]*))?(\.(?P<patch>0|[1-9][a-zA-Z0-9]*)){0,3}(?:-(?P<prerelease>\d*[.a-zA-Z-][.0-9a-zA-Z-]*)*)?(?:\+(?P<metadata>[.0-9a-zA-Z-]+(?:\.[.0-9a-zA-Z-]+)*))?)`)
|
||||
|
||||
unstableVersion = regexp.MustCompile(`-(?P<version>unstable-\d{4}-\d{2}-\d{2})$`)
|
||||
)
|
||||
|
||||
// checkout the package naming conventions here: https://nixos.org/manual/nixpkgs/stable/#sec-package-naming
|
||||
|
||||
type nixStorePath struct {
|
||||
path string
|
||||
hash string
|
||||
outputHash string
|
||||
name string
|
||||
version string
|
||||
output string
|
||||
// New field to indicate the type of Nix path
|
||||
pathType string // "package", "derivation", "source", or "other"
|
||||
}
|
||||
|
||||
func (p nixStorePath) isValidPackage() bool {
|
||||
return p.name != "" && p.version != ""
|
||||
}
|
||||
|
||||
func findParentNixStorePath(source string) string {
|
||||
source = strings.TrimRight(source, "/")
|
||||
indicator := "nix/store/"
|
||||
start := strings.Index(source, indicator)
|
||||
if start == -1 {
|
||||
return ""
|
||||
}
|
||||
|
||||
startOfHash := start + len(indicator)
|
||||
nextField := strings.Index(source[startOfHash:], "/")
|
||||
if nextField == -1 {
|
||||
return ""
|
||||
}
|
||||
startOfSubPath := startOfHash + nextField
|
||||
|
||||
return source[0:startOfSubPath]
|
||||
}
|
||||
|
||||
func parseNixStorePath(source string) *nixStorePath {
|
||||
if strings.HasSuffix(source, ".drv") {
|
||||
// ignore derivations
|
||||
// parseNixStorePath extracts package information from a Nix store path.
|
||||
// This is a more permissive implementation that accepts more formats.
|
||||
func parseNixStorePath(path string) *nixStorePath {
|
||||
// Extract the store path component - handle both standard and short forms
|
||||
// Standard form: /nix/store/[hash]-[name]
|
||||
// Short form: /[hash]-[name]
|
||||
storePathPattern := regexp.MustCompile(`(^|.*?)(?:/nix/store/|/)([a-z0-9]{32})-(.+?)(/.*)?$`)
|
||||
matches := storePathPattern.FindStringSubmatch(path)
|
||||
if len(matches) < 4 {
|
||||
return nil
|
||||
}
|
||||
|
||||
source = path.Base(source)
|
||||
// Extract hash and full package name
|
||||
hash := matches[2]
|
||||
fullName := matches[3]
|
||||
|
||||
versionStartIdx, versionIsh, prerelease := findVersionIsh(source)
|
||||
if versionStartIdx == -1 {
|
||||
return nil
|
||||
// Extract path to first / after the hash-name pattern
|
||||
basePath := "/nix/store/" + hash + "-" + fullName
|
||||
if len(matches) > 4 && matches[4] != "" {
|
||||
// We have a file within a store path
|
||||
basePath = "/nix/store/" + hash + "-" + fullName
|
||||
}
|
||||
|
||||
hashName := strings.TrimSuffix(source[0:versionStartIdx], "-")
|
||||
hashNameFields := strings.Split(hashName, "-")
|
||||
if len(hashNameFields) < 2 {
|
||||
return nil
|
||||
}
|
||||
hash, name := hashNameFields[0], strings.Join(hashNameFields[1:], "-")
|
||||
|
||||
prereleaseFields := strings.Split(prerelease, "-")
|
||||
lastPrereleaseField := prereleaseFields[len(prereleaseFields)-1]
|
||||
|
||||
var version = versionIsh
|
||||
var output string
|
||||
if !hasNumeric(lastPrereleaseField) {
|
||||
// this last prerelease field is probably a nix output
|
||||
version = strings.TrimSuffix(versionIsh, fmt.Sprintf("-%s", lastPrereleaseField))
|
||||
output = lastPrereleaseField
|
||||
}
|
||||
|
||||
return &nixStorePath{
|
||||
// Create the basic result
|
||||
result := &nixStorePath{
|
||||
path: basePath,
|
||||
hash: hash,
|
||||
outputHash: hash,
|
||||
name: name,
|
||||
version: version,
|
||||
output: output,
|
||||
}
|
||||
|
||||
// Determine the path type based on extension or patterns
|
||||
if strings.HasSuffix(fullName, ".drv") {
|
||||
result.pathType = "derivation"
|
||||
// Remove .drv suffix for name parsing
|
||||
fullName = strings.TrimSuffix(fullName, ".drv")
|
||||
} else if strings.Contains(fullName, ".tar.") ||
|
||||
strings.HasSuffix(fullName, ".tgz") ||
|
||||
strings.HasSuffix(fullName, ".tar.gz") ||
|
||||
strings.HasSuffix(fullName, ".tar.bz2") ||
|
||||
strings.HasSuffix(fullName, ".tar.xz") ||
|
||||
strings.HasSuffix(fullName, ".zip") ||
|
||||
strings.HasSuffix(fullName, ".patch") {
|
||||
result.pathType = "source"
|
||||
} else {
|
||||
result.pathType = "package"
|
||||
}
|
||||
|
||||
// Try to parse the output from the path (if present)
|
||||
if outputParts := strings.Split(fullName, "-"); len(outputParts) > 1 {
|
||||
lastPart := outputParts[len(outputParts)-1]
|
||||
// Common Nix output names
|
||||
outputs := map[string]bool{
|
||||
"bin": true, "dev": true, "lib": true, "man": true,
|
||||
"doc": true, "info": true, "out": true,
|
||||
}
|
||||
if outputs[lastPart] {
|
||||
result.output = lastPart
|
||||
fullName = strings.Join(outputParts[:len(outputParts)-1], "-")
|
||||
}
|
||||
}
|
||||
|
||||
// For non-drv files, try different version extraction patterns
|
||||
versionPatterns := []*regexp.Regexp{
|
||||
// Standard version: name-1.2.3
|
||||
regexp.MustCompile(`^(.+)-([0-9][0-9.]+(?:-[0-9]+)?)$`),
|
||||
|
||||
// Unstable version: name-unstable-2022-05-15
|
||||
regexp.MustCompile(`^(.+)-unstable-([0-9]{4}-[0-9]{2}-[0-9]{2})$`),
|
||||
|
||||
// Version with prefix: name-v1.2.3
|
||||
regexp.MustCompile(`^(.+)-(v[0-9][0-9.]+)$`),
|
||||
|
||||
// Year-based versions: name-2022.05
|
||||
regexp.MustCompile(`^(.+)-([0-9]{4}(?:\.[0-9]+)*)$`),
|
||||
|
||||
// Version with suffix: name-1.2.3-suffix
|
||||
regexp.MustCompile(`^(.+)-([0-9][0-9.]+(?:-[a-z0-9]+)*)$`),
|
||||
}
|
||||
|
||||
// Try each pattern to extract name and version
|
||||
for _, pattern := range versionPatterns {
|
||||
if nameVerMatches := pattern.FindStringSubmatch(fullName); len(nameVerMatches) == 3 {
|
||||
result.name = nameVerMatches[1]
|
||||
result.version = nameVerMatches[2]
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
// If no version pattern matched, consider the entire string the package name
|
||||
result.name = fullName
|
||||
result.version = ""
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func hasNumeric(s string) bool {
|
||||
return numericPattern.MatchString(s)
|
||||
// isLikelyOutput checks if a string is likely to be a Nix output name
|
||||
func isLikelyOutput(s string) bool {
|
||||
commonOutputs := map[string]bool{
|
||||
"bin": true, "lib": true, "dev": true, "out": true,
|
||||
"doc": true, "man": true, "info": true, "devdoc": true,
|
||||
}
|
||||
return commonOutputs[s]
|
||||
}
|
||||
|
||||
func findVersionIsh(input string) (int, string, string) {
|
||||
// we want to return the index of the start of the "version" group (the first capture group).
|
||||
// note that the match indices are in the form of [start, end, start, end, ...]. Also note that the
|
||||
// capture group for version in both regexes are the same index, but if the regexes are changed
|
||||
// this code will start to fail.
|
||||
versionGroup := 1
|
||||
// isLikelyVersion checks if a string looks like a version number
|
||||
func isLikelyVersion(s string) bool {
|
||||
// Simple check for common version patterns
|
||||
versionPattern := regexp.MustCompile(`^v?\d+(\.\d+)*(-\w+)*$`)
|
||||
return versionPattern.MatchString(s)
|
||||
}
|
||||
|
||||
match := unstableVersion.FindAllStringSubmatchIndex(input, -1)
|
||||
if len(match) > 0 && len(match[0]) > 0 {
|
||||
return match[0][versionGroup*2], input[match[0][versionGroup*2]:match[0][(versionGroup*2)+1]], ""
|
||||
// shouldIncludeAsPackage determines if this store path should be included as a package
|
||||
func (n *nixStorePath) shouldIncludeAsPackage() bool {
|
||||
// Only include actual packages, not derivations or source files
|
||||
if n.pathType == "derivation" || n.pathType == "source" {
|
||||
return false
|
||||
}
|
||||
|
||||
match = rightMostVersionIshPattern.FindAllStringSubmatchIndex(input, -1)
|
||||
if len(match) == 0 || len(match[0]) == 0 {
|
||||
return -1, "", ""
|
||||
// Skip paths that look like source archives even if not explicitly marked
|
||||
if strings.HasSuffix(n.path, ".tar.gz") ||
|
||||
strings.HasSuffix(n.path, ".tgz") ||
|
||||
strings.HasSuffix(n.path, ".tar.bz2") ||
|
||||
strings.HasSuffix(n.path, ".tar.xz") ||
|
||||
strings.HasSuffix(n.path, ".zip") ||
|
||||
strings.HasSuffix(n.path, ".drv") {
|
||||
return false
|
||||
}
|
||||
|
||||
var version string
|
||||
versionStart, versionStop := match[0][versionGroup*2], match[0][(versionGroup*2)+1]
|
||||
if versionStart != -1 || versionStop != -1 {
|
||||
version = input[versionStart:versionStop]
|
||||
}
|
||||
|
||||
prereleaseGroup := 7
|
||||
|
||||
var prerelease string
|
||||
prereleaseStart, prereleaseStop := match[0][prereleaseGroup*2], match[0][(prereleaseGroup*2)+1]
|
||||
if prereleaseStart != -1 && prereleaseStop != -1 {
|
||||
prerelease = input[prereleaseStart:prereleaseStop]
|
||||
}
|
||||
// Make sure it has a name (at minimum)
|
||||
return n.name != ""
|
||||
}
|
||||
|
||||
return versionStart,
|
||||
version,
|
||||
prerelease
|
||||
// isValidPackage should be much more permissive
|
||||
func (n *nixStorePath) isValidPackage() bool {
|
||||
// Any path with a hash and name is considered valid
|
||||
return n.hash != "" && n.name != ""
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ func parseOpamPackage(_ context.Context, _ file.Resolver, _ *generic.Environment
|
||||
// If name is inferred from file name/path
|
||||
var name, version string
|
||||
var licenses []string
|
||||
loc := reader.Location.LocationData.AccessPath
|
||||
loc := reader.AccessPath
|
||||
dir, file := path.Split(loc)
|
||||
|
||||
if file == "opam" {
|
||||
@@ -95,7 +95,7 @@ func parseOpamPackage(_ context.Context, _ file.Resolver, _ *generic.Environment
|
||||
pkgs,
|
||||
newOpamPackage(
|
||||
entry,
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package python
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"testing"
|
||||
@@ -15,264 +14,383 @@ import (
|
||||
|
||||
func Test_PackageCataloger(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fixtures []string
|
||||
expectedPackage pkg.Package
|
||||
name string
|
||||
fixture string
|
||||
expectedPackages []pkg.Package
|
||||
}{
|
||||
{
|
||||
name: "egg-file-no-version",
|
||||
fixtures: []string{"test-fixtures/no-version-py3.8.egg-info"},
|
||||
expectedPackage: pkg.Package{
|
||||
Name: "no-version",
|
||||
PURL: "pkg:pypi/no-version",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "no-version",
|
||||
SitePackagesRootPath: "test-fixtures",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "egg-info directory",
|
||||
fixtures: []string{
|
||||
"test-fixtures/egg-info/PKG-INFO",
|
||||
"test-fixtures/egg-info/RECORD",
|
||||
"test-fixtures/egg-info/top_level.txt",
|
||||
},
|
||||
expectedPackage: pkg.Package{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
PURL: "pkg:pypi/requests@2.22.0",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/egg-info/PKG-INFO")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
Platform: "UNKNOWN",
|
||||
Author: "Kenneth Reitz",
|
||||
AuthorEmail: "me@kennethreitz.org",
|
||||
SitePackagesRootPath: "test-fixtures",
|
||||
Files: []pkg.PythonFileRecord{
|
||||
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
|
||||
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
|
||||
{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
|
||||
{Path: "requests/__pycache__/utils.cpython-38.pyc"},
|
||||
{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
|
||||
{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
|
||||
name: "egg-file-no-version",
|
||||
fixture: "test-fixtures/site-packages/no-version",
|
||||
expectedPackages: []pkg.Package{
|
||||
{
|
||||
Name: "no-version",
|
||||
Locations: file.NewLocationSet(file.NewLocation("no-version-py3.8.egg-info")),
|
||||
PURL: "pkg:pypi/no-version",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "no-version",
|
||||
SitePackagesRootPath: ".", // requires scanning the grandparent directory to get a valid path
|
||||
},
|
||||
TopLevelPackages: []string{"requests"},
|
||||
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
|
||||
ProvidesExtra: []string{"security", "socks"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "egg-info directory case sensitive",
|
||||
fixtures: []string{
|
||||
"test-fixtures/casesensitive/EGG-INFO/PKG-INFO",
|
||||
"test-fixtures/casesensitive/EGG-INFO/RECORD",
|
||||
"test-fixtures/casesensitive/EGG-INFO/top_level.txt",
|
||||
},
|
||||
expectedPackage: pkg.Package{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
PURL: "pkg:pypi/requests@2.22.0",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/casesensitive/EGG-INFO/PKG-INFO")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
Platform: "UNKNOWN",
|
||||
Author: "Kenneth Reitz",
|
||||
AuthorEmail: "me@kennethreitz.org",
|
||||
SitePackagesRootPath: "test-fixtures/casesensitive",
|
||||
Files: []pkg.PythonFileRecord{
|
||||
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
|
||||
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
|
||||
{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
|
||||
{Path: "requests/__pycache__/utils.cpython-38.pyc"},
|
||||
{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
|
||||
{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
|
||||
},
|
||||
TopLevelPackages: []string{"requests"},
|
||||
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
|
||||
ProvidesExtra: []string{"security", "socks"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "dist-info directory",
|
||||
fixtures: []string{
|
||||
"test-fixtures/dist-info/METADATA",
|
||||
"test-fixtures/dist-info/RECORD",
|
||||
"test-fixtures/dist-info/top_level.txt",
|
||||
"test-fixtures/dist-info/direct_url.json",
|
||||
},
|
||||
expectedPackage: pkg.Package{
|
||||
Name: "pygments",
|
||||
Version: "2.6.1",
|
||||
PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/dist-info/METADATA")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Platform: "any",
|
||||
Author: "Georg Brandl",
|
||||
AuthorEmail: "georg@python.org",
|
||||
SitePackagesRootPath: "test-fixtures",
|
||||
Files: []pkg.PythonFileRecord{
|
||||
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
|
||||
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
|
||||
{Path: "Pygments-2.6.1.dist-info/RECORD"},
|
||||
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
|
||||
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
|
||||
name: "dist-info+egg-info site-packages directory",
|
||||
fixture: "test-fixtures/site-packages/nested",
|
||||
expectedPackages: []pkg.Package{
|
||||
{
|
||||
Name: "pygments",
|
||||
Version: "2.6.1",
|
||||
PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocation("dist-name/dist-info/METADATA"),
|
||||
file.NewLocation("dist-name/dist-info/RECORD"),
|
||||
file.NewLocation("dist-name/dist-info/direct_url.json"),
|
||||
file.NewLocation("dist-name/dist-info/top_level.txt"),
|
||||
),
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
// here we only used the license that was declared in the METADATA file, we did not go searching for other licenses
|
||||
// this is the better source of truth when there is no explicit LicenseFile given
|
||||
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("dist-name/dist-info/METADATA")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Platform: "any",
|
||||
Author: "Georg Brandl",
|
||||
AuthorEmail: "georg@python.org",
|
||||
SitePackagesRootPath: "dist-name",
|
||||
Files: []pkg.PythonFileRecord{
|
||||
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
|
||||
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
|
||||
{Path: "Pygments-2.6.1.dist-info/LICENSE.txt", Digest: &pkg.PythonFileDigest{Algorithm: "sha256", Value: "utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY"}, Size: "8449"},
|
||||
{Path: "Pygments-2.6.1.dist-info/RECORD"},
|
||||
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
|
||||
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
|
||||
|
||||
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
|
||||
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
|
||||
},
|
||||
TopLevelPackages: []string{"pygments", "something_else"},
|
||||
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
|
||||
RequiresPython: ">=3.5",
|
||||
RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"},
|
||||
ProvidesExtra: []string{"html5lib", "lxml"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
PURL: "pkg:pypi/requests@2.22.0",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocation("egg-name/egg-info/PKG-INFO"),
|
||||
file.NewLocation("egg-name/egg-info/RECORD"),
|
||||
file.NewLocation("egg-name/egg-info/top_level.txt"),
|
||||
),
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("egg-name/egg-info/PKG-INFO")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
Platform: "UNKNOWN",
|
||||
Author: "Kenneth Reitz",
|
||||
AuthorEmail: "me@kennethreitz.org",
|
||||
SitePackagesRootPath: "egg-name",
|
||||
Files: []pkg.PythonFileRecord{
|
||||
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
|
||||
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
|
||||
{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
|
||||
{Path: "requests/__pycache__/utils.cpython-38.pyc"},
|
||||
{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
|
||||
{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
|
||||
},
|
||||
TopLevelPackages: []string{"requests"},
|
||||
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
|
||||
ProvidesExtra: []string{"security", "socks"},
|
||||
},
|
||||
TopLevelPackages: []string{"pygments", "something_else"},
|
||||
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
|
||||
RequiresPython: ">=3.5",
|
||||
RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"},
|
||||
ProvidesExtra: []string{"html5lib", "lxml"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "dist-info directory case sensitive",
|
||||
fixtures: []string{
|
||||
"test-fixtures/casesensitive/DIST-INFO/METADATA",
|
||||
"test-fixtures/casesensitive/DIST-INFO/RECORD",
|
||||
"test-fixtures/casesensitive/DIST-INFO/top_level.txt",
|
||||
"test-fixtures/casesensitive/DIST-INFO/direct_url.json",
|
||||
},
|
||||
expectedPackage: pkg.Package{
|
||||
Name: "pygments",
|
||||
Version: "2.6.1",
|
||||
PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/casesensitive/DIST-INFO/METADATA")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Platform: "any",
|
||||
Author: "Georg Brandl",
|
||||
AuthorEmail: "georg@python.org",
|
||||
SitePackagesRootPath: "test-fixtures/casesensitive",
|
||||
Files: []pkg.PythonFileRecord{
|
||||
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
|
||||
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
|
||||
{Path: "Pygments-2.6.1.dist-info/RECORD"},
|
||||
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
|
||||
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
|
||||
name: "DIST-INFO+EGG-INFO site-packages directory (case insensitive)",
|
||||
fixture: "test-fixtures/site-packages/uppercase",
|
||||
expectedPackages: []pkg.Package{
|
||||
{
|
||||
Name: "pygments",
|
||||
Version: "2.6.1",
|
||||
PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocation("dist-name/DIST-INFO/METADATA"),
|
||||
file.NewLocation("dist-name/DIST-INFO/RECORD"),
|
||||
file.NewLocation("dist-name/DIST-INFO/direct_url.json"),
|
||||
file.NewLocation("dist-name/DIST-INFO/top_level.txt"),
|
||||
),
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
// here we only used the license that was declared in the METADATA file, we did not go searching for other licenses
|
||||
// this is the better source of truth when there is no explicit LicenseFile given
|
||||
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("dist-name/DIST-INFO/METADATA")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Platform: "any",
|
||||
Author: "Georg Brandl",
|
||||
AuthorEmail: "georg@python.org",
|
||||
SitePackagesRootPath: "dist-name",
|
||||
Files: []pkg.PythonFileRecord{
|
||||
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
|
||||
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
|
||||
{Path: "Pygments-2.6.1.dist-info/RECORD"},
|
||||
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
|
||||
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
|
||||
|
||||
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
|
||||
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
|
||||
},
|
||||
TopLevelPackages: []string{"pygments", "something_else"},
|
||||
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
|
||||
RequiresPython: ">=3.5",
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
PURL: "pkg:pypi/requests@2.22.0",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocation("egg-name/EGG-INFO/PKG-INFO"),
|
||||
file.NewLocation("egg-name/EGG-INFO/RECORD"),
|
||||
file.NewLocation("egg-name/EGG-INFO/top_level.txt"),
|
||||
),
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("egg-name/EGG-INFO/PKG-INFO")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
Platform: "UNKNOWN",
|
||||
Author: "Kenneth Reitz",
|
||||
AuthorEmail: "me@kennethreitz.org",
|
||||
SitePackagesRootPath: "egg-name",
|
||||
Files: []pkg.PythonFileRecord{
|
||||
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
|
||||
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
|
||||
{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
|
||||
{Path: "requests/__pycache__/utils.cpython-38.pyc"},
|
||||
{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
|
||||
{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
|
||||
},
|
||||
TopLevelPackages: []string{"requests"},
|
||||
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
|
||||
ProvidesExtra: []string{"security", "socks"},
|
||||
},
|
||||
TopLevelPackages: []string{"pygments", "something_else"},
|
||||
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
|
||||
RequiresPython: ">=3.5",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "malformed-record",
|
||||
fixtures: []string{
|
||||
"test-fixtures/malformed-record/dist-info/METADATA",
|
||||
"test-fixtures/malformed-record/dist-info/RECORD",
|
||||
name: "detect licenses",
|
||||
fixture: "test-fixtures/site-packages/license",
|
||||
expectedPackages: []pkg.Package{
|
||||
{
|
||||
Name: "pygments",
|
||||
Version: "2.6.1",
|
||||
PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocation("with-license-file-declared.dist-info/METADATA"), // the LicenseFile is declared in the METADATA file
|
||||
file.NewLocation("with-license-file-declared.dist-info/RECORD"),
|
||||
file.NewLocation("with-license-file-declared.dist-info/top_level.txt"),
|
||||
file.NewLocation("with-license-file-declared.dist-info/direct_url.json"),
|
||||
),
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.License{
|
||||
Value: "BSD-3-Clause",
|
||||
SPDXExpression: "BSD-3-Clause",
|
||||
Type: "concluded",
|
||||
// we read the path from the LicenseFile field in the METADATA file, then read the license file directly
|
||||
Locations: file.NewLocationSet(file.NewLocation("with-license-file-declared.dist-info/LICENSE.txt")),
|
||||
},
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Platform: "any",
|
||||
Author: "Georg Brandl",
|
||||
AuthorEmail: "georg@python.org",
|
||||
SitePackagesRootPath: ".",
|
||||
Files: []pkg.PythonFileRecord{
|
||||
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
|
||||
{Path: "with-license-file-declared.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
|
||||
{Path: "with-license-file-declared.dist-info/LICENSE.txt", Digest: &pkg.PythonFileDigest{Algorithm: "sha256", Value: "utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY"}, Size: "8449"},
|
||||
{Path: "with-license-file-declared.dist-info/RECORD"},
|
||||
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
|
||||
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
|
||||
|
||||
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
|
||||
},
|
||||
TopLevelPackages: []string{"pygments", "something_else"},
|
||||
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
|
||||
RequiresPython: ">=3.5",
|
||||
RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"},
|
||||
ProvidesExtra: []string{"html5lib", "lxml"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "pygments",
|
||||
Version: "2.6.1",
|
||||
PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocation("without-license-file-declared.dist-info/METADATA"), // the LicenseFile is declared in the METADATA file
|
||||
file.NewLocation("without-license-file-declared.dist-info/RECORD"),
|
||||
file.NewLocation("without-license-file-declared.dist-info/top_level.txt"),
|
||||
file.NewLocation("without-license-file-declared.dist-info/direct_url.json"),
|
||||
),
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.License{
|
||||
Value: "BSD-3-Clause",
|
||||
SPDXExpression: "BSD-3-Clause",
|
||||
Type: "concluded",
|
||||
// we discover license files automatically
|
||||
Locations: file.NewLocationSet(file.NewLocation("without-license-file-declared.dist-info/LICENSE.txt")),
|
||||
},
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Platform: "any",
|
||||
Author: "Georg Brandl",
|
||||
AuthorEmail: "georg@python.org",
|
||||
SitePackagesRootPath: ".",
|
||||
Files: []pkg.PythonFileRecord{
|
||||
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
|
||||
{Path: "without-license-file-declared.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
|
||||
{Path: "without-license-file-declared.dist-info/LICENSE.txt", Digest: &pkg.PythonFileDigest{Algorithm: "sha256", Value: "utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY"}, Size: "8449"},
|
||||
{Path: "without-license-file-declared.dist-info/RECORD"},
|
||||
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
|
||||
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
|
||||
|
||||
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
|
||||
},
|
||||
TopLevelPackages: []string{"pygments", "something_else"},
|
||||
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
|
||||
RequiresPython: ">=3.5",
|
||||
RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"},
|
||||
ProvidesExtra: []string{"html5lib", "lxml"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedPackage: pkg.Package{
|
||||
Name: "pygments",
|
||||
Version: "2.6.1",
|
||||
PURL: "pkg:pypi/pygments@2.6.1",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/malformed-record/dist-info/METADATA")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Platform: "any",
|
||||
Author: "Georg Brandl",
|
||||
AuthorEmail: "georg@python.org",
|
||||
SitePackagesRootPath: "test-fixtures/malformed-record",
|
||||
Files: []pkg.PythonFileRecord{
|
||||
{Path: "flask/json/tag.py", Digest: &pkg.PythonFileDigest{"sha256", "9ehzrmt5k7hxf7ZEK0NOs3swvQyU9fWNe-pnYe69N60"}, Size: "8223"},
|
||||
{Path: "../../Scripts/flask.exe", Digest: &pkg.PythonFileDigest{"sha256", "mPrbVeZCDX20himZ_bRai1nCs_tgr7jHIOGZlcgn-T4"}, Size: "93063"},
|
||||
{Path: "../../Scripts/flask.exe", Size: "89470", Digest: &pkg.PythonFileDigest{"sha256", "jvqh4N3qOqXLlq40i6ZOLCY9tAOwfwdzIpLDYhRjoqQ"}},
|
||||
{Path: "Flask-1.0.2.dist-info/INSTALLER", Size: "4", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}},
|
||||
},
|
||||
{
|
||||
name: "malformed-record",
|
||||
fixture: "test-fixtures/site-packages/malformed-record",
|
||||
expectedPackages: []pkg.Package{
|
||||
{
|
||||
Name: "pygments",
|
||||
Version: "2.6.1",
|
||||
PURL: "pkg:pypi/pygments@2.6.1",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocation("dist-info/METADATA"),
|
||||
file.NewLocation("dist-info/RECORD"),
|
||||
),
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("dist-info/METADATA")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Platform: "any",
|
||||
Author: "Georg Brandl",
|
||||
AuthorEmail: "georg@python.org",
|
||||
SitePackagesRootPath: ".",
|
||||
Files: []pkg.PythonFileRecord{
|
||||
{Path: "flask/json/tag.py", Digest: &pkg.PythonFileDigest{"sha256", "9ehzrmt5k7hxf7ZEK0NOs3swvQyU9fWNe-pnYe69N60"}, Size: "8223"},
|
||||
{Path: "../../Scripts/flask.exe", Digest: &pkg.PythonFileDigest{"sha256", "mPrbVeZCDX20himZ_bRai1nCs_tgr7jHIOGZlcgn-T4"}, Size: "93063"},
|
||||
{Path: "../../Scripts/flask.exe", Size: "89470", Digest: &pkg.PythonFileDigest{"sha256", "jvqh4N3qOqXLlq40i6ZOLCY9tAOwfwdzIpLDYhRjoqQ"}},
|
||||
{Path: "Flask-1.0.2.dist-info/INSTALLER", Size: "4", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}},
|
||||
},
|
||||
RequiresPython: ">=3.5",
|
||||
},
|
||||
RequiresPython: ">=3.5",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// in cases where the metadata file is available and the record is not we should still record there is a package
|
||||
// additionally empty top_level.txt files should not result in an error
|
||||
name: "partial dist-info directory",
|
||||
fixtures: []string{"test-fixtures/partial.dist-info/METADATA"},
|
||||
expectedPackage: pkg.Package{
|
||||
Name: "pygments",
|
||||
Version: "2.6.1",
|
||||
PURL: "pkg:pypi/pygments@2.6.1",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/partial.dist-info/METADATA")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Platform: "any",
|
||||
Author: "Georg Brandl",
|
||||
AuthorEmail: "georg@python.org",
|
||||
SitePackagesRootPath: "test-fixtures",
|
||||
RequiresPython: ">=3.5",
|
||||
name: "partial dist-info directory",
|
||||
fixture: "test-fixtures/site-packages/partial.dist-info",
|
||||
expectedPackages: []pkg.Package{
|
||||
{
|
||||
Name: "pygments",
|
||||
Version: "2.6.1",
|
||||
PURL: "pkg:pypi/pygments@2.6.1",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocation("METADATA"),
|
||||
),
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("METADATA")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Platform: "any",
|
||||
Author: "Georg Brandl",
|
||||
AuthorEmail: "georg@python.org",
|
||||
SitePackagesRootPath: ".",
|
||||
RequiresPython: ">=3.5",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "egg-info regular file",
|
||||
fixtures: []string{"test-fixtures/test.egg-info"},
|
||||
expectedPackage: pkg.Package{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
PURL: "pkg:pypi/requests@2.22.0",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/test.egg-info")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
Platform: "UNKNOWN",
|
||||
Author: "Kenneth Reitz",
|
||||
AuthorEmail: "me@kennethreitz.org",
|
||||
SitePackagesRootPath: "test-fixtures",
|
||||
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
|
||||
ProvidesExtra: []string{"security", "socks"},
|
||||
name: "egg-info regular file",
|
||||
fixture: "test-fixtures/site-packages/test",
|
||||
expectedPackages: []pkg.Package{
|
||||
{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
PURL: "pkg:pypi/requests@2.22.0",
|
||||
Type: pkg.PythonPkg,
|
||||
Language: pkg.Python,
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocation("test.egg-info"),
|
||||
),
|
||||
Licenses: pkg.NewLicenseSet(
|
||||
pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test.egg-info")),
|
||||
),
|
||||
FoundBy: "python-installed-package-cataloger",
|
||||
Metadata: pkg.PythonPackage{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
Platform: "UNKNOWN",
|
||||
Author: "Kenneth Reitz",
|
||||
AuthorEmail: "me@kennethreitz.org",
|
||||
SitePackagesRootPath: ".",
|
||||
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
|
||||
ProvidesExtra: []string{"security", "socks"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -280,16 +398,9 @@ func Test_PackageCataloger(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
resolver := file.NewMockResolverForPaths(test.fixtures...)
|
||||
|
||||
locations, err := resolver.FilesByPath(test.fixtures...)
|
||||
require.NoError(t, err)
|
||||
|
||||
test.expectedPackage.Locations = file.NewLocationSet(locations...)
|
||||
|
||||
pkgtest.NewCatalogTester().
|
||||
WithResolver(resolver).
|
||||
Expects([]pkg.Package{test.expectedPackage}, nil).
|
||||
FromDirectory(t, test.fixture).
|
||||
Expects(test.expectedPackages, nil).
|
||||
TestCataloger(t, NewInstalledPackageCataloger())
|
||||
})
|
||||
}
|
||||
@@ -311,7 +422,7 @@ func Test_PackageCataloger_IgnorePackage(t *testing.T) {
|
||||
t.Run(test.MetadataFixture, func(t *testing.T) {
|
||||
resolver := file.NewMockResolverForPaths(test.MetadataFixture)
|
||||
|
||||
actual, _, err := NewInstalledPackageCataloger().Catalog(context.Background(), resolver)
|
||||
actual, _, err := NewInstalledPackageCataloger().Catalog(pkgtest.Context(), resolver)
|
||||
require.NoError(t, err)
|
||||
|
||||
if len(actual) != 0 {
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
package python
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/packageurl-go"
|
||||
"github.com/anchore/syft/internal/licenses"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
@@ -92,38 +89,6 @@ func newPackageForPackage(m parsedData, licenses pkg.LicenseSet, sources ...file
|
||||
return p
|
||||
}
|
||||
|
||||
func findLicenses(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, m parsedData) pkg.LicenseSet {
|
||||
var licenseSet pkg.LicenseSet
|
||||
|
||||
switch {
|
||||
case m.LicenseExpression != "":
|
||||
licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(m.LicenseLocation, m.LicenseExpression)...)
|
||||
case m.Licenses != "":
|
||||
licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(m.LicenseLocation, m.Licenses)...)
|
||||
case m.LicenseLocation.Path() != "":
|
||||
// If we have a license file then resolve and parse it
|
||||
found, err := resolver.FilesByPath(m.LicenseLocation.Path())
|
||||
if err != nil {
|
||||
log.WithFields("error", err, "path", m.LicenseLocation.Path()).Trace("unable to resolve python license")
|
||||
}
|
||||
if len(found) > 0 {
|
||||
metadataContents, err := resolver.FileContentsByLocation(found[0])
|
||||
if err == nil {
|
||||
parsed, err := scanner.PkgSearch(ctx, file.NewLocationReadCloser(m.LicenseLocation, metadataContents))
|
||||
if err != nil {
|
||||
log.WithFields("error", err, "path", m.LicenseLocation.Path()).Trace("unable to parse a license from the file")
|
||||
}
|
||||
if len(parsed) > 0 {
|
||||
licenseSet = pkg.NewLicenseSet(parsed...)
|
||||
}
|
||||
} else {
|
||||
log.WithFields("error", err, "path", m.LicenseLocation.Path()).Trace("unable to read file contents")
|
||||
}
|
||||
}
|
||||
}
|
||||
return licenseSet
|
||||
}
|
||||
|
||||
func packageURL(name, version string, m *pkg.PythonPackage) string {
|
||||
// generate a purl from the package data
|
||||
pURL := packageurl.NewPackageURL(
|
||||
|
||||
@@ -99,7 +99,7 @@ func poetryLockPackages(reader file.LocationReadCloser) ([]pkg.Package, error) {
|
||||
p.Name,
|
||||
p.Version,
|
||||
newPythonPoetryLockEntry(p),
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -153,7 +153,7 @@ func (rp requirementsParser) parseRequirementsTxt(_ context.Context, _ file.Reso
|
||||
URL: parseURL(req.URL),
|
||||
Markers: req.Markers,
|
||||
},
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ func parseSetup(_ context.Context, _ file.Resolver, _ *generic.Environment, read
|
||||
newPackageForIndex(
|
||||
name,
|
||||
version,
|
||||
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -6,7 +6,12 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/licenses"
|
||||
@@ -247,3 +252,97 @@ func assembleEggOrWheelMetadata(resolver file.Resolver, metadataLocation file.Lo
|
||||
pd.DirectURLOrigin = d
|
||||
return &pd, sources, nil
|
||||
}
|
||||
|
||||
func findLicenses(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, m parsedData) pkg.LicenseSet {
|
||||
var licenseSet pkg.LicenseSet
|
||||
|
||||
licenseLocations := file.NewLocationSet()
|
||||
if m.LicenseFilePath != "" {
|
||||
locs, err := resolver.FilesByPath(m.LicenseFilePath)
|
||||
if err != nil {
|
||||
log.WithFields("error", err, "path", m.LicenseFilePath).Trace("unable to resolve python license file")
|
||||
} else {
|
||||
licenseLocations.Add(locs...)
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case m.LicenseExpression != "" || m.Licenses != "":
|
||||
licenseSet = getLicenseSetFromValues(licenseLocations.ToSlice(), m.LicenseExpression, m.Licenses)
|
||||
case !licenseLocations.Empty():
|
||||
licenseSet = getLicenseSetFromFiles(ctx, scanner, resolver, licenseLocations.ToSlice()...)
|
||||
|
||||
default:
|
||||
// search for known license paths from RECORDS file
|
||||
licenseNames := strset.New()
|
||||
for _, n := range licenses.FileNames() {
|
||||
licenseNames.Add(strings.ToLower(n))
|
||||
}
|
||||
parent := path.Base(path.Dir(m.DistInfoLocation.Path()))
|
||||
candidatePaths := strset.New()
|
||||
for _, f := range m.Files {
|
||||
if !strings.HasPrefix(f.Path, parent) || strings.Count(f.Path, "/") > 1 {
|
||||
continue
|
||||
}
|
||||
|
||||
if licenseNames.Has(strings.ToLower(filepath.Base(f.Path))) {
|
||||
candidatePaths.Add(path.Join(m.SitePackagesRootPath, f.Path))
|
||||
}
|
||||
}
|
||||
|
||||
paths := candidatePaths.List()
|
||||
sort.Strings(paths)
|
||||
locationSet := file.NewLocationSet()
|
||||
for _, p := range paths {
|
||||
locs, err := resolver.FilesByPath(p)
|
||||
if err != nil {
|
||||
log.WithFields("error", err, "path", p).Trace("unable to resolve python license in dist-info")
|
||||
continue
|
||||
}
|
||||
locationSet.Add(locs...)
|
||||
}
|
||||
|
||||
licenseSet = getLicenseSetFromFiles(ctx, scanner, resolver, locationSet.ToSlice()...)
|
||||
}
|
||||
return licenseSet
|
||||
}
|
||||
|
||||
func getLicenseSetFromValues(locations []file.Location, licenseValues ...string) pkg.LicenseSet {
|
||||
if len(locations) == 0 {
|
||||
return pkg.NewLicenseSet(pkg.NewLicensesFromValues(licenseValues...)...)
|
||||
}
|
||||
|
||||
licenseSet := pkg.NewLicenseSet()
|
||||
for _, value := range licenseValues {
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
licenseSet.Add(pkg.NewLicenseFromLocations(value, locations...))
|
||||
}
|
||||
return licenseSet
|
||||
}
|
||||
|
||||
func getLicenseSetFromFiles(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, locations ...file.Location) pkg.LicenseSet {
|
||||
licenseSet := pkg.NewLicenseSet()
|
||||
for _, loc := range locations {
|
||||
licenseSet.Add(getLicenseSetFromFile(ctx, scanner, resolver, loc)...)
|
||||
}
|
||||
return licenseSet
|
||||
}
|
||||
|
||||
func getLicenseSetFromFile(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, location file.Location) []pkg.License {
|
||||
metadataContents, err := resolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
log.WithFields("error", err, "path", location.Path()).Trace("unable to read file contents")
|
||||
return nil
|
||||
}
|
||||
defer internal.CloseAndLogError(metadataContents, location.Path())
|
||||
parsed, err := scanner.PkgSearch(ctx, file.NewLocationReadCloser(location, metadataContents))
|
||||
if err != nil {
|
||||
log.WithFields("error", err, "path", location.Path()).Trace("unable to parse a license from the file")
|
||||
return nil
|
||||
}
|
||||
|
||||
return parsed
|
||||
}
|
||||
|
||||
@@ -15,11 +15,18 @@ import (
|
||||
)
|
||||
|
||||
type parsedData struct {
|
||||
// core info
|
||||
|
||||
// DistInfoLocation is the location of the METADATA file within the .dist-info directory where we obtained the python package information
|
||||
DistInfoLocation file.Location
|
||||
pkg.PythonPackage `mapstructure:",squash"`
|
||||
|
||||
// license info
|
||||
|
||||
Licenses string `mapstructure:"License"`
|
||||
LicenseFile string `mapstructure:"LicenseFile"`
|
||||
LicenseExpression string `mapstructure:"LicenseExpression"`
|
||||
LicenseLocation file.Location
|
||||
pkg.PythonPackage `mapstructure:",squash"`
|
||||
LicenseFilePath string
|
||||
}
|
||||
|
||||
var pluralFields = map[string]bool{
|
||||
@@ -45,11 +52,13 @@ func parseWheelOrEggMetadata(locationReader file.LocationReadCloser) (parsedData
|
||||
|
||||
pd.SitePackagesRootPath = determineSitePackagesRootPath(path)
|
||||
if pd.Licenses != "" || pd.LicenseExpression != "" {
|
||||
pd.LicenseLocation = file.NewLocation(path)
|
||||
pd.LicenseFilePath = path
|
||||
} else if pd.LicenseFile != "" {
|
||||
pd.LicenseLocation = file.NewLocation(filepath.Join(filepath.Dir(path), pd.LicenseFile))
|
||||
pd.LicenseFilePath = filepath.Join(filepath.Dir(path), pd.LicenseFile)
|
||||
}
|
||||
|
||||
pd.DistInfoLocation = locationReader.Location
|
||||
|
||||
return pd, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -22,38 +22,40 @@ func TestParseWheelEggMetadata(t *testing.T) {
|
||||
ExpectedMetadata parsedData
|
||||
}{
|
||||
{
|
||||
Fixture: "test-fixtures/egg-info/PKG-INFO",
|
||||
Fixture: "test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO",
|
||||
ExpectedMetadata: parsedData{
|
||||
"Apache 2.0",
|
||||
"",
|
||||
"",
|
||||
file.NewLocation("test-fixtures/egg-info/PKG-INFO"),
|
||||
pkg.PythonPackage{
|
||||
DistInfoLocation: file.NewLocation("test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO"),
|
||||
Licenses: "Apache 2.0",
|
||||
LicenseFile: "",
|
||||
LicenseExpression: "",
|
||||
LicenseFilePath: "test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO",
|
||||
PythonPackage: pkg.PythonPackage{
|
||||
Name: "requests",
|
||||
Version: "2.22.0",
|
||||
Platform: "UNKNOWN",
|
||||
Author: "Kenneth Reitz",
|
||||
AuthorEmail: "me@kennethreitz.org",
|
||||
SitePackagesRootPath: "test-fixtures",
|
||||
SitePackagesRootPath: "test-fixtures/site-packages/nested/egg-name",
|
||||
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
|
||||
ProvidesExtra: []string{"security", "socks"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Fixture: "test-fixtures/dist-info/METADATA",
|
||||
Fixture: "test-fixtures/site-packages/nested/dist-name/dist-info/METADATA",
|
||||
ExpectedMetadata: parsedData{
|
||||
"BSD License",
|
||||
"",
|
||||
"",
|
||||
file.NewLocation("test-fixtures/dist-info/METADATA"),
|
||||
pkg.PythonPackage{
|
||||
DistInfoLocation: file.NewLocation("test-fixtures/site-packages/nested/dist-name/dist-info/METADATA"),
|
||||
Licenses: "BSD License",
|
||||
LicenseFile: "",
|
||||
LicenseExpression: "",
|
||||
LicenseFilePath: "test-fixtures/site-packages/nested/dist-name/dist-info/METADATA",
|
||||
PythonPackage: pkg.PythonPackage{
|
||||
Name: "Pygments",
|
||||
Version: "2.6.1",
|
||||
Platform: "any",
|
||||
Author: "Georg Brandl",
|
||||
AuthorEmail: "georg@python.org",
|
||||
SitePackagesRootPath: "test-fixtures",
|
||||
SitePackagesRootPath: "test-fixtures/site-packages/nested/dist-name",
|
||||
RequiresPython: ">=3.5",
|
||||
RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"},
|
||||
ProvidesExtra: []string{"html5lib", "lxml"},
|
||||
@@ -149,16 +151,17 @@ func TestParseWheelEggMetadataInvalid(t *testing.T) {
|
||||
ExpectedMetadata parsedData
|
||||
}{
|
||||
{
|
||||
Fixture: "test-fixtures/egg-info/PKG-INFO-INVALID",
|
||||
Fixture: "test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO-INVALID",
|
||||
ExpectedMetadata: parsedData{
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
file.Location{},
|
||||
pkg.PythonPackage{
|
||||
DistInfoLocation: file.NewLocation("test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO-INVALID"),
|
||||
Licenses: "",
|
||||
LicenseExpression: "",
|
||||
LicenseFile: "",
|
||||
LicenseFilePath: "",
|
||||
PythonPackage: pkg.PythonPackage{
|
||||
Name: "mxnet",
|
||||
Version: "1.8.0",
|
||||
SitePackagesRootPath: "test-fixtures",
|
||||
SitePackagesRootPath: "test-fixtures/site-packages/nested/egg-name",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -15,7 +15,7 @@ func TestParseWheelEggRecord(t *testing.T) {
|
||||
ExpectedMetadata []pkg.PythonFileRecord
|
||||
}{
|
||||
{
|
||||
Fixture: "test-fixtures/egg-info/RECORD",
|
||||
Fixture: "test-fixtures/site-packages/nested/egg-name/egg-info/RECORD",
|
||||
ExpectedMetadata: []pkg.PythonFileRecord{
|
||||
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
|
||||
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
|
||||
@@ -26,10 +26,11 @@ func TestParseWheelEggRecord(t *testing.T) {
|
||||
},
|
||||
},
|
||||
{
|
||||
Fixture: "test-fixtures/dist-info/RECORD",
|
||||
Fixture: "test-fixtures/site-packages/nested/dist-name/dist-info/RECORD",
|
||||
ExpectedMetadata: []pkg.PythonFileRecord{
|
||||
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
|
||||
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
|
||||
{Path: "Pygments-2.6.1.dist-info/LICENSE.txt", Digest: &pkg.PythonFileDigest{Algorithm: "sha256", Value: "utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY"}, Size: "8449"},
|
||||
{Path: "Pygments-2.6.1.dist-info/RECORD"},
|
||||
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
|
||||
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
Copyright 2010 Pallets
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
|
||||
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
||||
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
Metadata-Version: 2.1
|
||||
Name: Pygments
|
||||
Version: 2.6.1
|
||||
Summary: Pygments is a syntax highlighting package written in Python.
|
||||
Home-page: https://pygments.org/
|
||||
Author: Georg Brandl
|
||||
Author-email: georg@python.org
|
||||
LicenseFile: LICENSE.txt
|
||||
Keywords: syntax highlighting
|
||||
Platform: any
|
||||
Classifier: License :: OSI Approved :: BSD License
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: Intended Audience :: End Users/Desktop
|
||||
Classifier: Intended Audience :: System Administrators
|
||||
Classifier: Development Status :: 6 - Mature
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3.5
|
||||
Classifier: Programming Language :: Python :: 3.6
|
||||
Classifier: Programming Language :: Python :: 3.7
|
||||
Classifier: Programming Language :: Python :: 3.8
|
||||
Classifier: Programming Language :: Python :: Implementation :: CPython
|
||||
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
||||
Classifier: Operating System :: OS Independent
|
||||
Classifier: Topic :: Text Processing :: Filters
|
||||
Classifier: Topic :: Utilities
|
||||
Requires-Python: >=3.5
|
||||
Description-Content-Type: text/markdown
|
||||
Requires-Dist: soupsieve (>1.2)
|
||||
Provides-Extra: html5lib
|
||||
Requires-Dist: html5lib ; extra == 'html5lib'
|
||||
Provides-Extra: lxml
|
||||
Requires-Dist: lxml ; extra == 'lxml'
|
||||
|
||||
|
||||
Pygments
|
||||
~~~~~~~~
|
||||
|
||||
Pygments is a syntax highlighting package written in Python.
|
||||
|
||||
It is a generic syntax highlighter suitable for use in code hosting, forums,
|
||||
wikis or other applications that need to prettify source code. Highlights
|
||||
are:
|
||||
|
||||
* a wide range of over 500 languages and other text formats is supported
|
||||
* special attention is paid to details, increasing quality by a fair amount
|
||||
* support for new languages and formats are added easily
|
||||
* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image formats that PIL supports and ANSI sequences
|
||||
* it is usable as a command-line tool and as a library
|
||||
|
||||
:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
|
||||
:license: BSD, see LICENSE for details.
|
||||
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220
|
||||
with-license-file-declared.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449
|
||||
with-license-file-declared.dist-info/LICENSE.txt,sha256=utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY,8449
|
||||
with-license-file-declared.dist-info/RECORD,,
|
||||
pygments/__pycache__/__init__.cpython-38.pyc,,
|
||||
pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778
|
||||
pygments/x_util.py,sha256=qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI=,10778
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
Copyright 2010 Pallets
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
|
||||
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
||||
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
Metadata-Version: 2.1
|
||||
Name: Pygments
|
||||
Version: 2.6.1
|
||||
Summary: Pygments is a syntax highlighting package written in Python.
|
||||
Home-page: https://pygments.org/
|
||||
Author: Georg Brandl
|
||||
Author-email: georg@python.org
|
||||
Keywords: syntax highlighting
|
||||
Platform: any
|
||||
Classifier: License :: OSI Approved :: BSD License
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: Intended Audience :: End Users/Desktop
|
||||
Classifier: Intended Audience :: System Administrators
|
||||
Classifier: Development Status :: 6 - Mature
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3.5
|
||||
Classifier: Programming Language :: Python :: 3.6
|
||||
Classifier: Programming Language :: Python :: 3.7
|
||||
Classifier: Programming Language :: Python :: 3.8
|
||||
Classifier: Programming Language :: Python :: Implementation :: CPython
|
||||
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
||||
Classifier: Operating System :: OS Independent
|
||||
Classifier: Topic :: Text Processing :: Filters
|
||||
Classifier: Topic :: Utilities
|
||||
Requires-Python: >=3.5
|
||||
Description-Content-Type: text/markdown
|
||||
Requires-Dist: soupsieve (>1.2)
|
||||
Provides-Extra: html5lib
|
||||
Requires-Dist: html5lib ; extra == 'html5lib'
|
||||
Provides-Extra: lxml
|
||||
Requires-Dist: lxml ; extra == 'lxml'
|
||||
|
||||
|
||||
Pygments
|
||||
~~~~~~~~
|
||||
|
||||
Pygments is a syntax highlighting package written in Python.
|
||||
|
||||
It is a generic syntax highlighter suitable for use in code hosting, forums,
|
||||
wikis or other applications that need to prettify source code. Highlights
|
||||
are:
|
||||
|
||||
* a wide range of over 500 languages and other text formats is supported
|
||||
* special attention is paid to details, increasing quality by a fair amount
|
||||
* support for new languages and formats are added easily
|
||||
* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image formats that PIL supports and ANSI sequences
|
||||
* it is usable as a command-line tool and as a library
|
||||
|
||||
:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
|
||||
:license: BSD, see LICENSE for details.
|
||||
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220
|
||||
without-license-file-declared.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449
|
||||
without-license-file-declared.dist-info/LICENSE.txt,sha256=utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY,8449
|
||||
without-license-file-declared.dist-info/RECORD,,
|
||||
pygments/__pycache__/__init__.cpython-38.pyc,,
|
||||
pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778
|
||||
pygments/x_util.py,sha256=qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI=,10778
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
Copyright 2010 Pallets
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
|
||||
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
|
||||
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
+1
@@ -1,5 +1,6 @@
|
||||
../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220
|
||||
Pygments-2.6.1.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449
|
||||
Pygments-2.6.1.dist-info/LICENSE.txt,sha256=utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY,8449
|
||||
Pygments-2.6.1.dist-info/RECORD,,
|
||||
pygments/__pycache__/__init__.cpython-38.pyc,,
|
||||
pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"url": "https://github.com/python-test/test.git", "vcs_info": {"commit_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "vcs": "git"}}
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
pygments
|
||||
something_else
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user