Compare commits

...
Author SHA1 Message Date
Alan Pope 7c8aad9e1b testing 2025-04-09 14:54:35 +01:00
anchore-actions-token-generator[bot]andwagoodman 987ba83674 chore(deps): update CPE dictionary index (#3782)
Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: wagoodman <590471+wagoodman@users.noreply.github.com>
2025-04-07 10:01:11 -04:00
anchore-actions-token-generator[bot]andspiffcs f11377fe30 chore(deps): update tools to latest versions (#3775)
---------
Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
Co-authored-by: spiffcs <32073428+spiffcs@users.noreply.github.com>
2025-04-03 17:35:26 +00:00
Alex Goodman 12f36420dd Parse GitHub actions comments (#3776)
* add version comment parsing support to github actions

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* update json schema with github actions metadata

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* add originator processing for github actions type

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
2025-04-03 14:46:27 +00:00
Alex Goodman f851085668 Expand python license scanning to cover unclaimed files (#3779)
* expand python license scanning to cover unclaimed files

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* speed up tests using the license scanner

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
2025-04-03 10:31:02 -04:00
Christopher Angelo Phillips da62a82413 feat: adds the DirectoryTag to the r cataloger (#3774)
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
2025-04-01 11:46:51 -04:00
117 changed files with 4736 additions and 589 deletions
+2 -2
View File
@@ -26,7 +26,7 @@ tools:
# used for linting
- name: golangci-lint
version:
want: v1.64.8
want: v2.0.2
method: github-release
with:
repo: golangci/golangci-lint
@@ -58,7 +58,7 @@ tools:
# used to release all artifacts
- name: goreleaser
version:
want: v2.8.1
want: v2.8.2
method: github-release
with:
repo: goreleaser/goreleaser
+49 -41
View File
@@ -1,15 +1,8 @@
issues:
max-same-issues: 25
uniq-by-line: false
# TODO: enable this when we have coverage on docstring comments
# # The list of ids of default excludes to include or disable.
# include:
# - EXC0002 # disable excluding of issues about comments from golint
version: "2"
run:
tests: false
linters:
# inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
disable-all: true
default: none
enable:
- asciicheck
- bodyclose
@@ -22,11 +15,8 @@ linters:
- goconst
- gocritic
- gocyclo
- gofmt
- goimports
- goprintffuncname
- gosec
- gosimple
- govet
- ineffassign
- misspell
@@ -34,37 +24,35 @@ linters:
- nolintlint
- revive
- staticcheck
- stylecheck
- typecheck
- unconvert
- unparam
- unused
- whitespace
linters-settings:
funlen:
# Checks the number of lines in a function.
# If lower than 0, disable the check.
# Default: 60
lines: 70
# Checks the number of statements in a function.
# If lower than 0, disable the check.
# Default: 40
statements: 50
gocritic:
enabled-checks:
- deferInLoop
- ruleguard
settings:
ruleguard:
rules: "test/rules/rules.go"
gosec:
excludes:
- G115
run:
timeout: 10m
tests: false
settings:
funlen:
lines: 70
statements: 50
gocritic:
enabled-checks:
- deferInLoop
- ruleguard
settings:
ruleguard:
rules: test/rules/rules.go
gosec:
excludes:
- G115
exclusions:
generated: lax
presets:
- comments
- common-false-positives
- legacy
- std-error-handling
paths:
- third_party$
- builtin$
- examples$
# do not enable...
# - deadcode # The owner seems to have abandoned the linter. Replaced by "unused".
@@ -91,3 +79,23 @@ run:
# - testpackage
# - varcheck # The owner seems to have abandoned the linter. Replaced by "unused".
# - wsl # this doens't have an auto-fixer yet and is pretty noisy (https://github.com/bombsimon/wsl/issues/90)
issues:
max-same-issues: 25
uniq-by-line: false
# TODO: enable this when we have coverage on docstring comments
# # The list of ids of default excludes to include or disable.
# include:
# - EXC0002 # disable excluding of issues about comments from golint
formatters:
enable:
- gofmt
- goimports
exclusions:
generated: lax
paths:
- third_party$
- builtin$
- examples$
+2 -2
View File
@@ -52,7 +52,7 @@ func Attest(app clio.Application) *cobra.Command {
opts := defaultAttestOptions()
// template format explicitly not allowed
opts.Format.Template.Enabled = false
opts.Template.Enabled = false
return app.SetupCommand(&cobra.Command{
Use: "attest --output [FORMAT] <IMAGE>",
@@ -136,7 +136,7 @@ func writeSBOMToFormattedFile(s *sbom.SBOM, sbomFile io.Writer, opts *attestOpti
return fmt.Errorf("no output file provided")
}
encs, err := opts.Format.Encoders()
encs, err := opts.Encoders()
if err != nil {
return fmt.Errorf("unable to create encoders: %w", err)
}
+2 -2
View File
@@ -110,7 +110,7 @@ func (o *scanOptions) PostLoad() error {
}
func (o *scanOptions) validateLegacyOptionsNotUsed() error {
if len(fangs.Flatten(o.Config.ConfigFile)) == 0 {
if len(fangs.Flatten(o.ConfigFile)) == 0 {
return nil
}
@@ -122,7 +122,7 @@ func (o *scanOptions) validateLegacyOptionsNotUsed() error {
File any `yaml:"file" json:"file" mapstructure:"file"`
}
for _, f := range fangs.Flatten(o.Config.ConfigFile) {
for _, f := range fangs.Flatten(o.ConfigFile) {
by, err := os.ReadFile(f)
if err != nil {
return fmt.Errorf("unable to read config file during validations %q: %w", f, err)
+1 -1
View File
@@ -89,7 +89,7 @@ func (o Output) SBOMWriter() (sbom.Writer, error) {
usesTemplateOutput := names.Has(string(template.ID))
if usesTemplateOutput && o.Format.Template.Path == "" {
if usesTemplateOutput && o.Template.Path == "" {
return nil, fmt.Errorf(`must specify path to template file when using "template" output format`)
}
+1 -5
View File
@@ -36,11 +36,7 @@ func capture(target **os.File, writer io.Writer, bufSize int) func() {
}()
buf := make([]byte, bufSize)
for {
if original == nil {
break
}
for original != nil {
n, err := r.Read(buf)
if n > 0 {
_, _ = writer.Write(buf[0:n])
@@ -6,6 +6,7 @@ import (
"io"
"path"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -76,7 +77,6 @@ func newAlpineConfiguration(resolver file.Resolver) (*AlpineConfiguration, []fil
return &AlpineConfiguration{
APKKeys: keys,
}, locations, nil
}
func getVersion(resolver file.Resolver) (string, []file.Location, error) {
@@ -92,6 +92,7 @@ func getVersion(resolver file.Resolver) (string, []file.Location, error) {
if err != nil {
return "", nil, fmt.Errorf("unable to read alpine version: %w", err)
}
defer internal.CloseAndLogError(reader, locations[0].RealPath)
version, err := io.ReadAll(reader)
if err != nil {
@@ -111,7 +112,11 @@ func getAPKKeys(resolver file.Resolver) (map[string]string, []file.Location, err
}
for _, location := range locations {
basename := path.Base(location.RealPath)
//nolint:gocritic
reader, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, nil, fmt.Errorf("unable to resolve file contents by location at %s: %w", location.RealPath, err)
}
content, err := io.ReadAll(reader)
if err != nil {
return nil, nil, fmt.Errorf("unable to read apk key content at %s: %w", location.RealPath, err)
-1
View File
@@ -124,5 +124,4 @@ func showAlpineConfiguration(s sbom.SBOM) {
panic(err)
}
fmt.Println(string(meta))
}
-1
View File
@@ -30,7 +30,6 @@ func main() {
if err := enc.Encode(sbom.Descriptor.Configuration); err != nil {
panic(err)
}
}
func imageReference() string {
+1 -1
View File
@@ -13,5 +13,5 @@ func DefaultLocationComparer(x, y file.Location) bool {
}
func LocationComparerWithoutLayer(x, y file.Location) bool {
return cmp.Equal(x.Coordinates.RealPath, y.Coordinates.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath)
return cmp.Equal(x.RealPath, y.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath)
}
+1 -1
View File
@@ -3,5 +3,5 @@ package internal
const (
// JSONSchemaVersion is the current schema version output by the JSON encoder
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
JSONSchemaVersion = "16.0.24"
JSONSchemaVersion = "16.0.25"
)
+1 -1
View File
@@ -23,7 +23,7 @@ func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...strin
defer file.Close()
// ignore directories
if file.FileInfo.IsDir() {
if file.IsDir() {
return nil
}
+1 -1
View File
@@ -28,7 +28,7 @@ func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
}
}()
for _, file := range zipReader.Reader.File {
for _, file := range zipReader.File {
manifest.Add(file.Name, file.FileInfo())
}
return manifest, nil
+1 -1
View File
@@ -53,7 +53,7 @@ func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths
}
}()
for _, file := range zipReader.Reader.File {
for _, file := range zipReader.File {
// if no paths are given then assume that all files should be traversed
if len(paths) > 0 {
if _, ok := request[file.Name]; !ok {
+1 -1
View File
@@ -22,7 +22,7 @@ func less(i, j artifact.Relationship) bool {
jFrom, ok3 := j.From.(pkg.Package)
jTo, ok4 := j.To.(pkg.Package)
if !(ok1 && ok2 && ok3 && ok4) {
if !ok1 && !ok2 && !ok3 && !ok4 {
return false
}
+2 -2
View File
@@ -245,10 +245,10 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
}
for _, ref := range pathRefs {
if oldRef, ok := locations[ref.Coordinates.ID()]; ok {
if oldRef, ok := locations[ref.ID()]; ok {
log.Debugf("found path duplicate of %s", oldRef.RealPath)
}
locations[ref.Coordinates.ID()] = ref
locations[ref.ID()] = ref
}
}
+1 -1
View File
@@ -69,7 +69,7 @@ func DefaultPackageTaskFactories() Factories {
newSimplePackageTaskFactory(cpp.NewConanInfoCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "cpp", "conan"),
newSimplePackageTaskFactory(javascript.NewPackageCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, JavaScript, Node),
newSimplePackageTaskFactory(php.NewComposerInstalledCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "php", "composer"),
newSimplePackageTaskFactory(r.NewPackageCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "r"),
newSimplePackageTaskFactory(r.NewPackageCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, pkgcataloging.DirectoryTag, "r"),
newSimplePackageTaskFactory(ruby.NewInstalledGemSpecCataloger, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "ruby", "gem", "gemspec"),
newSimplePackageTaskFactory(rust.NewAuditBinaryCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.LanguageTag, "rust", "binary"),
File diff suppressed because it is too large Load Diff
+18 -1
View File
@@ -1,6 +1,6 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "anchore.io/schema/syft/json/16.0.24/document",
"$id": "anchore.io/schema/syft/json/16.0.25/document",
"$ref": "#/$defs/Document",
"$defs": {
"AlpmDbEntry": {
@@ -995,6 +995,20 @@
"size"
]
},
"GithubActionsUseStatement": {
"properties": {
"value": {
"type": "string"
},
"comment": {
"type": "string"
}
},
"type": "object",
"required": [
"value"
]
},
"GoModuleBuildinfoEntry": {
"properties": {
"goBuildSettings": {
@@ -1804,6 +1818,9 @@
{
"$ref": "#/$defs/ErlangRebarLockEntry"
},
{
"$ref": "#/$defs/GithubActionsUseStatement"
},
{
"$ref": "#/$defs/GoModuleBuildinfoEntry"
},
+4 -4
View File
@@ -48,15 +48,15 @@ func (m *LocationMetadata) merge(other LocationMetadata) error {
}
func (l Location) WithAnnotation(key, value string) Location {
if l.LocationMetadata.Annotations == nil {
l.LocationMetadata.Annotations = map[string]string{}
if l.Annotations == nil {
l.Annotations = map[string]string{}
}
l.LocationMetadata.Annotations[key] = value
l.Annotations[key] = value
return l
}
func (l Location) WithoutAnnotations() Location {
l.LocationMetadata.Annotations = map[string]string{}
l.Annotations = map[string]string{}
return l
}
@@ -361,6 +361,7 @@ func collectDocRelationships(spdxIDMap map[string]any, doc *spdx.Document) (out
from, fromOk := a.(pkg.Package)
toPackage, toPackageOk := b.(pkg.Package)
toLocation, toLocationOk := b.(file.Location)
//nolint:staticcheck
if !fromOk || !(toPackageOk || toLocationOk) {
log.Debugf("unable to find valid relationship mapping from SPDX, ignoring: (from: %+v) (to: %+v)", a, b)
continue
@@ -213,13 +213,15 @@ func reduceOuter(expression string) string {
func isBalanced(expression string) bool {
count := 0
for _, c := range expression {
if c == '(' {
switch c {
case '(':
count++
} else if c == ')' {
case ')':
count--
if count < 0 {
return false
}
default:
}
}
return count == 0
@@ -34,7 +34,7 @@ const (
//
// Available options are: <omit>, NOASSERTION, Person: <person>, Organization: <org>
// return values are: <type>, <value>
func Originator(p pkg.Package) (typ string, author string) { //nolint: funlen
func Originator(p pkg.Package) (typ string, author string) { //nolint: gocyclo,funlen
if !hasMetadata(p) {
return typ, author
}
@@ -57,6 +57,15 @@ func Originator(p pkg.Package) (typ string, author string) { //nolint: funlen
case pkg.DpkgArchiveEntry:
author = metadata.Maintainer
case pkg.GitHubActionsUseStatement:
typ = orgType
org := strings.Split(metadata.Value, "/")[0]
if org == "actions" {
// this is a GitHub action, so the org is GitHub
org = "GitHub"
}
author = org
case pkg.JavaArchive:
if metadata.Manifest != nil {
author = metadata.Manifest.Main.MustGet("Specification-Vendor")
@@ -45,6 +45,7 @@ func Test_OriginatorSupplier(t *testing.T) {
pkg.SwiplPackEntry{},
pkg.OpamPackage{},
pkg.YarnLockEntry{},
pkg.TerraformLockProviderEntry{},
)
tests := []struct {
name string
@@ -384,20 +385,24 @@ func Test_OriginatorSupplier(t *testing.T) {
supplier: "Person: me (me@auth.com)",
},
{
name: "from ocaml opam",
name: "from github actions workflow/action",
input: pkg.Package{
Metadata: pkg.OpamPackage{},
Metadata: pkg.GitHubActionsUseStatement{
Value: "actions/checkout@v4",
},
},
originator: "",
supplier: "",
originator: "Organization: GitHub",
supplier: "Organization: GitHub",
},
{
name: "from terraform lock",
name: "from github actions workflow/action",
input: pkg.Package{
Metadata: pkg.TerraformLockProviderEntry{},
Metadata: pkg.GitHubActionsUseStatement{
Value: "google/something@v6",
},
},
originator: "",
supplier: "",
originator: "Organization: google",
supplier: "Organization: google",
},
}
for _, test := range tests {
+1 -1
View File
@@ -228,7 +228,7 @@ func toSyftRelationships(doc *model.Document, catalog *pkg.Collection, relations
idMap[string(p.ID())] = p
locations := p.Locations.ToSlice()
for _, l := range locations {
idMap[string(l.Coordinates.ID())] = l.Coordinates
idMap[string(l.ID())] = l.Coordinates
}
}
@@ -57,7 +57,7 @@ func (r *ContainerImageAllLayers) fileByRef(ref stereoscopeFile.Reference, uniqu
return nil, fmt.Errorf("unable to fetch metadata (ref=%+v): %w", ref, err)
}
if entry.Metadata.Type == stereoscopeFile.TypeHardLink || entry.Metadata.Type == stereoscopeFile.TypeSymLink {
if entry.Type == stereoscopeFile.TypeHardLink || entry.Type == stereoscopeFile.TypeSymLink {
// a link may resolve in this layer or higher, assuming a squashed tree is used to search
// we should search all possible resolutions within the valid source
for _, subLayerIdx := range r.layers[layerIdx:] {
@@ -102,7 +102,7 @@ func (r *ContainerImageAllLayers) FilesByPath(paths ...string) ([]file.Location,
if err != nil {
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
}
if metadata.Metadata.IsDir() {
if metadata.IsDir() {
continue
}
}
@@ -146,7 +146,7 @@ func (r *ContainerImageAllLayers) FilesByGlob(patterns ...string) ([]file.Locati
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.RequestPath, err)
}
// don't consider directories
if metadata.Metadata.IsDir() {
if metadata.IsDir() {
continue
}
}
@@ -192,7 +192,7 @@ func (r *ContainerImageAllLayers) FileContentsByLocation(location file.Location)
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err)
}
switch entry.Metadata.Type {
switch entry.Type {
case stereoscopeFile.TypeSymLink, stereoscopeFile.TypeHardLink:
// the location we are searching may be a symlink, we should always work with the resolved file
newLocation := r.RelativeFileByPath(location, location.AccessPath)
@@ -58,7 +58,7 @@ func (r *ContainerImageSquash) FilesByPath(paths ...string) ([]file.Location, er
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
}
// don't consider directories
if metadata.Metadata.IsDir() {
if metadata.IsDir() {
continue
}
}
@@ -106,7 +106,7 @@ func (r *ContainerImageSquash) FilesByGlob(patterns ...string) ([]file.Location,
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.RequestPath, err)
}
// don't consider directories
if metadata.Metadata.IsDir() {
if metadata.IsDir() {
continue
}
}
@@ -151,7 +151,7 @@ func (r *ContainerImageSquash) FileContentsByLocation(location file.Location) (i
return nil, fmt.Errorf("unable to get metadata for path=%q from file catalog: %w", location.RealPath, err)
}
switch entry.Metadata.Type {
switch entry.Type {
case stereoscopeFile.TypeSymLink, stereoscopeFile.TypeHardLink:
// the location we are searching may be a symlink, we should always work with the resolved file
locations, err := r.FilesByPath(location.RealPath)
+1 -1
View File
@@ -59,7 +59,7 @@ func (r *Directory) buildIndex() error {
r.tree = tree
r.index = index
r.filetreeResolver.searchContext = filetree.NewSearchContext(tree, index)
r.searchContext = filetree.NewSearchContext(tree, index)
return nil
}
+1 -1
View File
@@ -51,7 +51,7 @@ func (r *File) buildIndex() error {
r.tree = tree
r.index = index
r.filetreeResolver.searchContext = filetree.NewSearchContext(tree, index)
r.searchContext = filetree.NewSearchContext(tree, index)
return nil
}
@@ -68,7 +68,7 @@ func (r filetreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err
}
// don't consider directories
if entry.Metadata.IsDir() {
if entry.IsDir() {
continue
}
@@ -114,17 +114,17 @@ func (r filetreeResolver) FilesByGlob(patterns ...string) ([]file.Location, erro
}
entry, err := r.index.Get(*refVia.Reference)
if err != nil {
return nil, fmt.Errorf("unable to get file metadata for reference %s: %w", refVia.Reference.RealPath, err)
return nil, fmt.Errorf("unable to get file metadata for reference %s: %w", refVia.RealPath, err)
}
// don't consider directories
if entry.Metadata.IsDir() {
if entry.IsDir() {
continue
}
loc := file.NewVirtualLocationFromDirectory(
r.responsePath(string(refVia.Reference.RealPath)), // the actual path relative to the resolver root
r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
r.responsePath(string(refVia.RealPath)), // the actual path relative to the resolver root
r.responsePath(string(refVia.RequestPath)), // the path used to access this file, relative to the resolver root
*refVia.Reference,
)
uniqueFileIDs.Add(*refVia.Reference)
@@ -217,7 +217,7 @@ func (r *filetreeResolver) FilesByMIMEType(types ...string) ([]file.Location, er
continue
}
location := file.NewVirtualLocationFromDirectory(
r.responsePath(string(refVia.Reference.RealPath)),
r.responsePath(string(refVia.RealPath)),
r.responsePath(string(refVia.RequestPath)),
*refVia.Reference,
)
@@ -25,6 +25,7 @@ func AllTypes() []any {
pkg.ELFBinaryPackageNoteJSONPayload{},
pkg.ElixirMixLockEntry{},
pkg.ErlangRebarLockEntry{},
pkg.GitHubActionsUseStatement{},
pkg.GolangBinaryBuildinfoEntry{},
pkg.GolangModuleEntry{},
pkg.HackageStackYamlEntry{},
+1
View File
@@ -78,6 +78,7 @@ var jsonTypes = makeJSONTypes(
jsonNames(pkg.DpkgDBEntry{}, "dpkg-db-entry", "DpkgMetadata"),
jsonNames(pkg.ELFBinaryPackageNoteJSONPayload{}, "elf-binary-package-note-json-payload"),
jsonNames(pkg.RubyGemspec{}, "ruby-gemspec", "GemMetadata"),
jsonNames(pkg.GitHubActionsUseStatement{}, "github-actions-use-statement"),
jsonNames(pkg.GolangBinaryBuildinfoEntry{}, "go-module-buildinfo-entry", "GolangBinMetadata", "GolangMetadata"),
jsonNames(pkg.GolangModuleEntry{}, "go-module-entry", "GolangModMetadata"),
jsonNames(pkg.HackageStackYamlLockEntry{}, "haskell-hackage-stack-lock-entry", "HackageMetadataType"),
+2 -2
View File
@@ -123,7 +123,7 @@ func parseApkDB(_ context.Context, resolver file.Resolver, env *generic.Environm
// This should get fixed with https://gitlab.alpinelinux.org/alpine/apk-tools/-/issues/10875
if r == nil {
// find the repositories file from the relative directory of the DB file
releases := findReleases(resolver, reader.Location.RealPath)
releases := findReleases(resolver, reader.RealPath)
if len(releases) > 0 {
r = &releases[0]
@@ -173,7 +173,7 @@ func parseReleasesFromAPKRepository(reader file.LocationReadCloser) []linux.Rele
reposB, err := io.ReadAll(reader)
if err != nil {
log.Tracef("unable to read APK repositories file %q: %+v", reader.Location.RealPath, err)
log.Tracef("unable to read APK repositories file %q: %+v", reader.RealPath, err)
return nil
}
+1 -1
View File
@@ -80,7 +80,7 @@ func parseAlpmDB(_ context.Context, resolver file.Resolver, env *generic.Environ
newPackage(
data,
env.LinuxRelease,
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
locs...,
),
}, nil, errs
+7 -7
View File
@@ -33,13 +33,13 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
}
if s == nil {
log.WithFields("path", reader.Location.RealPath).Trace("file is not an SBOM")
log.WithFields("path", reader.RealPath).Trace("file is not an SBOM")
return nil, nil, nil
}
// Bitnami exclusively uses SPDX JSON SBOMs
if sFormat != "spdx-json" {
log.WithFields("path", reader.Location.RealPath).Trace("file is not an SPDX JSON SBOM")
log.WithFields("path", reader.RealPath).Trace("file is not an SPDX JSON SBOM")
return nil, nil, nil
}
@@ -59,7 +59,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
// where there is evidence of this file, and the catalogers have not run against any file other than,
// the SBOM, this is the only location that is relevant for this cataloger.
p.Locations = file.NewLocationSet(
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
)
// Parse the Bitnami-specific metadata
@@ -70,7 +70,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
// Bitnami packages reported in a SPDX file are shipped under the same directory
// as the SPDX file itself.
metadata.Path = filepath.Dir(reader.Location.RealPath)
metadata.Path = filepath.Dir(reader.RealPath)
if p.ID() != mainPkgID {
metadata.Files = packageFiles(s.Relationships, p, metadata.Path)
secondaryPkgsFiles = append(secondaryPkgsFiles, metadata.Files...)
@@ -86,12 +86,12 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
}
// Resolve all files owned by the main package in the SBOM and update the metadata
if mainPkgFiles, err := mainPkgFiles(resolver, reader.Location.RealPath, secondaryPkgsFiles); err == nil {
if mainPkgFiles, err := mainPkgFiles(resolver, reader.RealPath, secondaryPkgsFiles); err == nil {
for i, p := range pkgs {
if p.ID() == mainPkgID {
metadata, ok := p.Metadata.(*pkg.BitnamiSBOMEntry)
if !ok {
log.WithFields("spdx-filepath", reader.Location.RealPath).Trace("main package in SBOM does not have Bitnami metadata")
log.WithFields("spdx-filepath", reader.RealPath).Trace("main package in SBOM does not have Bitnami metadata")
continue
}
@@ -100,7 +100,7 @@ func parseSBOM(_ context.Context, resolver file.Resolver, _ *generic.Environment
}
}
} else {
log.WithFields("spdx-filepath", reader.Location.RealPath, "error", err).Trace("unable to resolve owned files for main package in SBOM")
log.WithFields("spdx-filepath", reader.RealPath, "error", err).Trace("unable to resolve owned files for main package in SBOM")
}
return pkgs, filterRelationships(s.Relationships, pkgs), nil
+1 -1
View File
@@ -48,7 +48,7 @@ func parseConanfile(_ context.Context, _ file.Resolver, _ *generic.Environment,
p := newConanfilePackage(
m,
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
)
if p == nil {
continue
+3 -3
View File
@@ -60,7 +60,7 @@ func parseFullRequiresLine(line string, reader file.LocationReadCloser, pkgs *[]
p := newConaninfoPackage(
meta,
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
)
if p != nil {
*pkgs = append(*pkgs, *p)
@@ -83,7 +83,7 @@ func parseConaninfo(_ context.Context, _ file.Resolver, _ *generic.Environment,
// First set the base package info by checking the relative path
fullFilePath := string(reader.Location.LocationData.Reference().RealPath)
if len(fullFilePath) == 0 {
fullFilePath = reader.Location.LocationData.RealPath
fullFilePath = reader.RealPath
}
mainMetadata, err := parseConanMetadataFromFilePath(fullFilePath)
@@ -102,7 +102,7 @@ func parseConaninfo(_ context.Context, _ file.Resolver, _ *generic.Environment,
case errors.Is(err, io.EOF):
mainPackage := newConaninfoPackage(
mainMetadata,
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
)
mainPackageRef := *mainPackage
+2 -2
View File
@@ -91,7 +91,7 @@ func handleConanLockV1(cl conanLock, reader file.LocationReadCloser, parsedPkgRe
p := newConanlockPackage(
metadata,
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
)
if p != nil {
@@ -115,7 +115,7 @@ func handleConanLockV2(cl conanLock, reader file.LocationReadCloser, indexToPkgM
p := newConanReferencePackage(
reference,
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
)
if p != nil {
@@ -102,7 +102,7 @@ func parsePubspecLock(_ context.Context, _ file.Resolver, _ *generic.Environment
newPubspecLockPackage(
name,
pubPkg,
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
),
)
}
+1 -1
View File
@@ -36,7 +36,7 @@ func parseDpkgDB(ctx context.Context, resolver file.Resolver, env *generic.Envir
return nil, nil, fmt.Errorf("unable to catalog dpkg DB=%q: %w", reader.RealPath, err)
}
dbLoc := reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)
dbLoc := reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)
var pkgs []pkg.Package
_ = sync.CollectSlice(&ctx, cataloging.ExecutorFile, sync.ToSeq(metadata), func(m pkg.DpkgDBEntry) (pkg.Package, error) {
return newDpkgPackage(m, dbLoc, resolver, env.LinuxRelease, findDpkgInfoFiles(m.Package, resolver, reader.Location)...), nil
@@ -69,7 +69,7 @@ func parseDotnetPackagesLock(_ context.Context, _ file.Resolver, _ *generic.Envi
name, _ := extractNameAndVersion(nameVersion)
dep := allDependencies[nameVersion]
dotnetPkg := newDotnetPackagesLockPackage(name, dep, reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
dotnetPkg := newDotnetPackagesLockPackage(name, dep, reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
if dotnetPkg != nil {
pkgs = append(pkgs, *dotnetPkg)
pkgMap[nameVersion] = *dotnetPkg
+1 -1
View File
@@ -61,7 +61,7 @@ func parseMixLock(_ context.Context, _ file.Resolver, _ *generic.Environment, re
PkgHash: hash,
PkgHashExt: hashExt,
},
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
),
)
}
+1 -1
View File
@@ -35,7 +35,7 @@ func parseOTPApp(_ context.Context, _ file.Resolver, _ *generic.Environment, rea
p := newPackageFromOTP(
name, version,
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
)
packages = append(packages, p)
@@ -56,7 +56,7 @@ func parseRebarLock(_ context.Context, _ file.Resolver, _ *generic.Environment,
Name: name,
Version: version,
},
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
)
pkgMap[name] = &p
@@ -27,14 +27,14 @@ var (
// parses individual CONTENTS files from the portage flat-file store (e.g. /var/db/pkg/*/*/CONTENTS).
func parsePortageContents(_ context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
cpvMatch := cpvRe.FindStringSubmatch(reader.Location.RealPath)
cpvMatch := cpvRe.FindStringSubmatch(reader.RealPath)
if cpvMatch == nil {
return nil, nil, fmt.Errorf("failed to match package and version in %s", reader.Location.RealPath)
return nil, nil, fmt.Errorf("failed to match package and version in %s", reader.RealPath)
}
name, version := cpvMatch[1], cpvMatch[2]
if name == "" || version == "" {
log.WithFields("path", reader.Location.RealPath).Debug("failed to parse portage name and version")
log.WithFields("path", reader.RealPath).Debug("failed to parse portage name and version")
return nil, nil, fmt.Errorf("failed to parse portage name and version")
}
@@ -43,7 +43,7 @@ func parsePortageContents(_ context.Context, resolver file.Resolver, _ *generic.
Version: version,
PURL: packageURL(name, version),
Locations: file.NewLocationSet(
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
),
Type: pkg.PortagePkg,
Metadata: pkg.PortageEntry{
+32 -17
View File
@@ -2,6 +2,7 @@ package githubactions
import (
"fmt"
"regexp"
"strings"
"github.com/anchore/packageurl-go"
@@ -10,8 +11,8 @@ import (
"github.com/anchore/syft/syft/pkg"
)
func newPackageFromUsageStatement(use string, location file.Location) (*pkg.Package, error) {
name, version := parseStepUsageStatement(use)
func newPackageFromUsageStatement(use, comment string, location file.Location) (*pkg.Package, error) {
name, version := parseStepUsageStatement(use, comment)
if name == "" {
log.WithFields("file", location.RealPath, "statement", use).Trace("unable to parse github action usage statement")
@@ -19,19 +20,20 @@ func newPackageFromUsageStatement(use string, location file.Location) (*pkg.Pack
}
if strings.Contains(name, ".github/workflows/") {
return newGithubActionWorkflowPackageUsage(name, version, location), nil
return newGithubActionWorkflowPackageUsage(name, version, location, pkg.GitHubActionsUseStatement{Value: use, Comment: comment}), nil
}
return newGithubActionPackageUsage(name, version, location), nil
return newGithubActionPackageUsage(name, version, location, pkg.GitHubActionsUseStatement{Value: use, Comment: comment}), nil
}
func newGithubActionWorkflowPackageUsage(name, version string, workflowLocation file.Location) *pkg.Package {
func newGithubActionWorkflowPackageUsage(name, version string, workflowLocation file.Location, m pkg.GitHubActionsUseStatement) *pkg.Package {
p := &pkg.Package{
Name: name,
Version: version,
Locations: file.NewLocationSet(workflowLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
PURL: packageURL(name, version),
Type: pkg.GithubActionWorkflowPkg,
Metadata: m,
}
p.SetID()
@@ -39,13 +41,14 @@ func newGithubActionWorkflowPackageUsage(name, version string, workflowLocation
return p
}
func newGithubActionPackageUsage(name, version string, workflowLocation file.Location) *pkg.Package {
func newGithubActionPackageUsage(name, version string, workflowLocation file.Location, m pkg.GitHubActionsUseStatement) *pkg.Package {
p := &pkg.Package{
Name: name,
Version: version,
Locations: file.NewLocationSet(workflowLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
PURL: packageURL(name, version),
Type: pkg.GithubActionPkg,
Metadata: m,
}
p.SetID()
@@ -53,20 +56,32 @@ func newGithubActionPackageUsage(name, version string, workflowLocation file.Loc
return p
}
func parseStepUsageStatement(use string) (string, string) {
// from octo-org/another-repo/.github/workflows/workflow.yml@v1 get octo-org/another-repo/.github/workflows/workflow.yml and v1
// from ./.github/workflows/workflow-2.yml interpret as only the name
// from actions/cache@v3 get actions/cache and v3
func parseStepUsageStatement(use, comment string) (string, string) {
// from "octo-org/another-repo/.github/workflows/workflow.yml@v1" get octo-org/another-repo/.github/workflows/workflow.yml and v1
// from "./.github/workflows/workflow-2.yml" interpret as only the name
// from "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2" get actions/checkout and v4.2.2
// from "actions/cache@v3" get actions/cache and v3
fields := strings.Split(use, "@")
switch len(fields) {
case 1:
return use, ""
case 2:
return fields[0], fields[1]
name := use
version := ""
if len(fields) == 2 {
name = fields[0]
version = fields[1]
}
return "", ""
// if version looks like a commit hash and we have a comment, try to extract version from comment
if version != "" && regexp.MustCompile(`^[0-9a-f]{7,}$`).MatchString(version) && comment != "" {
versionRegex := regexp.MustCompile(`v?\d+\.\d+\.\d+`)
matches := versionRegex.FindStringSubmatch(comment)
if len(matches) >= 1 {
return name, matches[0]
}
}
return name, version
}
func packageURL(name, version string) string {
@@ -43,7 +43,7 @@ func parseCompositeActionForActionUsage(_ context.Context, _ file.Resolver, _ *g
continue
}
p, err := newPackageFromUsageStatement(step.Uses, reader.Location)
p, err := newPackageFromUsageStatement(step.Uses, step.UsesComment, reader.Location)
if err != nil {
errs = unknown.Append(errs, reader, err)
}
@@ -20,6 +20,7 @@ func Test_parseCompositeActionForActionUsage(t *testing.T) {
Type: pkg.GithubActionPkg,
Locations: fixtureLocationSet,
PURL: "pkg:github/actions/setup-go@v4",
Metadata: pkg.GitHubActionsUseStatement{Value: "actions/setup-go@v4"},
},
{
Name: "actions/cache",
@@ -27,6 +28,7 @@ func Test_parseCompositeActionForActionUsage(t *testing.T) {
Type: pkg.GithubActionPkg,
Locations: fixtureLocationSet,
PURL: "pkg:github/actions/cache@v3",
Metadata: pkg.GitHubActionsUseStatement{Value: "actions/cache@v3"},
},
}
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
"regexp"
"gopkg.in/yaml.v3"
@@ -24,14 +25,16 @@ type workflowDef struct {
}
type workflowJobDef struct {
Uses string `yaml:"uses"`
Steps []stepDef `yaml:"steps"`
Uses string `yaml:"uses"`
UsesComment string `yaml:"-"`
Steps []stepDef `yaml:"steps"`
}
type stepDef struct {
Name string `yaml:"name"`
Uses string `yaml:"uses"`
With struct {
Name string `yaml:"name"`
Uses string `yaml:"uses"`
UsesComment string `yaml:"-"`
With struct {
Path string `yaml:"path"`
Key string `yaml:"key"`
} `yaml:"with"`
@@ -43,17 +46,26 @@ func parseWorkflowForWorkflowUsage(_ context.Context, _ file.Resolver, _ *generi
return nil, nil, fmt.Errorf("unable to read yaml workflow file: %w", errs)
}
var wf workflowDef
if errs = yaml.Unmarshal(contents, &wf); errs != nil {
// parse the yaml file into a generic node to preserve comments
var node yaml.Node
if errs = yaml.Unmarshal(contents, &node); errs != nil {
return nil, nil, fmt.Errorf("unable to parse yaml workflow file: %w", errs)
}
// unmarshal the node into a workflowDef struct
var wf workflowDef
if errs = node.Decode(&wf); errs != nil {
return nil, nil, fmt.Errorf("unable to decode workflow: %w", errs)
}
attachUsageComments(&node, &wf)
// we use a collection to help with deduplication before raising to higher level processing
pkgs := pkg.NewCollection()
for _, job := range wf.Jobs {
if job.Uses != "" {
p, err := newPackageFromUsageStatement(job.Uses, reader.Location)
p, err := newPackageFromUsageStatement(job.Uses, job.UsesComment, reader.Location)
if err != nil {
errs = unknown.Append(errs, reader, err)
}
@@ -72,11 +84,20 @@ func parseWorkflowForActionUsage(_ context.Context, _ file.Resolver, _ *generic.
return nil, nil, fmt.Errorf("unable to read yaml workflow file: %w", errs)
}
var wf workflowDef
if errs = yaml.Unmarshal(contents, &wf); errs != nil {
// parse the yaml file into a generic node to preserve comments
var node yaml.Node
if errs = yaml.Unmarshal(contents, &node); errs != nil {
return nil, nil, fmt.Errorf("unable to parse yaml workflow file: %w", errs)
}
// unmarshal the node into a workflowDef struct
var wf workflowDef
if errs = node.Decode(&wf); errs != nil {
return nil, nil, fmt.Errorf("unable to decode workflow: %w", errs)
}
attachUsageComments(&node, &wf)
// we use a collection to help with deduplication before raising to higher level processing
pkgs := pkg.NewCollection()
@@ -85,7 +106,7 @@ func parseWorkflowForActionUsage(_ context.Context, _ file.Resolver, _ *generic.
if step.Uses == "" {
continue
}
p, err := newPackageFromUsageStatement(step.Uses, reader.Location)
p, err := newPackageFromUsageStatement(step.Uses, step.UsesComment, reader.Location)
if err != nil {
errs = unknown.Append(errs, reader, err)
}
@@ -97,3 +118,101 @@ func parseWorkflowForActionUsage(_ context.Context, _ file.Resolver, _ *generic.
return pkgs.Sorted(), nil, errs
}
// attachUsageComments traverses the yaml node tree and attaches usage comments to the workflowDef job strcuts and step structs.
// This is a best-effort approach to attach comments to the correct job or step.
func attachUsageComments(node *yaml.Node, wf *workflowDef) {
// for a document node, process its content (usually a single mapping node)
if node.Kind == yaml.DocumentNode && len(node.Content) > 0 {
processNode(node.Content[0], wf, nil, nil, nil)
} else {
processNode(node, wf, nil, nil, nil)
}
}
func processNode(node *yaml.Node, wf *workflowDef, currentJob *string, currentStep *int, inJobsSection *bool) {
switch node.Kind {
case yaml.MappingNode:
for i := 0; i < len(node.Content); i += 2 {
key := node.Content[i]
value := node.Content[i+1]
// track if we're in the jobs section...
if key.Value == "jobs" && inJobsSection == nil {
inJobs := true
inJobsSection = &inJobs
processNode(value, wf, nil, nil, inJobsSection)
continue
}
// if we're in jobs section, and this is a job key...
if inJobsSection != nil && *inJobsSection && currentJob == nil {
job := key.Value
currentJob = &job
processNode(value, wf, currentJob, nil, inJobsSection)
currentJob = nil
continue
}
// if this is a "uses" key...
if key.Value == "uses" {
processUsesNode(value, wf, currentJob, currentStep)
}
// if this is a "steps" key inside a job...
if key.Value == "steps" && currentJob != nil {
for j, stepNode := range value.Content {
stepIndex := j
processNode(stepNode, wf, currentJob, &stepIndex, inJobsSection)
}
continue
}
processNode(key, wf, currentJob, currentStep, inJobsSection)
processNode(value, wf, currentJob, currentStep, inJobsSection)
}
case yaml.SequenceNode:
for i, item := range node.Content {
idx := i
processNode(item, wf, currentJob, &idx, inJobsSection)
}
}
}
func processUsesNode(node *yaml.Node, wf *workflowDef, currentJob *string, currentStep *int) {
if node.Kind != yaml.ScalarNode {
return
}
comment := node.LineComment
if comment == "" {
comment = node.HeadComment
}
if comment == "" {
comment = node.FootComment
}
if comment != "" {
versionRegex := regexp.MustCompile(`v?\d+(\.\d+)*`)
versionMatch := versionRegex.FindString(comment)
if versionMatch != "" {
if currentJob != nil && currentStep == nil {
// this is a job level "uses"
if job, ok := wf.Jobs[*currentJob]; ok {
job.UsesComment = versionMatch
wf.Jobs[*currentJob] = job
}
} else if currentJob != nil && currentStep != nil {
// this is a step level "uses"
if job, ok := wf.Jobs[*currentJob]; ok {
if *currentStep < len(job.Steps) {
job.Steps[*currentStep].UsesComment = versionMatch
wf.Jobs[*currentJob] = job
}
}
}
}
}
}
@@ -20,6 +20,7 @@ func Test_parseWorkflowForActionUsage(t *testing.T) {
Type: pkg.GithubActionPkg,
Locations: fixtureLocationSet,
PURL: "", // don't have enough context without parsing the git origin, which still may not be accurate
Metadata: pkg.GitHubActionsUseStatement{Value: "./.github/actions/bootstrap"},
},
{
Name: "actions/cache",
@@ -27,6 +28,7 @@ func Test_parseWorkflowForActionUsage(t *testing.T) {
Type: pkg.GithubActionPkg,
Locations: fixtureLocationSet,
PURL: "pkg:github/actions/cache@v3",
Metadata: pkg.GitHubActionsUseStatement{Value: "actions/cache@v3"},
},
{
Name: "actions/cache/restore",
@@ -34,6 +36,7 @@ func Test_parseWorkflowForActionUsage(t *testing.T) {
Type: pkg.GithubActionPkg,
Locations: fixtureLocationSet,
PURL: "pkg:github/actions/cache@v3#restore",
Metadata: pkg.GitHubActionsUseStatement{Value: "actions/cache/restore@v3"},
},
{
Name: "actions/cache/save",
@@ -41,6 +44,7 @@ func Test_parseWorkflowForActionUsage(t *testing.T) {
Type: pkg.GithubActionPkg,
Locations: fixtureLocationSet,
PURL: "pkg:github/actions/cache@v3#save",
Metadata: pkg.GitHubActionsUseStatement{Value: "actions/cache/save@v3"},
},
{
Name: "actions/checkout",
@@ -48,6 +52,7 @@ func Test_parseWorkflowForActionUsage(t *testing.T) {
Type: pkg.GithubActionPkg,
Locations: fixtureLocationSet,
PURL: "pkg:github/actions/checkout@v4",
Metadata: pkg.GitHubActionsUseStatement{Value: "actions/checkout@v4"},
},
}
@@ -66,6 +71,9 @@ func Test_parseWorkflowForWorkflowUsage(t *testing.T) {
Type: pkg.GithubActionWorkflowPkg,
Locations: fixtureLocationSet,
PURL: "pkg:github/octo-org/this-repo@172239021f7ba04fe7327647b213799853a9eb89#.github/workflows/workflow-1.yml",
Metadata: pkg.GitHubActionsUseStatement{
Value: "octo-org/this-repo/.github/workflows/workflow-1.yml@172239021f7ba04fe7327647b213799853a9eb89",
},
},
{
Name: "./.github/workflows/workflow-2.yml",
@@ -73,6 +81,7 @@ func Test_parseWorkflowForWorkflowUsage(t *testing.T) {
Type: pkg.GithubActionWorkflowPkg,
Locations: fixtureLocationSet,
PURL: "", // don't have enough context without parsing the git origin, which still may not be accurate
Metadata: pkg.GitHubActionsUseStatement{Value: "./.github/workflows/workflow-2.yml"},
},
{
Name: "octo-org/another-repo/.github/workflows/workflow.yml",
@@ -80,6 +89,7 @@ func Test_parseWorkflowForWorkflowUsage(t *testing.T) {
Type: pkg.GithubActionWorkflowPkg,
Locations: fixtureLocationSet,
PURL: "pkg:github/octo-org/another-repo@v1#.github/workflows/workflow.yml",
Metadata: pkg.GitHubActionsUseStatement{Value: "octo-org/another-repo/.github/workflows/workflow.yml@v1"},
},
}
@@ -87,6 +97,38 @@ func Test_parseWorkflowForWorkflowUsage(t *testing.T) {
pkgtest.TestFileParser(t, fixture, parseWorkflowForWorkflowUsage, expected, expectedRelationships)
}
func Test_parseWorkflowForVersionComments(t *testing.T) {
fixture := "test-fixtures/workflow-with-version-comments.yaml"
fixtureLocationSet := file.NewLocationSet(file.NewLocation(fixture).WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
expected := []pkg.Package{
{
Name: "./.github/actions/bootstrap",
Version: "",
Type: pkg.GithubActionPkg,
Locations: fixtureLocationSet,
PURL: "", // don't have enough context without parsing the git origin, which still may not be accurate
Metadata: pkg.GitHubActionsUseStatement{
Value: "./.github/actions/bootstrap",
},
},
{
Name: "actions/checkout",
Version: "v4.2.2",
Type: pkg.GithubActionPkg,
Locations: fixtureLocationSet,
PURL: "pkg:github/actions/checkout@v4.2.2",
Metadata: pkg.GitHubActionsUseStatement{
Value: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683",
Comment: "v4.2.2",
},
},
}
var expectedRelationships []artifact.Relationship
pkgtest.TestFileParser(t, fixture, parseWorkflowForActionUsage, expected, expectedRelationships)
}
func Test_corruptActionWorkflow(t *testing.T) {
pkgtest.NewCatalogTester().
FromFile(t, "test-fixtures/corrupt/workflow-multi-job.yaml").
@@ -0,0 +1,28 @@
name: "Validations"
on:
workflow_dispatch:
pull_request:
push:
branches:
- main
permissions:
contents: read
jobs:
call-workflow-1-in-local-repo:
uses: octo-org/this-repo/.github/workflows/workflow-1.yml@172239021f7ba04fe7327647b213799853a9eb89 #v1.0.0
Static-Analysis:
name: "Static analysis"
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
- name: Bootstrap environment
uses: ./.github/actions/bootstrap
- name: Run static analysis
run: make static-analysis
+1 -1
View File
@@ -343,7 +343,7 @@ func getGOARCHFromBin(r io.ReaderAt) (string, error) {
if err != nil {
return "", fmt.Errorf("unrecognized file format: %w", err)
}
arch = fmt.Sprintf("%d", f.FileHeader.TargetMachine)
arch = fmt.Sprintf("%d", f.TargetMachine)
default:
return "", errUnrecognizedFormat
}
+3 -3
View File
@@ -61,7 +61,7 @@ func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resol
Name: m.Mod.Path,
Version: m.Mod.Version,
Licenses: pkg.NewLicenseSet(lics...),
Locations: file.NewLocationSet(reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
Locations: file.NewLocationSet(reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
PURL: packageURL(m.Mod.Path, m.Mod.Version),
Language: pkg.Go,
Type: pkg.GoModulePkg,
@@ -83,7 +83,7 @@ func (c *goModCataloger) parseGoModFile(ctx context.Context, resolver file.Resol
Name: m.New.Path,
Version: m.New.Version,
Licenses: pkg.NewLicenseSet(lics...),
Locations: file.NewLocationSet(reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
Locations: file.NewLocationSet(reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
PURL: packageURL(m.New.Path, m.New.Version),
Language: pkg.Go,
Type: pkg.GoModulePkg,
@@ -120,7 +120,7 @@ func parseGoSumFile(resolver file.Resolver, reader file.LocationReadCloser) (map
return out, fmt.Errorf("no resolver provided")
}
goSumPath := strings.TrimSuffix(reader.Location.RealPath, ".mod") + ".sum"
goSumPath := strings.TrimSuffix(reader.RealPath, ".mod") + ".sum"
goSumLocation := resolver.RelativeFileByPath(reader.Location, goSumPath)
if goSumLocation == nil {
return nil, fmt.Errorf("unable to resolve: %s", goSumPath)
@@ -10732,7 +10732,7 @@
"cpe:2.3:a:infolific:enhanced_plugin_admin:*:*:*:*:*:wordpress:*:*"
],
"enhanced-text-widget": [
"cpe:2.3:a:themecheck:enhanced_text_widget:*:*:*:*:*:wordpress:*:*"
"cpe:2.3:a:inisev:enhanced_text_widget:*:*:*:*:*:wordpress:*:*"
],
"enhanced-tooltipglossary": [
"cpe:2.3:a:cminds:tooltip_glossary:*:*:*:*:*:wordpress:*:*"
@@ -11570,6 +11570,9 @@
"gdpr-compliance-cookie-consent": [
"cpe:2.3:a:stylemixthemes:gdpr_compliance_\\\u0026_cookie_consent:*:*:*:*:*:wordpress:*:*"
],
"gdpr-cookie-compliance": [
"cpe:2.3:a:mooveagency:gdpr_cookie_compliance:*:*:*:*:*:wordpress:*:*"
],
"gdpr-cookie-consent": [
"cpe:2.3:a:wpeka:wp_cookie_consent:*:*:*:*:-:wordpress:*:*"
],
@@ -13058,6 +13061,9 @@
"login-as-customer-or-user": [
"cpe:2.3:a:wp-buy:login_as_user_or_customer_\\(user_switching\\):*:*:*:*:*:wordpress:*:*"
],
"login-as-users": [
"cpe:2.3:a:geekcodelab:login_as_users:*:*:*:*:*:wordpress:*:*"
],
"login-block-ips": [
"cpe:2.3:a:gunkastudios:login_block_ips:*:*:*:*:*:wordpress:*:*"
],
@@ -13784,6 +13790,9 @@
"my-favorites": [
"cpe:2.3:a:takashimatsuyama:my_favorites:*:*:*:*:*:wordpress:*:*"
],
"my-geo-posts-free": [
"cpe:2.3:a:mindstien:my_geo_posts_free:*:*:*:*:*:wordpress:*:*"
],
"my-tickets": [
"cpe:2.3:a:my_tickets_project:my_tickets:*:*:*:*:*:wordpress:*:*"
],
@@ -17489,7 +17498,7 @@
"cpe:2.3:a:wpxpo:postx:*:*:*:*:*:wordpress:*:*"
],
"ultimate-posts-widget": [
"cpe:2.3:a:themecheck:ultimate_posts_widget:*:*:*:*:*:wordpress:*:*"
"cpe:2.3:a:inisev:ultimate_posts_widget:*:*:*:*:*:wordpress:*:*"
],
"ultimate-product-catalogue": [
"cpe:2.3:a:etoilewebdesign:ultimate_product_catalog:*:*:*:*:*:wordpress:*:*"
@@ -18909,6 +18918,7 @@
"cpe:2.3:a:wpeasycart:wp_easycart:*:*:*:*:*:wordpress:*:*"
],
"wp-ecommerce-paypal": [
"cpe:2.3:a:wpplugin:easy_paypal_\\\u0026_stripe_buy_now_button:*:*:*:*:*:wordpress:*:*",
"cpe:2.3:a:wpplugin:easy_paypal_buy_now_button:*:*:*:*:*:wordpress:*:*"
],
"wp-edit-menu": [
@@ -19936,6 +19946,9 @@
"wpbulky-wp-bulk-edit-post-types": [
"cpe:2.3:a:villatheme:wpbulky:*:*:*:*:*:wordpress:*:*"
],
"wpc-badge-management": [
"cpe:2.3:a:wpclever:wpc_badge_management_for_woocommerce:*:*:*:*:*:wordpress:*:*"
],
"wpc-shop-as-customer": [
"cpe:2.3:a:wpclever:wpc_shop_as_a_customer_for_woocommerce:*:*:*:*:*:wordpress:*:*"
],
@@ -20743,6 +20756,9 @@
"yuki": [
"cpe:2.3:a:wpmoose:yuki:*:*:*:*:*:wordpress:*:*"
],
"zbench": [
"cpe:2.3:a:zww:zbench:*:*:*:*:*:wordpress:*:*"
],
"zigcy-baby": [
"cpe:2.3:a:accesspressthemes:zigcy_baby:*:*:*:*:*:wordpress:*:*"
],
@@ -7,15 +7,18 @@ import (
"os"
"sort"
"strings"
"sync"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/google/licensecheck"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/internal/cmptest"
"github.com/anchore/syft/internal/licenses"
"github.com/anchore/syft/internal/relationship"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
@@ -27,6 +30,11 @@ import (
"github.com/anchore/syft/syft/source/stereoscopesource"
)
var (
once sync.Once
licenseScanner *licenses.Scanner
)
type CatalogTester struct {
expectedPkgs []pkg.Package
expectedRelationships []artifact.Relationship
@@ -44,10 +52,26 @@ type CatalogTester struct {
licenseComparer cmptest.LicenseComparer
packageStringer func(pkg.Package) string
customAssertions []func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship)
context context.Context
}
func Context() context.Context {
once.Do(func() {
// most of the time in testing is initializing the scanner. Let's do that just once
sc := &licenses.ScannerConfig{Scanner: licensecheck.Scan, CoverageThreshold: 75}
scanner, err := licenses.NewScanner(sc)
if err != nil {
panic("unable to setup licences scanner for testing")
}
licenseScanner = &scanner
})
return licenses.SetContextLicenseScanner(context.Background(), *licenseScanner)
}
func NewCatalogTester() *CatalogTester {
return &CatalogTester{
context: Context(),
locationComparer: cmptest.DefaultLocationComparer,
licenseComparer: cmptest.DefaultLicenseComparer,
packageStringer: stringPackage,
@@ -64,6 +88,11 @@ func NewCatalogTester() *CatalogTester {
}
}
func (p *CatalogTester) WithContext(ctx context.Context) *CatalogTester {
p.context = ctx
return p
}
func (p *CatalogTester) FromDirectory(t *testing.T, path string) *CatalogTester {
t.Helper()
@@ -202,7 +231,7 @@ func (p *CatalogTester) IgnoreUnfulfilledPathResponses(paths ...string) *Catalog
func (p *CatalogTester) TestParser(t *testing.T, parser generic.Parser) {
t.Helper()
pkgs, relationships, err := parser(context.Background(), p.resolver, p.env, p.reader)
pkgs, relationships, err := parser(p.context, p.resolver, p.env, p.reader)
// only test for errors if explicitly requested
if p.wantErr != nil {
p.wantErr(t, err)
@@ -215,7 +244,7 @@ func (p *CatalogTester) TestCataloger(t *testing.T, cataloger pkg.Cataloger) {
resolver := NewObservingResolver(p.resolver)
pkgs, relationships, err := cataloger.Catalog(context.Background(), resolver)
pkgs, relationships, err := cataloger.Catalog(p.context, resolver)
// this is a minimum set, the resolver may return more that just this list
for _, path := range p.expectedPathResponses {
@@ -275,7 +275,7 @@ func (ni nativeImageElf) fetchPkgs() (pkgs []pkg.Package, relationships []artifa
if dataSection == nil {
return nil, nil, fmt.Errorf("no .data section found in binary: %w", err)
}
dataSectionBase := dataSection.SectionHeader.Addr
dataSectionBase := dataSection.Addr
data, err := dataSection.Data()
if err != nil {
return nil, nil, fmt.Errorf("cannot read the .data section: %w", err)
@@ -63,7 +63,7 @@ func parseGradleLockfile(_ context.Context, _ file.Resolver, _ *generic.Environm
Name: dep.Name,
Version: dep.Version,
Locations: file.NewLocationSet(
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
),
Language: pkg.Java,
Type: pkg.JavaPkg,
@@ -67,7 +67,7 @@ func parsePackageJSON(_ context.Context, _ file.Resolver, _ *generic.Environment
// a compliance filter later will remove these packages based on compliance rules
pkgs = append(
pkgs,
newPackageJSONPackage(p, reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
newPackageJSONPackage(p, reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)),
)
}
@@ -28,7 +28,7 @@ func parseLinuxKernelModuleFile(_ context.Context, _ file.Resolver, _ *generic.E
return nil, nil, nil
}
metadata.Path = reader.Location.RealPath
metadata.Path = reader.RealPath
return []pkg.Package{
newLinuxKernelModulePackage(
+1 -1
View File
@@ -74,7 +74,7 @@ func parseRockspec(_ context.Context, _ file.Resolver, _ *generic.Environment, r
Homepage: homepage,
Description: description,
},
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
)
return []pkg.Package{p}, nil, nil
+113 -6
View File
@@ -6,6 +6,9 @@ package nix
import (
"context"
"fmt"
"path/filepath"
"regexp"
"strings"
"github.com/bmatcuk/doublestar/v4"
@@ -28,6 +31,30 @@ func (c *storeCataloger) Name() string {
return catalogerName
}
// Find the parent Nix store path for a given path
func findParentNixStorePath(path string) string {
// Handle standard /nix/store/ paths
parts := strings.Split(path, "/")
for i := 0; i < len(parts)-2; i++ {
if parts[i] == "nix" && parts[i+1] == "store" && i+2 < len(parts) {
// Check if it matches the hash-name pattern
storeItem := parts[i+2]
if matched, _ := regexp.MatchString(`^[a-z0-9]{32}-.*`, storeItem); matched {
return filepath.Join("/", filepath.Join(parts[:i+3]...))
}
}
}
// Handle short-form Nix paths (without the /nix/store/ prefix)
// These would be paths like /[hash]-[package-name]/...
shortPathPattern := regexp.MustCompile(`^/([a-z0-9]{32}-[^/]+)`)
if matches := shortPathPattern.FindStringSubmatch(path); len(matches) == 2 {
return "/" + matches[1]
}
return ""
}
func (c *storeCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
// we want to search for only directories, which isn't possible via the stereoscope API, so we need to apply the glob manually on all returned paths
var pkgs []pkg.Package
@@ -37,9 +64,23 @@ func (c *storeCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([
for location := range resolver.AllLocations(ctx) {
matchesStorePath, err := doublestar.Match("**/nix/store/*", location.RealPath)
if err != nil {
log.Debugf("Error matching path %s: %v", location.RealPath, err)
return nil, nil, fmt.Errorf("failed to match nix store path: %w", err)
}
if !matchesStorePath {
// Check if this is a "short-form" Nix store path (without the /nix/store/ prefix)
// This pattern matches paths like /[hash]-[package-name]/...
shortPathPattern := regexp.MustCompile(`^/([a-z0-9]{32})-([^/]+)`)
if shortPathPattern.MatchString(location.RealPath) {
matchesStorePath = true
log.Debugf("Matched short-form Nix path: %s", location.RealPath)
}
}
if !matchesStorePath {
log.Debugf("Not a Nix store path: %s", location.RealPath)
continue
}
parentStorePath := findParentNixStorePath(location.RealPath)
if parentStorePath != "" {
if _, ok := filesByPath[parentStorePath]; !ok {
@@ -50,17 +91,24 @@ func (c *storeCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([
}
if !matchesStorePath {
log.Debugf("Not a Nix store path: %s", location.RealPath)
continue
}
storePath := parseNixStorePath(location.RealPath)
if storePath == nil || !storePath.isValidPackage() {
if storePath == nil {
log.Debugf("Failed to parse Nix path: %s", location.RealPath)
continue
}
p := newNixStorePackage(*storePath, location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
pkgs = append(pkgs, p)
// Only create packages for non-derivation/source paths
if storePath.shouldIncludeAsPackage() {
p := newNixStorePackage(*storePath, location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation))
pkgs = append(pkgs, p)
} else {
log.Debugf("Skipping non-package path: %s (type: %s)", location.RealPath, storePath.pathType)
}
}
// add file sets to packages
@@ -80,7 +128,64 @@ func (c *storeCataloger) Catalog(ctx context.Context, resolver file.Resolver) ([
appendFiles(p, files.ToSlice()...)
}
return pkgs, nil, nil
// After all packages are created, enrich them with derivation data
for i := range pkgs {
p := &pkgs[i]
metadata, ok := p.Metadata.(pkg.NixStoreEntry)
if !ok {
continue
}
// Find and parse the derivation file
derivPath := findDeriverPath(metadata.Path)
if derivPath != "" {
if deriv, err := ParseDerivation(derivPath); err == nil {
// Enrich package metadata with derivation data
if deriv.PName != "" {
p.Name = deriv.PName
}
if deriv.Version != "" {
p.Version = deriv.Version
}
// Add license, homepage, etc. to metadata
metadata.License = deriv.License
metadata.Homepage = deriv.Homepage
metadata.Description = deriv.Description
metadata.DeriverPath = derivPath
// Update the package metadata
p.Metadata = metadata
}
}
}
// Find and add relationships between packages
relationships, err := findDependencies(pkgs)
if err != nil {
log.Warnf("failed to resolve nix dependencies: %v", err)
}
// Deduplicate packages based on name and version
dedupedPkgs := deduplicateNixPackages(pkgs)
pkgs = dedupedPkgs
return pkgs, relationships, nil
}
// ensureNixStorePrefix makes sure the path has the /nix/store/ prefix
func ensureNixStorePrefix(path string) string {
// Check if the path already has the /nix/store/ prefix
if strings.Contains(path, "/nix/store/") {
return path
}
// Check if this is a short-form path starting with /[hash]-[name]
shortPathPattern := regexp.MustCompile(`^/([a-z0-9]{32}-[^/]+)(.*)$`)
if matches := shortPathPattern.FindStringSubmatch(path); len(matches) >= 3 {
// Convert short form path to full path with /nix/store/ prefix
return "/nix/store/" + matches[1] + matches[2]
}
return path
}
func appendFiles(p *pkg.Package, location ...file.Location) {
@@ -91,7 +196,9 @@ func appendFiles(p *pkg.Package, location ...file.Location) {
}
for _, l := range location {
metadata.Files = append(metadata.Files, l.RealPath)
// Normalize the path to ensure it has the /nix/store/ prefix
normalizedPath := ensureNixStorePrefix(l.RealPath)
metadata.Files = append(metadata.Files, normalizedPath)
}
if metadata.Files == nil {
+207
View File
@@ -0,0 +1,207 @@
package nix
import (
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
)
// findDependencies attempts to find relationships between Nix packages
// without using external commands
func findDependencies(pkgs []pkg.Package) ([]artifact.Relationship, error) {
var relationships []artifact.Relationship
packageByPath := make(map[string]*pkg.Package)
packageByStorePath := make(map[string]*pkg.Package)
// Index packages by store path and path for quick lookups
for i := range pkgs {
p := &pkgs[i]
metadata, ok := p.Metadata.(pkg.NixStoreEntry)
if !ok {
continue
}
packageByStorePath[metadata.Path] = p
// Also index by the package files
for _, filePath := range metadata.Files {
packageByPath[filePath] = p
}
}
// For each package, try to find references to other packages
for i := range pkgs {
p := &pkgs[i]
metadata, ok := p.Metadata.(pkg.NixStoreEntry)
if !ok {
continue
}
// Scan for references using the derivation file if available
if metadata.DeriverPath != "" {
refs, err := findReferencesInDerivation(metadata.DeriverPath, packageByStorePath)
if err == nil {
relationships = append(relationships, refs...)
}
}
// Scan for references within the package files
if len(metadata.Files) > 0 {
refs, err := findReferencesInFiles(metadata.Files, p, packageByStorePath)
if err == nil {
relationships = append(relationships, refs...)
}
}
}
return relationships, nil
}
// findReferencesInDerivation tries to extract dependencies from a derivation file
func findReferencesInDerivation(drvPath string, packageByPath map[string]*pkg.Package) ([]artifact.Relationship, error) {
var relationships []artifact.Relationship
// Read the derivation file contents
content, err := os.ReadFile(drvPath)
if err != nil {
return nil, err
}
// Look for store paths in the derivation file
storePathPattern := regexp.MustCompile(`/nix/store/[a-z0-9]{32}-[^"'\s]+`)
matches := storePathPattern.FindAllString(string(content), -1)
// Unique matches
uniquePaths := make(map[string]struct{})
for _, match := range matches {
uniquePaths[match] = struct{}{}
}
// Create relationships for found dependencies
for path := range uniquePaths {
depPkg, exists := packageByPath[path]
if !exists {
continue
}
// Skip self-references
fromMeta, ok := depPkg.Metadata.(pkg.NixStoreEntry)
if !ok || fromMeta.Path == drvPath {
continue
}
// Add the dependency relationship
pkg, exists := packageByPath[drvPath]
if !exists {
continue
}
relationships = append(relationships, artifact.Relationship{
From: *pkg,
To: *depPkg,
Type: artifact.DependencyOfRelationship,
Data: map[string]string{"dependencyType": "buildtime"},
})
}
return relationships, nil
}
// findReferencesInFiles looks for references to other packages within file contents
func findReferencesInFiles(files []string, fromPkg *pkg.Package, packageByPath map[string]*pkg.Package) ([]artifact.Relationship, error) {
var relationships []artifact.Relationship
uniqueRefs := make(map[string]struct{})
// Limit search to a reasonable number of files to avoid performance issues
maxFiles := 10
fileCount := 0
for _, file := range files {
if fileCount >= maxFiles {
break
}
// Skip files that are too large
info, err := os.Stat(file)
if err != nil || info.Size() > 1024*1024 {
continue
}
// Skip directories
if info.IsDir() {
continue
}
// Look for binary files that might have references
if isBinary(file) {
fileCount++
refs, err := findReferencesInBinary(file)
if err != nil {
continue
}
for _, ref := range refs {
uniqueRefs[ref] = struct{}{}
}
}
}
// Create relationships for found dependencies
for path := range uniqueRefs {
depPkg, exists := packageByPath[path]
if !exists {
continue
}
// Skip self-references
fromMeta, ok := fromPkg.Metadata.(pkg.NixStoreEntry)
if !ok || fromMeta.Path == path {
continue
}
relationships = append(relationships, artifact.Relationship{
From: *fromPkg,
To: *depPkg,
Type: artifact.DependencyOfRelationship,
Data: map[string]string{"dependencyType": "runtime"},
})
}
return relationships, nil
}
// Helper functions
// isBinary returns true if the file appears to be a binary
func isBinary(path string) bool {
ext := strings.ToLower(filepath.Ext(path))
return ext == ".so" || ext == ".dylib" || ext == ".a" ||
strings.HasSuffix(path, ".so.1") ||
strings.Contains(path, ".so.")
}
// findReferencesInBinary extracts Nix store references from binary files
func findReferencesInBinary(path string) ([]string, error) {
var references []string
// Use string search instead of executing nix-store
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
// Search for store paths in the binary
storePathPattern := regexp.MustCompile(`/nix/store/[a-z0-9]{32}-[^\\:\*\?"<>\|\x00-\x1F]+`)
matches := storePathPattern.FindAll(data, -1)
for _, match := range matches {
references = append(references, string(match))
}
return references, nil
}
+136
View File
@@ -0,0 +1,136 @@
package nix
import (
"bufio"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/anchore/syft/internal/log"
)
// NixDerivation represents a parsed Nix derivation file
type NixDerivation struct {
Path string
Name string
PName string
Version string
System string
Builder string
Args []string
EnvVars map[string]string
Outputs map[string]string
InputDrvs map[string][]string
InputSrcs []string
License string
Homepage string
Description string
OutputHash string
}
// ParseDerivation attempts to parse a Nix derivation file without external commands
func ParseDerivation(path string) (*NixDerivation, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
derivation := &NixDerivation{
Path: path,
EnvVars: make(map[string]string),
Outputs: make(map[string]string),
InputDrvs: make(map[string][]string),
}
// Simple parser for .drv files
scanner := bufio.NewScanner(file)
inEnvVars := false
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
// Process environment variables section
if strings.Contains(line, "envVars = {") {
inEnvVars = true
continue
} else if inEnvVars && strings.Contains(line, "};") {
inEnvVars = false
continue
} else if inEnvVars && strings.Contains(line, " = ") {
// Parse environment variables
parts := strings.SplitN(line, " = ", 2)
if len(parts) == 2 {
key := strings.Trim(parts[0], " \"")
value := strings.Trim(parts[1], " \";")
derivation.EnvVars[key] = value
}
}
}
// Extract package information from environment variables
derivation.Name = derivation.EnvVars["name"]
derivation.PName = derivation.EnvVars["pname"]
derivation.Version = derivation.EnvVars["version"]
derivation.System = derivation.EnvVars["system"]
derivation.License = derivation.EnvVars["license"]
derivation.Homepage = derivation.EnvVars["homepage"]
derivation.Description = derivation.EnvVars["description"]
// If pname is not set but name follows pname-version pattern, extract it
if derivation.PName == "" && derivation.Name != "" {
re := regexp.MustCompile(`^(.*?)-([0-9].*)$`)
if matches := re.FindStringSubmatch(derivation.Name); len(matches) == 3 {
derivation.PName = matches[1]
if derivation.Version == "" {
derivation.Version = matches[2]
}
} else {
derivation.PName = derivation.Name
}
}
// Check if this is a patched package with security fixes
for key := range derivation.EnvVars {
if strings.HasPrefix(key, "patches") {
value := derivation.EnvVars[key]
if strings.Contains(value, "CVE-") {
log.Debugf("Found security patch in %s: %s", derivation.Name, value)
}
}
}
return derivation, nil
}
// findDeriverPath attempts to find the derivation file for a store path
func findDeriverPath(storePath string) string {
// If the path is already a derivation, return it
if strings.HasSuffix(storePath, ".drv") {
return storePath
}
// Look for a .drv file with a similar name
drvPathGuess := storePath + ".drv"
if _, err := os.Stat(drvPathGuess); err == nil {
return drvPathGuess
}
// Fall back strategy using string manipulation
// This is an approximation without running nix-store commands
parts := strings.Split(storePath, "-")
if len(parts) >= 2 {
// Try to find the derivation file using the hash part
hash := strings.Split(filepath.Base(storePath), "-")[0]
if matched, _ := regexp.MatchString(`^[a-z0-9]{32}$`, hash); matched {
drvDir := filepath.Join("/nix/store", hash+"-*.drv")
matches, err := filepath.Glob(drvDir)
if err == nil && len(matches) > 0 {
return matches[0]
}
}
}
return ""
}
+64
View File
@@ -56,3 +56,67 @@ func packageURL(storePath nixStorePath) string {
"")
return pURL.ToString()
}
// deduplicateNixPackages combines information from packages with the same name and version
func deduplicateNixPackages(pkgs []pkg.Package) []pkg.Package {
if len(pkgs) == 0 {
return pkgs
}
// Use map to group by name+version
packageMap := make(map[string]*pkg.Package)
for i := range pkgs {
p := &pkgs[i]
key := p.Name + ":" + p.Version
existing, exists := packageMap[key]
if !exists {
// First time seeing this name+version
packageMap[key] = p
continue
}
// Merge information from this package into the existing one
mergePackageInfo(existing, p)
}
// Convert map back to slice
result := make([]pkg.Package, 0, len(packageMap))
for _, p := range packageMap {
result = append(result, *p)
}
return result
}
// mergePackageInfo combines information from src into dest
func mergePackageInfo(dest, src *pkg.Package) {
// Merge locations
for _, loc := range src.Locations.ToSlice() {
dest.Locations.Add(loc)
}
// Merge metadata if both have NixStoreEntry type
destMeta, destOk := dest.Metadata.(pkg.NixStoreEntry)
srcMeta, srcOk := src.Metadata.(pkg.NixStoreEntry)
if destOk && srcOk {
// Combine files lists
destMeta.Files = append(destMeta.Files, srcMeta.Files...)
// Prefer non-empty fields from src
if destMeta.License == "" && srcMeta.License != "" {
destMeta.License = srcMeta.License
}
if destMeta.Homepage == "" && srcMeta.Homepage != "" {
destMeta.Homepage = srcMeta.Homepage
}
if destMeta.Description == "" && srcMeta.Description != "" {
destMeta.Description = srcMeta.Description
}
// Update the metadata
dest.Metadata = destMeta
}
}
+122 -101
View File
@@ -1,134 +1,155 @@
package nix
import (
"fmt"
"path"
"regexp"
"strings"
)
var (
numericPattern = regexp.MustCompile(`\d`)
// attempts to find the right-most example of something that appears to be a version (semver or otherwise)
// example input: h0cnbmfcn93xm5dg2x27ixhag1cwndga-glibc-2.34-210-bin
// example output:
// version: "2.34-210"
// major: "2"
// minor: "34"
// patch: "210"
// (there are other capture groups, but they can be ignored)
rightMostVersionIshPattern = regexp.MustCompile(`-(?P<version>(?P<major>[0-9][a-zA-Z0-9]*)(\.(?P<minor>[0-9][a-zA-Z0-9]*))?(\.(?P<patch>0|[1-9][a-zA-Z0-9]*)){0,3}(?:-(?P<prerelease>\d*[.a-zA-Z-][.0-9a-zA-Z-]*)*)?(?:\+(?P<metadata>[.0-9a-zA-Z-]+(?:\.[.0-9a-zA-Z-]+)*))?)`)
unstableVersion = regexp.MustCompile(`-(?P<version>unstable-\d{4}-\d{2}-\d{2})$`)
)
// checkout the package naming conventions here: https://nixos.org/manual/nixpkgs/stable/#sec-package-naming
type nixStorePath struct {
path string
hash string
outputHash string
name string
version string
output string
// New field to indicate the type of Nix path
pathType string // "package", "derivation", "source", or "other"
}
func (p nixStorePath) isValidPackage() bool {
return p.name != "" && p.version != ""
}
func findParentNixStorePath(source string) string {
source = strings.TrimRight(source, "/")
indicator := "nix/store/"
start := strings.Index(source, indicator)
if start == -1 {
return ""
}
startOfHash := start + len(indicator)
nextField := strings.Index(source[startOfHash:], "/")
if nextField == -1 {
return ""
}
startOfSubPath := startOfHash + nextField
return source[0:startOfSubPath]
}
func parseNixStorePath(source string) *nixStorePath {
if strings.HasSuffix(source, ".drv") {
// ignore derivations
// parseNixStorePath extracts package information from a Nix store path.
// This is a more permissive implementation that accepts more formats.
func parseNixStorePath(path string) *nixStorePath {
// Extract the store path component - handle both standard and short forms
// Standard form: /nix/store/[hash]-[name]
// Short form: /[hash]-[name]
storePathPattern := regexp.MustCompile(`(^|.*?)(?:/nix/store/|/)([a-z0-9]{32})-(.+?)(/.*)?$`)
matches := storePathPattern.FindStringSubmatch(path)
if len(matches) < 4 {
return nil
}
source = path.Base(source)
// Extract hash and full package name
hash := matches[2]
fullName := matches[3]
versionStartIdx, versionIsh, prerelease := findVersionIsh(source)
if versionStartIdx == -1 {
return nil
// Extract path to first / after the hash-name pattern
basePath := "/nix/store/" + hash + "-" + fullName
if len(matches) > 4 && matches[4] != "" {
// We have a file within a store path
basePath = "/nix/store/" + hash + "-" + fullName
}
hashName := strings.TrimSuffix(source[0:versionStartIdx], "-")
hashNameFields := strings.Split(hashName, "-")
if len(hashNameFields) < 2 {
return nil
}
hash, name := hashNameFields[0], strings.Join(hashNameFields[1:], "-")
prereleaseFields := strings.Split(prerelease, "-")
lastPrereleaseField := prereleaseFields[len(prereleaseFields)-1]
var version = versionIsh
var output string
if !hasNumeric(lastPrereleaseField) {
// this last prerelease field is probably a nix output
version = strings.TrimSuffix(versionIsh, fmt.Sprintf("-%s", lastPrereleaseField))
output = lastPrereleaseField
}
return &nixStorePath{
// Create the basic result
result := &nixStorePath{
path: basePath,
hash: hash,
outputHash: hash,
name: name,
version: version,
output: output,
}
// Determine the path type based on extension or patterns
if strings.HasSuffix(fullName, ".drv") {
result.pathType = "derivation"
// Remove .drv suffix for name parsing
fullName = strings.TrimSuffix(fullName, ".drv")
} else if strings.Contains(fullName, ".tar.") ||
strings.HasSuffix(fullName, ".tgz") ||
strings.HasSuffix(fullName, ".tar.gz") ||
strings.HasSuffix(fullName, ".tar.bz2") ||
strings.HasSuffix(fullName, ".tar.xz") ||
strings.HasSuffix(fullName, ".zip") ||
strings.HasSuffix(fullName, ".patch") {
result.pathType = "source"
} else {
result.pathType = "package"
}
// Try to parse the output from the path (if present)
if outputParts := strings.Split(fullName, "-"); len(outputParts) > 1 {
lastPart := outputParts[len(outputParts)-1]
// Common Nix output names
outputs := map[string]bool{
"bin": true, "dev": true, "lib": true, "man": true,
"doc": true, "info": true, "out": true,
}
if outputs[lastPart] {
result.output = lastPart
fullName = strings.Join(outputParts[:len(outputParts)-1], "-")
}
}
// For non-drv files, try different version extraction patterns
versionPatterns := []*regexp.Regexp{
// Standard version: name-1.2.3
regexp.MustCompile(`^(.+)-([0-9][0-9.]+(?:-[0-9]+)?)$`),
// Unstable version: name-unstable-2022-05-15
regexp.MustCompile(`^(.+)-unstable-([0-9]{4}-[0-9]{2}-[0-9]{2})$`),
// Version with prefix: name-v1.2.3
regexp.MustCompile(`^(.+)-(v[0-9][0-9.]+)$`),
// Year-based versions: name-2022.05
regexp.MustCompile(`^(.+)-([0-9]{4}(?:\.[0-9]+)*)$`),
// Version with suffix: name-1.2.3-suffix
regexp.MustCompile(`^(.+)-([0-9][0-9.]+(?:-[a-z0-9]+)*)$`),
}
// Try each pattern to extract name and version
for _, pattern := range versionPatterns {
if nameVerMatches := pattern.FindStringSubmatch(fullName); len(nameVerMatches) == 3 {
result.name = nameVerMatches[1]
result.version = nameVerMatches[2]
return result
}
}
// If no version pattern matched, consider the entire string the package name
result.name = fullName
result.version = ""
return result
}
func hasNumeric(s string) bool {
return numericPattern.MatchString(s)
// isLikelyOutput checks if a string is likely to be a Nix output name
func isLikelyOutput(s string) bool {
commonOutputs := map[string]bool{
"bin": true, "lib": true, "dev": true, "out": true,
"doc": true, "man": true, "info": true, "devdoc": true,
}
return commonOutputs[s]
}
func findVersionIsh(input string) (int, string, string) {
// we want to return the index of the start of the "version" group (the first capture group).
// note that the match indices are in the form of [start, end, start, end, ...]. Also note that the
// capture group for version in both regexes are the same index, but if the regexes are changed
// this code will start to fail.
versionGroup := 1
// isLikelyVersion checks if a string looks like a version number
func isLikelyVersion(s string) bool {
// Simple check for common version patterns
versionPattern := regexp.MustCompile(`^v?\d+(\.\d+)*(-\w+)*$`)
return versionPattern.MatchString(s)
}
match := unstableVersion.FindAllStringSubmatchIndex(input, -1)
if len(match) > 0 && len(match[0]) > 0 {
return match[0][versionGroup*2], input[match[0][versionGroup*2]:match[0][(versionGroup*2)+1]], ""
// shouldIncludeAsPackage determines if this store path should be included as a package
func (n *nixStorePath) shouldIncludeAsPackage() bool {
// Only include actual packages, not derivations or source files
if n.pathType == "derivation" || n.pathType == "source" {
return false
}
match = rightMostVersionIshPattern.FindAllStringSubmatchIndex(input, -1)
if len(match) == 0 || len(match[0]) == 0 {
return -1, "", ""
// Skip paths that look like source archives even if not explicitly marked
if strings.HasSuffix(n.path, ".tar.gz") ||
strings.HasSuffix(n.path, ".tgz") ||
strings.HasSuffix(n.path, ".tar.bz2") ||
strings.HasSuffix(n.path, ".tar.xz") ||
strings.HasSuffix(n.path, ".zip") ||
strings.HasSuffix(n.path, ".drv") {
return false
}
var version string
versionStart, versionStop := match[0][versionGroup*2], match[0][(versionGroup*2)+1]
if versionStart != -1 || versionStop != -1 {
version = input[versionStart:versionStop]
}
prereleaseGroup := 7
var prerelease string
prereleaseStart, prereleaseStop := match[0][prereleaseGroup*2], match[0][(prereleaseGroup*2)+1]
if prereleaseStart != -1 && prereleaseStop != -1 {
prerelease = input[prereleaseStart:prereleaseStop]
}
// Make sure it has a name (at minimum)
return n.name != ""
}
return versionStart,
version,
prerelease
// isValidPackage should be much more permissive
func (n *nixStorePath) isValidPackage() bool {
// Any path with a hash and name is considered valid
return n.hash != "" && n.name != ""
}
+2 -2
View File
@@ -39,7 +39,7 @@ func parseOpamPackage(_ context.Context, _ file.Resolver, _ *generic.Environment
// If name is inferred from file name/path
var name, version string
var licenses []string
loc := reader.Location.LocationData.AccessPath
loc := reader.AccessPath
dir, file := path.Split(loc)
if file == "opam" {
@@ -95,7 +95,7 @@ func parseOpamPackage(_ context.Context, _ file.Resolver, _ *generic.Environment
pkgs,
newOpamPackage(
entry,
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
),
)
+349 -238
View File
@@ -1,7 +1,6 @@
package python
import (
"context"
"fmt"
"path"
"testing"
@@ -15,264 +14,383 @@ import (
func Test_PackageCataloger(t *testing.T) {
tests := []struct {
name string
fixtures []string
expectedPackage pkg.Package
name string
fixture string
expectedPackages []pkg.Package
}{
{
name: "egg-file-no-version",
fixtures: []string{"test-fixtures/no-version-py3.8.egg-info"},
expectedPackage: pkg.Package{
Name: "no-version",
PURL: "pkg:pypi/no-version",
Type: pkg.PythonPkg,
Language: pkg.Python,
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "no-version",
SitePackagesRootPath: "test-fixtures",
},
},
},
{
name: "egg-info directory",
fixtures: []string{
"test-fixtures/egg-info/PKG-INFO",
"test-fixtures/egg-info/RECORD",
"test-fixtures/egg-info/top_level.txt",
},
expectedPackage: pkg.Package{
Name: "requests",
Version: "2.22.0",
PURL: "pkg:pypi/requests@2.22.0",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/egg-info/PKG-INFO")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "requests",
Version: "2.22.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: "test-fixtures",
Files: []pkg.PythonFileRecord{
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
{Path: "requests/__pycache__/utils.cpython-38.pyc"},
{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
name: "egg-file-no-version",
fixture: "test-fixtures/site-packages/no-version",
expectedPackages: []pkg.Package{
{
Name: "no-version",
Locations: file.NewLocationSet(file.NewLocation("no-version-py3.8.egg-info")),
PURL: "pkg:pypi/no-version",
Type: pkg.PythonPkg,
Language: pkg.Python,
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "no-version",
SitePackagesRootPath: ".", // requires scanning the grandparent directory to get a valid path
},
TopLevelPackages: []string{"requests"},
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
ProvidesExtra: []string{"security", "socks"},
},
},
},
{
name: "egg-info directory case sensitive",
fixtures: []string{
"test-fixtures/casesensitive/EGG-INFO/PKG-INFO",
"test-fixtures/casesensitive/EGG-INFO/RECORD",
"test-fixtures/casesensitive/EGG-INFO/top_level.txt",
},
expectedPackage: pkg.Package{
Name: "requests",
Version: "2.22.0",
PURL: "pkg:pypi/requests@2.22.0",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/casesensitive/EGG-INFO/PKG-INFO")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "requests",
Version: "2.22.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: "test-fixtures/casesensitive",
Files: []pkg.PythonFileRecord{
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
{Path: "requests/__pycache__/utils.cpython-38.pyc"},
{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
},
TopLevelPackages: []string{"requests"},
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
ProvidesExtra: []string{"security", "socks"},
},
},
},
{
name: "dist-info directory",
fixtures: []string{
"test-fixtures/dist-info/METADATA",
"test-fixtures/dist-info/RECORD",
"test-fixtures/dist-info/top_level.txt",
"test-fixtures/dist-info/direct_url.json",
},
expectedPackage: pkg.Package{
Name: "pygments",
Version: "2.6.1",
PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/dist-info/METADATA")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "Pygments",
Version: "2.6.1",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "test-fixtures",
Files: []pkg.PythonFileRecord{
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "Pygments-2.6.1.dist-info/RECORD"},
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
name: "dist-info+egg-info site-packages directory",
fixture: "test-fixtures/site-packages/nested",
expectedPackages: []pkg.Package{
{
Name: "pygments",
Version: "2.6.1",
PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Type: pkg.PythonPkg,
Language: pkg.Python,
Locations: file.NewLocationSet(
file.NewLocation("dist-name/dist-info/METADATA"),
file.NewLocation("dist-name/dist-info/RECORD"),
file.NewLocation("dist-name/dist-info/direct_url.json"),
file.NewLocation("dist-name/dist-info/top_level.txt"),
),
Licenses: pkg.NewLicenseSet(
// here we only used the license that was declared in the METADATA file, we did not go searching for other licenses
// this is the better source of truth when there is no explicit LicenseFile given
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("dist-name/dist-info/METADATA")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "Pygments",
Version: "2.6.1",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "dist-name",
Files: []pkg.PythonFileRecord{
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "Pygments-2.6.1.dist-info/LICENSE.txt", Digest: &pkg.PythonFileDigest{Algorithm: "sha256", Value: "utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "Pygments-2.6.1.dist-info/RECORD"},
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
},
TopLevelPackages: []string{"pygments", "something_else"},
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
RequiresPython: ">=3.5",
RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"},
ProvidesExtra: []string{"html5lib", "lxml"},
},
},
{
Name: "requests",
Version: "2.22.0",
PURL: "pkg:pypi/requests@2.22.0",
Type: pkg.PythonPkg,
Language: pkg.Python,
Locations: file.NewLocationSet(
file.NewLocation("egg-name/egg-info/PKG-INFO"),
file.NewLocation("egg-name/egg-info/RECORD"),
file.NewLocation("egg-name/egg-info/top_level.txt"),
),
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("egg-name/egg-info/PKG-INFO")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "requests",
Version: "2.22.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: "egg-name",
Files: []pkg.PythonFileRecord{
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
{Path: "requests/__pycache__/utils.cpython-38.pyc"},
{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
},
TopLevelPackages: []string{"requests"},
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
ProvidesExtra: []string{"security", "socks"},
},
TopLevelPackages: []string{"pygments", "something_else"},
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
RequiresPython: ">=3.5",
RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"},
ProvidesExtra: []string{"html5lib", "lxml"},
},
},
},
{
name: "dist-info directory case sensitive",
fixtures: []string{
"test-fixtures/casesensitive/DIST-INFO/METADATA",
"test-fixtures/casesensitive/DIST-INFO/RECORD",
"test-fixtures/casesensitive/DIST-INFO/top_level.txt",
"test-fixtures/casesensitive/DIST-INFO/direct_url.json",
},
expectedPackage: pkg.Package{
Name: "pygments",
Version: "2.6.1",
PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/casesensitive/DIST-INFO/METADATA")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "Pygments",
Version: "2.6.1",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "test-fixtures/casesensitive",
Files: []pkg.PythonFileRecord{
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "Pygments-2.6.1.dist-info/RECORD"},
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
name: "DIST-INFO+EGG-INFO site-packages directory (case insensitive)",
fixture: "test-fixtures/site-packages/uppercase",
expectedPackages: []pkg.Package{
{
Name: "pygments",
Version: "2.6.1",
PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Type: pkg.PythonPkg,
Language: pkg.Python,
Locations: file.NewLocationSet(
file.NewLocation("dist-name/DIST-INFO/METADATA"),
file.NewLocation("dist-name/DIST-INFO/RECORD"),
file.NewLocation("dist-name/DIST-INFO/direct_url.json"),
file.NewLocation("dist-name/DIST-INFO/top_level.txt"),
),
Licenses: pkg.NewLicenseSet(
// here we only used the license that was declared in the METADATA file, we did not go searching for other licenses
// this is the better source of truth when there is no explicit LicenseFile given
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("dist-name/DIST-INFO/METADATA")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "Pygments",
Version: "2.6.1",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "dist-name",
Files: []pkg.PythonFileRecord{
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "Pygments-2.6.1.dist-info/RECORD"},
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
},
TopLevelPackages: []string{"pygments", "something_else"},
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
RequiresPython: ">=3.5",
},
},
{
Name: "requests",
Version: "2.22.0",
PURL: "pkg:pypi/requests@2.22.0",
Type: pkg.PythonPkg,
Language: pkg.Python,
Locations: file.NewLocationSet(
file.NewLocation("egg-name/EGG-INFO/PKG-INFO"),
file.NewLocation("egg-name/EGG-INFO/RECORD"),
file.NewLocation("egg-name/EGG-INFO/top_level.txt"),
),
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("egg-name/EGG-INFO/PKG-INFO")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "requests",
Version: "2.22.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: "egg-name",
Files: []pkg.PythonFileRecord{
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
{Path: "requests/__pycache__/utils.cpython-38.pyc"},
{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
},
TopLevelPackages: []string{"requests"},
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
ProvidesExtra: []string{"security", "socks"},
},
TopLevelPackages: []string{"pygments", "something_else"},
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
RequiresPython: ">=3.5",
},
},
},
{
name: "malformed-record",
fixtures: []string{
"test-fixtures/malformed-record/dist-info/METADATA",
"test-fixtures/malformed-record/dist-info/RECORD",
name: "detect licenses",
fixture: "test-fixtures/site-packages/license",
expectedPackages: []pkg.Package{
{
Name: "pygments",
Version: "2.6.1",
PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Type: pkg.PythonPkg,
Language: pkg.Python,
Locations: file.NewLocationSet(
file.NewLocation("with-license-file-declared.dist-info/METADATA"), // the LicenseFile is declared in the METADATA file
file.NewLocation("with-license-file-declared.dist-info/RECORD"),
file.NewLocation("with-license-file-declared.dist-info/top_level.txt"),
file.NewLocation("with-license-file-declared.dist-info/direct_url.json"),
),
Licenses: pkg.NewLicenseSet(
pkg.License{
Value: "BSD-3-Clause",
SPDXExpression: "BSD-3-Clause",
Type: "concluded",
// we read the path from the LicenseFile field in the METADATA file, then read the license file directly
Locations: file.NewLocationSet(file.NewLocation("with-license-file-declared.dist-info/LICENSE.txt")),
},
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "Pygments",
Version: "2.6.1",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: ".",
Files: []pkg.PythonFileRecord{
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "with-license-file-declared.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "with-license-file-declared.dist-info/LICENSE.txt", Digest: &pkg.PythonFileDigest{Algorithm: "sha256", Value: "utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "with-license-file-declared.dist-info/RECORD"},
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
},
TopLevelPackages: []string{"pygments", "something_else"},
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
RequiresPython: ">=3.5",
RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"},
ProvidesExtra: []string{"html5lib", "lxml"},
},
},
{
Name: "pygments",
Version: "2.6.1",
PURL: "pkg:pypi/pygments@2.6.1?vcs_url=git%2Bhttps%3A%2F%2Fgithub.com%2Fpython-test%2Ftest.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Type: pkg.PythonPkg,
Language: pkg.Python,
Locations: file.NewLocationSet(
file.NewLocation("without-license-file-declared.dist-info/METADATA"), // the LicenseFile is declared in the METADATA file
file.NewLocation("without-license-file-declared.dist-info/RECORD"),
file.NewLocation("without-license-file-declared.dist-info/top_level.txt"),
file.NewLocation("without-license-file-declared.dist-info/direct_url.json"),
),
Licenses: pkg.NewLicenseSet(
pkg.License{
Value: "BSD-3-Clause",
SPDXExpression: "BSD-3-Clause",
Type: "concluded",
// we discover license files automatically
Locations: file.NewLocationSet(file.NewLocation("without-license-file-declared.dist-info/LICENSE.txt")),
},
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "Pygments",
Version: "2.6.1",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: ".",
Files: []pkg.PythonFileRecord{
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "without-license-file-declared.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "without-license-file-declared.dist-info/LICENSE.txt", Digest: &pkg.PythonFileDigest{Algorithm: "sha256", Value: "utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "without-license-file-declared.dist-info/RECORD"},
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
},
TopLevelPackages: []string{"pygments", "something_else"},
DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
RequiresPython: ">=3.5",
RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"},
ProvidesExtra: []string{"html5lib", "lxml"},
},
},
},
expectedPackage: pkg.Package{
Name: "pygments",
Version: "2.6.1",
PURL: "pkg:pypi/pygments@2.6.1",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/malformed-record/dist-info/METADATA")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "Pygments",
Version: "2.6.1",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "test-fixtures/malformed-record",
Files: []pkg.PythonFileRecord{
{Path: "flask/json/tag.py", Digest: &pkg.PythonFileDigest{"sha256", "9ehzrmt5k7hxf7ZEK0NOs3swvQyU9fWNe-pnYe69N60"}, Size: "8223"},
{Path: "../../Scripts/flask.exe", Digest: &pkg.PythonFileDigest{"sha256", "mPrbVeZCDX20himZ_bRai1nCs_tgr7jHIOGZlcgn-T4"}, Size: "93063"},
{Path: "../../Scripts/flask.exe", Size: "89470", Digest: &pkg.PythonFileDigest{"sha256", "jvqh4N3qOqXLlq40i6ZOLCY9tAOwfwdzIpLDYhRjoqQ"}},
{Path: "Flask-1.0.2.dist-info/INSTALLER", Size: "4", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}},
},
{
name: "malformed-record",
fixture: "test-fixtures/site-packages/malformed-record",
expectedPackages: []pkg.Package{
{
Name: "pygments",
Version: "2.6.1",
PURL: "pkg:pypi/pygments@2.6.1",
Type: pkg.PythonPkg,
Language: pkg.Python,
Locations: file.NewLocationSet(
file.NewLocation("dist-info/METADATA"),
file.NewLocation("dist-info/RECORD"),
),
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("dist-info/METADATA")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "Pygments",
Version: "2.6.1",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: ".",
Files: []pkg.PythonFileRecord{
{Path: "flask/json/tag.py", Digest: &pkg.PythonFileDigest{"sha256", "9ehzrmt5k7hxf7ZEK0NOs3swvQyU9fWNe-pnYe69N60"}, Size: "8223"},
{Path: "../../Scripts/flask.exe", Digest: &pkg.PythonFileDigest{"sha256", "mPrbVeZCDX20himZ_bRai1nCs_tgr7jHIOGZlcgn-T4"}, Size: "93063"},
{Path: "../../Scripts/flask.exe", Size: "89470", Digest: &pkg.PythonFileDigest{"sha256", "jvqh4N3qOqXLlq40i6ZOLCY9tAOwfwdzIpLDYhRjoqQ"}},
{Path: "Flask-1.0.2.dist-info/INSTALLER", Size: "4", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}},
},
RequiresPython: ">=3.5",
},
RequiresPython: ">=3.5",
},
},
},
{
// in cases where the metadata file is available and the record is not we should still record there is a package
// additionally empty top_level.txt files should not result in an error
name: "partial dist-info directory",
fixtures: []string{"test-fixtures/partial.dist-info/METADATA"},
expectedPackage: pkg.Package{
Name: "pygments",
Version: "2.6.1",
PURL: "pkg:pypi/pygments@2.6.1",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/partial.dist-info/METADATA")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "Pygments",
Version: "2.6.1",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "test-fixtures",
RequiresPython: ">=3.5",
name: "partial dist-info directory",
fixture: "test-fixtures/site-packages/partial.dist-info",
expectedPackages: []pkg.Package{
{
Name: "pygments",
Version: "2.6.1",
PURL: "pkg:pypi/pygments@2.6.1",
Type: pkg.PythonPkg,
Language: pkg.Python,
Locations: file.NewLocationSet(
file.NewLocation("METADATA"),
),
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("BSD License", file.NewLocation("METADATA")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "Pygments",
Version: "2.6.1",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: ".",
RequiresPython: ">=3.5",
},
},
},
},
{
name: "egg-info regular file",
fixtures: []string{"test-fixtures/test.egg-info"},
expectedPackage: pkg.Package{
Name: "requests",
Version: "2.22.0",
PURL: "pkg:pypi/requests@2.22.0",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/test.egg-info")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "requests",
Version: "2.22.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: "test-fixtures",
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
ProvidesExtra: []string{"security", "socks"},
name: "egg-info regular file",
fixture: "test-fixtures/site-packages/test",
expectedPackages: []pkg.Package{
{
Name: "requests",
Version: "2.22.0",
PURL: "pkg:pypi/requests@2.22.0",
Type: pkg.PythonPkg,
Language: pkg.Python,
Locations: file.NewLocationSet(
file.NewLocation("test.egg-info"),
),
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test.egg-info")),
),
FoundBy: "python-installed-package-cataloger",
Metadata: pkg.PythonPackage{
Name: "requests",
Version: "2.22.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: ".",
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
ProvidesExtra: []string{"security", "socks"},
},
},
},
},
@@ -280,16 +398,9 @@ func Test_PackageCataloger(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resolver := file.NewMockResolverForPaths(test.fixtures...)
locations, err := resolver.FilesByPath(test.fixtures...)
require.NoError(t, err)
test.expectedPackage.Locations = file.NewLocationSet(locations...)
pkgtest.NewCatalogTester().
WithResolver(resolver).
Expects([]pkg.Package{test.expectedPackage}, nil).
FromDirectory(t, test.fixture).
Expects(test.expectedPackages, nil).
TestCataloger(t, NewInstalledPackageCataloger())
})
}
@@ -311,7 +422,7 @@ func Test_PackageCataloger_IgnorePackage(t *testing.T) {
t.Run(test.MetadataFixture, func(t *testing.T) {
resolver := file.NewMockResolverForPaths(test.MetadataFixture)
actual, _, err := NewInstalledPackageCataloger().Catalog(context.Background(), resolver)
actual, _, err := NewInstalledPackageCataloger().Catalog(pkgtest.Context(), resolver)
require.NoError(t, err)
if len(actual) != 0 {
-35
View File
@@ -1,14 +1,11 @@
package python
import (
"context"
"fmt"
"regexp"
"strings"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/internal/licenses"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
)
@@ -92,38 +89,6 @@ func newPackageForPackage(m parsedData, licenses pkg.LicenseSet, sources ...file
return p
}
func findLicenses(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, m parsedData) pkg.LicenseSet {
var licenseSet pkg.LicenseSet
switch {
case m.LicenseExpression != "":
licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(m.LicenseLocation, m.LicenseExpression)...)
case m.Licenses != "":
licenseSet = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(m.LicenseLocation, m.Licenses)...)
case m.LicenseLocation.Path() != "":
// If we have a license file then resolve and parse it
found, err := resolver.FilesByPath(m.LicenseLocation.Path())
if err != nil {
log.WithFields("error", err, "path", m.LicenseLocation.Path()).Trace("unable to resolve python license")
}
if len(found) > 0 {
metadataContents, err := resolver.FileContentsByLocation(found[0])
if err == nil {
parsed, err := scanner.PkgSearch(ctx, file.NewLocationReadCloser(m.LicenseLocation, metadataContents))
if err != nil {
log.WithFields("error", err, "path", m.LicenseLocation.Path()).Trace("unable to parse a license from the file")
}
if len(parsed) > 0 {
licenseSet = pkg.NewLicenseSet(parsed...)
}
} else {
log.WithFields("error", err, "path", m.LicenseLocation.Path()).Trace("unable to read file contents")
}
}
}
return licenseSet
}
func packageURL(name, version string, m *pkg.PythonPackage) string {
// generate a purl from the package data
pURL := packageurl.NewPackageURL(
@@ -99,7 +99,7 @@ func poetryLockPackages(reader file.LocationReadCloser) ([]pkg.Package, error) {
p.Name,
p.Version,
newPythonPoetryLockEntry(p),
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
),
)
}
@@ -153,7 +153,7 @@ func (rp requirementsParser) parseRequirementsTxt(_ context.Context, _ file.Reso
URL: parseURL(req.URL),
Markers: req.Markers,
},
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
),
)
}
+1 -1
View File
@@ -59,7 +59,7 @@ func parseSetup(_ context.Context, _ file.Resolver, _ *generic.Environment, read
newPackageForIndex(
name,
version,
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
),
)
}
@@ -6,7 +6,12 @@ import (
"encoding/json"
"fmt"
"io"
"path"
"path/filepath"
"sort"
"strings"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/licenses"
@@ -247,3 +252,97 @@ func assembleEggOrWheelMetadata(resolver file.Resolver, metadataLocation file.Lo
pd.DirectURLOrigin = d
return &pd, sources, nil
}
func findLicenses(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, m parsedData) pkg.LicenseSet {
var licenseSet pkg.LicenseSet
licenseLocations := file.NewLocationSet()
if m.LicenseFilePath != "" {
locs, err := resolver.FilesByPath(m.LicenseFilePath)
if err != nil {
log.WithFields("error", err, "path", m.LicenseFilePath).Trace("unable to resolve python license file")
} else {
licenseLocations.Add(locs...)
}
}
switch {
case m.LicenseExpression != "" || m.Licenses != "":
licenseSet = getLicenseSetFromValues(licenseLocations.ToSlice(), m.LicenseExpression, m.Licenses)
case !licenseLocations.Empty():
licenseSet = getLicenseSetFromFiles(ctx, scanner, resolver, licenseLocations.ToSlice()...)
default:
// search for known license paths from RECORDS file
licenseNames := strset.New()
for _, n := range licenses.FileNames() {
licenseNames.Add(strings.ToLower(n))
}
parent := path.Base(path.Dir(m.DistInfoLocation.Path()))
candidatePaths := strset.New()
for _, f := range m.Files {
if !strings.HasPrefix(f.Path, parent) || strings.Count(f.Path, "/") > 1 {
continue
}
if licenseNames.Has(strings.ToLower(filepath.Base(f.Path))) {
candidatePaths.Add(path.Join(m.SitePackagesRootPath, f.Path))
}
}
paths := candidatePaths.List()
sort.Strings(paths)
locationSet := file.NewLocationSet()
for _, p := range paths {
locs, err := resolver.FilesByPath(p)
if err != nil {
log.WithFields("error", err, "path", p).Trace("unable to resolve python license in dist-info")
continue
}
locationSet.Add(locs...)
}
licenseSet = getLicenseSetFromFiles(ctx, scanner, resolver, locationSet.ToSlice()...)
}
return licenseSet
}
func getLicenseSetFromValues(locations []file.Location, licenseValues ...string) pkg.LicenseSet {
if len(locations) == 0 {
return pkg.NewLicenseSet(pkg.NewLicensesFromValues(licenseValues...)...)
}
licenseSet := pkg.NewLicenseSet()
for _, value := range licenseValues {
if value == "" {
continue
}
licenseSet.Add(pkg.NewLicenseFromLocations(value, locations...))
}
return licenseSet
}
func getLicenseSetFromFiles(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, locations ...file.Location) pkg.LicenseSet {
licenseSet := pkg.NewLicenseSet()
for _, loc := range locations {
licenseSet.Add(getLicenseSetFromFile(ctx, scanner, resolver, loc)...)
}
return licenseSet
}
func getLicenseSetFromFile(ctx context.Context, scanner licenses.Scanner, resolver file.Resolver, location file.Location) []pkg.License {
metadataContents, err := resolver.FileContentsByLocation(location)
if err != nil {
log.WithFields("error", err, "path", location.Path()).Trace("unable to read file contents")
return nil
}
defer internal.CloseAndLogError(metadataContents, location.Path())
parsed, err := scanner.PkgSearch(ctx, file.NewLocationReadCloser(location, metadataContents))
if err != nil {
log.WithFields("error", err, "path", location.Path()).Trace("unable to parse a license from the file")
return nil
}
return parsed
}
@@ -15,11 +15,18 @@ import (
)
type parsedData struct {
// core info
// DistInfoLocation is the location of the METADATA file within the .dist-info directory where we obtained the python package information
DistInfoLocation file.Location
pkg.PythonPackage `mapstructure:",squash"`
// license info
Licenses string `mapstructure:"License"`
LicenseFile string `mapstructure:"LicenseFile"`
LicenseExpression string `mapstructure:"LicenseExpression"`
LicenseLocation file.Location
pkg.PythonPackage `mapstructure:",squash"`
LicenseFilePath string
}
var pluralFields = map[string]bool{
@@ -45,11 +52,13 @@ func parseWheelOrEggMetadata(locationReader file.LocationReadCloser) (parsedData
pd.SitePackagesRootPath = determineSitePackagesRootPath(path)
if pd.Licenses != "" || pd.LicenseExpression != "" {
pd.LicenseLocation = file.NewLocation(path)
pd.LicenseFilePath = path
} else if pd.LicenseFile != "" {
pd.LicenseLocation = file.NewLocation(filepath.Join(filepath.Dir(path), pd.LicenseFile))
pd.LicenseFilePath = filepath.Join(filepath.Dir(path), pd.LicenseFile)
}
pd.DistInfoLocation = locationReader.Location
return pd, nil
}
@@ -22,38 +22,40 @@ func TestParseWheelEggMetadata(t *testing.T) {
ExpectedMetadata parsedData
}{
{
Fixture: "test-fixtures/egg-info/PKG-INFO",
Fixture: "test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO",
ExpectedMetadata: parsedData{
"Apache 2.0",
"",
"",
file.NewLocation("test-fixtures/egg-info/PKG-INFO"),
pkg.PythonPackage{
DistInfoLocation: file.NewLocation("test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO"),
Licenses: "Apache 2.0",
LicenseFile: "",
LicenseExpression: "",
LicenseFilePath: "test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO",
PythonPackage: pkg.PythonPackage{
Name: "requests",
Version: "2.22.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: "test-fixtures",
SitePackagesRootPath: "test-fixtures/site-packages/nested/egg-name",
RequiresPython: ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
ProvidesExtra: []string{"security", "socks"},
},
},
},
{
Fixture: "test-fixtures/dist-info/METADATA",
Fixture: "test-fixtures/site-packages/nested/dist-name/dist-info/METADATA",
ExpectedMetadata: parsedData{
"BSD License",
"",
"",
file.NewLocation("test-fixtures/dist-info/METADATA"),
pkg.PythonPackage{
DistInfoLocation: file.NewLocation("test-fixtures/site-packages/nested/dist-name/dist-info/METADATA"),
Licenses: "BSD License",
LicenseFile: "",
LicenseExpression: "",
LicenseFilePath: "test-fixtures/site-packages/nested/dist-name/dist-info/METADATA",
PythonPackage: pkg.PythonPackage{
Name: "Pygments",
Version: "2.6.1",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "test-fixtures",
SitePackagesRootPath: "test-fixtures/site-packages/nested/dist-name",
RequiresPython: ">=3.5",
RequiresDist: []string{"soupsieve (>1.2)", "html5lib ; extra == 'html5lib'", "lxml ; extra == 'lxml'"},
ProvidesExtra: []string{"html5lib", "lxml"},
@@ -149,16 +151,17 @@ func TestParseWheelEggMetadataInvalid(t *testing.T) {
ExpectedMetadata parsedData
}{
{
Fixture: "test-fixtures/egg-info/PKG-INFO-INVALID",
Fixture: "test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO-INVALID",
ExpectedMetadata: parsedData{
"",
"",
"",
file.Location{},
pkg.PythonPackage{
DistInfoLocation: file.NewLocation("test-fixtures/site-packages/nested/egg-name/egg-info/PKG-INFO-INVALID"),
Licenses: "",
LicenseExpression: "",
LicenseFile: "",
LicenseFilePath: "",
PythonPackage: pkg.PythonPackage{
Name: "mxnet",
Version: "1.8.0",
SitePackagesRootPath: "test-fixtures",
SitePackagesRootPath: "test-fixtures/site-packages/nested/egg-name",
},
},
},
@@ -15,7 +15,7 @@ func TestParseWheelEggRecord(t *testing.T) {
ExpectedMetadata []pkg.PythonFileRecord
}{
{
Fixture: "test-fixtures/egg-info/RECORD",
Fixture: "test-fixtures/site-packages/nested/egg-name/egg-info/RECORD",
ExpectedMetadata: []pkg.PythonFileRecord{
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
@@ -26,10 +26,11 @@ func TestParseWheelEggRecord(t *testing.T) {
},
},
{
Fixture: "test-fixtures/dist-info/RECORD",
Fixture: "test-fixtures/site-packages/nested/dist-name/dist-info/RECORD",
ExpectedMetadata: []pkg.PythonFileRecord{
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "Pygments-2.6.1.dist-info/LICENSE.txt", Digest: &pkg.PythonFileDigest{Algorithm: "sha256", Value: "utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "Pygments-2.6.1.dist-info/RECORD"},
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
@@ -0,0 +1,28 @@
Copyright 2010 Pallets
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,53 @@
Metadata-Version: 2.1
Name: Pygments
Version: 2.6.1
Summary: Pygments is a syntax highlighting package written in Python.
Home-page: https://pygments.org/
Author: Georg Brandl
Author-email: georg@python.org
LicenseFile: LICENSE.txt
Keywords: syntax highlighting
Platform: any
Classifier: License :: OSI Approved :: BSD License
Classifier: Intended Audience :: Developers
Classifier: Intended Audience :: End Users/Desktop
Classifier: Intended Audience :: System Administrators
Classifier: Development Status :: 6 - Mature
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Operating System :: OS Independent
Classifier: Topic :: Text Processing :: Filters
Classifier: Topic :: Utilities
Requires-Python: >=3.5
Description-Content-Type: text/markdown
Requires-Dist: soupsieve (>1.2)
Provides-Extra: html5lib
Requires-Dist: html5lib ; extra == 'html5lib'
Provides-Extra: lxml
Requires-Dist: lxml ; extra == 'lxml'
Pygments
~~~~~~~~
Pygments is a syntax highlighting package written in Python.
It is a generic syntax highlighter suitable for use in code hosting, forums,
wikis or other applications that need to prettify source code. Highlights
are:
* a wide range of over 500 languages and other text formats is supported
* special attention is paid to details, increasing quality by a fair amount
* support for new languages and formats are added easily
* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image formats that PIL supports and ANSI sequences
* it is usable as a command-line tool and as a library
:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
@@ -0,0 +1,7 @@
../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220
with-license-file-declared.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449
with-license-file-declared.dist-info/LICENSE.txt,sha256=utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY,8449
with-license-file-declared.dist-info/RECORD,,
pygments/__pycache__/__init__.cpython-38.pyc,,
pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778
pygments/x_util.py,sha256=qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI=,10778
@@ -0,0 +1,28 @@
Copyright 2010 Pallets
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,52 @@
Metadata-Version: 2.1
Name: Pygments
Version: 2.6.1
Summary: Pygments is a syntax highlighting package written in Python.
Home-page: https://pygments.org/
Author: Georg Brandl
Author-email: georg@python.org
Keywords: syntax highlighting
Platform: any
Classifier: License :: OSI Approved :: BSD License
Classifier: Intended Audience :: Developers
Classifier: Intended Audience :: End Users/Desktop
Classifier: Intended Audience :: System Administrators
Classifier: Development Status :: 6 - Mature
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Operating System :: OS Independent
Classifier: Topic :: Text Processing :: Filters
Classifier: Topic :: Utilities
Requires-Python: >=3.5
Description-Content-Type: text/markdown
Requires-Dist: soupsieve (>1.2)
Provides-Extra: html5lib
Requires-Dist: html5lib ; extra == 'html5lib'
Provides-Extra: lxml
Requires-Dist: lxml ; extra == 'lxml'
Pygments
~~~~~~~~
Pygments is a syntax highlighting package written in Python.
It is a generic syntax highlighter suitable for use in code hosting, forums,
wikis or other applications that need to prettify source code. Highlights
are:
* a wide range of over 500 languages and other text formats is supported
* special attention is paid to details, increasing quality by a fair amount
* support for new languages and formats are added easily
* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image formats that PIL supports and ANSI sequences
* it is usable as a command-line tool and as a library
:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
@@ -0,0 +1,7 @@
../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220
without-license-file-declared.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449
without-license-file-declared.dist-info/LICENSE.txt,sha256=utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY,8449
without-license-file-declared.dist-info/RECORD,,
pygments/__pycache__/__init__.cpython-38.pyc,,
pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778
pygments/x_util.py,sha256=qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI=,10778
@@ -0,0 +1,28 @@
Copyright 2010 Pallets
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -1,5 +1,6 @@
../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220
Pygments-2.6.1.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449
Pygments-2.6.1.dist-info/LICENSE.txt,sha256=utiUvpzxqFPVpvuPnWG2_Oku6BGuay2I8-NIrqCvqUY,8449
Pygments-2.6.1.dist-info/RECORD,,
pygments/__pycache__/__init__.cpython-38.pyc,,
pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778
@@ -0,0 +1 @@
{"url": "https://github.com/python-test/test.git", "vcs_info": {"commit_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "vcs": "git"}}
@@ -0,0 +1,2 @@
pygments
something_else

Some files were not shown because too many files have changed in this diff Show More