Compare commits

...
9 Commits
Author SHA1 Message Date
Alex Goodman 6d4d083acc Filter out CPE product candidates that are asterisks (#513)
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-09-22 20:55:03 -04:00
Dan Luhring 316d4341c8 Use Anchore fork of packageurl lib without replace directive (#512)
Signed-off-by: Dan Luhring <dan.luhring@anchore.com>
2021-09-22 15:04:09 -04:00
Christopher Angelo Phillips 3e8afc5274 update log file permissions to 0644 (#511)
Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>
2021-09-21 10:34:10 -04:00
Christopher Angelo Phillips 93d00dc340 Populate Files and Relationship fields for spdx-json output (#507)
* update spdx22 Document model to include relationships field

Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>

* update document and relationship to match current JSON spec
https://github.com/spdx/spdx-spec/blob/development/v2.2.1/schemas/spdx-schema.json
https://github.com/spdx/spdx-spec/pull/528
https://github.com/spdx/spdx-spec/pull/528#issuecomment-904180177

Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>

* update File struct based on SPDX schema

Required fields:
[ "SPDXID", "fileName", "copyrightText", "licenseConcluded" ]
Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>
2021-09-17 09:06:12 -04:00
Alex Goodman 9fe1da8ee6 update go-rpmdb and restruct gomod references (#506)
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-09-14 14:51:19 -04:00
Christopher Angelo Phillips 95df7dd035 Add command examples to release documentation (#503)
* update release document with commands run
Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>

* small edits
Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>
2021-09-14 09:08:37 -04:00
Alex Goodman ddfc8e20c0 Revert "disable docker releases (workaround) (#493)" (#501)
This reverts commit 06dcd3261d.

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-09-13 13:06:23 -04:00
Christopher Angelo Phillips 3f305aa6cd (#342) - remove strong distro check (#496)
* add Type conversion to remove strong distro type limit
Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>

* update signatures to be correct variable from os-release
Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>
2021-09-09 13:05:22 -04:00
houdini91 2f99a35f51 Power user command support for directory scans (#467)
* Power-user directory source support
Signed-off-by: Mikey Strauss <mikey@scribe-security.com>

Signed-off-by: houdini91 <mdstrauss91@gmail.com>

* Remove newline

Signed-off-by: houdini91 <mdstrauss91@gmail.com>

* Shared filetree (#1)

* Shared directory resolver filetree

Signed-off-by: houdini91 <mdstrauss91@gmail.com>

* PR - change error ErrObserve to ErrPath

Signed-off-by: houdini91 <mdstrauss91@gmail.com>

* PR - share directory resolver
* Use pointer to source struct

Signed-off-by: houdini91 <mdstrauss91@gmail.com>

* Fix Lint

Signed-off-by: houdini91 <mdstrauss91@gmail.com>
2021-09-08 09:18:53 -04:00
57 changed files with 491 additions and 156 deletions
+2 -2
View File
@@ -6,7 +6,7 @@ permit:
- ISC
ignore-packages:
# packageurl-go is released under the MIT license located in the root of the repo at /mit.LICENSE
- github.com/package-url/packageurl-go
- github.com/anchore/packageurl-go
# from: https://github.com/spdx/tools-golang/blob/main/LICENSE.code
# The tools-golang source code is provided and may be used, at your option,
@@ -14,4 +14,4 @@ ignore-packages:
# * Apache License, version 2.0 (Apache-2.0), OR
# * GNU General Public License, version 2.0 or later (GPL-2.0-or-later).
# (we choose Apache-2.0)
- github.com/spdx/tools-golang
- github.com/spdx/tools-golang
+4 -8
View File
@@ -94,15 +94,11 @@ jobs:
needs: [quality-gate]
runs-on: macos-latest # Due to our code signing process, it's vital that we run our release steps on macOS.
steps:
- uses: docker-practice/actions-setup-docker@v1
# we are having an infinite loop of installing docker https://github.com/anchore/syft/actions/runs/1199137004
# we are still looking for a fix
# - uses: docker-practice/actions-setup-docker@v1
#
# # note, it is important to always be auth'd into docker.io to prevent rate limiting issues
# - name: Login to Docker Hub
# run: echo ${{ secrets.TOOLBOX_DOCKER_PASS }} | docker login docker.io -u ${{ secrets.TOOLBOX_DOCKER_USER }} --password-stdin
# note, it is important to always be auth'd into docker.io to prevent rate limiting issues
- name: Login to Docker Hub
run: echo ${{ secrets.TOOLBOX_DOCKER_PASS }} | docker login docker.io -u ${{ secrets.TOOLBOX_DOCKER_USER }} --password-stdin
- uses: actions/setup-go@v2
with:
+13 -16
View File
@@ -91,19 +91,16 @@ brews:
homepage: *website
description: *description
# we are having an infinite loop of installing docker https://github.com/anchore/syft/actions/runs/1199137004
# we are still looking for a fix
#dockers:
# - dockerfile: Dockerfile
# image_templates:
# - "anchore/syft:latest"
# - "anchore/syft:{{ .Tag }}"
# - "anchore/syft:v{{ .Major }}"
# - "anchore/syft:v{{ .Major }}.{{ .Minor }}"
# build_flag_templates:
# - "--build-arg=BUILD_DATE={{.Date}}"
# - "--build-arg=BUILD_VERSION={{.Version}}"
# - "--build-arg=VCS_REF={{.FullCommit}}"
# - "--build-arg=VCS_URL={{.GitURL}}"
# use_buildx: true
dockers:
- dockerfile: Dockerfile
image_templates:
- "anchore/syft:latest"
- "anchore/syft:{{ .Tag }}"
- "anchore/syft:v{{ .Major }}"
- "anchore/syft:v{{ .Major }}.{{ .Minor }}"
build_flag_templates:
- "--build-arg=BUILD_DATE={{.Date}}"
- "--build-arg=BUILD_VERSION={{.Version}}"
- "--build-arg=VCS_REF={{.FullCommit}}"
- "--build-arg=VCS_URL={{.GitURL}}"
use_buildx: true
+5 -2
View File
@@ -87,9 +87,12 @@ This release process itself should be as automated as possible, and have only a
1. Tag the main branch with a full semantic-version, prefixed with a `v`. If there is a
milestone with a partial version, the full version should be used for the git tag (e.g.
with a Milestone of `v0.1` the tag should be `v0.1.0`).
with a Milestone of `v0.1` the tag should be `v0.1.0`). You can determine the changes going
into a release by running `make changelog-unreleased`. Use this change list to determine the
release increment. After determining the release increment (major, minor, patch), create the tag.
Given the above example the command to create the tag would be `git tag v0.1.0`.
1. Push the tag.
1. Push the tag. Given the above example the command to push the tag would be `git push origin v0.1.0`.
1. A release admin must approve the release on the GitHub Actions release pipeline run page.
Once approved, the release pipeline will generate all assets and draft a GitHub Release.
+1 -1
View File
@@ -237,7 +237,7 @@ func packagesExecWorker(userInput string) <-chan error {
return errs
}
func runPackageSbomUpload(src source.Source, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, scope source.Scope) error {
func runPackageSbomUpload(src *source.Source, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, scope source.Scope) error {
log.Infof("uploading results to %s", appConfig.Anchore.Host)
if src.Metadata.Scheme != source.ImageScheme {
-5
View File
@@ -102,11 +102,6 @@ func powerUserExecWorker(userInput string) <-chan error {
}
defer cleanup()
if src.Metadata.Scheme != source.ImageScheme {
errs <- fmt.Errorf("the power-user subcommand only allows for 'image' schemes, given %q", src.Metadata.Scheme)
return
}
analysisResults := poweruser.JSONDocumentConfig{
SourceMetadata: src.Metadata,
ApplicationConfig: *appConfig,
+7 -7
View File
@@ -10,7 +10,7 @@ import (
"github.com/anchore/syft/syft/source"
)
type powerUserTask func(*poweruser.JSONDocumentConfig, source.Source) error
type powerUserTask func(*poweruser.JSONDocumentConfig, *source.Source) error
func powerUserTasks() ([]powerUserTask, error) {
var tasks []powerUserTask
@@ -42,7 +42,7 @@ func catalogPackagesTask() (powerUserTask, error) {
return nil, nil
}
task := func(results *poweruser.JSONDocumentConfig, src source.Source) error {
task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
packageCatalog, theDistro, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt)
if err != nil {
return err
@@ -64,7 +64,7 @@ func catalogFileMetadataTask() (powerUserTask, error) {
metadataCataloger := file.NewMetadataCataloger()
task := func(results *poweruser.JSONDocumentConfig, src source.Source) error {
task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.FileMetadata.Cataloger.ScopeOpt)
if err != nil {
return err
@@ -110,7 +110,7 @@ func catalogFileDigestsTask() (powerUserTask, error) {
return nil, err
}
task := func(results *poweruser.JSONDocumentConfig, src source.Source) error {
task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.FileMetadata.Cataloger.ScopeOpt)
if err != nil {
return err
@@ -142,7 +142,7 @@ func catalogSecretsTask() (powerUserTask, error) {
return nil, err
}
task := func(results *poweruser.JSONDocumentConfig, src source.Source) error {
task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.Secrets.Cataloger.ScopeOpt)
if err != nil {
return err
@@ -170,7 +170,7 @@ func catalogFileClassificationsTask() (powerUserTask, error) {
return nil, err
}
task := func(results *poweruser.JSONDocumentConfig, src source.Source) error {
task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.FileClassification.Cataloger.ScopeOpt)
if err != nil {
return err
@@ -197,7 +197,7 @@ func catalogContentsTask() (powerUserTask, error) {
return nil, err
}
task := func(results *poweruser.JSONDocumentConfig, src source.Source) error {
task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.FileContents.Cataloger.ScopeOpt)
if err != nil {
return err
+2 -4
View File
@@ -7,9 +7,10 @@ require (
github.com/adrg/xdg v0.2.1
github.com/alecthomas/jsonschema v0.0.0-20210301060011-54c507b6f074
github.com/anchore/client-go v0.0.0-20210222170800-9c70f9b80bcf
github.com/anchore/go-rpmdb v0.0.0-20210602151223-1f0f707a2894
github.com/anchore/go-rpmdb v0.0.0-20210914181456-a9c52348da63
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29
github.com/anchore/stereoscope v0.0.0-20210817160504-0f4abc2a5a5a
github.com/antihax/optional v1.0.0
github.com/bmatcuk/doublestar/v2 v2.0.4
@@ -24,7 +25,6 @@ require (
github.com/mitchellh/go-homedir v1.1.0
github.com/mitchellh/mapstructure v1.3.1
github.com/olekukonko/tablewriter v0.0.4
github.com/package-url/packageurl-go v0.1.0
github.com/pelletier/go-toml v1.8.0
github.com/pkg/profile v1.5.0
github.com/scylladb/go-set v1.0.2
@@ -47,5 +47,3 @@ require (
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
gopkg.in/yaml.v2 v2.3.0
)
replace github.com/package-url/packageurl-go v0.1.0 => github.com/anchore/packageurl-go v0.1.0-fixed
+6 -6
View File
@@ -109,14 +109,14 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/anchore/client-go v0.0.0-20210222170800-9c70f9b80bcf h1:DYssiUV1pBmKqzKsm4mqXx8artqC0Q8HgZsVI3lMsAg=
github.com/anchore/client-go v0.0.0-20210222170800-9c70f9b80bcf/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
github.com/anchore/go-rpmdb v0.0.0-20210602151223-1f0f707a2894 h1:VvCq7fFNU8dwWkCTGUykm4p64nVaDCYkKrj87x350Sk=
github.com/anchore/go-rpmdb v0.0.0-20210602151223-1f0f707a2894/go.mod h1:8jNYOxCJC5kyD/Ct4MbzsDN2hOhRoCAzQcb/7KdYYGw=
github.com/anchore/go-rpmdb v0.0.0-20210914181456-a9c52348da63 h1:C9W/LAydEz/qdUhx1MdjO9l8NEcFKYknkxDVyo9LAoM=
github.com/anchore/go-rpmdb v0.0.0-20210914181456-a9c52348da63/go.mod h1:6qH8c6U/3CBVvDDDBZnPSTbTINq3cIdADUYTaVf75EM=
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8=
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
github.com/anchore/packageurl-go v0.1.0-fixed h1:2QJUTALDF7m2WHPOS1NyHguSX4ciG0xD3idaBQwHJS8=
github.com/anchore/packageurl-go v0.1.0-fixed/go.mod h1:C/ApiuWpmbpni4DIOECf6WCjFUZV7O1Fx7VAzrZHgBw=
github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29 h1:K9LfnxwhqvihqU0+MF325FNy7fsKV9EGaUxdfR4gnWk=
github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29/go.mod h1:Oc1UkGaJwY6ND6vtAqPSlYrptKRJngHwkwB6W7l1uP0=
github.com/anchore/stereoscope v0.0.0-20210817160504-0f4abc2a5a5a h1:RQb+Gft1MKxjDfJCnHP/f1mwfy0Jz50Kp9QGgSWKQiY=
github.com/anchore/stereoscope v0.0.0-20210817160504-0f4abc2a5a5a/go.mod h1:165DfE5jApgEkHTWwu7Bijeml9fofudrgcpuWaD9+tk=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
@@ -267,8 +267,8 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8
github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-restruct/restruct v0.0.0-20191227155143-5734170a48a1 h1:LoN2wx/aN8JPGebG+2DaUyk4M+xRcqJXfuIbs8AWHdE=
github.com/go-restruct/restruct v0.0.0-20191227155143-5734170a48a1/go.mod h1:KqrpKpn4M8OLznErihXTGLlsXFGeLxHUrLRRI/1YjGk=
github.com/go-restruct/restruct v1.2.0-alpha h1:2Lp474S/9660+SJjpVxoKuWX09JsXHSrdV7Nv3/gkvc=
github.com/go-restruct/restruct v1.2.0-alpha/go.mod h1:KqrpKpn4M8OLznErihXTGLlsXFGeLxHUrLRRI/1YjGk=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+24
View File
@@ -1,7 +1,9 @@
package internal
import (
"fmt"
"io"
"os"
"github.com/anchore/syft/internal/log"
)
@@ -12,3 +14,25 @@ func CloseAndLogError(closer io.Closer, location string) {
log.Warnf("unable to close file for location=%q: %+v", location, err)
}
}
type ErrPath struct {
Path string
Err error
}
func (e ErrPath) Error() string {
return fmt.Sprintf("unable to observe contents of %+v: %v", e.Path, e.Err)
}
func IsErrPath(err error) bool {
_, ok := err.(ErrPath)
return ok
}
func IsErrPathPermission(err error) bool {
pathErr, ok := err.(ErrPath)
if ok {
return os.IsPermission(pathErr.Err)
}
return ok
}
+5 -2
View File
@@ -3,6 +3,7 @@ package logger
import (
"fmt"
"io"
"io/fs"
"io/ioutil"
"os"
@@ -10,6 +11,8 @@ import (
prefixed "github.com/x-cray/logrus-prefixed-formatter"
)
const defaultLogFilePermissions fs.FileMode = 0644
// LogrusConfig contains all configurable values for the Logrus logger
type LogrusConfig struct {
EnableConsole bool
@@ -38,7 +41,7 @@ func NewLogrusLogger(cfg LogrusConfig) *LogrusLogger {
var output io.Writer
switch {
case cfg.EnableConsole && cfg.EnableFile:
logFile, err := os.OpenFile(cfg.FileLocation, os.O_WRONLY|os.O_CREATE, 0755)
logFile, err := os.OpenFile(cfg.FileLocation, os.O_WRONLY|os.O_CREATE, defaultLogFilePermissions)
if err != nil {
panic(fmt.Errorf("unable to setup log file: %w", err))
}
@@ -46,7 +49,7 @@ func NewLogrusLogger(cfg LogrusConfig) *LogrusLogger {
case cfg.EnableConsole:
output = os.Stderr
case cfg.EnableFile:
logFile, err := os.OpenFile(cfg.FileLocation, os.O_WRONLY|os.O_CREATE, 0755)
logFile, err := os.OpenFile(cfg.FileLocation, os.O_WRONLY|os.O_CREATE, defaultLogFilePermissions)
if err != nil {
panic(fmt.Errorf("unable to setup log file: %w", err))
}
@@ -40,4 +40,6 @@ type Document struct {
Files []File `json:"files,omitempty"`
// Snippets referenced in the SPDX document
Snippets []Snippet `json:"snippets,omitempty"`
// Relationships referenced in the SPDX document
Relationships []Relationship `json:"relationships,omitempty"`
}
@@ -20,22 +20,22 @@ type File struct {
Item
// (At least one is required.) The checksum property provides a mechanism that can be used to verify that the
// contents of a File or Package have not changed.
Checksums []Checksum `json:"checksums"`
Checksums []Checksum `json:"checksums,omitempty"`
// This field provides a place for the SPDX file creator to record file contributors. Contributors could include
// names of copyright holders and/or authors who may not be copyright holders yet contributed to the file content.
FileContributors []string `json:"fileContributors"`
FileContributors []string `json:"fileContributors,omitempty"`
// Each element is a SPDX ID for a File.
FileDependencies []string `json:"fileDependencies"`
FileDependencies []string `json:"fileDependencies,omitempty"`
// The name of the file relative to the root of the package.
FileName string `json:"fileName"`
// The type of the file
FileTypes []string `json:"fileTypes"`
FileTypes []string `json:"fileTypes,omitempty"`
// This field provides a place for the SPDX file creator to record potential legal notices found in the file.
// This may or may not include copyright statements.
NoticeText string `json:"noticeText,omitempty"`
// Indicates the project in which the SpdxElement originated. Tools must preserve doap:homepage and doap:name
// properties and the URI (if one is known) of doap:Project resources that are values of this property. All other
// properties of doap:Projects are not directly supported by SPDX and may be dropped when translating to or
// from some SPDX formats.
ArtifactOf []string `json:"artifactOf"`
// from some SPDX formats(deprecated).
ArtifactOf []string `json:"artifactOf,omitempty"`
}
@@ -1,11 +1,13 @@
package spdx22
type Relationship struct {
// SPDX ID for SpdxElement. A related SpdxElement.
RelatedSpdxElement string `json:"relatedSpdxElement"`
// Id to which the SPDX element is related
SpdxElementID string `json:"spdxElementId"`
// Describes the type of relationship between two SPDX elements.
RelationshipType RelationshipType `json:"relationshipType"`
Comment string `json:"comment,omitempty"`
// SPDX ID for SpdxElement. A related SpdxElement.
RelatedSpdxElement string `json:"relatedSpdxElement"`
Comment string `json:"comment,omitempty"`
}
// source: https://spdx.github.io/spdx-spec/7-relationships-between-SPDX-elements/
@@ -1,7 +1,9 @@
package packages
import (
"crypto/sha256"
"fmt"
"path/filepath"
"strings"
"github.com/anchore/syft/internal/presenter/packages/model/spdx22"
@@ -29,6 +31,43 @@ func getSPDXExternalRefs(p *pkg.Package) (externalRefs []spdx22.ExternalRef) {
return externalRefs
}
func getSPDXFiles(packageSpdxID string, p *pkg.Package) (files []spdx22.File, fileIDs []string, relationships []spdx22.Relationship) {
files = make([]spdx22.File, 0)
fileIDs = make([]string, 0)
relationships = make([]spdx22.Relationship, 0)
pkgFileOwner, ok := p.Metadata.(pkg.FileOwner)
if !ok {
return files, fileIDs, relationships
}
for _, ownedFilePath := range pkgFileOwner.OwnedFiles() {
baseFileName := filepath.Base(ownedFilePath)
pathHash := sha256.Sum256([]byte(ownedFilePath))
fileSpdxID := spdx22.ElementID(fmt.Sprintf("File-%s-%x", p.Name, pathHash)).String()
fileIDs = append(fileIDs, fileSpdxID)
files = append(files, spdx22.File{
FileName: ownedFilePath,
Item: spdx22.Item{
Element: spdx22.Element{
SPDXID: fileSpdxID,
Name: baseFileName,
},
},
})
relationships = append(relationships, spdx22.Relationship{
SpdxElementID: packageSpdxID,
RelationshipType: spdx22.ContainsRelationship,
RelatedSpdxElement: fileSpdxID,
})
}
return files, fileIDs, relationships
}
func getSPDXLicense(p *pkg.Package) string {
// source: https://spdx.github.io/spdx-spec/3-package-information/#313-concluded-license
// The options to populate this field are limited to:
@@ -39,6 +39,7 @@ func (pres *SPDXJsonPresenter) Present(output io.Writer) error {
return enc.Encode(&doc)
}
// newSPDXJsonDocument creates and populates a new JSON document struct that follows the SPDX 2.2 spec from the given cataloging results.
func newSPDXJsonDocument(catalog *pkg.Catalog, srcMetadata source.Metadata) spdx22.Document {
var name string
switch srcMetadata.Scheme {
@@ -48,6 +49,8 @@ func newSPDXJsonDocument(catalog *pkg.Catalog, srcMetadata source.Metadata) spdx
name = srcMetadata.Path
}
packages, files, relationships := newSPDXJsonElements(catalog)
return spdx22.Document{
Element: spdx22.Element{
SPDXID: spdx22.ElementID("DOCUMENT").String(),
@@ -65,22 +68,34 @@ func newSPDXJsonDocument(catalog *pkg.Catalog, srcMetadata source.Metadata) spdx
},
DataLicense: "CC0-1.0",
DocumentNamespace: fmt.Sprintf("https://anchore.com/syft/image/%s", srcMetadata.ImageMetadata.UserInput),
Packages: newSPDXJsonPackages(catalog),
Packages: packages,
Files: files,
Relationships: relationships,
}
}
func newSPDXJsonPackages(catalog *pkg.Catalog) []spdx22.Package {
results := make([]spdx22.Package, 0)
func newSPDXJsonElements(catalog *pkg.Catalog) ([]spdx22.Package, []spdx22.File, []spdx22.Relationship) {
packages := make([]spdx22.Package, 0)
relationships := make([]spdx22.Relationship, 0)
files := make([]spdx22.File, 0)
for _, p := range catalog.Sorted() {
license := getSPDXLicense(p)
packageSpdxID := spdx22.ElementID(fmt.Sprintf("Package-%+v-%s-%s", p.Type, p.Name, p.Version)).String()
packageFiles, fileIDs, packageFileRelationships := getSPDXFiles(packageSpdxID, p)
files = append(files, packageFiles...)
relationships = append(relationships, packageFileRelationships...)
// note: the license concluded and declared should be the same since we are collecting license information
// from the project data itself (the installed package files).
results = append(results, spdx22.Package{
packages = append(packages, spdx22.Package{
Description: getSPDXDescription(p),
DownloadLocation: getSPDXDownloadLocation(p),
ExternalRefs: getSPDXExternalRefs(p),
FilesAnalyzed: false,
HasFiles: fileIDs,
Homepage: getSPDXHomepage(p),
LicenseDeclared: license, // The Declared License is what the authors of a project believe govern the package
Originator: getSPDXOriginator(p),
@@ -89,11 +104,12 @@ func newSPDXJsonPackages(catalog *pkg.Catalog) []spdx22.Package {
Item: spdx22.Item{
LicenseConcluded: license, // The Concluded License field is the license the SPDX file creator believes governs the package
Element: spdx22.Element{
SPDXID: spdx22.ElementID(fmt.Sprintf("Package-%+v-%s-%s", p.Type, p.Name, p.Version)).String(),
SPDXID: packageSpdxID,
Name: p.Name,
},
},
})
}
return results
return packages, files, relationships
}
@@ -27,6 +27,11 @@
"author": "",
"authorEmail": "",
"platform": "",
"files": [
{
"path": "/some/path/pkg1/depedencies/foo"
}
],
"sitePackagesRootPath": ""
}
},
@@ -9,7 +9,7 @@
"locations": [
{
"path": "/somefile-1.txt",
"layerID": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59"
"layerID": "sha256:ffb5e9eaa453a002110719d12c294960117ca2903953d1faa40f01dc3f77045c"
}
],
"licenses": [
@@ -40,7 +40,7 @@
"locations": [
{
"path": "/somefile-2.txt",
"layerID": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec"
"layerID": "sha256:8463854829fc53d47b9dcdf7ee79fe7eb4ca7933c910f67f8521412f7a2f5c21"
}
],
"licenses": [],
@@ -67,7 +67,7 @@
"type": "image",
"target": {
"userInput": "user-image-input",
"imageID": "sha256:5900c94a5bc1e083aa24ad1a223bf6eb9910dc8a6b01cb979ec306cb91709ea1",
"imageID": "sha256:112851310e48e604f7379e2a3acddab50e91ce926edacb598a532e60ff6b776a",
"manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"tags": [
@@ -77,17 +77,17 @@
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59",
"digest": "sha256:ffb5e9eaa453a002110719d12c294960117ca2903953d1faa40f01dc3f77045c",
"size": 22
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec",
"digest": "sha256:8463854829fc53d47b9dcdf7ee79fe7eb4ca7933c910f67f8521412f7a2f5c21",
"size": 16
}
],
"manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NjUsImRpZ2VzdCI6InNoYTI1Njo1OTAwYzk0YTViYzFlMDgzYWEyNGFkMWEyMjNiZjZlYjk5MTBkYzhhNmIwMWNiOTc5ZWMzMDZjYjkxNzA5ZWExIn0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjpmYjZiZWVjYjc1YjM5ZjRiYjgxM2RiZjE3N2U1MDFlZGQ1ZGRiM2U2OWJiNDVjZWRlYjc4YzY3NmVlMWI3YTU5In0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2OjMxOWI1ODhjZTY0MjUzYTg3YjUzM2M4ZWQwMWNmMDAyNWUwZWFjOThlN2I1MTZlMTI1MzI5NTdlMTI0NGZkZWMifV19",
"config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8iLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjEtMDQtMDZUMTk6MTM6NTIuNTI0Mzc4WiIsImhpc3RvcnkiOlt7ImNyZWF0ZWQiOiIyMDIxLTA0LTA2VDE5OjEzOjUyLjQ1ODIwNjFaIiwiY3JlYXRlZF9ieSI6IkFERCBmaWxlLTEudHh0IC9zb21lZmlsZS0xLnR4dCAjIGJ1aWxka2l0IiwiY29tbWVudCI6ImJ1aWxka2l0LmRvY2tlcmZpbGUudjAifSx7ImNyZWF0ZWQiOiIyMDIxLTA0LTA2VDE5OjEzOjUyLjUyNDM3OFoiLCJjcmVhdGVkX2J5IjoiQUREIGZpbGUtMi50eHQgL3NvbWVmaWxlLTIudHh0ICMgYnVpbGRraXQiLCJjb21tZW50IjoiYnVpbGRraXQuZG9ja2VyZmlsZS52MCJ9XSwib3MiOiJsaW51eCIsInJvb3RmcyI6eyJ0eXBlIjoibGF5ZXJzIiwiZGlmZl9pZHMiOlsic2hhMjU2OmZiNmJlZWNiNzViMzlmNGJiODEzZGJmMTc3ZTUwMWVkZDVkZGIzZTY5YmI0NWNlZGViNzhjNjc2ZWUxYjdhNTkiLCJzaGEyNTY6MzE5YjU4OGNlNjQyNTNhODdiNTMzYzhlZDAxY2YwMDI1ZTBlYWM5OGU3YjUxNmUxMjUzMjk1N2UxMjQ0ZmRlYyJdfX0=",
"manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NzMsImRpZ2VzdCI6InNoYTI1NjoxMTI4NTEzMTBlNDhlNjA0ZjczNzllMmEzYWNkZGFiNTBlOTFjZTkyNmVkYWNiNTk4YTUzMmU2MGZmNmI3NzZhIn0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjpmZmI1ZTllYWE0NTNhMDAyMTEwNzE5ZDEyYzI5NDk2MDExN2NhMjkwMzk1M2QxZmFhNDBmMDFkYzNmNzcwNDVjIn0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2Ojg0NjM4NTQ4MjlmYzUzZDQ3YjlkY2RmN2VlNzlmZTdlYjRjYTc5MzNjOTEwZjY3Zjg1MjE0MTJmN2EyZjVjMjEifV19",
"config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8iLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjEtMDktMDhUMTc6MjE6NTguODk2NTI5MTkyWiIsImhpc3RvcnkiOlt7ImNyZWF0ZWQiOiIyMDIxLTA5LTA4VDE3OjIxOjU4Ljg3OTY5MDgyNFoiLCJjcmVhdGVkX2J5IjoiQUREIGZpbGUtMS50eHQgL3NvbWVmaWxlLTEudHh0ICMgYnVpbGRraXQiLCJjb21tZW50IjoiYnVpbGRraXQuZG9ja2VyZmlsZS52MCJ9LHsiY3JlYXRlZCI6IjIwMjEtMDktMDhUMTc6MjE6NTguODk2NTI5MTkyWiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0yLnR4dCAvc29tZWZpbGUtMi50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6ZmZiNWU5ZWFhNDUzYTAwMjExMDcxOWQxMmMyOTQ5NjAxMTdjYTI5MDM5NTNkMWZhYTQwZjAxZGMzZjc3MDQ1YyIsInNoYTI1Njo4NDYzODU0ODI5ZmM1M2Q0N2I5ZGNkZjdlZTc5ZmU3ZWI0Y2E3OTMzYzkxMGY2N2Y4NTIxNDEyZjdhMmY1YzIxIl19fQ==",
"repoDigests": [],
"scope": "Squashed"
}
@@ -3,12 +3,12 @@
"name": "/some/path",
"spdxVersion": "SPDX-2.2",
"creationInfo": {
"created": "2021-06-23T17:48:32.734847Z",
"created": "2021-09-16T20:44:35.198887Z",
"creators": [
"Organization: Anchore, Inc",
"Tool: syft-[not provided]"
],
"licenseListVersion": "3.13"
"licenseListVersion": "3.14"
},
"dataLicense": "CC0-1.0",
"documentNamespace": "https://anchore.com/syft/image/",
@@ -31,6 +31,9 @@
}
],
"filesAnalyzed": false,
"hasFiles": [
"SPDXRef-File-package-1-04cd22424378dcd6c77fce08beb52493b5494a60ea5e1f9bdf9b16dc0cacffe9"
],
"licenseDeclared": "MIT",
"sourceInfo": "acquired package info from installed python package manifest file: /some/path/pkg1",
"versionInfo": "1.0.1"
@@ -57,5 +60,20 @@
"sourceInfo": "acquired package info from DPKG DB: /some/path/pkg1",
"versionInfo": "2.0.1"
}
],
"files": [
{
"SPDXID": "SPDXRef-File-package-1-04cd22424378dcd6c77fce08beb52493b5494a60ea5e1f9bdf9b16dc0cacffe9",
"name": "foo",
"licenseConcluded": "",
"fileName": "/some/path/pkg1/depedencies/foo"
}
],
"relationships": [
{
"spdxElementId": "SPDXRef-Package-python-package-1-1.0.1",
"relationshipType": "CONTAINS",
"relatedSpdxElement": "SPDXRef-File-package-1-04cd22424378dcd6c77fce08beb52493b5494a60ea5e1f9bdf9b16dc0cacffe9"
}
]
}
@@ -3,12 +3,12 @@
"name": "user-image-input",
"spdxVersion": "SPDX-2.2",
"creationInfo": {
"created": "2021-06-23T17:48:32.7379Z",
"created": "2021-09-16T20:44:35.203911Z",
"creators": [
"Organization: Anchore, Inc",
"Tool: syft-[not provided]"
],
"licenseListVersion": "3.13"
"licenseListVersion": "3.14"
},
"dataLicense": "CC0-1.0",
"documentNamespace": "https://anchore.com/syft/image/user-image-input",
@@ -1,11 +1,11 @@
[Image]
Layer: 0
Digest: sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59
Digest: sha256:ffb5e9eaa453a002110719d12c294960117ca2903953d1faa40f01dc3f77045c
Size: 22
MediaType: application/vnd.docker.image.rootfs.diff.tar.gzip
Layer: 1
Digest: sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec
Digest: sha256:8463854829fc53d47b9dcdf7ee79fe7eb4ca7933c910f67f8521412f7a2f5c21
Size: 16
MediaType: application/vnd.docker.image.rootfs.diff.tar.gzip
@@ -158,6 +158,11 @@ func presenterDirectoryInput(t testing.TB) (*pkg.Catalog, source.Metadata, *dist
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
Files: []pkg.PythonFileRecord{
{
Path: "/some/path/pkg1/depedencies/foo",
},
},
},
PURL: "a-purl-2",
CPEs: []pkg.CPE{
@@ -1,3 +1,4 @@
//go:build ignore
// +build ignore
package main
+3 -1
View File
@@ -19,10 +19,12 @@ func NewDistro(t Type, ver, like string) (Distro, error) {
if ver == "" {
return Distro{Type: t}, nil
}
verObj, err := hashiVer.NewVersion(ver)
if err != nil {
return Distro{}, fmt.Errorf("could not create distro version: %w", err)
}
return Distro{
Type: t,
Version: verObj,
@@ -39,7 +41,7 @@ func (d Distro) Name() string {
// MajorVersion returns the major version value from the pseudo-semantically versioned distro version value.
func (d Distro) MajorVersion() string {
if d.Version == nil {
return fmt.Sprint("(version unknown)")
return "(version unknown)"
}
return fmt.Sprintf("%d", d.Version.Segments()[0])
}
+12 -10
View File
@@ -97,11 +97,11 @@ identifyLoop:
return distro
}
func assemble(name, version, like string) *Distro {
distroType, ok := IDMapping[name]
func assemble(id, version, like string) *Distro {
distroType, ok := IDMapping[id]
// Both distro and version must be present
if len(name) == 0 && len(version) == 0 {
if len(id) == 0 && len(version) == 0 {
return nil
}
@@ -110,15 +110,17 @@ func assemble(name, version, like string) *Distro {
distroType, ok = IDMapping[like]
}
if ok {
distro, err := NewDistro(distroType, version, like)
if err != nil {
return nil
}
return &distro
// If we still can't match allow name to be used in constructor
if !ok {
distroType = Type(id)
}
return nil
distro, err := NewDistro(distroType, version, like)
if err != nil {
return nil
}
return &distro
}
func parseOsRelease(contents string) *Distro {
+7 -3
View File
@@ -2,16 +2,19 @@ package distro
import (
"fmt"
hashiVer "github.com/hashicorp/go-version"
"io/ioutil"
"os"
"testing"
hashiVer "github.com/hashicorp/go-version"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/source"
"github.com/stretchr/testify/assert"
)
const CustomDistro Type = "scientific"
func TestIdentifyDistro(t *testing.T) {
tests := []struct {
fixture string
@@ -68,8 +71,9 @@ func TestIdentifyDistro(t *testing.T) {
Type: UnknownDistroType,
},
{
fixture: "test-fixtures/os/unmatchable",
Type: UnknownDistroType,
fixture: "test-fixtures/os/custom",
Type: CustomDistro,
Version: "8.0.0",
},
{
fixture: "test-fixtures/os/opensuse-leap",
+9 -2
View File
@@ -118,11 +118,18 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
loc := source.NewLocation(test.location)
if _, ok := actualResults[loc]; !ok {
ok := false
for actual_loc, actual_classification := range actualResults {
if loc.RealPath == actual_loc.RealPath {
ok = true
assert.Equal(t, test.expected, actual_classification)
}
}
if !ok {
t.Fatalf("could not find test location=%q", test.location)
}
assert.Equal(t, test.expected, actualResults[loc])
})
}
}
+5 -3
View File
@@ -3,7 +3,6 @@ package file
import (
"bytes"
"encoding/base64"
"fmt"
"io"
"github.com/anchore/syft/internal"
@@ -32,7 +31,6 @@ func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Lo
if err != nil {
return nil, err
}
for _, location := range locations {
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
@@ -44,6 +42,10 @@ func (i *ContentsCataloger) Catalog(resolver source.FileResolver) (map[source.Lo
}
result, err := i.catalogLocation(resolver, location)
if internal.IsErrPathPermission(err) {
log.Debugf("file contents cataloger skipping - %+v", err)
continue
}
if err != nil {
return nil, err
}
@@ -63,7 +65,7 @@ func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, locati
buf := &bytes.Buffer{}
if _, err = io.Copy(base64.NewEncoder(base64.StdEncoding, buf), contentReader); err != nil {
return "", fmt.Errorf("unable to observe contents of %+v: %w", location.RealPath, err)
return "", internal.ErrPath{Path: location.RealPath, Err: err}
}
return buf.String(), nil
+6 -1
View File
@@ -39,6 +39,11 @@ func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Loc
for _, location := range locations {
stage.Current = location.RealPath
result, err := i.catalogLocation(resolver, location)
if internal.IsErrPathPermission(err) {
log.Debugf("file digests cataloger skipping - %+v", err)
continue
}
if err != nil {
return nil, err
}
@@ -67,7 +72,7 @@ func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, locatio
size, err := io.Copy(io.MultiWriter(writers...), contentReader)
if err != nil {
return nil, fmt.Errorf("unable to observe contents of %+v: %+v", location.RealPath, err)
return nil, internal.ErrPath{Path: location.RealPath, Err: err}
}
if size == 0 {
+6 -1
View File
@@ -50,6 +50,11 @@ func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Loc
for _, location := range locations {
stage.Current = location.RealPath
result, err := i.catalogLocation(resolver, location)
if internal.IsErrPathPermission(err) {
log.Debugf("secrets cataloger skipping - %+v", err)
continue
}
if err != nil {
return nil, err
}
@@ -77,7 +82,7 @@ func (i *SecretsCataloger) catalogLocation(resolver source.FileResolver, locatio
// TODO: in the future we can swap out search strategies here
secrets, err := catalogLocationByLine(resolver, location, i.patterns)
if err != nil {
return nil, err
return nil, internal.ErrPath{Path: location.RealPath, Err: err}
}
if i.revealValues {
+1 -1
View File
@@ -32,7 +32,7 @@ import (
// CatalogPackages takes an inventory of packages from the given image from a particular perspective
// (e.g. squashed source, all-layers source). Returns the discovered set of packages, the identified Linux
// distribution, and the source object used to wrap the data source.
func CatalogPackages(src source.Source, scope source.Scope) (*pkg.Catalog, *distro.Distro, error) {
func CatalogPackages(src *source.Source, scope source.Scope) (*pkg.Catalog, *distro.Distro, error) {
resolver, err := src.FileResolver(scope)
if err != nil {
return nil, nil, fmt.Errorf("unable to determine resolver while cataloging packages: %w", err)
+3 -3
View File
@@ -5,13 +5,13 @@ import (
"github.com/anchore/syft/syft/file"
"github.com/package-url/packageurl-go"
"github.com/anchore/packageurl-go"
"github.com/scylladb/go-set/strset"
)
const ApkDbGlob = "**/lib/apk/db/installed"
var _ fileOwner = (*ApkMetadata)(nil)
var _ FileOwner = (*ApkMetadata)(nil)
// ApkMetadata represents all captured data for a Alpine DB package entry.
// See the following sources for more information:
@@ -63,7 +63,7 @@ func (m ApkMetadata) PackageURL() string {
return pURL.ToString()
}
func (m ApkMetadata) ownedFiles() (result []string) {
func (m ApkMetadata) OwnedFiles() (result []string) {
s := strset.New()
for _, f := range m.Files {
if f.Path != "" {
+3 -3
View File
@@ -4,8 +4,8 @@ import (
"strings"
"testing"
"github.com/anchore/packageurl-go"
"github.com/go-test/deep"
"github.com/package-url/packageurl-go"
"github.com/sergi/go-diff/diffmatchpatch"
)
@@ -73,7 +73,7 @@ func TestApkMetadata_pURL(t *testing.T) {
}
}
func TestApkMetadata_fileOwner(t *testing.T) {
func TestApkMetadata_FileOwner(t *testing.T) {
tests := []struct {
metadata ApkMetadata
expected []string
@@ -107,7 +107,7 @@ func TestApkMetadata_fileOwner(t *testing.T) {
t.Run(strings.Join(test.expected, ","), func(t *testing.T) {
var i interface{}
i = test.metadata
actual := i.(fileOwner).ownedFiles()
actual := i.(FileOwner).OwnedFiles()
for _, d := range deep.Equal(test.expected, actual) {
t.Errorf("diff: %+v", d)
}
@@ -46,6 +46,16 @@ func (s fieldCandidateSet) add(candidates ...fieldCandidate) {
}
}
func (s fieldCandidateSet) removeByValue(values ...string) {
for _, value := range values {
for candidate := range s {
if candidate.value == value {
delete(s, candidate)
}
}
}
}
func (s fieldCandidateSet) clear() {
for k := range s {
delete(s, k)
@@ -261,3 +261,33 @@ func Test_cpeFieldCandidateSet_uniqueValues(t *testing.T) {
assert.ElementsMatch(t, []string{"1", "2", "3"}, set.uniqueValues())
}
func Test_cpeFieldCandidateSet_removeByValue(t *testing.T) {
s := newFieldCandidateSet()
// should be removed
s.add(fieldCandidate{
value: "1",
disallowSubSelections: true,
disallowDelimiterVariations: true,
})
s.add(fieldCandidate{
value: "1",
disallowSubSelections: true,
})
s.add(fieldCandidate{
value: "1",
disallowDelimiterVariations: true,
})
s.add(fieldCandidate{
value: "1",
})
// should not be removed
s.add(fieldCandidate{
value: "2",
})
assert.Len(t, s.values(), 5)
s.removeByValue("1")
assert.Len(t, s.values(), 1)
}
@@ -151,6 +151,9 @@ func candidateProducts(p pkg.Package) []string {
products.addValue(prod)
}
}
// it is never OK to have candidates with these values ["" and "*"] (since CPEs will match any other value)
products.removeByValue("")
products.removeByValue("*")
// try swapping hyphens for underscores, vice versa, and removing separators altogether
addDelimiterVariations(products)
@@ -511,10 +511,12 @@ func TestGeneratePackageCPEs(t *testing.T) {
func TestCandidateProducts(t *testing.T) {
tests := []struct {
name string
p pkg.Package
expected []string
}{
{
name: "springframework",
p: pkg.Package{
Name: "springframework",
Type: pkg.JavaPkg,
@@ -522,6 +524,7 @@ func TestCandidateProducts(t *testing.T) {
expected: []string{"spring_framework", "springsource_spring_framework" /* <-- known good names | default guess --> */, "springframework"},
},
{
name: "java",
p: pkg.Package{
Name: "some-java-package-with-group-id",
Type: pkg.JavaPkg,
@@ -535,6 +538,21 @@ func TestCandidateProducts(t *testing.T) {
expected: []string{"itunes", "some-java-package-with-group-id", "some_java_package_with_group_id"},
},
{
name: "java-with-asterisk",
p: pkg.Package{
Name: "some-java-package-with-group-id",
Type: pkg.JavaPkg,
Language: pkg.Java,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
GroupID: "com.apple.itunes.*",
},
},
},
expected: []string{"itunes", "some-java-package-with-group-id", "some_java_package_with_group_id"},
},
{
name: "jenkins-plugin",
p: pkg.Package{
Name: "some-jenkins-plugin",
Type: pkg.JenkinsPluginPkg,
@@ -548,6 +566,7 @@ func TestCandidateProducts(t *testing.T) {
expected: []string{"some-jenkins-plugin", "some_jenkins_plugin", "jenkins"},
},
{
name: "javascript",
p: pkg.Package{
Name: "handlebars.js",
Type: pkg.NpmPkg,
@@ -555,6 +574,7 @@ func TestCandidateProducts(t *testing.T) {
expected: []string{"handlebars" /* <-- known good names | default guess --> */, "handlebars.js"},
},
{
name: "gem",
p: pkg.Package{
Name: "RedCloth",
Type: pkg.GemPkg,
@@ -562,6 +582,7 @@ func TestCandidateProducts(t *testing.T) {
expected: []string{"redcloth_library" /* <-- known good names | default guess --> */, "RedCloth"},
},
{
name: "python",
p: pkg.Package{
Name: "python-rrdtool",
Type: pkg.PythonPkg,
+1 -1
View File
@@ -4,9 +4,9 @@ import (
"regexp"
"strings"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/package-url/packageurl-go"
)
// generatePackageURL returns a package-URL representation of the given package (see https://github.com/package-url/purl-spec)
+4 -3
View File
@@ -5,14 +5,14 @@ import (
"github.com/anchore/syft/syft/file"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/distro"
"github.com/package-url/packageurl-go"
"github.com/scylladb/go-set/strset"
)
const DpkgDbGlob = "**/var/lib/dpkg/{status,status.d/**}"
var _ fileOwner = (*DpkgMetadata)(nil)
var _ FileOwner = (*DpkgMetadata)(nil)
// DpkgMetadata represents all captured data for a Debian package DB entry; available fields are described
// at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html in the --showformat section.
@@ -41,6 +41,7 @@ func (m DpkgMetadata) PackageURL(d *distro.Distro) string {
}
pURL := packageurl.NewPackageURL(
// TODO: replace with `packageurl.TypeDebian` upon merge of https://github.com/package-url/packageurl-go/pull/21
// TODO: or, since we're now using an Anchore fork of this module, we could do this sooner.
"deb",
d.Type.String(),
m.Package,
@@ -55,7 +56,7 @@ func (m DpkgMetadata) PackageURL(d *distro.Distro) string {
return pURL.ToString()
}
func (m DpkgMetadata) ownedFiles() (result []string) {
func (m DpkgMetadata) OwnedFiles() (result []string) {
s := strset.New()
for _, f := range m.Files {
if f.Path != "" {
+2 -2
View File
@@ -54,7 +54,7 @@ func TestDpkgMetadata_pURL(t *testing.T) {
}
}
func TestDpkgMetadata_fileOwner(t *testing.T) {
func TestDpkgMetadata_FileOwner(t *testing.T) {
tests := []struct {
metadata DpkgMetadata
expected []string
@@ -88,7 +88,7 @@ func TestDpkgMetadata_fileOwner(t *testing.T) {
t.Run(strings.Join(test.expected, ","), func(t *testing.T) {
var i interface{}
i = test.metadata
actual := i.(fileOwner).ownedFiles()
actual := i.(FileOwner).OwnedFiles()
for _, d := range deep.Equal(test.expected, actual) {
t.Errorf("diff: %+v", d)
}
+6 -2
View File
@@ -1,5 +1,9 @@
package pkg
type fileOwner interface {
ownedFiles() []string
// FileOwner is the interface that wraps OwnedFiles method.
//
// OwnedFiles returns a list of files that a piece of
// package Metadata indicates are owned by the package.
type FileOwner interface {
OwnedFiles() []string
}
+1 -1
View File
@@ -3,8 +3,8 @@ package pkg
import (
"strings"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/internal"
"github.com/package-url/packageurl-go"
)
var JenkinsPluginPomPropertiesGroupIDs = []string{
+2 -2
View File
@@ -55,11 +55,11 @@ func findOwnershipByFilesRelationships(catalog *Catalog) map[ID]map[ID]*strset.S
}
// check to see if this is a file owner
pkgFileOwner, ok := candidateOwnerPkg.Metadata.(fileOwner)
pkgFileOwner, ok := candidateOwnerPkg.Metadata.(FileOwner)
if !ok {
continue
}
for _, ownedFilePath := range pkgFileOwner.ownedFiles() {
for _, ownedFilePath := range pkgFileOwner.OwnedFiles() {
if matchesAny(ownedFilePath, globsForbiddenFromBeingOwned) {
// we skip over known exceptions to file ownership, such as the RPM package owning
// the RPM DB path, otherwise the RPM package would "own" all RPMs, which is not intended
+2 -2
View File
@@ -6,7 +6,7 @@ import (
"github.com/scylladb/go-set/strset"
)
var _ fileOwner = (*PythonPackageMetadata)(nil)
var _ FileOwner = (*PythonPackageMetadata)(nil)
// PythonFileDigest represents the file metadata for a single file attributed to a python package.
type PythonFileDigest struct {
@@ -34,7 +34,7 @@ type PythonPackageMetadata struct {
TopLevelPackages []string `json:"topLevelPackages,omitempty"`
}
func (m PythonPackageMetadata) ownedFiles() (result []string) {
func (m PythonPackageMetadata) OwnedFiles() (result []string) {
s := strset.New()
for _, f := range m.Files {
if f.Path != "" {
+2 -2
View File
@@ -7,7 +7,7 @@ import (
"github.com/go-test/deep"
)
func TestPythonMetadata_fileOwner(t *testing.T) {
func TestPythonMetadata_FileOwner(t *testing.T) {
tests := []struct {
metadata PythonPackageMetadata
expected []string
@@ -41,7 +41,7 @@ func TestPythonMetadata_fileOwner(t *testing.T) {
t.Run(strings.Join(test.expected, ","), func(t *testing.T) {
var i interface{}
i = test.metadata
actual := i.(fileOwner).ownedFiles()
actual := i.(FileOwner).OwnedFiles()
for _, d := range deep.Equal(test.expected, actual) {
t.Errorf("diff: %+v", d)
}
+3 -3
View File
@@ -9,13 +9,13 @@ import (
"github.com/scylladb/go-set/strset"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/distro"
"github.com/package-url/packageurl-go"
)
const RpmDbGlob = "**/var/lib/rpm/Packages"
var _ fileOwner = (*RpmdbMetadata)(nil)
var _ FileOwner = (*RpmdbMetadata)(nil)
// RpmdbMetadata represents all captured data for a RPM DB package entry.
type RpmdbMetadata struct {
@@ -79,7 +79,7 @@ func (m RpmdbMetadata) PackageURL(d *distro.Distro) string {
return pURL.ToString()
}
func (m RpmdbMetadata) ownedFiles() (result []string) {
func (m RpmdbMetadata) OwnedFiles() (result []string) {
s := strset.New()
for _, f := range m.Files {
if f.Path != "" {
+2 -2
View File
@@ -56,7 +56,7 @@ func TestRpmMetadata_pURL(t *testing.T) {
}
}
func TestRpmMetadata_fileOwner(t *testing.T) {
func TestRpmMetadata_FileOwner(t *testing.T) {
tests := []struct {
metadata RpmdbMetadata
expected []string
@@ -90,7 +90,7 @@ func TestRpmMetadata_fileOwner(t *testing.T) {
t.Run(strings.Join(test.expected, ","), func(t *testing.T) {
var i interface{}
i = test.metadata
actual := i.(fileOwner).ownedFiles()
actual := i.(FileOwner).OwnedFiles()
for _, d := range deep.Equal(test.expected, actual) {
t.Errorf("diff: %+v", d)
}
+1 -1
View File
@@ -1,6 +1,6 @@
package pkg
import "github.com/package-url/packageurl-go"
import "github.com/anchore/packageurl-go"
// Type represents a Package Type for or within a language ecosystem (there may be multiple package types within a language ecosystem)
type Type string
+28 -7
View File
@@ -8,6 +8,7 @@ import (
"path"
"path/filepath"
"strings"
"syscall"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree"
@@ -218,7 +219,12 @@ func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
continue
}
references = append(references, NewLocation(r.responsePath(userStrPath)))
exists, ref, err := r.fileTree.File(file.Path(userStrPath))
if err == nil && exists {
references = append(references, NewLocationFromDirectory(r.responsePath(userStrPath), *ref))
} else {
log.Warnf("path (%s) not found in file tree: Exists: %t Err:%+v", userStrPath, exists, err)
}
}
return references, nil
@@ -234,7 +240,7 @@ func (r directoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
return nil, err
}
for _, globResult := range globResults {
result = append(result, NewLocation(r.responsePath(string(globResult.MatchPath))))
result = append(result, NewLocationFromDirectory(r.responsePath(string(globResult.MatchPath)), globResult.Reference))
}
}
@@ -267,7 +273,7 @@ func (r *directoryResolver) AllLocations() <-chan Location {
go func() {
defer close(results)
for _, ref := range r.fileTree.AllFiles() {
results <- NewLocation(r.responsePath(string(ref.RealPath)))
results <- NewLocationFromDirectory(r.responsePath(string(ref.RealPath)), ref)
}
}()
return results
@@ -276,15 +282,22 @@ func (r *directoryResolver) AllLocations() <-chan Location {
func (r *directoryResolver) FileMetadataByLocation(location Location) (FileMetadata, error) {
info, exists := r.infos[location.ref.ID()]
if !exists {
return FileMetadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrExist)
return FileMetadata{}, fmt.Errorf("location: %+v : %w", location, os.ErrNotExist)
}
uid := -1
gid := -1
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
uid = int(stat.Uid)
gid = int(stat.Gid)
}
return FileMetadata{
Mode: info.Mode(),
Type: newFileTypeFromMode(info.Mode()),
// unsupported across platforms
UserID: -1,
GroupID: -1,
UserID: uid,
GroupID: gid,
}, nil
}
@@ -297,6 +310,8 @@ func indexAllRoots(root string, indexer func(string, *progress.Stage) ([]string,
// in which case we need to additionally index where the link resolves to. it's for this reason why the filetree
// must be relative to the root of the filesystem (and not just relative to the given path).
pathsToIndex := []string{root}
fullPathsMap := map[string]struct{}{}
stager, prog := indexingProgress(root)
defer prog.SetCompleted()
loop:
@@ -315,7 +330,13 @@ loop:
if err != nil {
return fmt.Errorf("unable to index filesystem path=%q: %w", currentPath, err)
}
pathsToIndex = append(pathsToIndex, additionalRoots...)
for _, newRoot := range additionalRoots {
if _, ok := fullPathsMap[newRoot]; !ok {
fullPathsMap[newRoot] = struct{}{}
pathsToIndex = append(pathsToIndex, newRoot)
}
}
}
return nil
+11 -3
View File
@@ -178,9 +178,17 @@ func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) {
assert.Len(t, refs, 6)
// ensure that symlink indexing outside of root worked
assert.Contains(t, refs, Location{
RealPath: "test-fixtures/system_paths/outside_root/link_target/place",
})
ok := false
test_location := "test-fixtures/system_paths/outside_root/link_target/place"
for _, actual_loc := range refs {
if test_location == actual_loc.RealPath {
ok = true
}
}
if !ok {
t.Fatalf("could not find test location=%q", test_location)
}
}
func TestDirectoryResolverUsesPathFilterFunction(t *testing.T) {
+8
View File
@@ -45,6 +45,14 @@ func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Ima
}
}
// NewLocationFromDirectory creates a new Location representing the given path (extracted from the ref) relative to the given directory.
func NewLocationFromDirectory(responsePath string, ref file.Reference) Location {
return Location{
RealPath: responsePath,
ref: ref,
}
}
func NewLocationFromReference(ref file.Reference) Location {
return Location{
VirtualPath: string(ref.RealPath),
+27 -14
View File
@@ -7,6 +7,7 @@ package source
import (
"fmt"
"sync"
"github.com/anchore/stereoscope"
"github.com/anchore/stereoscope/pkg/image"
@@ -16,58 +17,61 @@ import (
// Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used
// in cataloging (based on the data source and configuration)
type Source struct {
Image *image.Image // the image object to be cataloged (image only)
Metadata Metadata
Image *image.Image // the image object to be cataloged (image only)
DirectoryResolver *directoryResolver
Metadata Metadata
Mutex *sync.Mutex
}
type sourceDetector func(string) (image.Source, string, error)
// New produces a Source based on userInput like dir: or image:tag
func New(userInput string, registryOptions *image.RegistryOptions) (Source, func(), error) {
func New(userInput string, registryOptions *image.RegistryOptions) (*Source, func(), error) {
fs := afero.NewOsFs()
parsedScheme, imageSource, location, err := detectScheme(fs, image.DetectSource, userInput)
if err != nil {
return Source{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err)
return &Source{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err)
}
switch parsedScheme {
case DirectoryScheme:
fileMeta, err := fs.Stat(location)
if err != nil {
return Source{}, func() {}, fmt.Errorf("unable to stat dir=%q: %w", location, err)
return &Source{}, func() {}, fmt.Errorf("unable to stat dir=%q: %w", location, err)
}
if !fileMeta.IsDir() {
return Source{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err)
return &Source{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err)
}
s, err := NewFromDirectory(location)
if err != nil {
return Source{}, func() {}, fmt.Errorf("could not populate source from path=%q: %w", location, err)
return &Source{}, func() {}, fmt.Errorf("could not populate source from path=%q: %w", location, err)
}
return s, func() {}, nil
return &s, func() {}, nil
case ImageScheme:
img, err := stereoscope.GetImageFromSource(location, imageSource, registryOptions)
cleanup := stereoscope.Cleanup
if err != nil || img == nil {
return Source{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err)
return &Source{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err)
}
s, err := NewFromImage(img, location)
if err != nil {
return Source{}, cleanup, fmt.Errorf("could not populate source with image: %w", err)
return &Source{}, cleanup, fmt.Errorf("could not populate source with image: %w", err)
}
return s, cleanup, nil
return &s, cleanup, nil
}
return Source{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
return &Source{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
}
// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively.
func NewFromDirectory(path string) (Source, error) {
return Source{
Mutex: &sync.Mutex{},
Metadata: Metadata{
Scheme: DirectoryScheme,
Path: path,
@@ -91,10 +95,19 @@ func NewFromImage(img *image.Image, userImageStr string) (Source, error) {
}, nil
}
func (s Source) FileResolver(scope Scope) (FileResolver, error) {
func (s *Source) FileResolver(scope Scope) (FileResolver, error) {
switch s.Metadata.Scheme {
case DirectoryScheme:
return newDirectoryResolver(s.Metadata.Path)
s.Mutex.Lock()
defer s.Mutex.Unlock()
if s.DirectoryResolver == nil {
directoryResolver, err := newDirectoryResolver(s.Metadata.Path)
if err != nil {
return nil, err
}
s.DirectoryResolver = directoryResolver
}
return s.DirectoryResolver, nil
case ImageScheme:
switch scope {
case SquashedScope:
+62
View File
@@ -89,6 +89,68 @@ func TestNewFromDirectory(t *testing.T) {
}
}
func TestNewFromDirectoryShared(t *testing.T) {
testCases := []struct {
desc string
input string
expString string
notExist string
inputPaths []string
expRefs int
}{
{
desc: "path detected",
input: "test-fixtures",
notExist: "foobar/",
inputPaths: []string{"test-fixtures/path-detected/.vimrc"},
expRefs: 1,
},
{
desc: "directory ignored",
input: "test-fixtures",
notExist: "foobar/",
inputPaths: []string{"test-fixtures/path-detected"},
expRefs: 0,
},
{
desc: "no files-by-path detected",
input: "test-fixtures",
notExist: "foobar/",
inputPaths: []string{"test-fixtures/no-path-detected"},
expRefs: 0,
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
src, err := NewFromDirectory(test.input)
if err != nil {
t.Errorf("could not create NewDirScope: %+v", err)
}
if src.Metadata.Path != test.input {
t.Errorf("mismatched stringer: '%s' != '%s'", src.Metadata.Path, test.input)
}
_, err = src.FileResolver(SquashedScope)
assert.NoError(t, err)
src.Metadata.Path = test.notExist
resolver2, err := src.FileResolver(SquashedScope)
assert.NoError(t, err)
refs, err := resolver2.FilesByPath(test.inputPaths...)
if err != nil {
t.Errorf("FilesByPath call produced an error: %+v", err)
}
if len(refs) != test.expRefs {
t.Errorf("unexpected number of refs returned: %d != %d", len(refs), test.expRefs)
}
})
}
}
func TestFilesByPathDoesNotExist(t *testing.T) {
testCases := []struct {
desc string
+23
View File
@@ -71,6 +71,29 @@ func TestPowerUserCmdFlags(t *testing.T) {
assertSuccessfulReturnCode,
},
},
{
name: "default-dir-results-w-pkg-coverage",
args: []string{"power-user", "dir:test-fixtures/image-pkg-coverage"},
assertions: []traitAssertion{
assertNotInOutput(" command is deprecated"), // only the root command should be deprecated
assertInOutput(`"type": "RegularFile"`), // proof of file-metadata data
assertInOutput(`"algorithm": "sha256"`), // proof of file-metadata default digest algorithm of sha256
assertInOutput(`"metadataType": "ApkMetadata"`), // proof of package artifacts data
assertSuccessfulReturnCode,
},
},
{
name: "defaut-secrets-dir-results-w-reveal-values",
env: map[string]string{
"SYFT_SECRETS_REVEAL_VALUES": "true",
},
args: []string{"power-user", "dir:test-fixtures/image-secrets"},
assertions: []traitAssertion{
assertInOutput(`"classification": "generic-api-key"`), // proof of the secrets cataloger finding something
assertInOutput(`"12345A7a901b345678901234567890123456789012345678901234567890"`), // proof of the secrets cataloger finding the api key
assertSuccessfulReturnCode,
},
},
}
for _, test := range tests {
+2 -2
View File
@@ -10,7 +10,7 @@ import (
"github.com/anchore/syft/syft/source"
)
func catalogFixtureImage(t *testing.T, fixtureImageName string) (*pkg.Catalog, *distro.Distro, source.Source) {
func catalogFixtureImage(t *testing.T, fixtureImageName string) (*pkg.Catalog, *distro.Distro, *source.Source) {
imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName)
tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
@@ -28,7 +28,7 @@ func catalogFixtureImage(t *testing.T, fixtureImageName string) (*pkg.Catalog, *
return pkgCatalog, actualDistro, theSource
}
func catalogDirectory(t *testing.T, dir string) (*pkg.Catalog, *distro.Distro, source.Source) {
func catalogDirectory(t *testing.T, dir string) (*pkg.Catalog, *distro.Distro, *source.Source) {
theSource, cleanupSource, err := source.New("dir:"+dir, nil)
t.Cleanup(cleanupSource)
if err != nil {