oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-04-05 22:30:35 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #266 from anchore/document-import

Browse Source 
Add JSON document import
This commit is contained in:
Alex Goodman 2020-11-17 13:21:57 -05:00 committed by GitHub
commit 71939557e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
107 changed files with 1712 additions and 1253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/cmd.go
View File

@ -4,14 +4,15 @@ import (
"fmt"
"os"
"github.com/gookit/color"
"github.com/spf13/cobra"
"github.com/anchore/syft/syft/presenter"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
"github.com/anchore/stereoscope"
"github.com/anchore/syft/internal/config"
"github.com/anchore/syft/internal/format"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/logger"
"github.com/anchore/syft/syft"
@ -49,8 +50,8 @@ func setGlobalCliOptions() {
// scan options
flag := "scope"
rootCmd.Flags().StringP(
"scope", "s", scope.SquashedScope.String(),
fmt.Sprintf("selection of layers to catalog, options=%v", scope.Options))
"scope", "s", source.SquashedScope.String(),
fmt.Sprintf("selection of layers to catalog, options=%v", source.AllScopes))
if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil {
fmt.Printf("unable to bind flag '%s': %+v", flag, err)
os.Exit(1)
@ -111,7 +112,7 @@ func logAppConfig() {
if err != nil {
log.Debugf("Could not display application config: %+v", err)
} else {
log.Debugf("Application config:\n%+v", format.Magenta.Format(string(appCfgStr)))
log.Debugf("Application config:\n%+v", color.Magenta.Sprint(string(appCfgStr)))
}
}

4
cmd/root.go
View File

@ -92,7 +92,7 @@ func startWorker(userInput string) <-chan error {
}
}
catalog, scope, distro, err := syft.Catalog(userInput, appConfig.ScopeOpt)
src, catalog, distro, err := syft.Catalog(userInput, appConfig.ScopeOpt)
if err != nil {
errs <- fmt.Errorf("failed to catalog input: %+v", err)
return
@ -100,7 +100,7 @@ func startWorker(userInput string) <-chan error {
bus.Publish(partybus.Event{
Type: event.CatalogerFinished,
Value: presenter.GetPresenter(appConfig.PresenterOpt, *scope, catalog, distro),
Value: presenter.GetPresenter(appConfig.PresenterOpt, src.Metadata, catalog, distro),
})
}()
return errs

8
go.sum
View File

@ -124,16 +124,12 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e h1:kty6r0R2JeaNPeWKSYDC+HW3hkqwFh4PP5TQ8pUPYFw=
github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e/go.mod h1:iYuIG0Nai8dR0ri3LhZQKUyO1loxUWAGvoWhXDmjy1A=
github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ=
github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12/go.mod h1:juoyWXIj7sJ1IDl4E/KIfyLtovbs5XQVSIdaQifFQT8=
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8=
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe h1:m4NSyTo2fVUoUHAV/ZVqE/PFMr/y8oz9HRrhWLk9It0=
github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409 h1:xKSpDRjmYrEFrdMeDh4AuSUAFc99pdro6YFBKxy2um0=
github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
@ -164,8 +160,6 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bmatcuk/doublestar v1.3.1 h1:rT8rxDPsavp9G+4ZULzqhhUSaI/OPsTZNG88Z3i0xvY=
github.com/bmatcuk/doublestar v1.3.1/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
github.com/bmatcuk/doublestar v1.3.3 h1:pVP1d49CcQQaNOl+PI6sPybIrIOD/6sux31PFdmhTH0=
github.com/bmatcuk/doublestar v1.3.3/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U=
@ -301,8 +295,6 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8=
github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=

10
internal/config/config.go
View File

@ -8,7 +8,7 @@ import (
"github.com/adrg/xdg"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/presenter"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
"github.com/mitchellh/go-homedir"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
@ -23,7 +23,7 @@ type Application struct {
ConfigPath string
PresenterOpt presenter.Option
Output string `mapstructure:"output"`
ScopeOpt scope.Option
ScopeOpt source.Scope
Scope string `mapstructure:"scope"`
Quiet bool `mapstructure:"quiet"`
Log Logging `mapstructure:"log"`
@ -79,9 +79,9 @@ func (cfg *Application) Build() error {
}
cfg.PresenterOpt = presenterOption
// set the scope
scopeOption := scope.ParseOption(cfg.Scope)
if scopeOption == scope.UnknownScope {
// set the source
scopeOption := source.ParseScope(cfg.Scope)
if scopeOption == source.UnknownScope {
return fmt.Errorf("bad --scope value '%s'", cfg.Scope)
}
cfg.ScopeOpt = scopeOption

2
internal/constants.go
View File

@ -1,4 +1,4 @@
package internal
// note: do not change this
// ApplicationName is the non-capitalized name of the application (do not change this)
const ApplicationName = "syft"

4
internal/docs.go Normal file
View File

@ -0,0 +1,4 @@
/*
Package internal contains miscellaneous functions and objects useful within syft but should not be used externally.
*/
package internal

1
internal/file/glob_match.go
View File

@ -1,5 +1,6 @@
package file
// GlobMatch evaluates the given glob pattern against the given "name" string, indicating if there is a match or not.
// Source: https://research.swtch.com/glob.go
func GlobMatch(pattern, name string) bool {
px := 0

2
internal/file/opener.go
View File

@ -5,10 +5,12 @@ import (
"os"
)
// Opener is an object that stores a path to later be opened as a file.
type Opener struct {
path string
}
// Open the stored path as a io.ReadCloser.
func (o Opener) Open() (io.ReadCloser, error) {
return os.Open(o.path)
}

6
internal/file/zip_file_manifest.go
View File

@ -12,16 +12,20 @@ import (
"github.com/anchore/syft/internal/log"
)
// ZipFileManifest is a collection of paths and their file metadata.
type ZipFileManifest map[string]os.FileInfo
// newZipManifest creates an empty ZipFileManifest.
func newZipManifest() ZipFileManifest {
return make(ZipFileManifest)
}
// Add a new path and it's file metadata to the collection.
func (z ZipFileManifest) Add(entry string, info os.FileInfo) {
z[entry] = info
}
// GlobMatch returns the path keys that match the given value(s).
func (z ZipFileManifest) GlobMatch(patterns ...string) []string {
uniqueMatches := internal.NewStringSet()
@ -43,6 +47,7 @@ func (z ZipFileManifest) GlobMatch(patterns ...string) []string {
return results
}
// NewZipFileManifest creates and returns a new ZipFileManifest populated with path and metadata from the given zip archive path.
func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
zipReader, err := zip.OpenReader(archivePath)
manifest := newZipManifest()
@ -62,6 +67,7 @@ func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
return manifest, nil
}
// normalizeZipEntryName takes the given path entry and ensures it is prefixed with "/".
func normalizeZipEntryName(entry string) string {
if !strings.HasPrefix(entry, "/") {
return "/" + entry

5
internal/file/zip_file_traversal.go
View File

@ -15,6 +15,7 @@ import (
)
const (
// represents the order of bytes
_ = iota
KB = 1 << (10 * iota)
MB
@ -33,6 +34,7 @@ func newZipTraverseRequest(paths ...string) zipTraversalRequest {
return results
}
// TraverseFilesInZip enumerates all paths stored within a zip archive using the visitor pattern.
func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths ...string) error {
request := newZipTraverseRequest(paths...)
@ -63,6 +65,7 @@ func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths
return nil
}
// ExtractFromZipToUniqueTempFile extracts select paths for the given archive to a temporary directory, returning file openers for each file extracted.
func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]Opener, error) {
results := make(map[string]Opener)
@ -121,6 +124,7 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m
return results, TraverseFilesInZip(archivePath, visitor, paths...)
}
// ContentsFromZip extracts select paths for the given archive and returns a set of string contents for each path.
func ContentsFromZip(archivePath string, paths ...string) (map[string]string, error) {
results := make(map[string]string)
@ -162,6 +166,7 @@ func ContentsFromZip(archivePath string, paths ...string) (map[string]string, er
return results, TraverseFilesInZip(archivePath, visitor, paths...)
}
// UnzipToDir extracts a zip archive to a target directory.
func UnzipToDir(archivePath, targetDir string) error {
visitor := func(file *zip.File) error {
// the zip-slip attack protection is still being erroneously detected

21
internal/format/color.go
View File

@ -1,21 +0,0 @@
package format
import "fmt"
const (
DefaultColor Color = iota + 30
Red
Green
Yellow
Blue
Magenta
Cyan
White
)
type Color uint8
// TODO: not cross platform (windows...)
func (c Color) Format(s string) string {
return fmt.Sprintf("\x1b[%dm%s\x1b[0m", c, s)
}

12
internal/log/log.go
View File

@ -1,37 +1,49 @@
/*
Package log contains the singleton object and helper functions for facilitating logging within the syft library.
*/
package log
import "github.com/anchore/syft/syft/logger"
// Log is the singleton used to facilitate logging internally within syft
var Log logger.Logger = &nopLogger{}
// Errorf takes a formatted template string and template arguments for the error logging level.
func Errorf(format string, args ...interface{}) {
Log.Errorf(format, args...)
}
// Error logs the given arguments at the error logging level.
func Error(args ...interface{}) {
Log.Error(args...)
}
// Warnf takes a formatted template string and template arguments for the warning logging level.
func Warnf(format string, args ...interface{}) {
Log.Warnf(format, args...)
}
// Warn logs the given arguments at the warning logging level.
func Warn(args ...interface{}) {
Log.Warn(args...)
}
// Infof takes a formatted template string and template arguments for the info logging level.
func Infof(format string, args ...interface{}) {
Log.Infof(format, args...)
}
// Info logs the given arguments at the info logging level.
func Info(args ...interface{}) {
Log.Info(args...)
}
// Debugf takes a formatted template string and template arguments for the debug logging level.
func Debugf(format string, args ...interface{}) {
Log.Debugf(format, args...)
}
// Debug logs the given arguments at the debug logging level.
func Debug(args ...interface{}) {
Log.Debug(args...)
}

4
internal/logger/doc.go Normal file
View File

@ -0,0 +1,4 @@
/*
Package logger contains implementations for the syft.logger.Logger interface.
*/
package logger

20
internal/logger/logrus.go
View File

@ -10,6 +10,7 @@ import (
prefixed "github.com/x-cray/logrus-prefixed-formatter"
)
// LogrusConfig contains all configurable values for the Logrus logger
type LogrusConfig struct {
EnableConsole bool
EnableFile bool
@ -18,16 +19,19 @@ type LogrusConfig struct {
FileLocation string
}
// LogrusLogger contains all runtime values for using Logrus with the configured output target and input configuration values.
type LogrusLogger struct {
Config LogrusConfig
Logger *logrus.Logger
Output io.Writer
}
// LogrusNestedLogger is a wrapper for Logrus to enable nested logging configuration (loggers that always attach key-value pairs to all log entries)
type LogrusNestedLogger struct {
Logger *logrus.Entry
}
// NewLogrusLogger creates a new LogrusLogger with the given configuration
func NewLogrusLogger(cfg LogrusConfig) *LogrusLogger {
appLogger := logrus.New()
@ -76,66 +80,82 @@ func NewLogrusLogger(cfg LogrusConfig) *LogrusLogger {
}
}
// Debugf takes a formatted template string and template arguments for the debug logging level.
func (l *LogrusLogger) Debugf(format string, args ...interface{}) {
l.Logger.Debugf(format, args...)
}
// Infof takes a formatted template string and template arguments for the info logging level.
func (l *LogrusLogger) Infof(format string, args ...interface{}) {
l.Logger.Infof(format, args...)
}
// Warnf takes a formatted template string and template arguments for the warning logging level.
func (l *LogrusLogger) Warnf(format string, args ...interface{}) {
l.Logger.Warnf(format, args...)
}
// Errorf takes a formatted template string and template arguments for the error logging level.
func (l *LogrusLogger) Errorf(format string, args ...interface{}) {
l.Logger.Errorf(format, args...)
}
// Debug logs the given arguments at the debug logging level.
func (l *LogrusLogger) Debug(args ...interface{}) {
l.Logger.Debug(args...)
}
// Info logs the given arguments at the info logging level.
func (l *LogrusLogger) Info(args ...interface{}) {
l.Logger.Info(args...)
}
// Warn logs the given arguments at the warning logging level.
func (l *LogrusLogger) Warn(args ...interface{}) {
l.Logger.Warn(args...)
}
// Error logs the given arguments at the error logging level.
func (l *LogrusLogger) Error(args ...interface{}) {
l.Logger.Error(args...)
}
// Debugf takes a formatted template string and template arguments for the debug logging level.
func (l *LogrusNestedLogger) Debugf(format string, args ...interface{}) {
l.Logger.Debugf(format, args...)
}
// Infof takes a formatted template string and template arguments for the info logging level.
func (l *LogrusNestedLogger) Infof(format string, args ...interface{}) {
l.Logger.Infof(format, args...)
}
// Warnf takes a formatted template string and template arguments for the warning logging level.
func (l *LogrusNestedLogger) Warnf(format string, args ...interface{}) {
l.Logger.Warnf(format, args...)
}
// Errorf takes a formatted template string and template arguments for the error logging level.
func (l *LogrusNestedLogger) Errorf(format string, args ...interface{}) {
l.Logger.Errorf(format, args...)
}
// Debug logs the given arguments at the debug logging level.
func (l *LogrusNestedLogger) Debug(args ...interface{}) {
l.Logger.Debug(args...)
}
// Info logs the given arguments at the info logging level.
func (l *LogrusNestedLogger) Info(args ...interface{}) {
l.Logger.Info(args...)
}
// Warn logs the given arguments at the warning logging level.
func (l *LogrusNestedLogger) Warn(args ...interface{}) {
l.Logger.Warn(args...)
}
// Error logs the given arguments at the error logging level.
func (l *LogrusNestedLogger) Error(args ...interface{}) {
l.Logger.Error(args...)
}

10
internal/stringset.go
View File

@ -1,11 +1,16 @@
package internal
import "sort"
// StringSet represents a set of string types.
type StringSet map[string]struct{}
// NewStringSet creates a new empty StringSet.
func NewStringSet() StringSet {
return make(StringSet)
}
// NewStringSetFromSlice creates a StringSet populated with values from the given slice.
func NewStringSetFromSlice(start []string) StringSet {
ret := make(StringSet)
for _, s := range start {
@ -14,19 +19,23 @@ func NewStringSetFromSlice(start []string) StringSet {
return ret
}
// Add a string to the set.
func (s StringSet) Add(i string) {
s[i] = struct{}{}
}
// Remove a string from the set.
func (s StringSet) Remove(i string) {
delete(s, i)
}
// Contains indicates if the given string is contained within the set.
func (s StringSet) Contains(i string) bool {
_, ok := s[i]
return ok
}
// ToSlice returns a sorted slice of strings that are contained within the set.
func (s StringSet) ToSlice() []string {
ret := make([]string, len(s))
idx := 0
@ -34,5 +43,6 @@ func (s StringSet) ToSlice() []string {
ret[idx] = v
idx++
}
sort.Strings(ret)
return ret
}

3
internal/version/build.go
View File

@ -1,3 +1,6 @@
/*
Package version contains all build time metadata (version, build time, git commit, etc).
*/
package version
import (

1
internal/version/update.go
View File

@ -18,6 +18,7 @@ var latestAppVersionURL = struct {
path: fmt.Sprintf("/%s/releases/latest/VERSION", internal.ApplicationName),
}
// IsUpdateAvailable indicates if there is a newer application version available, and if so, what the new version is.
func IsUpdateAvailable() (bool, string, error) {
currentVersionStr := FromBuild().Version
currentVersion, err := hashiVersion.NewVersion(currentVersionStr)

67
schema/json/schema.json
View File

@ -5,10 +5,10 @@
"items": {
"properties": {
"foundBy": {
"items": {
"type": "string"
},
"type": "array"
"language": {
"type": "string"
},
"licenses": {
"anyOf": [
@ -25,26 +25,18 @@
},
"locations": {
"items": {
"anyOf": [
{
"type": "string"
},
{
"properties": {
"layerIndex": {
"type": "integer"
"layerID": {
"type": "string"
},
"path": {
"type": "string"
}
},
"required": [
"layerIndex",
"path"
],
"type": "object"
}
]
},
"type": "array"
},
@ -315,6 +307,9 @@
},
"type": "object"
},
"metadataType": {
"type": "string"
},
"name": {
"type": "string"
},
@ -327,8 +322,10 @@
},
"required": [
"foundBy",
"language",
"licenses",
"locations",
"metadataType",
"name",
"type",
"version"
@ -337,6 +334,40 @@
},
"type": "array"
},
"descriptor": {
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
}
},
"required": [
"name",
"version"
],
"type": "object"
},
"distro": {
"properties": {
"idLike": {
"type": "string"
},
"name": {
"type": "string"
},
"version": {
"type": "string"
}
},
"required": [
"idLike",
"name",
"version"
],
"type": "object"
},
"source": {
"properties": {
"target": {
@ -374,6 +405,9 @@
"mediaType": {
"type": "string"
},
"scope": {
"type": "string"
},
"size": {
"type": "integer"
},
@ -382,14 +416,19 @@
"type": "string"
},
"type": "array"
},
"userInput": {
"type": "string"
}
},
"required": [
"digest",
"layers",
"mediaType",
"scope",
"size",
"tags"
"tags",
"userInput"
],
"type": "object"
}
@ -408,6 +447,8 @@
},
"required": [
"artifacts",
"descriptor",
"distro",
"source"
],
"type": "object"

6
syft/cataloger/catalog.go
View File

@ -5,7 +5,7 @@ import (
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
"github.com/hashicorp/go-multierror"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
@ -32,11 +32,11 @@ func newMonitor() (*progress.Manual, *progress.Manual) {
return &filesProcessed, &packagesDiscovered
}
// Catalog a given scope (container image or filesystem) with the given catalogers, returning all discovered packages.
// Catalog a given source (container image or filesystem) with the given catalogers, returning all discovered packages.
// In order to efficiently retrieve contents from a underlying container image the content fetch requests are
// done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single
// request.
func Catalog(resolver scope.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) {
func Catalog(resolver source.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) {
catalog := pkg.NewCatalog()
filesProcessed, packagesDiscovered := newMonitor()

4
syft/cataloger/cataloger.go
View File

@ -15,7 +15,7 @@ import (
"github.com/anchore/syft/syft/cataloger/rpmdb"
"github.com/anchore/syft/syft/cataloger/ruby"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
// Cataloger describes behavior for an object to participate in parsing container image or file system
@ -25,7 +25,7 @@ type Cataloger interface {
// Name returns a string that uniquely describes a cataloger
Name() string
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
Catalog(resolver scope.Resolver) ([]pkg.Package, error)
Catalog(resolver source.Resolver) ([]pkg.Package, error)
}
// ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.

39
syft/cataloger/common/generic_cataloger.go
View File

@ -6,10 +6,9 @@ package common
import (
"strings"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
// GenericCataloger implements the Catalog interface and is responsible for dispatching the proper parser function for
@ -17,8 +16,8 @@ import (
type GenericCataloger struct {
globParsers map[string]ParserFn
pathParsers map[string]ParserFn
selectedFiles []file.Reference
parsers map[file.Reference]ParserFn
selectedFiles []source.Location
parsers map[source.Location]ParserFn
upstreamCataloger string
}
@ -27,8 +26,8 @@ func NewGenericCataloger(pathParsers map[string]ParserFn, globParsers map[string
return &GenericCataloger{
globParsers: globParsers,
pathParsers: pathParsers,
selectedFiles: make([]file.Reference, 0),
parsers: make(map[file.Reference]ParserFn),
selectedFiles: make([]source.Location, 0),
parsers: make(map[source.Location]ParserFn),
upstreamCataloger: upstreamCataloger,
}
}
@ -39,7 +38,7 @@ func (c *GenericCataloger) Name() string {
}
// register pairs a set of file references with a parser function for future cataloging (when the file contents are resolved)
func (c *GenericCataloger) register(files []file.Reference, parser ParserFn) {
func (c *GenericCataloger) register(files []source.Location, parser ParserFn) {
c.selectedFiles = append(c.selectedFiles, files...)
for _, f := range files {
c.parsers[f] = parser
@ -48,14 +47,14 @@ func (c *GenericCataloger) register(files []file.Reference, parser ParserFn) {
// clear deletes all registered file-reference-to-parser-function pairings from former SelectFiles() and register() calls
func (c *GenericCataloger) clear() {
c.selectedFiles = make([]file.Reference, 0)
c.parsers = make(map[file.Reference]ParserFn)
c.selectedFiles = make([]source.Location, 0)
c.parsers = make(map[source.Location]ParserFn)
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
func (c *GenericCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
func (c *GenericCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
fileSelection := c.selectFiles(resolver)
contents, err := resolver.MultipleFileContentsByRef(fileSelection...)
contents, err := resolver.MultipleFileContentsByLocation(fileSelection)
if err != nil {
return nil, err
}
@ -63,10 +62,10 @@ func (c *GenericCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, erro
}
// SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging
func (c *GenericCataloger) selectFiles(resolver scope.FileResolver) []file.Reference {
func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []source.Location {
// select by exact path
for path, parser := range c.pathParsers {
files, err := resolver.FilesByPath(file.Path(path))
files, err := resolver.FilesByPath(path)
if err != nil {
log.Warnf("cataloger failed to select files by path: %+v", err)
}
@ -90,28 +89,28 @@ func (c *GenericCataloger) selectFiles(resolver scope.FileResolver) []file.Refer
}
// catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
func (c *GenericCataloger) catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.Package, error) {
defer c.clear()
packages := make([]pkg.Package, 0)
for reference, parser := range c.parsers {
content, ok := contents[reference]
for location, parser := range c.parsers {
content, ok := contents[location]
if !ok {
log.Warnf("cataloger '%s' missing file content: %+v", c.upstreamCataloger, reference)
log.Warnf("cataloger '%s' missing file content: %+v", c.upstreamCataloger, location)
continue
}
entries, err := parser(string(reference.Path), strings.NewReader(content))
entries, err := parser(location.Path, strings.NewReader(content))
if err != nil {
// TODO: should we fail? or only log?
log.Warnf("cataloger '%s' failed to parse entries (reference=%+v): %+v", c.upstreamCataloger, reference, err)
log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)
continue
}
for _, entry := range entries {
entry.FoundBy = c.upstreamCataloger
entry.Source = []file.Reference{reference}
entry.Locations = []source.Location{location}
packages = append(packages, entry)
}

36
syft/cataloger/common/generic_cataloger_test.go
View File

@ -6,48 +6,50 @@ import (
"io/ioutil"
"testing"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/pkg"
)
type testResolverMock struct {
contents map[file.Reference]string
contents map[source.Location]string
}
func newTestResolver() *testResolverMock {
return &testResolverMock{
contents: make(map[file.Reference]string),
contents: make(map[source.Location]string),
}
}
func (r *testResolverMock) FileContentsByRef(_ file.Reference) (string, error) {
func (r *testResolverMock) FileContentsByLocation(_ source.Location) (string, error) {
return "", fmt.Errorf("not implemented")
}
func (r *testResolverMock) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) {
func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]string, error) {
return r.contents, nil
}
func (r *testResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
results := make([]file.Reference, len(paths))
func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, error) {
results := make([]source.Location, len(paths))
for idx, p := range paths {
results[idx] = file.NewFileReference(p)
results[idx] = source.NewLocation(p)
r.contents[results[idx]] = fmt.Sprintf("%s file contents!", p)
}
return results, nil
}
func (r *testResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) {
func (r *testResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) {
path := "/a-path.txt"
ref := file.NewFileReference(file.Path(path))
r.contents[ref] = fmt.Sprintf("%s file contents!", path)
return []file.Reference{ref}, nil
location := source.NewLocation(path)
r.contents[location] = fmt.Sprintf("%s file contents!", path)
return []source.Location{location}, nil
}
func (r *testResolverMock) RelativeFileByPath(_ file.Reference, _ string) (*file.Reference, error) {
return nil, fmt.Errorf("not implemented")
func (r *testResolverMock) RelativeFileByPath(_ source.Location, _ string) *source.Location {
panic(fmt.Errorf("not implemented"))
return nil
}
func parser(_ string, reader io.Reader) ([]pkg.Package, error) {
@ -94,8 +96,8 @@ func TestGenericCataloger(t *testing.T) {
}
for _, p := range actualPkgs {
ref := p.Source[0]
exP, ok := expectedPkgs[string(ref.Path)]
ref := p.Locations[0]
exP, ok := expectedPkgs[ref.Path]
if !ok {
t.Errorf("missing expected pkg: ref=%+v", ref)
continue
@ -106,7 +108,7 @@ func TestGenericCataloger(t *testing.T) {
}
if exP.Name != p.Name {
t.Errorf("bad contents mapping: %+v", p.Source)
t.Errorf("bad contents mapping: %+v", p.Locations)
}
}
}

93
syft/cataloger/deb/cataloger.go
View File

@ -7,11 +7,11 @@ import (
"fmt"
"io"
"path"
"path/filepath"
"strings"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
const (
@ -33,30 +33,30 @@ func (c *Cataloger) Name() string {
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files.
func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
dbFileMatches, err := resolver.FilesByGlob(dpkgStatusGlob)
if err != nil {
return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err)
}
var pkgs []pkg.Package
for _, dbRef := range dbFileMatches {
dbContents, err := resolver.FileContentsByRef(dbRef)
for _, dbLocation := range dbFileMatches {
dbContents, err := resolver.FileContentsByLocation(dbLocation)
if err != nil {
return nil, err
}
pkgs, err = parseDpkgStatus(strings.NewReader(dbContents))
if err != nil {
return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbRef.Path, err)
return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err)
}
md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbRef, pkgs)
md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbLocation, pkgs)
if err != nil {
return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err)
}
copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbRef, pkgs)
copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
if err != nil {
return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err)
}
@ -64,7 +64,7 @@ func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
for i := range pkgs {
p := &pkgs[i]
p.FoundBy = c.Name()
p.Source = []file.Reference{dbRef}
p.Locations = []source.Location{dbLocation}
if md5Reader, ok := md5ContentsByName[md5Key(*p)]; ok {
// attach the file list
@ -74,7 +74,7 @@ func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
// keep a record of the file where this was discovered
if ref, ok := md5RefsByName[md5Key(*p)]; ok {
p.Source = append(p.Source, ref)
p.Locations = append(p.Locations, ref)
}
}
@ -85,7 +85,7 @@ func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
// keep a record of the file where this was discovered
if ref, ok := copyrightRefsByName[p.Name]; ok {
p.Source = append(p.Source, ref)
p.Locations = append(p.Locations, ref)
}
}
}
@ -93,93 +93,82 @@ func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
return pkgs, nil
}
func fetchMd5Contents(resolver scope.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) {
func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {
// fetch all MD5 file contents. This approach is more efficient than fetching each MD5 file one at a time
var md5FileMatches []file.Reference
var nameByRef = make(map[file.Reference]string)
parentPath, err := dbRef.Path.ParentPath()
if err != nil {
return nil, nil, fmt.Errorf("unable to find parent of path=%+v: %w", dbRef.Path, err)
}
var md5FileMatches []source.Location
var nameByRef = make(map[source.Location]string)
parentPath := filepath.Dir(dbLocation.Path)
for _, p := range pkgs {
// look for /var/lib/dpkg/info/NAME:ARCH.md5sums
name := md5Key(p)
md5sumPath := path.Join(string(parentPath), "info", name+md5sumsExt)
md5SumRef, err := resolver.RelativeFileByPath(dbRef, md5sumPath)
if err != nil {
return nil, nil, fmt.Errorf("unable to find relative md5sum from path=%+v: %w", dbRef.Path, err)
}
md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
md5SumLocation := resolver.RelativeFileByPath(dbLocation, md5sumPath)
if md5SumRef == nil {
if md5SumLocation == nil {
// the most specific key did not work, fallback to just the name
// look for /var/lib/dpkg/info/NAME.md5sums
name := p.Name
md5sumPath := path.Join(string(parentPath), "info", name+md5sumsExt)
md5SumRef, err = resolver.RelativeFileByPath(dbRef, md5sumPath)
if err != nil {
return nil, nil, fmt.Errorf("unable to find relative md5sum from path=%+v: %w", dbRef.Path, err)
}
md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
md5SumLocation = resolver.RelativeFileByPath(dbLocation, md5sumPath)
}
// we should have at least one reference
if md5SumRef != nil {
md5FileMatches = append(md5FileMatches, *md5SumRef)
nameByRef[*md5SumRef] = name
if md5SumLocation != nil {
md5FileMatches = append(md5FileMatches, *md5SumLocation)
nameByRef[*md5SumLocation] = name
}
}
// fetch the md5 contents
md5ContentsByRef, err := resolver.MultipleFileContentsByRef(md5FileMatches...)
md5ContentsByLocation, err := resolver.MultipleFileContentsByLocation(md5FileMatches)
if err != nil {
return nil, nil, err
}
// organize content results and refs by a combination of name and architecture
var contentsByName = make(map[string]io.Reader)
var refsByName = make(map[string]file.Reference)
for ref, contents := range md5ContentsByRef {
name := nameByRef[ref]
var refsByName = make(map[string]source.Location)
for location, contents := range md5ContentsByLocation {
name := nameByRef[location]
contentsByName[name] = strings.NewReader(contents)
refsByName[name] = ref
refsByName[name] = location
}
return contentsByName, refsByName, nil
}
func fetchCopyrightContents(resolver scope.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) {
func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {
// fetch all copyright file contents. This approach is more efficient than fetching each copyright file one at a time
var copyrightFileMatches []file.Reference
var nameByRef = make(map[file.Reference]string)
var copyrightFileMatches []source.Location
var nameByLocation = make(map[source.Location]string)
for _, p := range pkgs {
// look for /usr/share/docs/NAME/copyright files
name := p.Name
copyrightPath := path.Join(docsPath, name, "copyright")
copyrightRef, err := resolver.RelativeFileByPath(dbRef, copyrightPath)
if err != nil {
return nil, nil, fmt.Errorf("unable to find relative copyright from path=%+v: %w", dbRef.Path, err)
}
copyrightLocation := resolver.RelativeFileByPath(dbLocation, copyrightPath)
// we may not have a copyright file for each package, ignore missing files
if copyrightRef != nil {
copyrightFileMatches = append(copyrightFileMatches, *copyrightRef)
nameByRef[*copyrightRef] = name
if copyrightLocation != nil {
copyrightFileMatches = append(copyrightFileMatches, *copyrightLocation)
nameByLocation[*copyrightLocation] = name
}
}
// fetch the copyright contents
copyrightContentsByRef, err := resolver.MultipleFileContentsByRef(copyrightFileMatches...)
copyrightContentsByLocation, err := resolver.MultipleFileContentsByLocation(copyrightFileMatches)
if err != nil {
return nil, nil, err
}
// organize content results and refs by package name
var contentsByName = make(map[string]io.Reader)
var refsByName = make(map[string]file.Reference)
for ref, contents := range copyrightContentsByRef {
name := nameByRef[ref]
var refsByName = make(map[string]source.Location)
for location, contents := range copyrightContentsByLocation {
name := nameByLocation[location]
contentsByName[name] = strings.NewReader(contents)
refsByName[name] = ref
refsByName[name] = location
}
return contentsByName, refsByName, nil

12
syft/cataloger/deb/cataloger_test.go
View File

@ -5,7 +5,7 @@ import (
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
"github.com/go-test/deep"
)
@ -54,7 +54,7 @@ func TestDpkgCataloger(t *testing.T) {
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg")
defer cleanup()
s, err := scope.NewScopeFromImage(img, scope.AllLayersScope)
s, err := source.NewFromImage(img, source.AllLayersScope, "")
if err != nil {
t.Fatal(err)
}
@ -77,11 +77,11 @@ func TestDpkgCataloger(t *testing.T) {
for idx := range actual {
a := &actual[idx]
// we will test the sources separately
var sourcesList = make([]string, len(a.Source))
for i, s := range a.Source {
sourcesList[i] = string(s.Path)
var sourcesList = make([]string, len(a.Locations))
for i, s := range a.Locations {
sourcesList[i] = s.Path
}
a.Source = nil
a.Locations = nil
for _, d := range deep.Equal(sourcesList, test.sources[a.Name]) {
t.Errorf("diff: %+v", d)

57
syft/cataloger/python/package_cataloger.go
View File

@ -8,11 +8,9 @@ import (
"github.com/anchore/syft/internal/log"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
const (
@ -33,9 +31,9 @@ func (c *PackageCataloger) Name() string {
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations.
func (c *PackageCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
func (c *PackageCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
// nolint:prealloc
var fileMatches []file.Reference
var fileMatches []source.Location
for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob} {
matches, err := resolver.FilesByGlob(glob)
@ -46,10 +44,10 @@ func (c *PackageCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, erro
}
var pkgs []pkg.Package
for _, ref := range fileMatches {
p, err := c.catalogEggOrWheel(resolver, ref)
for _, location := range fileMatches {
p, err := c.catalogEggOrWheel(resolver, location)
if err != nil {
return nil, fmt.Errorf("unable to catalog python package=%+v: %w", ref.Path, err)
return nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.Path, err)
}
if p != nil {
pkgs = append(pkgs, *p)
@ -59,8 +57,8 @@ func (c *PackageCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, erro
}
// catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents.
func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRef file.Reference) (*pkg.Package, error) {
metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataRef)
func (c *PackageCataloger) catalogEggOrWheel(resolver source.Resolver, metadataLocation source.Location) (*pkg.Package, error) {
metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataLocation)
if err != nil {
return nil, err
}
@ -74,7 +72,7 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRe
Name: metadata.Name,
Version: metadata.Version,
FoundBy: c.Name(),
Source: sources,
Locations: sources,
Licenses: licenses,
Language: pkg.Python,
Type: pkg.PythonPkg,
@ -84,22 +82,19 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRe
}
// fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained.
func (c *PackageCataloger) fetchRecordFiles(resolver scope.Resolver, metadataRef file.Reference) (files []pkg.PythonFileRecord, sources []file.Reference, err error) {
func (c *PackageCataloger) fetchRecordFiles(resolver source.Resolver, metadataLocation source.Location) (files []pkg.PythonFileRecord, sources []source.Location, err error) {
// we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory
// or for an image... for an image the METADATA file may be present within multiple layers, so it is important
// to reconcile the RECORD path to the same layer (or the next adjacent lower layer).
// lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure)
recordPath := filepath.Join(filepath.Dir(string(metadataRef.Path)), "RECORD")
recordRef, err := resolver.RelativeFileByPath(metadataRef, recordPath)
if err != nil {
return nil, nil, err
}
recordPath := filepath.Join(filepath.Dir(metadataLocation.Path), "RECORD")
recordRef := resolver.RelativeFileByPath(metadataLocation, recordPath)
if recordRef != nil {
sources = append(sources, *recordRef)
recordContents, err := resolver.FileContentsByRef(*recordRef)
recordContents, err := resolver.FileContentsByLocation(*recordRef)
if err != nil {
return nil, nil, err
}
@ -116,22 +111,20 @@ func (c *PackageCataloger) fetchRecordFiles(resolver scope.Resolver, metadataRef
}
// fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained.
func (c *PackageCataloger) fetchTopLevelPackages(resolver scope.Resolver, metadataRef file.Reference) (pkgs []string, sources []file.Reference, err error) {
func (c *PackageCataloger) fetchTopLevelPackages(resolver source.Resolver, metadataLocation source.Location) (pkgs []string, sources []source.Location, err error) {
// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
parentDir := filepath.Dir(string(metadataRef.Path))
parentDir := filepath.Dir(metadataLocation.Path)
topLevelPath := filepath.Join(parentDir, "top_level.txt")
topLevelRef, err := resolver.RelativeFileByPath(metadataRef, topLevelPath)
if err != nil {
return nil, nil, err
}
topLevelRef := resolver.RelativeFileByPath(metadataLocation, topLevelPath)
if topLevelRef == nil {
log.Warnf("missing python package top_level.txt (package=%q)", string(metadataRef.Path))
log.Warnf("missing python package top_level.txt (package=%q)", metadataLocation.Path)
return nil, nil, nil
}
sources = append(sources, *topLevelRef)
topLevelContents, err := resolver.FileContentsByRef(*topLevelRef)
topLevelContents, err := resolver.FileContentsByLocation(*topLevelRef)
if err != nil {
return nil, nil, err
}
@ -149,21 +142,21 @@ func (c *PackageCataloger) fetchTopLevelPackages(resolver scope.Resolver, metada
}
// assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver scope.Resolver, metadataRef file.Reference) (*pkg.PythonPackageMetadata, []file.Reference, error) {
var sources = []file.Reference{metadataRef}
func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.Resolver, metadataLocation source.Location) (*pkg.PythonPackageMetadata, []source.Location, error) {
var sources = []source.Location{metadataLocation}
metadataContents, err := resolver.FileContentsByRef(metadataRef)
metadataContents, err := resolver.FileContentsByLocation(metadataLocation)
if err != nil {
return nil, nil, err
}
metadata, err := parseWheelOrEggMetadata(metadataRef.Path, strings.NewReader(metadataContents))
metadata, err := parseWheelOrEggMetadata(metadataLocation.Path, strings.NewReader(metadataContents))
if err != nil {
return nil, nil, err
}
// attach any python files found for the given wheel/egg installation
r, s, err := c.fetchRecordFiles(resolver, metadataRef)
r, s, err := c.fetchRecordFiles(resolver, metadataLocation)
if err != nil {
return nil, nil, err
}
@ -171,7 +164,7 @@ func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver scope.Resolver, m
metadata.Files = r
// attach any top-level package names found for the given wheel/egg installation
p, s, err := c.fetchTopLevelPackages(resolver, metadataRef)
p, s, err := c.fetchTopLevelPackages(resolver, metadataLocation)
if err != nil {
return nil, nil, err
}

58
syft/cataloger/python/package_cataloger_test.go
View File

@ -8,7 +8,7 @@ import (
"strings"
"testing"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/pkg"
"github.com/go-test/deep"
@ -18,10 +18,10 @@ type pythonTestResolverMock struct {
metadataReader io.Reader
recordReader io.Reader
topLevelReader io.Reader
metadataRef *file.Reference
recordRef *file.Reference
topLevelRef *file.Reference
contents map[file.Reference]string
metadataRef *source.Location
recordRef *source.Location
topLevelRef *source.Location
contents map[source.Location]string
}
func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMock {
@ -46,17 +46,17 @@ func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMo
}
}
var recordRef *file.Reference
var recordRef *source.Location
if recordReader != nil {
ref := file.NewFileReference("test-fixtures/dist-info/RECORD")
ref := source.NewLocation("test-fixtures/dist-info/RECORD")
recordRef = &ref
}
var topLevelRef *file.Reference
var topLevelRef *source.Location
if topLevelReader != nil {
ref := file.NewFileReference("test-fixtures/dist-info/top_level.txt")
ref := source.NewLocation("test-fixtures/dist-info/top_level.txt")
topLevelRef = &ref
}
metadataRef := file.NewFileReference("test-fixtures/dist-info/METADATA")
metadataRef := source.NewLocation("test-fixtures/dist-info/METADATA")
return &pythonTestResolverMock{
recordReader: recordReader,
metadataReader: metadataReader,
@ -64,11 +64,11 @@ func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMo
metadataRef: &metadataRef,
recordRef: recordRef,
topLevelRef: topLevelRef,
contents: make(map[file.Reference]string),
contents: make(map[source.Location]string),
}
}
func (r *pythonTestResolverMock) FileContentsByRef(ref file.Reference) (string, error) {
func (r *pythonTestResolverMock) FileContentsByLocation(ref source.Location) (string, error) {
switch {
case r.topLevelRef != nil && ref.Path == r.topLevelRef.Path:
b, err := ioutil.ReadAll(r.topLevelReader)
@ -92,25 +92,25 @@ func (r *pythonTestResolverMock) FileContentsByRef(ref file.Reference) (string,
return "", fmt.Errorf("invalid value given")
}
func (r *pythonTestResolverMock) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) {
func (r *pythonTestResolverMock) MultipleFileContentsByLocation(_ []source.Location) (map[source.Location]string, error) {
return nil, fmt.Errorf("not implemented")
}
func (r *pythonTestResolverMock) FilesByPath(_ ...file.Path) ([]file.Reference, error) {
func (r *pythonTestResolverMock) FilesByPath(_ ...string) ([]source.Location, error) {
return nil, fmt.Errorf("not implemented")
}
func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) {
func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) {
return nil, fmt.Errorf("not implemented")
}
func (r *pythonTestResolverMock) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) {
func (r *pythonTestResolverMock) RelativeFileByPath(_ source.Location, path string) *source.Location {
switch {
case strings.Contains(path, "RECORD"):
return r.recordRef, nil
return r.recordRef
case strings.Contains(path, "top_level.txt"):
return r.topLevelRef, nil
return r.topLevelRef
default:
return nil, fmt.Errorf("invalid RelativeFileByPath value given: %q", path)
panic(fmt.Errorf("invalid RelativeFileByPath value given: %q", path))
}
}
@ -142,12 +142,12 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: "test-fixtures",
Files: []pkg.PythonFileRecord{
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
{Path: "requests/__init__.py", Digest: &pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
{Path: "requests/__pycache__/utils.cpython-38.pyc"},
{Path: "requests/__version__.py", Digest: &pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
{Path: "requests/utils.py", Digest: &pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
},
TopLevelPackages: []string{"requests"},
},
@ -174,11 +174,11 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "test-fixtures",
Files: []pkg.PythonFileRecord{
{Path: "../../../bin/pygmentize", Digest: &pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "Pygments-2.6.1.dist-info/RECORD"},
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
{Path: "pygments/util.py", Digest: &pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
},
TopLevelPackages: []string{"pygments", "something_else"},
},
@ -214,13 +214,13 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
resolver := newTestResolver(test.MetadataFixture, test.RecordFixture, test.TopLevelFixture)
// note that the source is the record ref created by the resolver mock... attach the expected values
test.ExpectedPackage.Source = []file.Reference{*resolver.metadataRef}
test.ExpectedPackage.Locations = []source.Location{*resolver.metadataRef}
if resolver.recordRef != nil {
test.ExpectedPackage.Source = append(test.ExpectedPackage.Source, *resolver.recordRef)
test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.recordRef)
}
if resolver.topLevelRef != nil {
test.ExpectedPackage.Source = append(test.ExpectedPackage.Source, *resolver.topLevelRef)
test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.topLevelRef)
}
// end patching expected values with runtime data...

6
syft/cataloger/python/parse_wheel_egg_metadata.go
View File

@ -7,8 +7,6 @@ import (
"path/filepath"
"strings"
"github.com/anchore/stereoscope/pkg/file"
"github.com/mitchellh/mapstructure"
"github.com/anchore/syft/syft/pkg"
@ -16,7 +14,7 @@ import (
// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
// returning all Python packages listed.
func parseWheelOrEggMetadata(path file.Path, reader io.Reader) (pkg.PythonPackageMetadata, error) {
func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMetadata, error) {
fields := make(map[string]string)
var key string
@ -73,7 +71,7 @@ func parseWheelOrEggMetadata(path file.Path, reader io.Reader) (pkg.PythonPackag
// add additional metadata not stored in the egg/wheel metadata file
sitePackagesRoot := filepath.Clean(filepath.Join(filepath.Dir(string(path)), ".."))
sitePackagesRoot := filepath.Clean(filepath.Join(filepath.Dir(path), ".."))
metadata.SitePackagesRootPath = sitePackagesRoot
return metadata, nil

4
syft/cataloger/python/parse_wheel_egg_metadata_test.go
View File

@ -4,8 +4,6 @@ import (
"os"
"testing"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/pkg"
"github.com/go-test/deep"
)
@ -48,7 +46,7 @@ func TestParseWheelEggMetadata(t *testing.T) {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseWheelOrEggMetadata(file.Path(test.Fixture), fixture)
actual, err := parseWheelOrEggMetadata(test.Fixture, fixture)
if err != nil {
t.Fatalf("failed to parse: %+v", err)
}

2
syft/cataloger/python/parse_wheel_egg_record.go
View File

@ -44,7 +44,7 @@ func parseWheelOrEggRecord(reader io.Reader) ([]pkg.PythonFileRecord, error) {
return nil, fmt.Errorf("unexpected python record digest: %q", item)
}
record.Digest = &pkg.Digest{
record.Digest = &pkg.PythonFileDigest{
Algorithm: fields[0],
Value: fields[1],
}

14
syft/cataloger/python/parse_wheel_egg_record_test.go
View File

@ -16,22 +16,22 @@ func TestParseWheelEggRecord(t *testing.T) {
{
Fixture: "test-fixtures/egg-info/RECORD",
ExpectedMetadata: []pkg.PythonFileRecord{
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
{Path: "requests/__init__.py", Digest: &pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
{Path: "requests/__pycache__/utils.cpython-38.pyc"},
{Path: "requests/__version__.py", Digest: &pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
{Path: "requests/utils.py", Digest: &pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
},
},
{
Fixture: "test-fixtures/dist-info/RECORD",
ExpectedMetadata: []pkg.PythonFileRecord{
{Path: "../../../bin/pygmentize", Digest: &pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "Pygments-2.6.1.dist-info/RECORD"},
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
{Path: "pygments/util.py", Digest: &pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
},
},
}

15
syft/cataloger/rpmdb/cataloger.go
View File

@ -8,11 +8,12 @@ import (
"strings"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
const (
packagesGlob = "**/var/lib/rpm/Packages"
catalogerName = "rpmdb-cataloger"
)
type Cataloger struct{}
@ -24,26 +25,26 @@ func NewRpmdbCataloger() *Cataloger {
// Name returns a string that uniquely describes a cataloger
func (c *Cataloger) Name() string {
return "rpmdb-cataloger"
return catalogerName
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
fileMatches, err := resolver.FilesByGlob(packagesGlob)
if err != nil {
return nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err)
}
var pkgs []pkg.Package
for _, dbRef := range fileMatches {
dbContents, err := resolver.FileContentsByRef(dbRef)
for _, location := range fileMatches {
dbContents, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, err
}
pkgs, err = parseRpmDB(resolver, dbRef, strings.NewReader(dbContents))
pkgs, err = parseRpmDB(resolver, location, strings.NewReader(dbContents))
if err != nil {
return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", dbRef.Path, err)
return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.Path, err)
}
}
return pkgs, nil

17
syft/cataloger/rpmdb/parse_rpmdb.go
View File

@ -6,18 +6,15 @@ import (
"io/ioutil"
"os"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/scope"
rpmdb "github.com/anchore/go-rpmdb/pkg"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
// parseApkDb parses an "Packages" RPM DB and returns the Packages listed within it.
func parseRpmDB(resolver scope.FileResolver, dbRef file.Reference, reader io.Reader) ([]pkg.Package, error) {
func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader io.Reader) ([]pkg.Package, error) {
f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb")
if err != nil {
return nil, fmt.Errorf("failed to create temp rpmdb file: %w", err)
@ -55,9 +52,9 @@ func parseRpmDB(resolver scope.FileResolver, dbRef file.Reference, reader io.Rea
p := pkg.Package{
Name: entry.Name,
Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does
//Version: fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch),
Source: []file.Reference{dbRef},
Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does, instead of fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch)
Locations: []source.Location{dbLocation},
FoundBy: catalogerName,
Type: pkg.RpmPkg,
MetadataType: pkg.RpmdbMetadataType,
Metadata: pkg.RpmdbMetadata{
@ -80,11 +77,11 @@ func parseRpmDB(resolver scope.FileResolver, dbRef file.Reference, reader io.Rea
return allPkgs, nil
}
func extractRpmdbFileRecords(resolver scope.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) {
func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) {
var records = make([]pkg.RpmdbFileRecord, 0)
for _, record := range entry.Files {
refs, err := resolver.FilesByPath(file.Path(record.Path))
refs, err := resolver.FilesByPath(record.Path)
if err != nil {
return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err)
}

28
syft/cataloger/rpmdb/parse_rpmdb_test.go
View File

@ -5,7 +5,8 @@ import (
"os"
"testing"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/pkg"
"github.com/go-test/deep"
)
@ -20,28 +21,29 @@ func newTestFileResolver(ignorePaths bool) *rpmdbTestFileResolverMock {
}
}
func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...string) ([]source.Location, error) {
if r.ignorePaths {
// act as if no paths exist
return nil, nil
}
// act as if all files exist
var refs = make([]file.Reference, len(paths))
var locations = make([]source.Location, len(paths))
for i, p := range paths {
refs[i] = file.NewFileReference(p)
locations[i] = source.NewLocation(p)
}
return refs, nil
return locations, nil
}
func (r *rpmdbTestFileResolverMock) FilesByGlob(...string) ([]file.Reference, error) {
func (r *rpmdbTestFileResolverMock) FilesByGlob(...string) ([]source.Location, error) {
return nil, fmt.Errorf("not implemented")
}
func (r *rpmdbTestFileResolverMock) RelativeFileByPath(file.Reference, string) (*file.Reference, error) {
return nil, fmt.Errorf("not implemented")
func (r *rpmdbTestFileResolverMock) RelativeFileByPath(source.Location, string) *source.Location {
panic(fmt.Errorf("not implemented"))
return nil
}
func TestParseRpmDB(t *testing.T) {
dbRef := file.NewFileReference("test-path")
dbLocation := source.NewLocation("test-path")
tests := []struct {
fixture string
@ -56,7 +58,8 @@ func TestParseRpmDB(t *testing.T) {
"dive": {
Name: "dive",
Version: "0.9.2-1",
Source: []file.Reference{dbRef},
Locations: []source.Location{dbLocation},
FoundBy: catalogerName,
Type: pkg.RpmPkg,
MetadataType: pkg.RpmdbMetadataType,
Metadata: pkg.RpmdbMetadata{
@ -82,7 +85,8 @@ func TestParseRpmDB(t *testing.T) {
"dive": {
Name: "dive",
Version: "0.9.2-1",
Source: []file.Reference{dbRef},
Locations: []source.Location{dbLocation},
FoundBy: catalogerName,
Type: pkg.RpmPkg,
MetadataType: pkg.RpmdbMetadataType,
Metadata: pkg.RpmdbMetadata{
@ -118,7 +122,7 @@ func TestParseRpmDB(t *testing.T) {
fileResolver := newTestFileResolver(test.ignorePaths)
actual, err := parseRpmDB(fileResolver, dbRef, fixture)
actual, err := parseRpmDB(fileResolver, dbLocation, fixture)
if err != nil {
t.Fatalf("failed to parse rpmdb: %+v", err)
}

15
syft/distro/distro.go
View File

@ -6,6 +6,7 @@ import (
hashiVer "github.com/hashicorp/go-version"
)
// Distro represents a Linux Distribution.
type Distro struct {
Type Type
Version *hashiVer.Version
@ -20,6 +21,7 @@ func NewUnknownDistro() Distro {
}
}
// NewDistro creates a new Distro object populated with the given values.
func NewDistro(t Type, ver, like string) (Distro, error) {
if ver == "" {
return Distro{Type: t}, nil
@ -36,6 +38,12 @@ func NewDistro(t Type, ver, like string) (Distro, error) {
}, nil
}
// Name provides a string repr of the distro
func (d Distro) Name() string {
return string(d.Type)
}
// MajorVersion returns the major version value from the pseudo-semantically versioned distro version value.
func (d Distro) MajorVersion() string {
if d.Version == nil {
return fmt.Sprint("(version unknown)")
@ -43,10 +51,12 @@ func (d Distro) MajorVersion() string {
return fmt.Sprintf("%d", d.Version.Segments()[0])
}
// FullVersion returns the original user version value.
func (d Distro) FullVersion() string {
return d.RawVersion
}
// String returns a human-friendly representation of the Linux distribution.
func (d Distro) String() string {
versionStr := "(version unknown)"
if d.RawVersion != "" {
@ -54,8 +64,3 @@ func (d Distro) String() string {
}
return fmt.Sprintf("%s %s", d.Type, versionStr)
}
// Name provides a string repr of the distro
func (d Distro) Name() string {
return d.Type.String()
}

23
syft/distro/identify.go
View File

@ -4,21 +4,20 @@ import (
"regexp"
"strings"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
// returns a distro or nil
type parseFunc func(string) *Distro
type parseEntry struct {
path file.Path
path string
fn parseFunc
}
// Identify parses distro-specific files to determine distro metadata like version and release.
func Identify(resolver scope.Resolver) Distro {
func Identify(resolver source.Resolver) Distro {
distro := NewUnknownDistro()
identityFiles := []parseEntry{
@ -41,25 +40,19 @@ func Identify(resolver scope.Resolver) Distro {
identifyLoop:
for _, entry := range identityFiles {
refs, err := resolver.FilesByPath(entry.path)
locations, err := resolver.FilesByPath(entry.path)
if err != nil {
log.Errorf("unable to get path refs from %s: %s", entry.path, err)
log.Errorf("unable to get path locations from %s: %s", entry.path, err)
break
}
if len(refs) == 0 {
if len(locations) == 0 {
log.Debugf("No Refs found from path: %s", entry.path)
continue
}
for _, ref := range refs {
contents, err := resolver.MultipleFileContentsByRef(ref)
content, ok := contents[ref]
if !ok {
log.Infof("no content present for ref: %s", ref)
continue
}
for _, location := range locations {
content, err := resolver.FileContentsByLocation(location)
if err != nil {
log.Debugf("unable to get contents from %s: %s", entry.path, err)

10
syft/distro/identify_test.go
View File

@ -8,7 +8,7 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
func TestIdentifyDistro(t *testing.T) {
@ -78,15 +78,15 @@ func TestIdentifyDistro(t *testing.T) {
observedDistros := internal.NewStringSet()
definedDistros := internal.NewStringSet()
for _, d := range All {
definedDistros.Add(d.String())
for _, distroType := range All {
definedDistros.Add(string(distroType))
}
for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) {
s, err := scope.NewScopeFromDir(test.fixture)
s, err := source.NewFromDirectory(test.fixture)
if err != nil {
t.Fatalf("unable to produce a new scope for testing: %s", test.fixture)
t.Fatalf("unable to produce a new source for testing: %s", test.fixture)
}
d := Identify(s.Resolver)

59
syft/distro/type.go
View File

@ -1,37 +1,25 @@
package distro
// Type represents the different Linux distribution options
type Type string
const (
UnknownDistroType Type = iota
Debian
Ubuntu
RedHat
CentOS
Fedora
Alpine
Busybox
AmazonLinux
OracleLinux
ArchLinux
OpenSuseLeap
// represents the set of valid/supported Linux Distributions
UnknownDistroType Type = "UnknownDistroType"
Debian Type = "debian"
Ubuntu Type = "ubuntu"
RedHat Type = "redhat"
CentOS Type = "centos"
Fedora Type = "fedora"
Alpine Type = "alpine"
Busybox Type = "busybox"
AmazonLinux Type = "amazonlinux"
OracleLinux Type = "oraclelinux"
ArchLinux Type = "archlinux"
OpenSuseLeap Type = "opensuseleap"
)
type Type int
var distroStr = []string{
"UnknownDistroType",
"debian",
"ubuntu",
"redhat",
"centos",
"fedora",
"alpine",
"busybox",
"amazn",
"oraclelinux",
"archlinux",
"opensuse-leap",
}
// All contains all Linux distribution options
var All = []Type{
Debian,
Ubuntu,
@ -46,14 +34,6 @@ var All = []Type{
OpenSuseLeap,
}
func (t Type) String() string {
if int(t) >= len(distroStr) || t < 0 {
return distroStr[0]
}
return distroStr[t]
}
// IDMapping connects a distro ID like "ubuntu" to a Distro type
var IDMapping = map[string]Type{
"debian": Debian,
@ -68,3 +48,8 @@ var IDMapping = map[string]Type{
"arch": ArchLinux,
"opensuse-leap": OpenSuseLeap,
}
// String returns the string representation of the given Linux distribution.
func (t Type) String() string {
return string(t)
}

5
syft/event/event.go
View File

@ -7,7 +7,12 @@ package event
import "github.com/wagoodman/go-partybus"
const (
// AppUpdateAvailable is a partybus event that occurs when an application update is available
AppUpdateAvailable partybus.EventType = "syft-app-update-available"
// CatalogerStarted is a partybus event that occurs when the package cataloging has begun
CatalogerStarted partybus.EventType = "syft-cataloger-started-event"
// CatalogerFinished is a partybus event that occurs when the package cataloging has completed
CatalogerFinished partybus.EventType = "syft-cataloger-finished-event"
)

69
syft/lib.go
View File

@ -7,8 +7,8 @@ Here is what the main execution path for syft does:
2. Invoke all catalogers to catalog the image, adding discovered packages to a single catalog object
3. Invoke a single presenter to show the contents of the catalog
A Scope object encapsulates the image object to be cataloged and the user options (catalog all layers vs. squashed layer),
providing a way to inspect paths and file content within the image. The Scope object, not the image object, is used
A Source object encapsulates the image object to be cataloged and the user options (catalog all layers vs. squashed layer),
providing a way to inspect paths and file content within the image. The Source object, not the image object, is used
throughout the main execution path. This abstraction allows for decoupling of what is cataloged (a docker image, an OCI
image, a filesystem, etc) and how it is cataloged (the individual catalogers).
@ -17,7 +17,9 @@ Similar to the cataloging process, Linux distribution identification is also per
package syft
import (
"encoding/json"
"fmt"
"io"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
@ -25,33 +27,34 @@ import (
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/logger"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
jsonPresenter "github.com/anchore/syft/syft/presenter/json"
"github.com/anchore/syft/syft/source"
"github.com/wagoodman/go-partybus"
)
// Catalog the given image from a particular perspective (e.g. squashed scope, all-layers scope). Returns the discovered
// set of packages, the identified Linux distribution, and the scope object used to wrap the data source.
func Catalog(userInput string, scoptOpt scope.Option) (*pkg.Catalog, *scope.Scope, *distro.Distro, error) {
// Catalog the given image from a particular perspective (e.g. squashed source, all-layers source). Returns the discovered
// set of packages, the identified Linux distribution, and the source object used to wrap the data source.
func Catalog(userInput string, scope source.Scope) (source.Source, *pkg.Catalog, distro.Distro, error) {
log.Info("cataloging image")
s, cleanup, err := scope.NewScope(userInput, scoptOpt)
s, cleanup, err := source.New(userInput, scope)
defer cleanup()
if err != nil {
return nil, nil, nil, err
return source.Source{}, nil, distro.Distro{}, err
}
d := IdentifyDistro(s)
catalog, err := CatalogFromScope(s)
if err != nil {
return nil, nil, nil, err
return source.Source{}, nil, distro.Distro{}, err
}
return catalog, &s, &d, nil
return s, catalog, d, nil
}
// IdentifyDistro attempts to discover what the underlying Linux distribution may be from the available flat files
// provided by the given scope object. If results are inconclusive a "UnknownDistro" Type is returned.
func IdentifyDistro(s scope.Scope) distro.Distro {
// provided by the given source object. If results are inconclusive a "UnknownDistro" Type is returned.
func IdentifyDistro(s source.Source) distro.Distro {
d := distro.Identify(s.Resolver)
if d.Type != distro.UnknownDistroType {
log.Infof("identified distro: %s", d.String())
@ -61,24 +64,54 @@ func IdentifyDistro(s scope.Scope) distro.Distro {
return d
}
// Catalog the given scope, which may represent a container image or filesystem. Returns the discovered set of packages.
func CatalogFromScope(s scope.Scope) (*pkg.Catalog, error) {
// Catalog the given source, which may represent a container image or filesystem. Returns the discovered set of packages.
func CatalogFromScope(s source.Source) (*pkg.Catalog, error) {
log.Info("building the catalog")
// conditionally have two sets of catalogers
var catalogers []cataloger.Cataloger
switch s.Scheme {
case scope.ImageScheme:
switch s.Metadata.Scheme {
case source.ImageScheme:
catalogers = cataloger.ImageCatalogers()
case scope.DirectoryScheme:
case source.DirectoryScheme:
catalogers = cataloger.DirectoryCatalogers()
default:
return nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", s.Scheme)
return nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", s.Metadata.Scheme)
}
return cataloger.Catalog(s.Resolver, catalogers...)
}
// CatalogFromJSON takes an existing syft report and generates native syft objects.
func CatalogFromJSON(reader io.Reader) (source.Metadata, *pkg.Catalog, distro.Distro, error) {
var doc jsonPresenter.Document
decoder := json.NewDecoder(reader)
if err := decoder.Decode(&doc); err != nil {
return source.Metadata{}, nil, distro.Distro{}, err
}
var pkgs = make([]pkg.Package, len(doc.Artifacts))
for i, a := range doc.Artifacts {
pkgs[i] = a.ToPackage()
}
catalog := pkg.NewCatalog(pkgs...)
var distroType distro.Type
if doc.Distro.Name == "" {
distroType = distro.UnknownDistroType
} else {
distroType = distro.Type(doc.Distro.Name)
}
theDistro, err := distro.NewDistro(distroType, doc.Distro.Version, doc.Distro.IDLike)
if err != nil {
return source.Metadata{}, nil, distro.Distro{}, err
}
return doc.Source.ToSourceMetadata(), catalog, theDistro, nil
}
// SetLogger sets the logger object used for all syft logging calls.
func SetLogger(logger logger.Logger) {
log.Log = logger

3
syft/logger/logger.go
View File

@ -1,8 +1,9 @@
/*
Defines the logging interface which is used throughout the syft library.
Package logger defines the logging interface which is used throughout the syft library.
*/
package logger
// Logger represents the behavior for logging within the syft library.
type Logger interface {
Errorf(format string, args ...interface{})
Error(args ...interface{})

1
syft/pkg/apk_metadata.go
View File

@ -35,6 +35,7 @@ type ApkFileRecord struct {
Checksum string `json:"checksum,omitempty"`
}
// PackageURL returns the PURL for the specific Alpine package (see https://github.com/package-url/purl-spec)
func (m ApkMetadata) PackageURL() string {
pURL := packageurl.NewPackageURL(
// note: this is currently a candidate and not technically within spec

26
syft/pkg/catalog.go
View File

@ -4,7 +4,8 @@ import (
"sort"
"sync"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/internal/log"
)
@ -14,17 +15,23 @@ var nextPackageID int64
type Catalog struct {
byID map[ID]*Package
byType map[Type][]*Package
byFile map[file.Reference][]*Package
byFile map[source.Location][]*Package
lock sync.RWMutex
}
// NewCatalog returns a new empty Catalog
func NewCatalog() *Catalog {
return &Catalog{
func NewCatalog(pkgs ...Package) *Catalog {
catalog := Catalog{
byID: make(map[ID]*Package),
byType: make(map[Type][]*Package),
byFile: make(map[file.Reference][]*Package),
byFile: make(map[source.Location][]*Package),
}
for _, p := range pkgs {
catalog.Add(p)
}
return &catalog
}
// PackageCount returns the total number of packages that have been added.
@ -38,8 +45,8 @@ func (c *Catalog) Package(id ID) *Package {
}
// PackagesByFile returns all packages that were discovered from the given source file reference.
func (c *Catalog) PackagesByFile(ref file.Reference) []*Package {
return c.byFile[ref]
func (c *Catalog) PackagesByFile(location source.Location) []*Package {
return c.byFile[location]
}
// Add a package to the Catalog.
@ -65,7 +72,7 @@ func (c *Catalog) Add(p Package) {
c.byType[p.Type] = append(c.byType[p.Type], &p)
// store by file references
for _, s := range p.Source {
for _, s := range p.Locations {
_, ok := c.byFile[s]
if !ok {
c.byFile[s] = make([]*Package, 0)
@ -111,6 +118,9 @@ func (c *Catalog) Sorted(types ...Type) []*Package {
sort.SliceStable(pkgs, func(i, j int) bool {
if pkgs[i].Name == pkgs[j].Name {
if pkgs[i].Version == pkgs[j].Version {
return pkgs[i].Type < pkgs[j].Type
}
return pkgs[i].Version < pkgs[j].Version
}
return pkgs[i].Name < pkgs[j].Name

2
syft/pkg/dpkg_metadata.go
View File

@ -17,11 +17,13 @@ type DpkgMetadata struct {
Files []DpkgFileRecord `json:"files"`
}
// DpkgFileRecord represents a single file attributed to a debian package.
type DpkgFileRecord struct {
Path string `json:"path"`
MD5 string `json:"md5"`
}
// PackageURL returns the PURL for the specific Debian package (see https://github.com/package-url/purl-spec)
func (m DpkgMetadata) PackageURL(d distro.Distro) string {
pURL := packageurl.NewPackageURL(
// TODO: replace with `packageurl.TypeDebian` upon merge of https://github.com/package-url/packageurl-go/pull/21

1
syft/pkg/gem_metadata.go
View File

@ -1,5 +1,6 @@
package pkg
// GemMetadata represents all metadata parsed from the gemspec file
type GemMetadata struct {
Name string `mapstructure:"name" json:"name"`
Version string `mapstructure:"version" json:"version"`

1
syft/pkg/java_metadata.go
View File

@ -26,6 +26,7 @@ type JavaManifest struct {
NamedSections map[string]map[string]string `json:"namedSections,omitempty"`
}
// PackageURL returns the PURL for the specific Alpine package (see https://github.com/package-url/purl-spec)
func (m JavaMetadata) PackageURL() string {
if m.PomProperties != nil {
pURL := packageurl.NewPackageURL(

36
syft/pkg/language.go
View File

@ -1,25 +1,19 @@
package pkg
// Language represents a single programming language.
type Language string
const (
UnknownLanguage Language = iota
Java
JavaScript
Python
Ruby
Go
// the full set of supported programming languages
UnknownLanguage Language = "UnknownLanguage"
Java Language = "java"
JavaScript Language = "javascript"
Python Language = "python"
Ruby Language = "ruby"
Go Language = "go"
)
type Language uint
var languageStr = []string{
"UnknownLanguage",
"java",
"javascript",
"python",
"ruby",
"go",
}
// AllLanguages is a set of all programming languages detected by syft.
var AllLanguages = []Language{
Java,
JavaScript,
@ -28,9 +22,7 @@ var AllLanguages = []Language{
Go,
}
func (t Language) String() string {
if int(t) >= len(languageStr) {
return languageStr[0]
}
return languageStr[t]
// String returns the string representation of the language.
func (l Language) String() string {
return string(l)
}

16
syft/pkg/metadata.go
View File

@ -1,14 +1,16 @@
package pkg
// MetadataType represents the data shape stored within pkg.Package.Metadata.
type MetadataType string
const (
// this is the full set of data shapes that can be represented within the pkg.Package.Metadata field
UnknownMetadataType MetadataType = "UnknownMetadata"
ApkMetadataType MetadataType = "apk-metadata"
DpkgMetadataType MetadataType = "dpkg-metadata"
GemMetadataType MetadataType = "gem-metadata"
JavaMetadataType MetadataType = "java-metadata"
NpmPackageJSONMetadataType MetadataType = "npm-package-json-metadata"
RpmdbMetadataType MetadataType = "rpmdb-metadata"
PythonPackageMetadataType MetadataType = "python-package-metadata"
ApkMetadataType MetadataType = "ApkMetadata"
DpkgMetadataType MetadataType = "DpkgMetadata"
GemMetadataType MetadataType = "GemMetadata"
JavaMetadataType MetadataType = "JavaMetadata"
NpmPackageJSONMetadataType MetadataType = "NpmPackageJsonMetadata"
RpmdbMetadataType MetadataType = "RpmdbMetadata"
PythonPackageMetadataType MetadataType = "PythonPackageMetadata"
)

22
syft/pkg/package.go
View File

@ -8,26 +8,28 @@ import (
"regexp"
"strings"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/distro"
"github.com/package-url/packageurl-go"
)
// ID represents a unique value for each package added to a package catalog.
type ID int64
// Package represents an application or library that has been bundled into a distributable format.
type Package struct {
id ID // uniquely identifies a package, set by the cataloger
Name string `json:"manifest"` // the package name
Version string `json:"version"` // the version of the package
FoundBy string `json:"foundBy"` // the specific cataloger that discovered this package
Source []file.Reference `json:"sources"` // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package)
Name string // the package name
Version string // the version of the package
FoundBy string // the specific cataloger that discovered this package
Locations []source.Location // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package)
// TODO: should we move licenses into metadata?
Licenses []string `json:"licenses"` // licenses discovered with the package metadata
Language Language `json:"language"` // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc)
Type Type `json:"type"` // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc)
MetadataType MetadataType `json:"metadataType"` // the shape of the additional data in the "metadata" field
Metadata interface{} `json:"metadata,omitempty"` // additional data found while parsing the package source
Licenses []string // licenses discovered with the package metadata
Language Language // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc)
Type Type // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc)
MetadataType MetadataType // the shape of the additional data in the "metadata" field
Metadata interface{} // additional data found while parsing the package source
}
// ID returns the package ID, which is unique relative to a package catalog.

5
syft/pkg/python_package_metadata.go
View File

@ -1,6 +1,7 @@
package pkg
type Digest struct {
// PythonFileDigest represents the file metadata for a single file attributed to a python package.
type PythonFileDigest struct {
Algorithm string `json:"algorithm"`
Value string `json:"value"`
}
@ -8,7 +9,7 @@ type Digest struct {
// PythonFileRecord represents a single entry within a RECORD file for a python wheel or egg package
type PythonFileRecord struct {
Path string `json:"path"`
Digest *Digest `json:"digest,omitempty"`
Digest *PythonFileDigest `json:"digest,omitempty"`
Size string `json:"size,omitempty"`
}

3
syft/pkg/rpmdb_metadata.go
View File

@ -21,6 +21,7 @@ type RpmdbMetadata struct {
Files []RpmdbFileRecord `json:"files"`
}
// RpmdbFileRecord represents the file metadata for a single file attributed to a RPM package.
type RpmdbFileRecord struct {
Path string `json:"path"`
Mode RpmdbFileMode `json:"mode"`
@ -28,8 +29,10 @@ type RpmdbFileRecord struct {
SHA256 string `json:"sha256"`
}
// RpmdbFileMode is the raw file mode for a single file. This can be interpreted as the linux stat.h mode (see https://pubs.opengroup.org/onlinepubs/007908799/xsh/sysstat.h.html)
type RpmdbFileMode uint16
// PackageURL returns the PURL for the specific RHEL package (see https://github.com/package-url/purl-spec)
func (m RpmdbMetadata) PackageURL(d distro.Distro) string {
pURL := packageurl.NewPackageURL(
packageurl.TypeRPM,

3
syft/pkg/type.go
View File

@ -6,6 +6,7 @@ import "github.com/package-url/packageurl-go"
type Type string
const (
// the full set of supported packages
UnknownPkg Type = "UnknownPackage"
ApkPkg Type = "apk"
GemPkg Type = "gem"
@ -18,6 +19,7 @@ const (
GoModulePkg Type = "go-module"
)
// AllPkgs represents all supported package types
var AllPkgs = []Type{
ApkPkg,
GemPkg,
@ -30,6 +32,7 @@ var AllPkgs = []Type{
GoModulePkg,
}
// PackageURLType returns the PURL package type for the current package.
func (t Type) PackageURLType() string {
switch t {
case ApkPkg:

32
syft/presenter/cyclonedx/bom-extension.go
View File

@ -4,8 +4,7 @@ import (
"encoding/xml"
"time"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/version"
"github.com/anchore/syft/syft/source"
)
// Source: https://cyclonedx.org/ext/bom-descriptor/
@ -35,15 +34,34 @@ type BdComponent struct {
}
// NewBomDescriptor returns a new BomDescriptor tailored for the current time and "syft" tool details.
func NewBomDescriptor() *BomDescriptor {
versionInfo := version.FromBuild()
return &BomDescriptor{
func NewBomDescriptor(name, version string, srcMetadata source.Metadata) *BomDescriptor {
descriptor := BomDescriptor{
XMLName: xml.Name{},
Timestamp: time.Now().Format(time.RFC3339),
Tool: &BdTool{
Vendor: "anchore",
Name: internal.ApplicationName,
Version: versionInfo.Version,
Name: name,
Version: version,
},
}
switch srcMetadata.Scheme {
case source.ImageScheme:
descriptor.Component = &BdComponent{
Component: Component{
Type: "container",
Name: srcMetadata.ImageMetadata.UserInput,
Version: srcMetadata.ImageMetadata.Digest,
},
}
case source.DirectoryScheme:
descriptor.Component = &BdComponent{
Component: Component{
Type: "file",
Name: srcMetadata.Path,
},
}
}
return &descriptor
}

2
syft/presenter/cyclonedx/component.go
View File

@ -15,7 +15,7 @@ type Component struct {
Description string `xml:"description,omitempty"` // A description of the component
Licenses *[]License `xml:"licenses>license"` // A node describing zero or more license names, SPDX license IDs or expressions
PackageURL string `xml:"purl,omitempty"` // Specifies the package-url (PackageURL). The purl, if specified, must be valid and conform to the specification defined at: https://github.com/package-url/purl-spec
// TODO: scope, hashes, copyright, cpe, purl, swid, modified, pedigree, externalReferences
// TODO: source, hashes, copyright, cpe, purl, swid, modified, pedigree, externalReferences
// TODO: add user-defined parameters for syft-specific values (image layer index, cataloger, location path, etc.)
}

24
syft/presenter/cyclonedx/document.go
View File

@ -3,9 +3,11 @@ package cyclonedx
import (
"encoding/xml"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/version"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
"github.com/google/uuid"
)
@ -22,19 +24,19 @@ type Document struct {
BomDescriptor *BomDescriptor `xml:"bd:metadata"` // The BOM descriptor extension
}
// NewDocument returns an empty CycloneDX Document object.
func NewDocument() Document {
return Document{
// NewDocumentFromCatalog returns a CycloneDX Document object populated with the catalog contents.
func NewDocument(catalog *pkg.Catalog, d distro.Distro, srcMetadata source.Metadata) Document {
versionInfo := version.FromBuild()
doc := Document{
XMLNs: "http://cyclonedx.org/schema/bom/1.2",
XMLNsBd: "http://cyclonedx.org/schema/ext/bom-descriptor/1.0",
Version: 1,
SerialNumber: uuid.New().URN(),
BomDescriptor: NewBomDescriptor(internal.ApplicationName, versionInfo.Version, srcMetadata),
}
}
// NewDocumentFromCatalog returns a CycloneDX Document object populated with the catalog contents.
func NewDocumentFromCatalog(catalog *pkg.Catalog, d distro.Distro) Document {
bom := NewDocument()
// attach components
for p := range catalog.Enumerate() {
component := Component{
Type: "library", // TODO: this is not accurate
@ -51,10 +53,8 @@ func NewDocumentFromCatalog(catalog *pkg.Catalog, d distro.Distro) Document {
if len(licenses) > 0 {
component.Licenses = &licenses
}
bom.Components = append(bom.Components, component)
doc.Components = append(doc.Components, component)
}
bom.BomDescriptor = NewBomDescriptor()
return bom
return doc
}

44
syft/presenter/cyclonedx/presenter.go
View File

@ -5,63 +5,33 @@ package cyclonedx
import (
"encoding/xml"
"fmt"
"io"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
// Presenter writes a CycloneDX report from the given Catalog and Scope contents
// Presenter writes a CycloneDX report from the given Catalog and Locations contents
type Presenter struct {
catalog *pkg.Catalog
scope scope.Scope
srcMetadata source.Metadata
distro distro.Distro
}
// NewPresenter creates a CycloneDX presenter from the given Catalog and Scope objects.
func NewPresenter(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) *Presenter {
// NewPresenter creates a CycloneDX presenter from the given Catalog and Locations objects.
func NewPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Distro) *Presenter {
return &Presenter{
catalog: catalog,
scope: s,
srcMetadata: srcMetadata,
distro: d,
}
}
// Present writes the CycloneDX report to the given io.Writer.
func (pres *Presenter) Present(output io.Writer) error {
bom := NewDocumentFromCatalog(pres.catalog, pres.distro)
switch src := pres.scope.Source.(type) {
case scope.DirSource:
bom.BomDescriptor.Component = &BdComponent{
Component: Component{
Type: "file",
Name: src.Path,
Version: "",
},
}
case scope.ImageSource:
var imageID string
var versionStr string
if len(src.Img.Metadata.Tags) > 0 {
imageID = src.Img.Metadata.Tags[0].Context().Name()
versionStr = src.Img.Metadata.Tags[0].TagStr()
} else {
imageID = src.Img.Metadata.Digest
}
bom.BomDescriptor.Component = &BdComponent{
Component: Component{
Type: "container",
Name: imageID,
Version: versionStr,
},
}
default:
return fmt.Errorf("unsupported source: %T", src)
}
bom := NewDocument(pres.catalog, pres.distro, pres.srcMetadata)
encoder := xml.NewEncoder(output)
encoder.Indent("", " ")

33
syft/presenter/cyclonedx/presenter_test.go
View File

@ -10,9 +10,8 @@ import (
"github.com/anchore/syft/syft/distro"
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
"github.com/sergi/go-diff/diffmatchpatch"
)
@ -29,7 +28,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
Version: "1.0.1",
Type: pkg.DebPkg,
FoundBy: "the-cataloger-1",
Source: []file.Reference{
Locations: []source.Location{
{Path: "/some/path/pkg1"},
},
Metadata: pkg.DpkgMetadata{
@ -43,7 +42,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
Version: "2.0.1",
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Source: []file.Reference{
Locations: []source.Location{
{Path: "/some/path/pkg1"},
},
Licenses: []string{
@ -57,7 +56,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
},
})
s, err := scope.NewScopeFromDir("/some/path")
s, err := source.NewFromDirectory("/some/path")
if err != nil {
t.Fatal(err)
}
@ -67,7 +66,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
t.Fatal(err)
}
pres := NewPresenter(catalog, s, d)
pres := NewPresenter(catalog, s.Metadata, d)
// run presenter
err = pres.Present(&buffer)
@ -105,8 +104,8 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
catalog.Add(pkg.Package{
Name: "package1",
Version: "1.0.1",
Source: []file.Reference{
*img.SquashedTree().File("/somefile-1.txt"),
Locations: []source.Location{
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
},
Type: pkg.RpmPkg,
FoundBy: "the-cataloger-1",
@ -125,8 +124,8 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
catalog.Add(pkg.Package{
Name: "package2",
Version: "2.0.1",
Source: []file.Reference{
*img.SquashedTree().File("/somefile-2.txt"),
Locations: []source.Location{
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
},
Type: pkg.RpmPkg,
FoundBy: "the-cataloger-2",
@ -147,7 +146,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
},
})
s, err := scope.NewScopeFromImage(img, scope.AllLayersScope)
s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
if err != nil {
t.Fatal(err)
}
@ -157,7 +156,15 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
t.Fatal(err)
}
pres := NewPresenter(catalog, s, d)
// This accounts for the non-deterministic digest value that we end up with when
// we build a container image dynamically during testing. Ultimately, we should
// use a golden image as a test fixture in place of building this image during
// testing. At that time, this line will no longer be necessary.
//
// This value is sourced from the "version" node in "./test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden"
s.Metadata.ImageMetadata.Digest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
pres := NewPresenter(catalog, s.Metadata, d)
// run presenter
err = pres.Present(&buffer)
@ -178,7 +185,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
if !bytes.Equal(expected, actual) {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(string(actual), string(expected), true)
diffs := dmp.DiffMain(string(expected), string(actual), true)
t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs))
}
}

4
syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:2bbada20-3e87-44ea-9a56-1aa0e4dd01a0">
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:815fdd6b-917e-423d-8c91-1fe648141505">
<components>
<component type="library">
<name>package1</name>
@ -21,7 +21,7 @@
</component>
</components>
<bd:metadata>
<bd:timestamp>2020-09-23T18:26:58-04:00</bd:timestamp>
<bd:timestamp>2020-11-16T08:45:54-05:00</bd:timestamp>
<bd:tool>
<bd:vendor>anchore</bd:vendor>
<bd:name>syft</bd:name>

8
syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:94dae829-4d5d-482f-afab-27f43f919e2c">
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:3cb10332-1645-44f6-be4a-4f8be5a60cf8">
<components>
<component type="library">
<name>package1</name>
@ -21,15 +21,15 @@
</component>
</components>
<bd:metadata>
<bd:timestamp>2020-09-23T18:26:58-04:00</bd:timestamp>
<bd:timestamp>2020-11-16T08:45:54-05:00</bd:timestamp>
<bd:tool>
<bd:vendor>anchore</bd:vendor>
<bd:name>syft</bd:name>
<bd:version>[not provided]</bd:version>
</bd:tool>
<bd:component type="container">
<name>index.docker.io/library/stereoscope-fixture-image-simple</name>
<version>04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7</version>
<name>user-image-input</name>
<version>sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368</version>
</bd:component>
</bd:metadata>
</bom>

136
syft/presenter/json/artifact.go
View File

@ -1,33 +1,141 @@
package json
import (
"encoding/json"
"fmt"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
type Artifact struct {
// Package represents a pkg.Package object specialized for JSON marshaling and unmarshaling.
type Package struct {
packageBasicMetadata
packageCustomMetadata
}
// packageBasicMetadata contains non-ambiguous values (type-wise) from pkg.Package.
type packageBasicMetadata struct {
Name string `json:"name"`
Version string `json:"version"`
Type string `json:"type"`
FoundBy []string `json:"foundBy"`
Locations Locations `json:"locations,omitempty"`
Type pkg.Type `json:"type"`
FoundBy string `json:"foundBy"`
Locations []source.Location `json:"locations"`
Licenses []string `json:"licenses"`
Language pkg.Language `json:"language"`
}
// packageCustomMetadata contains ambiguous values (type-wise) from pkg.Package.
type packageCustomMetadata struct {
MetadataType pkg.MetadataType `json:"metadataType"`
Metadata interface{} `json:"metadata,omitempty"`
}
func NewArtifact(p *pkg.Package, s scope.Scope) (Artifact, error) {
locations, err := NewLocations(p, s)
if err != nil {
return Artifact{}, err
}
// packageMetadataUnpacker is all values needed from Package to disambiguate ambiguous fields during json unmarshaling.
type packageMetadataUnpacker struct {
MetadataType string `json:"metadataType"`
Metadata json.RawMessage `json:"metadata"`
}
return Artifact{
// NewPackage crates a new Package from the given pkg.Package.
func NewPackage(p *pkg.Package) (Package, error) {
return Package{
packageBasicMetadata: packageBasicMetadata{
Name: p.Name,
Version: p.Version,
Type: string(p.Type),
FoundBy: []string{p.FoundBy},
Locations: locations,
Type: p.Type,
FoundBy: p.FoundBy,
Locations: p.Locations,
Licenses: p.Licenses,
Language: p.Language,
},
packageCustomMetadata: packageCustomMetadata{
MetadataType: p.MetadataType,
Metadata: p.Metadata,
},
}, nil
}
// ToPackage generates a pkg.Package from the current Package.
func (a Package) ToPackage() pkg.Package {
return pkg.Package{
// does not include found-by and locations
Name: a.Name,
Version: a.Version,
FoundBy: a.FoundBy,
Licenses: a.Licenses,
Language: a.Language,
Locations: a.Locations,
Type: a.Type,
MetadataType: a.MetadataType,
Metadata: a.Metadata,
}
}
// UnmarshalJSON is a custom unmarshaller for handling basic values and values with ambiguous types.
// nolint:funlen
func (a *Package) UnmarshalJSON(b []byte) error {
var basic packageBasicMetadata
if err := json.Unmarshal(b, &basic); err != nil {
return err
}
a.packageBasicMetadata = basic
var unpacker packageMetadataUnpacker
if err := json.Unmarshal(b, &unpacker); err != nil {
return err
}
a.MetadataType = pkg.MetadataType(unpacker.MetadataType)
switch a.MetadataType {
case pkg.RpmdbMetadataType:
var payload pkg.RpmdbMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
a.Metadata = payload
case pkg.PythonPackageMetadataType:
var payload pkg.PythonPackageMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
a.Metadata = payload
case pkg.DpkgMetadataType:
var payload pkg.DpkgMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
a.Metadata = payload
case pkg.ApkMetadataType:
var payload pkg.ApkMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
a.Metadata = payload
case pkg.JavaMetadataType:
var payload pkg.JavaMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
a.Metadata = payload
case pkg.NpmPackageJSONMetadataType:
var payload pkg.NpmPackageJSONMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
a.Metadata = payload
case pkg.GemMetadataType:
var payload pkg.GemMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
a.Metadata = payload
case "":
// there may be packages with no metadata, which is OK
default:
return fmt.Errorf("unsupported package metadata type: %+v", a.MetadataType)
}
return nil
}

7
syft/presenter/json/descriptor.go Normal file
View File

@ -0,0 +1,7 @@
package json
// Descriptor describes what created the document as well as surrounding metadata
type Descriptor struct {
Name string `json:"name"`
Version string `json:"version"`
}

24
syft/presenter/json/distribution.go Normal file
View File

@ -0,0 +1,24 @@
package json
import "github.com/anchore/syft/syft/distro"
// Distribution provides information about a detected Linux Distribution.
type Distribution struct {
Name string `json:"name"` // Name of the Linux distribution
Version string `json:"version"` // Version of the Linux distribution (major or major.minor version)
IDLike string `json:"idLike"` // the ID_LIKE field found within the /etc/os-release file
}
// NewDistribution creates a struct with the Linux distribution to be represented in JSON.
func NewDistribution(d distro.Distro) Distribution {
distroName := d.Name()
if distroName == "UnknownDistroType" {
distroName = ""
}
return Distribution{
Name: distroName,
Version: d.FullVersion(),
IDLike: d.IDLike,
}
}

48
syft/presenter/json/document.go
View File

@ -1,46 +1,40 @@
package json
import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/version"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
// Document represents the syft cataloging findings as a JSON document
type Document struct {
Artifacts []Artifact `json:"artifacts"`
Source Source `json:"source"`
Distro Distribution `json:"distro"`
Artifacts []Package `json:"artifacts"` // Artifacts is the list of packages discovered and placed into the catalog
Source Source `json:"source"` // Source represents the original object that was cataloged
Distro Distribution `json:"distro"` // Distro represents the Linux distribution that was detected from the source
Descriptor Descriptor `json:"descriptor"` // Descriptor is a block containing self-describing information about syft
}
// Distritbution provides information about a detected Linux Distribution
type Distribution struct {
Name string `json:"name"`
Version string `json:"version"`
IDLike string `json:"idLike"`
}
func NewDocument(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) (Document, error) {
doc := Document{
Artifacts: make([]Artifact, 0),
}
src, err := NewSource(s)
// NewDocument creates and populates a new JSON document struct from the given cataloging results.
func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Distro) (Document, error) {
src, err := NewSource(srcMetadata)
if err != nil {
return Document{}, nil
}
doc.Source = src
distroName := d.Name()
if distroName == "UnknownDistroType" {
distroName = ""
}
doc.Distro = Distribution{
Name: distroName,
Version: d.FullVersion(),
IDLike: d.IDLike,
doc := Document{
Artifacts: make([]Package, 0),
Source: src,
Distro: NewDistribution(d),
Descriptor: Descriptor{
Name: internal.ApplicationName,
Version: version.FromBuild().Version,
},
}
for _, p := range catalog.Sorted() {
art, err := NewArtifact(p, s)
art, err := NewPackage(p)
if err != nil {
return Document{}, err
}

44
syft/presenter/json/image.go
View File

@ -1,44 +0,0 @@
package json
import (
"github.com/anchore/syft/syft/scope"
)
type Image struct {
Layers []Layer `json:"layers"`
Size int64 `json:"size"`
Digest string `json:"digest"`
MediaType string `json:"mediaType"`
Tags []string `json:"tags"`
}
type Layer struct {
MediaType string `json:"mediaType"`
Digest string `json:"digest"`
Size int64 `json:"size"`
}
func NewImage(src scope.ImageSource) *Image {
// populate artifacts...
tags := make([]string, len(src.Img.Metadata.Tags))
for idx, tag := range src.Img.Metadata.Tags {
tags[idx] = tag.String()
}
img := Image{
Digest: src.Img.Metadata.Digest,
Size: src.Img.Metadata.Size,
MediaType: string(src.Img.Metadata.MediaType),
Tags: tags,
Layers: make([]Layer, len(src.Img.Layers)),
}
// populate image metadata
for idx, l := range src.Img.Layers {
img.Layers[idx] = Layer{
MediaType: string(l.Metadata.MediaType),
Digest: l.Metadata.Digest,
Size: l.Metadata.Size,
}
}
return &img
}

45
syft/presenter/json/location.go
View File

@ -1,45 +0,0 @@
package json
import (
"fmt"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
)
type Locations interface{}
type ImageLocation struct {
Path string `json:"path"`
LayerIndex uint `json:"layerIndex"`
}
func NewLocations(p *pkg.Package, s scope.Scope) (Locations, error) {
switch src := s.Source.(type) {
case scope.ImageSource:
locations := make([]ImageLocation, len(p.Source))
for idx := range p.Source {
entry, err := src.Img.FileCatalog.Get(p.Source[idx])
if err != nil {
return nil, fmt.Errorf("unable to find layer index for source-idx=%d package=%s", idx, p.Name)
}
artifactSource := ImageLocation{
LayerIndex: entry.Source.Metadata.Index,
Path: string(p.Source[idx].Path),
}
locations[idx] = artifactSource
}
return locations, nil
case scope.DirSource:
locations := make([]string, len(p.Source))
for idx := range p.Source {
locations[idx] = string(p.Source[idx].Path)
}
return locations, nil
default:
return nil, fmt.Errorf("unable to determine source: %T", src)
}
}

13
syft/presenter/json/presenter.go
View File

@ -6,25 +6,28 @@ import (
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
// Presenter is a JSON presentation object for the syft results
type Presenter struct {
catalog *pkg.Catalog
scope scope.Scope
srcMetadata source.Metadata
distro distro.Distro
}
func NewPresenter(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) *Presenter {
// NewPresenter creates a new JSON presenter object for the given cataloging results.
func NewPresenter(catalog *pkg.Catalog, s source.Metadata, d distro.Distro) *Presenter {
return &Presenter{
catalog: catalog,
scope: s,
srcMetadata: s,
distro: d,
}
}
// Present the catalog results to the given writer.
func (pres *Presenter) Present(output io.Writer) error {
doc, err := NewDocument(pres.catalog, pres.scope, pres.distro)
doc, err := NewDocument(pres.catalog, pres.srcMetadata, pres.distro)
if err != nil {
return err
}

55
syft/presenter/json/presenter_test.go
View File

@ -6,11 +6,10 @@ import (
"testing"
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
"github.com/sergi/go-diff/diffmatchpatch"
)
@ -25,27 +24,39 @@ func TestJsonDirsPresenter(t *testing.T) {
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Type: pkg.DebPkg,
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Source: []file.Reference{
Locations: []source.Location{
{Path: "/some/path/pkg1"},
},
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Licenses: []string{"MIT"},
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
},
})
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Source: []file.Reference{
Locations: []source.Location{
{Path: "/some/path/pkg1"},
},
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "package-2",
Version: "2.0.1",
},
})
d := distro.NewUnknownDistro()
s, err := scope.NewScopeFromDir("/some/path")
s, err := source.NewFromDirectory("/some/path")
if err != nil {
t.Fatal(err)
}
pres := NewPresenter(catalog, s, d)
pres := NewPresenter(catalog, s.Metadata, d)
// run presenter
err = pres.Present(&buffer)
@ -62,7 +73,7 @@ func TestJsonDirsPresenter(t *testing.T) {
if !bytes.Equal(expected, actual) {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(string(actual), string(expected), true)
diffs := dmp.DiffMain(string(expected), string(actual), true)
t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs))
}
@ -84,25 +95,37 @@ func TestJsonImgsPresenter(t *testing.T) {
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Source: []file.Reference{
*img.SquashedTree().File("/somefile-1.txt"),
Locations: []source.Location{
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
},
Type: pkg.DebPkg,
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Licenses: []string{"MIT"},
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
},
})
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Source: []file.Reference{
*img.SquashedTree().File("/somefile-2.txt"),
Locations: []source.Location{
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
},
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "package-2",
Version: "2.0.1",
},
})
s, err := scope.NewScopeFromImage(img, scope.AllLayersScope)
s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
d := distro.NewUnknownDistro()
pres := NewPresenter(catalog, s, d)
pres := NewPresenter(catalog, s.Metadata, d)
// run presenter
err = pres.Present(&buffer)
@ -119,7 +142,7 @@ func TestJsonImgsPresenter(t *testing.T) {
if !bytes.Equal(expected, actual) {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(string(actual), string(expected), true)
diffs := dmp.DiffMain(string(expected), string(actual), true)
t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs))
}
}

58
syft/presenter/json/source.go
View File

@ -1,24 +1,33 @@
package json
import (
"encoding/json"
"fmt"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
// Source object represents the thing that was cataloged
type Source struct {
Type string `json:"type"`
Target interface{} `json:"target"`
}
func NewSource(s scope.Scope) (Source, error) {
switch src := s.Source.(type) {
case scope.ImageSource:
// sourceUnpacker is used to unmarshal Source objects
type sourceUnpacker struct {
Type string `json:"type"`
Target json.RawMessage `json:"target"`
}
// NewSource creates a new source object to be represented into JSON.
func NewSource(src source.Metadata) (Source, error) {
switch src.Scheme {
case source.ImageScheme:
return Source{
Type: "image",
Target: NewImage(src),
Target: src.ImageMetadata,
}, nil
case scope.DirSource:
case source.DirectoryScheme:
return Source{
Type: "directory",
Target: src.Path,
@ -27,3 +36,40 @@ func NewSource(s scope.Scope) (Source, error) {
return Source{}, fmt.Errorf("unsupported source: %T", src)
}
}
// UnmarshalJSON populates a source object from JSON bytes.
func (s *Source) UnmarshalJSON(b []byte) error {
var unpacker sourceUnpacker
if err := json.Unmarshal(b, &unpacker); err != nil {
return err
}
s.Type = unpacker.Type
switch s.Type {
case "image":
var payload source.ImageMetadata
if err := json.Unmarshal(unpacker.Target, &payload); err != nil {
return err
}
s.Target = payload
default:
return fmt.Errorf("unsupported package metadata type: %+v", s.Type)
}
return nil
}
// ToSourceMetadata takes a source object represented from JSON and creates a source.Metadata object.
func (s *Source) ToSourceMetadata() source.Metadata {
var metadata source.Metadata
switch s.Type {
case "directory":
metadata.Scheme = source.DirectoryScheme
metadata.Path = s.Target.(string)
case "image":
metadata.Scheme = source.ImageScheme
metadata.ImageMetadata = s.Target.(source.ImageMetadata)
}
return metadata
}

50
syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden
View File

@ -3,26 +3,50 @@
{
"name": "package-1",
"version": "1.0.1",
"type": "deb",
"foundBy": [
"the-cataloger-1"
],
"type": "python",
"foundBy": "the-cataloger-1",
"locations": [
"/some/path/pkg1"
{
"path": "/some/path/pkg1"
}
],
"licenses": null
"licenses": [
"MIT"
],
"language": "python",
"metadataType": "PythonPackageMetadata",
"metadata": {
"name": "package-1",
"version": "1.0.1",
"license": "",
"author": "",
"authorEmail": "",
"platform": "",
"sitePackagesRootPath": ""
}
},
{
"name": "package-2",
"version": "2.0.1",
"type": "deb",
"foundBy": [
"the-cataloger-2"
],
"foundBy": "the-cataloger-2",
"locations": [
"/some/path/pkg1"
{
"path": "/some/path/pkg1"
}
],
"licenses": null
"licenses": null,
"language": "",
"metadataType": "DpkgMetadata",
"metadata": {
"package": "package-2",
"source": "",
"version": "2.0.1",
"architecture": "",
"maintainer": "",
"installedSize": 0,
"files": null
}
}
],
"source": {
@ -33,5 +57,9 @@
"name": "",
"version": "",
"idLike": ""
},
"descriptor": {
"name": "syft",
"version": "[not provided]"
}
}

56
syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden
View File

@ -3,56 +3,78 @@
{
"name": "package-1",
"version": "1.0.1",
"type": "deb",
"foundBy": [
"the-cataloger-1"
],
"type": "python",
"foundBy": "the-cataloger-1",
"locations": [
{
"path": "/somefile-1.txt",
"layerIndex": 0
"layerID": "sha256:e158b57d6f5a96ef5fd22f2fe76c70b5ba6ff5b2619f9d83125b2aad0492ac7b"
}
],
"licenses": null
"licenses": [
"MIT"
],
"language": "python",
"metadataType": "PythonPackageMetadata",
"metadata": {
"name": "package-1",
"version": "1.0.1",
"license": "",
"author": "",
"authorEmail": "",
"platform": "",
"sitePackagesRootPath": ""
}
},
{
"name": "package-2",
"version": "2.0.1",
"type": "deb",
"foundBy": [
"the-cataloger-2"
],
"foundBy": "the-cataloger-2",
"locations": [
{
"path": "/somefile-2.txt",
"layerIndex": 1
"layerID": "sha256:da21056e7bf4308ecea0c0836848a7fe92f38fdcf35bc09ee6d98e7ab7beeebf"
}
],
"licenses": null
"licenses": null,
"language": "",
"metadataType": "DpkgMetadata",
"metadata": {
"package": "package-2",
"source": "",
"version": "2.0.1",
"architecture": "",
"maintainer": "",
"installedSize": 0,
"files": null
}
}
],
"source": {
"type": "image",
"target": {
"userInput": "user-image-input",
"scope": "AllLayers",
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:78783bfc74fef84f899b4977561ad1172f87753f82cc2157b06bf097e56dfbce",
"digest": "sha256:e158b57d6f5a96ef5fd22f2fe76c70b5ba6ff5b2619f9d83125b2aad0492ac7b",
"size": 22
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:54ec7f643dafbf9f27032a5e60afe06248c0e99b50aed54bb0fe28ea4825ccaf",
"digest": "sha256:da21056e7bf4308ecea0c0836848a7fe92f38fdcf35bc09ee6d98e7ab7beeebf",
"size": 16
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:ec4775a139c45b1ddf9ea8e1cb43385e92e5c0bf6ec2e3f4192372785b18c106",
"digest": "sha256:f0e18aa6032c24659a9c741fc36ca56f589782ea132061ccf6f52b952403da94",
"size": 27
}
],
"size": 65,
"digest": "sha256:fedd7bcc0b90f071501b662d8e7c9ac7548b88daba6b3deedfdf33f22ed8d95b",
"digest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"tags": [
"stereoscope-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7"
@ -63,5 +85,9 @@
"name": "",
"version": "",
"idLike": ""
},
"descriptor": {
"name": "syft",
"version": "[not provided]"
}
}

BIN
syft/presenter/json/test-fixtures/snapshot/anchore-fixture-image-simple.golden
View File

Binary file not shown.

BIN
syft/presenter/json/test-fixtures/snapshot/stereoscope-fixture-image-simple.golden
View File

Binary file not shown.

12
syft/presenter/presenter.go
View File

@ -15,7 +15,7 @@ import (
"github.com/anchore/syft/syft/presenter/json"
"github.com/anchore/syft/syft/presenter/table"
"github.com/anchore/syft/syft/presenter/text"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
// Presenter defines the expected behavior for an object responsible for displaying arbitrary input and processed data
@ -25,16 +25,16 @@ type Presenter interface {
}
// GetPresenter returns a presenter for images or directories
func GetPresenter(option Option, s scope.Scope, catalog *pkg.Catalog, d *distro.Distro) Presenter {
func GetPresenter(option Option, srcMetadata source.Metadata, catalog *pkg.Catalog, d distro.Distro) Presenter {
switch option {
case JSONPresenter:
return json.NewPresenter(catalog, s, *d)
return json.NewPresenter(catalog, srcMetadata, d)
case TextPresenter:
return text.NewPresenter(catalog, s)
return text.NewPresenter(catalog, srcMetadata)
case TablePresenter:
return table.NewPresenter(catalog, s)
return table.NewPresenter(catalog)
case CycloneDxPresenter:
return cyclonedx.NewPresenter(catalog, s, *d)
return cyclonedx.NewPresenter(catalog, srcMetadata, d)
default:
return nil
}

5
syft/presenter/table/presenter.go
View File

@ -9,18 +9,15 @@ import (
"github.com/olekukonko/tablewriter"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
)
type Presenter struct {
catalog *pkg.Catalog
scope scope.Scope
}
func NewPresenter(catalog *pkg.Catalog, s scope.Scope) *Presenter {
func NewPresenter(catalog *pkg.Catalog) *Presenter {
return &Presenter{
catalog: catalog,
scope: s,
}
}

19
syft/presenter/table/presenter_test.go
View File

@ -3,14 +3,14 @@ package table
import (
"bytes"
"flag"
"github.com/go-test/deep"
"testing"
"github.com/go-test/deep"
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
"github.com/sergi/go-diff/diffmatchpatch"
)
@ -29,25 +29,24 @@ func TestTablePresenter(t *testing.T) {
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Source: []file.Reference{
*img.SquashedTree().File("/somefile-1.txt"),
Locations: []source.Location{
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
},
Type: pkg.DebPkg,
})
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Source: []file.Reference{
*img.SquashedTree().File("/somefile-2.txt"),
Locations: []source.Location{
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
},
Type: pkg.DebPkg,
})
s, err := scope.NewScopeFromImage(img, scope.AllLayersScope)
pres := NewPresenter(catalog, s)
pres := NewPresenter(catalog)
// run presenter
err = pres.Present(&buffer)
err := pres.Present(&buffer)
if err != nil {
t.Fatal(err)
}

28
syft/presenter/text/presenter.go
View File

@ -7,18 +7,20 @@ import (
"text/tabwriter"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
)
// Presenter is a human-friendly text presenter to represent package and source data.
type Presenter struct {
catalog *pkg.Catalog
scope scope.Scope
srcMetadata source.Metadata
}
func NewPresenter(catalog *pkg.Catalog, s scope.Scope) *Presenter {
// NewPresenter creates a new presenter for the given set of catalog and image data.
func NewPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata) *Presenter {
return &Presenter{
catalog: catalog,
scope: s,
srcMetadata: srcMetadata,
}
}
@ -28,22 +30,22 @@ func (pres *Presenter) Present(output io.Writer) error {
w := new(tabwriter.Writer)
w.Init(output, 0, 8, 0, '\t', tabwriter.AlignRight)
switch src := pres.scope.Source.(type) {
case scope.DirSource:
fmt.Fprintln(w, fmt.Sprintf("[Path: %s]", src.Path))
case scope.ImageSource:
switch pres.srcMetadata.Scheme {
case source.DirectoryScheme:
fmt.Fprintln(w, fmt.Sprintf("[Path: %s]", pres.srcMetadata.Path))
case source.ImageScheme:
fmt.Fprintln(w, "[Image]")
for idx, l := range src.Img.Layers {
for idx, l := range pres.srcMetadata.ImageMetadata.Layers {
fmt.Fprintln(w, " Layer:\t", idx)
fmt.Fprintln(w, " Digest:\t", l.Metadata.Digest)
fmt.Fprintln(w, " Size:\t", l.Metadata.Size)
fmt.Fprintln(w, " MediaType:\t", l.Metadata.MediaType)
fmt.Fprintln(w, " Digest:\t", l.Digest)
fmt.Fprintln(w, " Size:\t", l.Size)
fmt.Fprintln(w, " MediaType:\t", l.MediaType)
fmt.Fprintln(w)
w.Flush()
}
default:
return fmt.Errorf("unsupported source: %T", src)
return fmt.Errorf("unsupported source: %T", pres.srcMetadata.Scheme)
}
// populate artifacts...

21
syft/presenter/text/presenter_test.go
View File

@ -6,10 +6,9 @@ import (
"testing"
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/syft/syft/source"
"github.com/sergi/go-diff/diffmatchpatch"
)
@ -32,11 +31,11 @@ func TestTextDirPresenter(t *testing.T) {
Type: pkg.DebPkg,
})
s, err := scope.NewScopeFromDir("/some/path")
s, err := source.NewFromDirectory("/some/path")
if err != nil {
t.Fatalf("unable to create scope: %+v", err)
t.Fatalf("unable to create source: %+v", err)
}
pres := NewPresenter(catalog, s)
pres := NewPresenter(catalog, s.Metadata)
// run presenter
err = pres.Present(&buffer)
@ -75,8 +74,8 @@ func TestTextImgPresenter(t *testing.T) {
catalog.Add(pkg.Package{
Name: "package-1",
Version: "1.0.1",
Source: []file.Reference{
*img.SquashedTree().File("/somefile-1.txt"),
Locations: []source.Location{
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
},
FoundBy: "dpkg",
Type: pkg.DebPkg,
@ -84,8 +83,8 @@ func TestTextImgPresenter(t *testing.T) {
catalog.Add(pkg.Package{
Name: "package-2",
Version: "2.0.1",
Source: []file.Reference{
*img.SquashedTree().File("/somefile-2.txt"),
Locations: []source.Location{
source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
},
FoundBy: "dpkg",
Metadata: PackageInfo{Name: "package-2", Version: "1.0.2"},
@ -98,11 +97,11 @@ func TestTextImgPresenter(t *testing.T) {
l.Metadata.Digest = "sha256:ad8ecdc058976c07e7e347cb89fa9ad86a294b5ceaae6d09713fb035f84115abf3c4a2388a4af3aa60f13b94f4c6846930bdf53"
}
s, err := scope.NewScopeFromImage(img, scope.AllLayersScope)
s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
if err != nil {
t.Fatal(err)
}
pres := NewPresenter(catalog, s)
pres := NewPresenter(catalog, s.Metadata)
// run presenter
err = pres.Present(&buffer)
if err != nil {

40
syft/scope/option.go
View File

@ -1,40 +0,0 @@
package scope
import "strings"
const (
UnknownScope Option = iota
SquashedScope
AllLayersScope
)
type Option int
var optionStr = []string{
"UnknownScope",
"Squashed",
"AllLayers",
}
var Options = []Option{
SquashedScope,
AllLayersScope,
}
func ParseOption(userStr string) Option {
switch strings.ToLower(userStr) {
case strings.ToLower(SquashedScope.String()):
return SquashedScope
case "all-layers", strings.ToLower(AllLayersScope.String()):
return AllLayersScope
}
return UnknownScope
}
func (o Option) String() string {
if int(o) >= len(optionStr) || o < 0 {
return optionStr[0]
}
return optionStr[o]
}

17
syft/scope/option_test.go
View File

@ -1,17 +0,0 @@
package scope
import (
"fmt"
"testing"
)
func TestOptionStringerBoundary(t *testing.T) {
var _ fmt.Stringer = Option(0)
for _, c := range []int{-1, 0, 3} {
option := Option(c)
if option.String() != UnknownScope.String() {
t.Errorf("expected Option(%d) to be unknown, found '%+v'", c, option)
}
}
}

127
syft/scope/resolvers/directory_resolver.go
View File

@ -1,127 +0,0 @@
package resolvers
import (
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal/log"
"github.com/bmatcuk/doublestar"
)
// DirectoryResolver implements path and content access for the directory data source.
type DirectoryResolver struct {
Path string
}
// Stringer to represent a directory path data source
func (s DirectoryResolver) String() string {
return fmt.Sprintf("dir:%s", s.Path)
}
// FilesByPath returns all file.References that match the given paths from the directory.
func (s DirectoryResolver) FilesByPath(userPaths ...file.Path) ([]file.Reference, error) {
var references = make([]file.Reference, 0)
for _, userPath := range userPaths {
userStrPath := string(userPath)
if filepath.IsAbs(userStrPath) {
// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
userStrPath = path.Join(s.Path, userStrPath)
}
fileMeta, err := os.Stat(userStrPath)
if os.IsNotExist(err) {
continue
} else if err != nil {
log.Errorf("path (%s) is not valid: %v", userStrPath, err)
}
// don't consider directories
if fileMeta.IsDir() {
continue
}
references = append(references, file.NewFileReference(file.Path(userStrPath)))
}
return references, nil
}
func fileContents(path file.Path) ([]byte, error) {
contents, err := ioutil.ReadFile(string(path))
if err != nil {
return nil, err
}
return contents, nil
}
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
result := make([]file.Reference, 0)
for _, pattern := range patterns {
pathPattern := path.Join(s.Path, pattern)
matches, err := doublestar.Glob(pathPattern)
if err != nil {
return result, err
}
for _, match := range matches {
fileMeta, err := os.Stat(match)
if err != nil {
continue
}
// don't consider directories
if fileMeta.IsDir() {
continue
}
matchedPath := file.Path(match)
result = append(result, file.NewFileReference(matchedPath))
}
}
return result, nil
}
func (s *DirectoryResolver) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) {
paths, err := s.FilesByPath(file.Path(path))
if err != nil {
return nil, err
}
if len(paths) == 0 {
return nil, nil
}
return &paths[0], nil
}
// MultipleFileContentsByRef returns the file contents for all file.References relative a directory.
func (s DirectoryResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) {
refContents := make(map[file.Reference]string)
for _, fileRef := range f {
contents, err := fileContents(fileRef.Path)
if err != nil {
return nil, fmt.Errorf("could not read contents of file: %s", fileRef.Path)
}
refContents[fileRef] = string(contents)
}
return refContents, nil
}
// FileContentsByRef fetches file contents for a single file reference relative to a directory.
// If the path does not exist an error is returned.
func (s DirectoryResolver) FileContentsByRef(reference file.Reference) (string, error) {
contents, err := fileContents(reference.Path)
if err != nil {
return "", fmt.Errorf("could not read contents of file: %s", reference.Path)
}
return string(contents), nil
}

4
syft/scope/resolvers/docs.go
View File

@ -1,4 +0,0 @@
/*
Package resolvers provides concrete implementations for the scope.Resolver interface for all supported data sources and scope options.
*/
package resolvers

166
syft/scope/scope.go
View File

@ -1,166 +0,0 @@
/*
Package scope provides an abstraction to allow a user to loosely define a data source to catalog and expose a common interface that
catalogers and use explore and analyze data from the data source. All valid (cataloggable) data sources are defined
within this package.
*/
package scope
import (
"fmt"
"strings"
"github.com/mitchellh/go-homedir"
"github.com/spf13/afero"
"github.com/anchore/stereoscope"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/syft/scope/resolvers"
)
const (
UnknownScheme Scheme = "unknown-scheme"
DirectoryScheme Scheme = "directory-scheme"
ImageScheme Scheme = "image-scheme"
)
type Scheme string
// ImageSource represents a data source that is a container image
type ImageSource struct {
Img *image.Image // the image object to be cataloged
}
// DirSource represents a data source that is a filesystem directory tree
type DirSource struct {
Path string // the root path to be cataloged
}
// Scope is an object that captures the data source to be cataloged, configuration, and a specific resolver used
// in cataloging (based on the data source and configuration)
type Scope struct {
Option Option // specific perspective to catalog
Resolver Resolver // a Resolver object to use in file path/glob resolution and file contents resolution
Source interface{} // the specific source object to be cataloged
Scheme Scheme // the source data scheme type (directory or image)
}
// NewScope produces a Scope based on userInput like dir: or image:tag
func NewScope(userInput string, o Option) (Scope, func(), error) {
fs := afero.NewOsFs()
parsedScheme, location, err := detectScheme(fs, image.DetectSource, userInput)
if err != nil {
return Scope{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err)
}
switch parsedScheme {
case DirectoryScheme:
fileMeta, err := fs.Stat(location)
if err != nil {
return Scope{}, func() {}, fmt.Errorf("unable to stat dir=%q: %w", location, err)
}
if !fileMeta.IsDir() {
return Scope{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err)
}
s, err := NewScopeFromDir(location)
if err != nil {
return Scope{}, func() {}, fmt.Errorf("could not populate scope from path=%q: %w", location, err)
}
return s, func() {}, nil
case ImageScheme:
img, err := stereoscope.GetImage(location)
cleanup := func() {
stereoscope.Cleanup()
}
if err != nil || img == nil {
return Scope{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err)
}
s, err := NewScopeFromImage(img, o)
if err != nil {
return Scope{}, cleanup, fmt.Errorf("could not populate scope with image: %w", err)
}
return s, cleanup, nil
}
return Scope{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
}
// NewScopeFromDir creates a new scope object tailored to catalog a given filesystem directory recursively.
func NewScopeFromDir(path string) (Scope, error) {
return Scope{
Resolver: &resolvers.DirectoryResolver{
Path: path,
},
Source: DirSource{
Path: path,
},
Scheme: DirectoryScheme,
}, nil
}
// NewScopeFromImage creates a new scope object tailored to catalog a given container image, relative to the
// option given (e.g. all-layers, squashed, etc)
func NewScopeFromImage(img *image.Image, option Option) (Scope, error) {
if img == nil {
return Scope{}, fmt.Errorf("no image given")
}
resolver, err := getImageResolver(img, option)
if err != nil {
return Scope{}, fmt.Errorf("could not determine file resolver: %w", err)
}
return Scope{
Option: option,
Resolver: resolver,
Source: ImageSource{
Img: img,
},
Scheme: ImageScheme,
}, nil
}
type sourceDetector func(string) (image.Source, string, error)
func detectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, string, error) {
if strings.HasPrefix(userInput, "dir:") {
// blindly trust the user's scheme
dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:"))
if err != nil {
return UnknownScheme, "", fmt.Errorf("unable to expand directory path: %w", err)
}
return DirectoryScheme, dirLocation, nil
}
// we should attempt to let stereoscope determine what the source is first --just because the source is a valid directory
// doesn't mean we yet know if it is an OCI layout directory (to be treated as an image) or if it is a generic filesystem directory.
source, imageSpec, err := imageDetector(userInput)
if err != nil {
return UnknownScheme, "", fmt.Errorf("unable to detect the scheme from %q: %w", userInput, err)
}
if source == image.UnknownSource {
dirLocation, err := homedir.Expand(userInput)
if err != nil {
return UnknownScheme, "", fmt.Errorf("unable to expand potential directory path: %w", err)
}
fileMeta, err := fs.Stat(dirLocation)
if err != nil {
return UnknownScheme, "", nil
}
if fileMeta.IsDir() {
return DirectoryScheme, dirLocation, nil
}
return UnknownScheme, "", nil
}
return ImageScheme, imageSpec, nil
}

94
syft/scope/resolvers/all_layers_resolver.go → syft/source/all_layers_resolver.go
View File

@ -1,4 +1,4 @@
package resolvers
package source
import (
"archive/tar"
@ -8,7 +8,9 @@ import (
"github.com/anchore/stereoscope/pkg/image"
)
// AllLayersResolver implements path and content access for the AllLayers scope option for container image data sources.
var _ Resolver = (*AllLayersResolver)(nil)
// AllLayersResolver implements path and content access for the AllLayers source option for container image data sources.
type AllLayersResolver struct {
img *image.Image
layers []int
@ -41,7 +43,7 @@ func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.Ref
if entry.Metadata.TypeFlag == tar.TypeLink || entry.Metadata.TypeFlag == tar.TypeSymlink {
// a link may resolve in this layer or higher, assuming a squashed tree is used to search
// we should search all possible resolutions within the valid scope
// we should search all possible resolutions within the valid source
for _, subLayerIdx := range r.layers[layerIdx:] {
resolvedRef, err := r.img.ResolveLinkByLayerSquash(ref, subLayerIdx)
if err != nil {
@ -61,14 +63,14 @@ func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.Ref
}
// FilesByPath returns all file.References that match the given paths from any layer in the image.
func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueFiles := make([]file.Reference, 0)
uniqueLocations := make([]Location, 0)
for _, path := range paths {
for idx, layerIdx := range r.layers {
tree := r.img.Layers[layerIdx].Tree
ref := tree.File(path)
ref := tree.File(file.Path(path))
if ref == nil {
// no file found, keep looking through layers
continue
@ -91,17 +93,19 @@ func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, e
if err != nil {
return nil, err
}
uniqueFiles = append(uniqueFiles, results...)
for _, result := range results {
uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img))
}
}
return uniqueFiles, nil
}
return uniqueLocations, nil
}
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
// nolint:gocognit
func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueFiles := make([]file.Reference, 0)
uniqueLocations := make([]Location, 0)
for _, pattern := range patterns {
for idx, layerIdx := range r.layers {
@ -128,31 +132,65 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, e
if err != nil {
return nil, err
}
uniqueFiles = append(uniqueFiles, results...)
for _, result := range results {
uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img))
}
}
}
}
return uniqueFiles, nil
return uniqueLocations, nil
}
func (r *AllLayersResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) {
entry, err := r.img.FileCatalog.Get(reference)
// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
// This is helpful when attempting to find a file that is in the same layer or lower as another file.
func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *Location {
entry, err := r.img.FileCatalog.Get(location.ref)
if err != nil {
return nil
}
relativeRef := entry.Source.SquashedTree.File(file.Path(path))
if relativeRef == nil {
return nil
}
relativeLocation := NewLocationFromImage(*relativeRef, r.img)
return &relativeLocation
}
// MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
// file.Reference is a path relative to a particular layer.
func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
}
// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
// If the path does not exist an error is returned.
func (r *AllLayersResolver) FileContentsByLocation(location Location) (string, error) {
return r.img.FileContentsByRef(location.ref)
}
type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]string, error)
func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]string, error) {
var fileRefs = make([]file.Reference, len(locations))
var locationByRefs = make(map[file.Reference]Location)
var results = make(map[Location]string)
for i, location := range locations {
locationByRefs[location.ref] = location
fileRefs[i] = location.ref
}
contentsByRef, err := callback(fileRefs...)
if err != nil {
return nil, err
}
return entry.Source.SquashedTree.File(file.Path(path)), nil
}
// MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a
// file.Reference is a path relative to a particular layer.
func (r *AllLayersResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) {
return r.img.MultipleFileContentsByRef(f...)
}
// FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer.
// If the path does not exist an error is returned.
func (r *AllLayersResolver) FileContentsByRef(ref file.Reference) (string, error) {
return r.img.FileContentsByRef(ref)
for ref, content := range contentsByRef {
results[locationByRefs[ref]] = content
}
return results, nil
}

14
syft/scope/resolvers/all_layers_resolver_test.go → syft/source/all_layers_resolver_test.go
View File

@ -1,11 +1,9 @@
package resolvers
package source
import (
"testing"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/stereoscope/pkg/file"
)
type resolution struct {
@ -97,7 +95,7 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
t.Fatalf("could not create resolver: %+v", err)
}
refs, err := resolver.FilesByPath(file.Path(c.linkPath))
refs, err := resolver.FilesByPath(c.linkPath)
if err != nil {
t.Fatalf("could not use resolver: %+v", err)
}
@ -109,11 +107,11 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
for idx, actual := range refs {
expected := c.resolutions[idx]
if actual.Path != file.Path(expected.path) {
if actual.Path != expected.path {
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
}
entry, err := img.FileCatalog.Get(actual)
entry, err := img.FileCatalog.Get(actual.ref)
if err != nil {
t.Fatalf("failed to get metadata: %+v", err)
}
@ -222,11 +220,11 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
for idx, actual := range refs {
expected := c.resolutions[idx]
if actual.Path != file.Path(expected.path) {
if actual.Path != expected.path {
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
}
entry, err := img.FileCatalog.Get(actual)
entry, err := img.FileCatalog.Get(actual.ref)
if err != nil {
t.Fatalf("failed to get metadata: %+v", err)
}

130
syft/source/directory_resolver.go Normal file
View File

@ -0,0 +1,130 @@
package source
import (
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"github.com/anchore/syft/internal/log"
"github.com/bmatcuk/doublestar"
)
var _ Resolver = (*DirectoryResolver)(nil)
// DirectoryResolver implements path and content access for the directory data source.
type DirectoryResolver struct {
Path string
}
// Stringer to represent a directory path data source
func (s DirectoryResolver) String() string {
return fmt.Sprintf("dir:%s", s.Path)
}
// FilesByPath returns all file.References that match the given paths from the directory.
func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) {
var references = make([]Location, 0)
for _, userPath := range userPaths {
userStrPath := userPath
if filepath.IsAbs(userStrPath) {
// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
userStrPath = path.Join(s.Path, userStrPath)
}
fileMeta, err := os.Stat(userStrPath)
if os.IsNotExist(err) {
continue
} else if err != nil {
log.Errorf("path (%s) is not valid: %v", userStrPath, err)
}
// don't consider directories
if fileMeta.IsDir() {
continue
}
references = append(references, NewLocation(userStrPath))
}
return references, nil
}
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
result := make([]Location, 0)
for _, pattern := range patterns {
pathPattern := path.Join(s.Path, pattern)
pathMatches, err := doublestar.Glob(pathPattern)
if err != nil {
return nil, err
}
for _, matchedPath := range pathMatches {
fileMeta, err := os.Stat(matchedPath)
if err != nil {
continue
}
// don't consider directories
if fileMeta.IsDir() {
continue
}
result = append(result, NewLocation(matchedPath))
}
}
return result, nil
}
// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
// This is helpful when attempting to find a file that is in the same layer or lower as another file. For the
// DirectoryResolver, this is a simple path lookup.
func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location {
paths, err := s.FilesByPath(path)
if err != nil {
return nil
}
if len(paths) == 0 {
return nil
}
return &paths[0]
}
// MultipleFileContentsByLocation returns the file contents for all file.References relative a directory.
func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
refContents := make(map[Location]string)
for _, location := range locations {
contents, err := fileContents(location.Path)
if err != nil {
return nil, fmt.Errorf("could not read contents of file: %s", location.Path)
}
refContents[location] = string(contents)
}
return refContents, nil
}
// FileContentsByLocation fetches file contents for a single file reference relative to a directory.
// If the path does not exist an error is returned.
func (s DirectoryResolver) FileContentsByLocation(location Location) (string, error) {
contents, err := fileContents(location.Path)
if err != nil {
return "", fmt.Errorf("could not read contents of file: %s", location.Path)
}
return string(contents), nil
}
func fileContents(path string) ([]byte, error) {
contents, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
return contents, nil
}

34
syft/scope/resolvers/directory_resolver_test.go → syft/source/directory_resolver_test.go
View File

@ -1,9 +1,7 @@
package resolvers
package source
import (
"testing"
"github.com/anchore/stereoscope/pkg/file"
)
func TestDirectoryResolver_FilesByPath(t *testing.T) {
@ -58,7 +56,7 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resolver := DirectoryResolver{c.root}
refs, err := resolver.FilesByPath(file.Path(c.input))
refs, err := resolver.FilesByPath(c.input)
if err != nil {
t.Fatalf("could not use resolver: %+v, %+v", err, refs)
}
@ -68,7 +66,7 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
}
for _, actual := range refs {
if actual.Path != file.Path(c.expected) {
if actual.Path != c.expected {
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.expected)
}
}
@ -79,22 +77,22 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
cases := []struct {
name string
input []file.Path
input []string
refCount int
}{
{
name: "finds multiple files",
input: []file.Path{file.Path("test-fixtures/image-symlinks/file-1.txt"), file.Path("test-fixtures/image-symlinks/file-2.txt")},
input: []string{"test-fixtures/image-symlinks/file-1.txt", "test-fixtures/image-symlinks/file-2.txt"},
refCount: 2,
},
{
name: "skips non-existing files",
input: []file.Path{file.Path("test-fixtures/image-symlinks/bogus.txt"), file.Path("test-fixtures/image-symlinks/file-1.txt")},
input: []string{"test-fixtures/image-symlinks/bogus.txt", "test-fixtures/image-symlinks/file-1.txt"},
refCount: 1,
},
{
name: "does not return anything for non-existing directories",
input: []file.Path{file.Path("test-fixtures/non-existing/bogus.txt"), file.Path("test-fixtures/non-existing/file-1.txt")},
input: []string{"test-fixtures/non-existing/bogus.txt", "test-fixtures/non-existing/file-1.txt"},
refCount: 0,
},
}
@ -117,47 +115,47 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
func TestDirectoryResolver_MultipleFileContentsByRef(t *testing.T) {
cases := []struct {
name string
input []file.Path
input []string
refCount int
contents []string
}{
{
name: "gets multiple file contents",
input: []file.Path{file.Path("test-fixtures/image-symlinks/file-1.txt"), file.Path("test-fixtures/image-symlinks/file-2.txt")},
input: []string{"test-fixtures/image-symlinks/file-1.txt", "test-fixtures/image-symlinks/file-2.txt"},
refCount: 2,
},
{
name: "skips non-existing files",
input: []file.Path{file.Path("test-fixtures/image-symlinks/bogus.txt"), file.Path("test-fixtures/image-symlinks/file-1.txt")},
input: []string{"test-fixtures/image-symlinks/bogus.txt", "test-fixtures/image-symlinks/file-1.txt"},
refCount: 1,
},
{
name: "does not return anything for non-existing directories",
input: []file.Path{file.Path("test-fixtures/non-existing/bogus.txt"), file.Path("test-fixtures/non-existing/file-1.txt")},
input: []string{"test-fixtures/non-existing/bogus.txt", "test-fixtures/non-existing/file-1.txt"},
refCount: 0,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
refs := make([]file.Reference, 0)
locations := make([]Location, 0)
resolver := DirectoryResolver{"test-fixtures"}
for _, p := range c.input {
newRefs, err := resolver.FilesByPath(p)
if err != nil {
t.Errorf("could not generate refs: %+v", err)
t.Errorf("could not generate locations: %+v", err)
}
for _, ref := range newRefs {
refs = append(refs, ref)
locations = append(locations, ref)
}
}
contents, err := resolver.MultipleFileContentsByRef(refs...)
contents, err := resolver.MultipleFileContentsByLocation(locations)
if err != nil {
t.Fatalf("unable to generate file contents by ref: %+v", err)
}
if len(contents) != c.refCount {
t.Errorf("unexpected number of refs produced: %d != %d", len(contents), c.refCount)
t.Errorf("unexpected number of locations produced: %d != %d", len(contents), c.refCount)
}
})

50
syft/source/image_metadata.go Normal file
View File

@ -0,0 +1,50 @@
package source
import "github.com/anchore/stereoscope/pkg/image"
// ImageMetadata represents all static metadata that defines what a container image is. This is useful to later describe
// "what" was cataloged without needing the more complicated stereoscope Image objects or Resolver objects.
type ImageMetadata struct {
UserInput string `json:"userInput"`
Scope Scope `json:"scope"` // specific perspective to catalog
Layers []LayerMetadata `json:"layers"`
Size int64 `json:"size"`
Digest string `json:"digest"`
MediaType string `json:"mediaType"`
Tags []string `json:"tags"`
}
// LayerMetadata represents all static metadata that defines what a container image layer is.
type LayerMetadata struct {
MediaType string `json:"mediaType"`
Digest string `json:"digest"`
Size int64 `json:"size"`
}
// NewImageMetadata creates a new ImageMetadata object populated from the given stereoscope Image object and user configuration.
func NewImageMetadata(img *image.Image, userInput string, scope Scope) ImageMetadata {
// populate artifacts...
tags := make([]string, len(img.Metadata.Tags))
for idx, tag := range img.Metadata.Tags {
tags[idx] = tag.String()
}
theImg := ImageMetadata{
UserInput: userInput,
Scope: scope,
Digest: img.Metadata.Digest,
Size: img.Metadata.Size,
MediaType: string(img.Metadata.MediaType),
Tags: tags,
Layers: make([]LayerMetadata, len(img.Layers)),
}
// populate image metadata
for idx, l := range img.Layers {
theImg.Layers[idx] = LayerMetadata{
MediaType: string(l.Metadata.MediaType),
Digest: l.Metadata.Digest,
Size: l.Metadata.Size,
}
}
return theImg
}

57
syft/scope/resolvers/image_squash_resolver.go → syft/source/image_squash_resolver.go
View File

@ -1,4 +1,4 @@
package resolvers
package source
import (
"fmt"
@ -7,7 +7,9 @@ import (
"github.com/anchore/stereoscope/pkg/image"
)
// ImageSquashResolver implements path and content access for the Squashed scope option for container image data sources.
var _ Resolver = (*ImageSquashResolver)(nil)
// ImageSquashResolver implements path and content access for the Squashed source option for container image data sources.
type ImageSquashResolver struct {
img *image.Image
}
@ -21,13 +23,13 @@ func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) {
}
// FilesByPath returns all file.References that match the given paths within the squashed representation of the image.
func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueFiles := make([]file.Reference, 0)
uniqueLocations := make([]Location, 0)
for _, path := range paths {
tree := r.img.SquashedTree()
ref := tree.File(path)
ref := tree.File(file.Path(path))
if ref == nil {
// no file found, keep looking through layers
continue
@ -54,17 +56,17 @@ func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference,
if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
uniqueFileIDs.Add(*resolvedRef)
uniqueFiles = append(uniqueFiles, *resolvedRef)
uniqueLocations = append(uniqueLocations, NewLocationFromImage(*resolvedRef, r.img))
}
}
return uniqueFiles, nil
return uniqueLocations, nil
}
// FilesByGlob returns all file.References that match the given path glob pattern within the squashed representation of the image.
func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error) {
uniqueFileIDs := file.NewFileReferenceSet()
uniqueFiles := make([]file.Reference, 0)
uniqueLocations := make([]Location, 0)
for _, pattern := range patterns {
refs, err := r.img.SquashedTree().FilesByGlob(pattern)
@ -86,42 +88,45 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference,
}
}
resolvedRefs, err := r.FilesByPath(ref.Path)
resolvedLocations, err := r.FilesByPath(string(ref.Path))
if err != nil {
return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err)
}
for _, resolvedRef := range resolvedRefs {
if !uniqueFileIDs.Contains(resolvedRef) {
uniqueFileIDs.Add(resolvedRef)
uniqueFiles = append(uniqueFiles, resolvedRef)
for _, resolvedLocation := range resolvedLocations {
if !uniqueFileIDs.Contains(resolvedLocation.ref) {
uniqueFileIDs.Add(resolvedLocation.ref)
uniqueLocations = append(uniqueLocations, resolvedLocation)
}
}
}
}
return uniqueFiles, nil
return uniqueLocations, nil
}
func (r *ImageSquashResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) {
paths, err := r.FilesByPath(file.Path(path))
// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
// This is helpful when attempting to find a file that is in the same layer or lower as another file. For the
// ImageSquashResolver, this is a simple path lookup.
func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Location {
paths, err := r.FilesByPath(path)
if err != nil {
return nil, err
return nil
}
if len(paths) == 0 {
return nil, nil
return nil
}
return &paths[0], nil
return &paths[0]
}
// MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a
// MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
// file.Reference is a path relative to a particular layer, in this case only from the squashed representation.
func (r *ImageSquashResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) {
return r.img.MultipleFileContentsByRef(f...)
func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
}
// FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer.
// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
// If the path does not exist an error is returned.
func (r *ImageSquashResolver) FileContentsByRef(ref file.Reference) (string, error) {
return r.img.FileContentsByRef(ref)
func (r *ImageSquashResolver) FileContentsByLocation(location Location) (string, error) {
return r.img.FileContentsByRef(location.ref)
}

14
syft/scope/resolvers/image_squash_resolver_test.go → syft/source/image_squash_resolver_test.go
View File

@ -1,11 +1,9 @@
package resolvers
package source
import (
"testing"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/stereoscope/pkg/file"
)
func TestImageSquashResolver_FilesByPath(t *testing.T) {
@ -61,7 +59,7 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
t.Fatalf("could not create resolver: %+v", err)
}
refs, err := resolver.FilesByPath(file.Path(c.linkPath))
refs, err := resolver.FilesByPath(c.linkPath)
if err != nil {
t.Fatalf("could not use resolver: %+v", err)
}
@ -82,11 +80,11 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
actual := refs[0]
if actual.Path != file.Path(c.resolvePath) {
if actual.Path != c.resolvePath {
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
}
entry, err := img.FileCatalog.Get(actual)
entry, err := img.FileCatalog.Get(actual.ref)
if err != nil {
t.Fatalf("failed to get metadata: %+v", err)
}
@ -172,11 +170,11 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
actual := refs[0]
if actual.Path != file.Path(c.resolvePath) {
if actual.Path != c.resolvePath {
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
}
entry, err := img.FileCatalog.Get(actual)
entry, err := img.FileCatalog.Get(actual.ref)
if err != nil {
t.Fatalf("failed to get metadata: %+v", err)
}

40
syft/source/location.go Normal file
View File

@ -0,0 +1,40 @@
package source
import (
"github.com/anchore/syft/internal/log"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/image"
)
// Location represents a path relative to a particular filesystem.
type Location struct {
Path string `json:"path"` // The string path of the location (e.g. /etc/hosts)
FileSystemID string `json:"layerID,omitempty"` // An ID representing the filesystem. For container images this is a layer digest, directories or root filesystem this is blank.
ref file.Reference // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
}
// NewLocation creates a new Location representing a path without denoting a filesystem or FileCatalog reference.
func NewLocation(path string) Location {
return Location{
Path: path,
}
}
// NewLocationFromImage creates a new Location representing the given path (extracted from the ref) relative to the given image.
func NewLocationFromImage(ref file.Reference, img *image.Image) Location {
entry, err := img.FileCatalog.Get(ref)
if err != nil {
log.Warnf("unable to find file catalog entry for ref=%+v", ref)
return Location{
Path: string(ref.Path),
ref: ref,
}
}
return Location{
Path: string(ref.Path),
FileSystemID: entry.Source.Metadata.Digest,
ref: ref,
}
}

8
syft/source/metadata.go Normal file
View File

@ -0,0 +1,8 @@
package source
// Metadata represents any static source data that helps describe "what" was cataloged.
type Metadata struct {
Scheme Scheme // the source data scheme type (directory or image)
ImageMetadata ImageMetadata // all image info (image only)
Path string // the root path to be cataloged (directory only)
}

26
syft/scope/resolver.go → syft/source/resolver.go
View File

@ -1,11 +1,9 @@
package scope
package source
import (
"fmt"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/syft/scope/resolvers"
)
// Resolver is an interface that encompasses how to get specific file references and file contents for a generic data source.
@ -16,30 +14,30 @@ type Resolver interface {
// ContentResolver knows how to get file content for given file.References
type ContentResolver interface {
FileContentsByRef(ref file.Reference) (string, error)
MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error)
FileContentsByLocation(Location) (string, error)
MultipleFileContentsByLocation([]Location) (map[Location]string, error)
// TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering).
}
// FileResolver knows how to get file.References for given string paths and globs
type FileResolver interface {
// FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches)
FilesByPath(paths ...file.Path) ([]file.Reference, error)
FilesByPath(paths ...string) ([]Location, error)
// FilesByGlob fetches a set of file references which the given glob matches
FilesByGlob(patterns ...string) ([]file.Reference, error)
FilesByGlob(patterns ...string) ([]Location, error)
// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
// This is helpful when attempting to find a file that is in the same layer or lower as another file.
RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error)
RelativeFileByPath(_ Location, path string) *Location
}
// getImageResolver returns the appropriate resolve for a container image given the scope option
func getImageResolver(img *image.Image, option Option) (Resolver, error) {
switch option {
// getImageResolver returns the appropriate resolve for a container image given the source option
func getImageResolver(img *image.Image, scope Scope) (Resolver, error) {
switch scope {
case SquashedScope:
return resolvers.NewImageSquashResolver(img)
return NewImageSquashResolver(img)
case AllLayersScope:
return resolvers.NewAllLayersResolver(img)
return NewAllLayersResolver(img)
default:
return nil, fmt.Errorf("bad option provided: %+v", option)
return nil, fmt.Errorf("bad scope provided: %+v", scope)
}
}

59
syft/source/scheme.go Normal file
View File

@ -0,0 +1,59 @@
package source
import (
"fmt"
"strings"
"github.com/anchore/stereoscope/pkg/image"
"github.com/mitchellh/go-homedir"
"github.com/spf13/afero"
)
// Scheme represents the optional prefixed string at the beginning of a user request (e.g. "docker:").
type Scheme string
const (
// UnknownScheme is the default scheme
UnknownScheme Scheme = "UnknownScheme"
// DirectoryScheme indicates the source being cataloged is a directory on the root filesystem
DirectoryScheme Scheme = "DirectoryScheme"
// ImageScheme indicates the source being cataloged is a container image
ImageScheme Scheme = "ImageScheme"
)
func detectScheme(fs afero.Fs, imageDetector sourceDetector, userInput string) (Scheme, string, error) {
if strings.HasPrefix(userInput, "dir:") {
// blindly trust the user's scheme
dirLocation, err := homedir.Expand(strings.TrimPrefix(userInput, "dir:"))
if err != nil {
return UnknownScheme, "", fmt.Errorf("unable to expand directory path: %w", err)
}
return DirectoryScheme, dirLocation, nil
}
// we should attempt to let stereoscope determine what the source is first --just because the source is a valid directory
// doesn't mean we yet know if it is an OCI layout directory (to be treated as an image) or if it is a generic filesystem directory.
source, imageSpec, err := imageDetector(userInput)
if err != nil {
return UnknownScheme, "", fmt.Errorf("unable to detect the scheme from %q: %w", userInput, err)
}
if source == image.UnknownSource {
dirLocation, err := homedir.Expand(userInput)
if err != nil {
return UnknownScheme, "", fmt.Errorf("unable to expand potential directory path: %w", err)
}
fileMeta, err := fs.Stat(dirLocation)
if err != nil {
return UnknownScheme, "", nil
}
if fileMeta.IsDir() {
return DirectoryScheme, dirLocation, nil
}
return UnknownScheme, "", nil
}
return ImageScheme, imageSpec, nil
}

36
syft/source/scope.go Normal file
View File

@ -0,0 +1,36 @@
package source
import "strings"
// Scope indicates "how" or from "which perspectives" the source object should be cataloged from.
type Scope string
const (
// UnknownScope is the default scope
UnknownScope Scope = "UnknownScope"
// SquashedScope indicates to only catalog content visible from the squashed filesystem representation (what can be seen only within the container at runtime)
SquashedScope Scope = "Squashed"
// AllLayersScope indicates to catalog content on all layers, irregardless if it is visible from the container at runtime.
AllLayersScope Scope = "AllLayers"
)
// AllScopes is a slice containing all possible scope options
var AllScopes = []Scope{
SquashedScope,
AllLayersScope,
}
// ParseScope returns a scope as indicated from the given string.
func ParseScope(userStr string) Scope {
switch strings.ToLower(userStr) {
case strings.ToLower(SquashedScope.String()):
return SquashedScope
case "all-layers", strings.ToLower(AllLayersScope.String()):
return AllLayersScope
}
return UnknownScope
}
func (o Scope) String() string {
return string(o)
}

106
syft/source/source.go Normal file
View File

@ -0,0 +1,106 @@
/*
Package source provides an abstraction to allow a user to loosely define a data source to catalog and expose a common interface that
catalogers and use explore and analyze data from the data source. All valid (cataloggable) data sources are defined
within this package.
*/
package source
import (
"fmt"
"github.com/spf13/afero"
"github.com/anchore/stereoscope"
"github.com/anchore/stereoscope/pkg/image"
)
// Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used
// in cataloging (based on the data source and configuration)
type Source struct {
Resolver Resolver // a Resolver object to use in file path/glob resolution and file contents resolution
Image *image.Image // the image object to be cataloged (image only)
Metadata Metadata
}
type sourceDetector func(string) (image.Source, string, error)
// New produces a Source based on userInput like dir: or image:tag
func New(userInput string, o Scope) (Source, func(), error) {
fs := afero.NewOsFs()
parsedScheme, location, err := detectScheme(fs, image.DetectSource, userInput)
if err != nil {
return Source{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err)
}
switch parsedScheme {
case DirectoryScheme:
fileMeta, err := fs.Stat(location)
if err != nil {
return Source{}, func() {}, fmt.Errorf("unable to stat dir=%q: %w", location, err)
}
if !fileMeta.IsDir() {
return Source{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err)
}
s, err := NewFromDirectory(location)
if err != nil {
return Source{}, func() {}, fmt.Errorf("could not populate source from path=%q: %w", location, err)
}
return s, func() {}, nil
case ImageScheme:
img, err := stereoscope.GetImage(location)
cleanup := func() {
stereoscope.Cleanup()
}
if err != nil || img == nil {
return Source{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err)
}
s, err := NewFromImage(img, o, location)
if err != nil {
return Source{}, cleanup, fmt.Errorf("could not populate source with image: %w", err)
}
return s, cleanup, nil
}
return Source{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
}
// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively.
func NewFromDirectory(path string) (Source, error) {
return Source{
Resolver: &DirectoryResolver{
Path: path,
},
Metadata: Metadata{
Scheme: DirectoryScheme,
Path: path,
},
}, nil
}
// NewFromImage creates a new source object tailored to catalog a given container image, relative to the
// option given (e.g. all-layers, squashed, etc)
func NewFromImage(img *image.Image, scope Scope, userImageStr string) (Source, error) {
if img == nil {
return Source{}, fmt.Errorf("no image given")
}
resolver, err := getImageResolver(img, scope)
if err != nil {
return Source{}, fmt.Errorf("could not determine file resolver: %w", err)
}
return Source{
Resolver: resolver,
Image: img,
Metadata: Metadata{
Scheme: ImageScheme,
ImageMetadata: NewImageMetadata(img, userImageStr, scope),
},
}, nil
}

73
syft/scope/scope_test.go → syft/source/source_test.go
View File

@ -1,96 +1,95 @@
package scope
package source
import (
"os"
"testing"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/image"
"github.com/mitchellh/go-homedir"
"github.com/spf13/afero"
)
func TestNewScopeFromImageFails(t *testing.T) {
func TestNewFromImageFails(t *testing.T) {
t.Run("no image given", func(t *testing.T) {
_, err := NewScopeFromImage(nil, AllLayersScope)
_, err := NewFromImage(nil, AllLayersScope, "")
if err == nil {
t.Errorf("expected an error condition but none was given")
}
})
}
func TestNewScopeFromImageUnknownOption(t *testing.T) {
func TestNewFromImageUnknownOption(t *testing.T) {
img := image.Image{}
t.Run("unknown option is an error", func(t *testing.T) {
_, err := NewScopeFromImage(&img, UnknownScope)
_, err := NewFromImage(&img, UnknownScope, "")
if err == nil {
t.Errorf("expected an error condition but none was given")
}
})
}
func TestNewScopeFromImage(t *testing.T) {
func TestNewFromImage(t *testing.T) {
layer := image.NewLayer(nil)
img := image.Image{
Layers: []*image.Layer{layer},
}
t.Run("create a new Scope object from image", func(t *testing.T) {
_, err := NewScopeFromImage(&img, AllLayersScope)
t.Run("create a new source object from image", func(t *testing.T) {
_, err := NewFromImage(&img, AllLayersScope, "")
if err != nil {
t.Errorf("unexpected error when creating a new Scope from img: %w", err)
t.Errorf("unexpected error when creating a new Locations from img: %+v", err)
}
})
}
func TestDirectoryScope(t *testing.T) {
func TestNewFromDirectory(t *testing.T) {
testCases := []struct {
desc string
input string
expString string
inputPaths []file.Path
inputPaths []string
expRefs int
}{
{
desc: "no paths exist",
input: "foobar/",
inputPaths: []file.Path{file.Path("/opt/"), file.Path("/other")},
inputPaths: []string{"/opt/", "/other"},
expRefs: 0,
},
{
desc: "path detected",
input: "test-fixtures",
inputPaths: []file.Path{file.Path("test-fixtures/path-detected/.vimrc")},
inputPaths: []string{"test-fixtures/path-detected/.vimrc"},
expRefs: 1,
},
{
desc: "directory ignored",
input: "test-fixtures",
inputPaths: []file.Path{file.Path("test-fixtures/path-detected")},
inputPaths: []string{"test-fixtures/path-detected"},
expRefs: 0,
},
{
desc: "no files-by-path detected",
input: "test-fixtures",
inputPaths: []file.Path{file.Path("test-fixtures/no-path-detected")},
inputPaths: []string{"test-fixtures/no-path-detected"},
expRefs: 0,
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
p, err := NewScopeFromDir(test.input)
src, err := NewFromDirectory(test.input)
if err != nil {
t.Errorf("could not create NewDirScope: %w", err)
t.Errorf("could not create NewDirScope: %+v", err)
}
if p.Source.(DirSource).Path != test.input {
t.Errorf("mismatched stringer: '%s' != '%s'", p.Source.(DirSource).Path, test.input)
if src.Metadata.Path != test.input {
t.Errorf("mismatched stringer: '%s' != '%s'", src.Metadata.Path, test.input)
}
refs, err := p.Resolver.FilesByPath(test.inputPaths...)
refs, err := src.Resolver.FilesByPath(test.inputPaths...)
if err != nil {
t.Errorf("FilesByPath call produced an error: %w", err)
t.Errorf("FilesByPath call produced an error: %+v", err)
}
if len(refs) != test.expRefs {
t.Errorf("unexpected number of refs returned: %d != %d", len(refs), test.expRefs)
@ -101,7 +100,7 @@ func TestDirectoryScope(t *testing.T) {
}
}
func TestMultipleFileContentsByRefContents(t *testing.T) {
func TestMultipleFileContentsByLocation(t *testing.T) {
testCases := []struct {
desc string
input string
@ -123,22 +122,22 @@ func TestMultipleFileContentsByRefContents(t *testing.T) {
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
p, err := NewScopeFromDir(test.input)
p, err := NewFromDirectory(test.input)
if err != nil {
t.Errorf("could not create NewDirScope: %w", err)
t.Errorf("could not create NewDirScope: %+v", err)
}
refs, err := p.Resolver.FilesByPath(file.Path(test.path))
locations, err := p.Resolver.FilesByPath(test.path)
if err != nil {
t.Errorf("could not get file references from path: %s, %v", test.path, err)
}
if len(refs) != 1 {
t.Fatalf("expected a single ref to be generated but got: %d", len(refs))
if len(locations) != 1 {
t.Fatalf("expected a single location to be generated but got: %d", len(locations))
}
ref := refs[0]
location := locations[0]
contents, err := p.Resolver.MultipleFileContentsByRef(ref)
content := contents[ref]
contents, err := p.Resolver.MultipleFileContentsByLocation([]Location{location})
content := contents[location]
if content != test.expected {
t.Errorf("unexpected contents from file: '%s' != '%s'", content, test.expected)
@ -148,7 +147,7 @@ func TestMultipleFileContentsByRefContents(t *testing.T) {
}
}
func TestMultipleFileContentsByRefNoContents(t *testing.T) {
func TestFilesByPathDoesNotExist(t *testing.T) {
testCases := []struct {
desc string
input string
@ -163,11 +162,11 @@ func TestMultipleFileContentsByRefNoContents(t *testing.T) {
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
p, err := NewScopeFromDir(test.input)
p, err := NewFromDirectory(test.input)
if err != nil {
t.Errorf("could not create NewDirScope: %w", err)
t.Errorf("could not create NewDirScope: %+v", err)
}
refs, err := p.Resolver.FilesByPath(file.Path(test.path))
refs, err := p.Resolver.FilesByPath(test.path)
if err != nil {
t.Errorf("could not get file references from path: %s, %v", test.path, err)
}
@ -208,9 +207,9 @@ func TestFilesByGlob(t *testing.T) {
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
p, err := NewScopeFromDir(test.input)
p, err := NewFromDirectory(test.input)
if err != nil {
t.Errorf("could not create NewDirScope: %w", err)
t.Errorf("could not create NewDirScope: %+v", err)
}
contents, err := p.Resolver.FilesByGlob(test.glob)

0
syft/scope/resolvers/test-fixtures/image-symlinks/Dockerfile → syft/source/test-fixtures/image-symlinks/Dockerfile
View File

0
syft/scope/resolvers/test-fixtures/image-symlinks/file-1.txt → syft/source/test-fixtures/image-symlinks/file-1.txt
View File

0
syft/scope/resolvers/test-fixtures/image-symlinks/file-2.txt → syft/source/test-fixtures/image-symlinks/file-2.txt
View File

0
syft/scope/resolvers/test-fixtures/image-symlinks/nested/nested/file-3.txt → syft/source/test-fixtures/image-symlinks/nested/nested/file-3.txt
View File

0
syft/scope/resolvers/test-fixtures/image-symlinks/new-file-2.txt → syft/source/test-fixtures/image-symlinks/new-file-2.txt
View File

Some files were not shown because too many files have changed in this diff Show More